From 87f028a368c4f03f09160f2c41c373ee62f2ea6a Mon Sep 17 00:00:00 2001 From: pablochacin Date: Wed, 31 Jul 2024 14:20:11 +0200 Subject: [PATCH] add cache server command (#20) * add cache server command --------- Signed-off-by: Pablo Chacin --- README.md | 63 +++++++++++++++++++++++++- cmd/cache.go | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/cmd.go | 1 + cmd/server.go | 40 ++++++++++------- 4 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 cmd/cache.go diff --git a/README.md b/README.md index 0316283..5f7308d 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,70 @@ Build k6 using one of the supported builders. ## Commands +* [k6build cache](#k6build-cache) - k6 cache server * [k6build client](#k6build-client) - build k6 using a remote build server * [k6build local](#k6build-local) - build using a local builder * [k6build server](#k6build-server) - k6 build service +--- +# k6build cache + +k6 cache server + +## Synopsis + + +Starts a k6build cache server. + +The cache server offers a REST API for storing and downloading objects. + +Objects can be retrieved by a download url returned when the object is stored. + +The --download-url specifies the base URL for downloading objects. This is necessary to allow +downloading the objects from different machines. + + +``` +k6build cache [flags] +``` + +## Examples + +``` + +# start the cache server serving an external url +k6build cache --download0url http://external.url + +# store object from same host +curl -x POST http://localhost:9000/cache/objectID -d "object content" | jq . +{ + "Error": "", + "Object": { + "ID": "objectID", + "Checksum": "17d3eb873fe4b1aac4f9d2505aefbb5b53b9a7f34a6aadd561be104c0e9d678b", + "URL": "http://external.url:9000/cache/objectID/download" + } + } + +# download object from another machine using the external url +curl http://external.url:9000/cache/objectID/download + +``` + +## Flags + +``` + -c, --cache-dir string cache directory (default "/tmp/cache/objectstore") + -d, --download-url string base url used for downloading objects. If not specified http://localhost: is used + -h, --help help for cache + -l, --log-level string log level (default "INFO") + -p, --port int port server will listen (default 9000) +``` + +## SEE ALSO + +* [k6build](#k6build) - Build k6 with various builders. + --- # k6build client @@ -161,7 +221,7 @@ k6 build service ## Synopsis -starts a k6build server that server +starts a k6build server ``` @@ -183,6 +243,7 @@ k6build server -e GOPROXY=http://localhost:80 ``` -f, --cache-dir string cache dir (default "/tmp/buildservice") + --cache-url string cache server url. If not specified, a local cache server is started -c, --catalog string dependencies catalog (default "catalog.json") -g, --copy-go-env copy go environment (default true) -e, --env stringToString build environment variables (default []) diff --git a/cmd/cache.go b/cmd/cache.go new file mode 100644 index 0000000..c414b8d --- /dev/null +++ b/cmd/cache.go @@ -0,0 +1,121 @@ +package cmd + +import ( + "fmt" + "log/slog" + "net/http" + "os" + + "github.com/grafana/k6build" + "github.com/spf13/cobra" +) + +const ( + cacheLong = ` +Starts a k6build cache server. + +The cache server offers a REST API for storing and downloading objects. + +Objects can be retrieved by a download url returned when the object is stored. + +The --download-url specifies the base URL for downloading objects. This is necessary to allow +downloading the objects from different machines. +` + + cacheExample = ` +# start the cache server serving an external url +k6build cache --download0url http://external.url + +# store object from same host +curl -x POST http://localhost:9000/cache/objectID -d "object content" | jq . +{ + "Error": "", + "Object": { + "ID": "objectID", + "Checksum": "17d3eb873fe4b1aac4f9d2505aefbb5b53b9a7f34a6aadd561be104c0e9d678b", + "URL": "http://external.url:9000/cache/objectID/download" + } + } + +# download object from another machine using the external url +curl http://external.url:9000/cache/objectID/download +` +) + +// NewCache creates new cobra command for cache command. +func NewCache() *cobra.Command { + var ( + cacheDir string + cacheSrvURL string + port int + logLevel string + ) + + cmd := &cobra.Command{ + Use: "cache", + Short: "k6 cache server", + Long: cacheLong, + Example: cacheExample, + // prevent the usage help to printed to stderr when an error is reported by a subcommand + SilenceUsage: true, + // this is needed to prevent cobra to print errors reported by subcommands in the stderr + SilenceErrors: true, + RunE: func(_ *cobra.Command, _ []string) error { + // set log + ll, err := k6build.ParseLogLevel(logLevel) + if err != nil { + return fmt.Errorf("parsing log level %w", err) + } + + log := slog.New( + slog.NewTextHandler( + os.Stderr, + &slog.HandlerOptions{ + Level: ll, + }, + ), + ) + + cache, err := k6build.NewFileCache(cacheDir) + if err != nil { + return fmt.Errorf("creating cache %w", err) + } + + // FIXME: this will not work across machines + if cacheSrvURL == "" { + cacheSrvURL = fmt.Sprintf("http://localhost:%d/cache", port) + } + config := k6build.CacheServerConfig{ + BaseURL: cacheSrvURL, + Cache: cache, + Log: log, + } + cacheSrv := k6build.NewCacheServer(config) + + srv := http.NewServeMux() + srv.Handle("/cache/", http.StripPrefix("/cache", cacheSrv)) + + listerAddr := fmt.Sprintf("localhost:%d", port) + log.Info("starting server", "address", listerAddr) + err = http.ListenAndServe(listerAddr, srv) //nolint:gosec + if err != nil { + log.Info("server ended", "error", err.Error()) + } + log.Info("ending server") + + return nil + }, + } + + cmd.Flags().StringVarP(&cacheDir, "cache-dir", "c", "/tmp/cache/objectstore", "cache directory") + cmd.Flags().IntVarP(&port, "port", "p", 9000, "port server will listen") + cmd.Flags().StringVarP(&cacheSrvURL, + "download-url", + "d", + "", + "base url used for downloading objects. If not specified http://localhost: is used", + ) + cmd.Flags().StringVarP(&logLevel, "log-level", "l", "INFO", "log level") + + return cmd +} diff --git a/cmd/cmd.go b/cmd/cmd.go index 8de794a..891a87d 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -17,6 +17,7 @@ func New() *cobra.Command { root.AddCommand(NewLocal()) root.AddCommand(NewServer()) + root.AddCommand(NewCache()) root.AddCommand(NewClient()) return root diff --git a/cmd/server.go b/cmd/server.go index c516e40..66e1773 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -28,13 +28,14 @@ k6build server -e GOPROXY=http://localhost:80` // NewServer creates new cobra command for server command. func NewServer() *cobra.Command { //nolint:funlen var ( - buildEnv map[string]string - cacheDir string - catalog string - copyGoEnv bool - port int - verbose bool - logLevel string + buildEnv map[string]string + cacheDir string + cacheSrvURL string + catalog string + copyGoEnv bool + port int + verbose bool + logLevel string ) cmd := &cobra.Command{ @@ -89,14 +90,19 @@ func NewServer() *cobra.Command { //nolint:funlen return fmt.Errorf("creating cache %w", err) } + srv := http.NewServeMux() + // FIXME: this will not work across machines - cacheSrvURL := fmt.Sprintf("http://localhost:%d/cache", port) - config := k6build.CacheServerConfig{ - BaseURL: cacheSrvURL, - Cache: cache, - Log: log, + if cacheSrvURL == "" { + cacheSrvURL = fmt.Sprintf("http://localhost:%d/cache", port) + config := k6build.CacheServerConfig{ + BaseURL: cacheSrvURL, + Cache: cache, + Log: log, + } + cacheSrv := k6build.NewCacheServer(config) + srv.Handle("/cache/", http.StripPrefix("/cache", cacheSrv)) } - cacheSrv := k6build.NewCacheServer(config) cacheClientConfig := k6build.CacheClientConfig{ Server: cacheSrvURL, @@ -118,9 +124,7 @@ func NewServer() *cobra.Command { //nolint:funlen } buildAPI := k6build.NewAPIServer(apiConfig) - srv := http.NewServeMux() srv.Handle("POST /build/", http.StripPrefix("/build", buildAPI)) - srv.Handle("/cache/", http.StripPrefix("/cache", cacheSrv)) listerAddr := fmt.Sprintf("localhost:%d", port) log.Info("starting server", "address", listerAddr) @@ -135,6 +139,12 @@ func NewServer() *cobra.Command { //nolint:funlen } cmd.Flags().StringVarP(&catalog, "catalog", "c", "catalog.json", "dependencies catalog") + cmd.Flags().StringVar( + &cacheSrvURL, + "cache-url", + "", + "cache server url. If not specified, a local cache server is started", + ) cmd.Flags().StringVarP(&cacheDir, "cache-dir", "f", "/tmp/buildservice", "cache dir") cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "print build process output") cmd.Flags().BoolVarP(©GoEnv, "copy-go-env", "g", true, "copy go environment")