From 61b5cb3e2233471884507b2ad8a2e48d44446ebc Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 5 Jul 2023 14:35:31 -0500 Subject: [PATCH] feat: add `LAYER_CACHE_DIR` for quick layer iteration (#22) --- README.md | 9 +++++ envbuilder.go | 77 +++++++++++++++++++++++++++++++++--- go.mod | 14 +++++-- go.sum | 35 +++++++++++----- registrytest/registrytest.go | 28 +++++++++++-- 5 files changed, 141 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1ed3a81c..c3929f28 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,15 @@ Cache layers in a container registry to speed up builds. To enable caching, [aut CACHE_REPO=ghcr.io/coder/repo-cache ``` +To experiment without setting up a registry, use `LAYER_CACHE_DIR`: + +```bash +docker run -it --rm \ + -v /tmp/envbuilder-cache:/cache \ + -e LAYER_CACHE_DIR=/cache + ... +``` + Each layer is stored in the registry as a separate image. The image tag is the hash of the layer's contents. The image digest is the hash of the image tag. The image digest is used to pull the layer from the registry. The performance improvement of builds depends on the complexity of your Dockerfile. For [`coder/coder`](https://github.com/coder/coder/blob/main/.devcontainer/Dockerfile), uncached builds take 36m while cached builds take 40s (~98% improvement). diff --git a/envbuilder.go b/envbuilder.go index 2b792b24..b790e11b 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -10,6 +10,8 @@ import ( "errors" "fmt" "io" + "net" + "net/http" "net/url" "os" "os/exec" @@ -21,11 +23,16 @@ import ( "syscall" "time" + dcontext "github.com/distribution/distribution/v3/context" + "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/executor" "github.com/coder/coder/codersdk" "github.com/coder/envbuilder/devcontainer" "github.com/containerd/containerd/platforms" + "github.com/distribution/distribution/v3/configuration" + "github.com/distribution/distribution/v3/registry/handlers" + _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" "github.com/docker/cli/cli/config/configfile" "github.com/fatih/color" "github.com/go-git/go-billy/v5" @@ -65,10 +72,17 @@ type Options struct { // will not be pushed. CacheRepo string `env:"CACHE_REPO"` - // CacheDir is the path to the directory where the cache - // will be stored. If this is empty, the cache will not - // be used. - CacheDir string `env:"CACHE_DIR"` + // BaseImageCacheDir is the path to a directory where the base + // image can be found. This should be a read-only directory + // solely mounted for the purpose of caching the base image. + BaseImageCacheDir string `env:"BASE_IMAGECACHE_DIR"` + + // LayerCacheDir is the path to a directory where built layers + // will be stored. This spawns an in-memory registry to serve + // the layers from. + // + // It will override CacheRepo if both are specified. + LayerCacheDir string `env:"LAYER_CACHE_DIR"` // DockerfilePath is a relative path to the workspace // folder that will be used to build the workspace. @@ -345,6 +359,54 @@ func Run(ctx context.Context, options Options) error { } }) + var closeAfterBuild func() + // Allows quick testing of layer caching using a local directory! + if options.LayerCacheDir != "" { + cfg := &configuration.Configuration{ + Storage: configuration.Storage{ + "filesystem": configuration.Parameters{ + "rootdirectory": options.LayerCacheDir, + }, + }, + } + + // Disable all logging from the registry... + logger := logrus.New() + logger.SetOutput(io.Discard) + entry := logrus.NewEntry(logger) + dcontext.SetDefaultLogger(entry) + ctx = dcontext.WithLogger(ctx, entry) + + // Spawn an in-memory registry to cache built layers... + registry := handlers.NewApp(ctx, cfg) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return err + } + tcpAddr, ok := listener.Addr().(*net.TCPAddr) + if !ok { + return fmt.Errorf("listener addr was of wrong type: %T", listener.Addr()) + } + srv := &http.Server{ + Handler: registry, + } + go func() { + err := srv.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logf(codersdk.LogLevelError, "Failed to serve registry: %s", err.Error()) + } + }() + closeAfterBuild = func() { + _ = srv.Close() + _ = listener.Close() + } + if options.CacheRepo != "" { + logf(codersdk.LogLevelWarn, "Overriding cache repo with local registry...") + } + options.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port) + } + build := func() (v1.Image, error) { endStage := startStage("🏗️ Building image...") // At this point we have all the context, we can now build! @@ -355,12 +417,13 @@ func Run(ctx context.Context, options Options) error { RunV2: true, Destinations: []string{"local"}, CacheRunLayers: true, + IgnorePaths: []string{options.LayerCacheDir, options.WorkspaceFolder, MagicDir}, CacheCopyLayers: true, CompressedCaching: true, CacheOptions: config.CacheOptions{ // Cache for a week by default! CacheTTL: time.Hour * 24 * 7, - CacheDir: options.CacheDir, + CacheDir: options.BaseImageCacheDir, }, ForceUnpack: true, BuildArgs: buildParams.BuildArgs, @@ -415,6 +478,10 @@ func Run(ctx context.Context, options Options) error { return fmt.Errorf("build with kaniko: %w", err) } + if closeAfterBuild != nil { + closeAfterBuild() + } + configFile, err := image.ConfigFile() if err != nil { return fmt.Errorf("get image config: %w", err) diff --git a/go.mod b/go.mod index dd8ca9e8..ea2c5f7f 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/breml/rootcerts v0.2.10 github.com/coder/coder v0.24.2-0.20230630184129-6015319e9d30 github.com/containerd/containerd v1.7.0 + github.com/distribution/distribution/v3 v3.0.0-20230629214736-bac7f02e02a1 github.com/docker/cli v23.0.1+incompatible github.com/docker/docker v23.0.3+incompatible github.com/fatih/color v1.15.0 @@ -23,7 +24,7 @@ require ( github.com/google/go-containerregistry v0.14.0 github.com/mattn/go-isatty v0.0.19 github.com/sirupsen/logrus v1.9.2 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 muzzammil.xyz/jsonc v1.0.0 @@ -93,8 +94,10 @@ require ( github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect github.com/ePirat/docker-credential-gitlabci v1.0.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-chi/chi/v5 v5.0.8 // indirect @@ -111,9 +114,12 @@ require ( github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/gomodule/redigo v1.8.2 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/handlers v1.5.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect @@ -143,7 +149,7 @@ require ( github.com/jsimonetti/rtnetlink v1.1.2-0.20220408201609-d380b505068b // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.16.3 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a // indirect github.com/lib/pq v1.10.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -173,7 +179,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/open-policy-agent/opa v0.51.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect github.com/opencontainers/selinux v1.11.0 // indirect @@ -202,7 +208,7 @@ require ( github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa // indirect github.com/u-root/uio v0.0.0-20221213070652-c3537552635f // indirect github.com/valyala/fasthttp v1.48.0 // indirect - github.com/vbatts/tar-split v0.11.2 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect github.com/vishvananda/netlink v1.2.1-beta.2 // indirect github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect diff --git a/go.sum b/go.sum index 28cbf956..db640a2a 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,7 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -269,6 +270,8 @@ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+ github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/distribution/distribution/v3 v3.0.0-20230629214736-bac7f02e02a1 h1:yRwt9RluqBtKyDLRY7J0Cf/TVqvG56vKx2Eyndy8qNQ= +github.com/distribution/distribution/v3 v3.0.0-20230629214736-bac7f02e02a1/go.mod h1:+fqBJ4vPYo4Uu1ZE4d+bUtTLRXfdSL3NvCZIZ9GHv58= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/docker/cli v23.0.1+incompatible h1:LRyWITpGzl2C9e9uGxzisptnxAn1zfZKXy13Ul2Q5oM= github.com/docker/cli v23.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -288,6 +291,8 @@ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHz github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/ePirat/docker-credential-gitlabci v1.0.0 h1:YRkUSvkON6rT88vtscClAmPEYWhtltGEAuRVYtz1/+Y= github.com/ePirat/docker-credential-gitlabci v1.0.0/go.mod h1:Ptmh+D0lzBQtgb6+QHjXl9HqOn3T1P8fKUHldiSQQGA= @@ -307,6 +312,9 @@ github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:Pjfxu github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -434,6 +442,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= @@ -486,6 +496,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -547,7 +561,6 @@ github.com/illarion/gonotify v1.0.1 h1:F1d+0Fgbq/sDWjj/r66ekjDG+IDeecQKUFH4wNwso github.com/illarion/gonotify v1.0.1/go.mod h1:zt5pmDofZpU1f8aqlK0+95eQhoEAn/d4G4B/FjVW4jE= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/insomniacslk/dhcp v0.0.0-20221215072855-de60144f33f8 h1:Z72DOke2yOK0Ms4Z2LK1E1OrRJXOxSj5DllTz2FYTRg= @@ -590,8 +603,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= -github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -713,8 +726,8 @@ github.com/open-policy-agent/opa v0.51.0 h1:2hS5xhos8HtkN+mgpqMhNJSFtn/1n/h3wh+A github.com/open-policy-agent/opa v0.51.0/go.mod h1:OjmwLfXdeR7skSxrt8Yd3ScXTqPxyJn7GeTRJrcEerU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -783,6 +796,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0= @@ -793,8 +807,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -849,12 +863,12 @@ github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFsk= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= -github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= -github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -1155,6 +1169,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/registrytest/registrytest.go b/registrytest/registrytest.go index e835b324..4c7e1e13 100644 --- a/registrytest/registrytest.go +++ b/registrytest/registrytest.go @@ -3,35 +3,57 @@ package registrytest import ( "archive/tar" "bytes" + "context" "crypto" "encoding/hex" "encoding/json" "io" - "log" "net/http/httptest" "net/url" "strings" "testing" "time" + "github.com/distribution/distribution/v3/configuration" + "github.com/distribution/distribution/v3/registry/handlers" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" + + // needed by the registry + _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" ) // New creates a new registry server that discards all logs. func New(t *testing.T) string { - srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) + cfg := &configuration.Configuration{ + Storage: configuration.Storage{ + "inmemory": configuration.Parameters{}, + }, + } + logrus.SetOutput(io.Discard) + app := handlers.NewApp(context.Background(), cfg) + srv := httptest.NewServer(app) t.Cleanup(srv.Close) return srv.URL } +type logrusFormatter struct { + callback func(entry *logrus.Entry) + empty []byte +} + +func (f *logrusFormatter) Format(entry *logrus.Entry) ([]byte, error) { + f.callback(entry) + return f.empty, nil +} + // WriteContainer uploads a container to the registry server. // It returns the reference to the uploaded container. func WriteContainer(t *testing.T, serverURL, containerRef, mediaType string, files map[string]any) string {