feat: add LAYER_CACHE_DIR for quick layer iteration (#22)

README.md

@@ -115,6 +115,15 @@ Cache layers in a container registry to speed up builds. To enable caching, [aut
 CACHE_REPO=ghcr.io/coder/repo-cache
 ```
 
+To experiment without setting up a registry, use `LAYER_CACHE_DIR`:
+
+```bash
+docker run -it --rm \
+ -v /tmp/envbuilder-cache:/cache \
+ -e LAYER_CACHE_DIR=/cache
+ ...
+```
+
 Each layer is stored in the registry as a separate image. The image tag is the hash of the layer's contents. The image digest is the hash of the image tag. The image digest is used to pull the layer from the registry.
 
 The performance improvement of builds depends on the complexity of your Dockerfile. For [`coder/coder`](https://github.com/coder/coder/blob/main/.devcontainer/Dockerfile), uncached builds take 36m while cached builds take 40s (~98% improvement).

envbuilder.go

@@ -10,6 +10,8 @@ import (
  "errors"
  "fmt"
  "io"
+ "net"
+ "net/http"
  "net/url"
  "os"
  "os/exec"
@@ -21,11 +23,16 @@ import (
  "syscall"
  "time"
 
+ dcontext "github.com/distribution/distribution/v3/context"
+
  "github.com/GoogleContainerTools/kaniko/pkg/config"
  "github.com/GoogleContainerTools/kaniko/pkg/executor"
  "github.com/coder/coder/codersdk"
  "github.com/coder/envbuilder/devcontainer"
  "github.com/containerd/containerd/platforms"
+ "github.com/distribution/distribution/v3/configuration"
+ "github.com/distribution/distribution/v3/registry/handlers"
+ _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem"
  "github.com/docker/cli/cli/config/configfile"
  "github.com/fatih/color"
  "github.com/go-git/go-billy/v5"
@@ -65,10 +72,17 @@ type Options struct {
  // will not be pushed.
  CacheRepo string `env:"CACHE_REPO"`
 
- // CacheDir is the path to the directory where the cache
- // will be stored. If this is empty, the cache will not
- // be used.
- CacheDir string `env:"CACHE_DIR"`
+ // BaseImageCacheDir is the path to a directory where the base
+ // image can be found. This should be a read-only directory
+ // solely mounted for the purpose of caching the base image.
+ BaseImageCacheDir string `env:"BASE_IMAGECACHE_DIR"`
+
+ // LayerCacheDir is the path to a directory where built layers
+ // will be stored. This spawns an in-memory registry to serve
+ // the layers from.
+ //
+ // It will override CacheRepo if both are specified.
+ LayerCacheDir string `env:"LAYER_CACHE_DIR"`
 
  // DockerfilePath is a relative path to the workspace
  // folder that will be used to build the workspace.
@@ -345,6 +359,54 @@ func Run(ctx context.Context, options Options) error {
  }
  })
 
+ var closeAfterBuild func()
+ // Allows quick testing of layer caching using a local directory!
+ if options.LayerCacheDir != "" {
+ cfg := &configuration.Configuration{
+ Storage: configuration.Storage{
+ "filesystem": configuration.Parameters{
+ "rootdirectory": options.LayerCacheDir,
+ },
+ },
+ }
+
+ // Disable all logging from the registry...
+ logger := logrus.New()
+ logger.SetOutput(io.Discard)
+ entry := logrus.NewEntry(logger)
+ dcontext.SetDefaultLogger(entry)
+ ctx = dcontext.WithLogger(ctx, entry)
+
+ // Spawn an in-memory registry to cache built layers...
+ registry := handlers.NewApp(ctx, cfg)
+
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ return err
+ }
+ tcpAddr, ok := listener.Addr().(*net.TCPAddr)
+ if !ok {
+ return fmt.Errorf("listener addr was of wrong type: %T", listener.Addr())
+ }
+ srv := &http.Server{
+ Handler: registry,
+ }
+ go func() {
+ err := srv.Serve(listener)
+ if err != nil && !errors.Is(err, http.ErrServerClosed) {
+ logf(codersdk.LogLevelError, "Failed to serve registry: %s", err.Error())
+ }
+ }()
+ closeAfterBuild = func() {
+ _ = srv.Close()
+ _ = listener.Close()
+ }
+ if options.CacheRepo != "" {
+ logf(codersdk.LogLevelWarn, "Overriding cache repo with local registry...")
+ }
+ options.CacheRepo = fmt.Sprintf("localhost:%d/local/cache", tcpAddr.Port)
+ }
+
  build := func() (v1.Image, error) {
  endStage := startStage("🏗️ Building image...")
  // At this point we have all the context, we can now build!
@@ -355,12 +417,13 @@ func Run(ctx context.Context, options Options) error {
  RunV2: true,
  Destinations: []string{"local"},
  CacheRunLayers: true,
+ IgnorePaths: []string{options.LayerCacheDir, options.WorkspaceFolder, MagicDir},
  CacheCopyLayers: true,
  CompressedCaching: true,
  CacheOptions: config.CacheOptions{
  // Cache for a week by default!
  CacheTTL: time.Hour * 24 * 7,
- CacheDir: options.CacheDir,
+ CacheDir: options.BaseImageCacheDir,
  },
  ForceUnpack: true,
  BuildArgs: buildParams.BuildArgs,
@@ -415,6 +478,10 @@ func Run(ctx context.Context, options Options) error {
  return fmt.Errorf("build with kaniko: %w", err)
  }
 
+ if closeAfterBuild != nil {
+ closeAfterBuild()
+ }
+
  configFile, err := image.ConfigFile()
  if err != nil {
  return fmt.Errorf("get image config: %w", err)

go.mod

@@ -15,6 +15,7 @@ require (
  github.com/breml/rootcerts v0.2.10
  github.com/coder/coder v0.24.2-0.20230630184129-6015319e9d30
  github.com/containerd/containerd v1.7.0
+ github.com/distribution/distribution/v3 v3.0.0-20230629214736-bac7f02e02a1
  github.com/docker/cli v23.0.1+incompatible
  github.com/docker/docker v23.0.3+incompatible
  github.com/fatih/color v1.15.0
@@ -23,7 +24,7 @@ require (
  github.com/google/go-containerregistry v0.14.0
  github.com/mattn/go-isatty v0.0.19
  github.com/sirupsen/logrus v1.9.2
- github.com/spf13/cobra v1.6.1
+ github.com/spf13/cobra v1.7.0
  github.com/stretchr/testify v1.8.4
  golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
  muzzammil.xyz/jsonc v1.0.0
@@ -93,8 +94,10 @@ require (
  github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
  github.com/docker/go-metrics v0.0.1 // indirect
  github.com/docker/go-units v0.5.0 // indirect
+ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
  github.com/ePirat/docker-credential-gitlabci v1.0.0 // indirect
  github.com/emirpasic/gods v1.18.1 // indirect
+ github.com/felixge/httpsnoop v1.0.3 // indirect
  github.com/fxamacker/cbor/v2 v2.4.0 // indirect
  github.com/ghodss/yaml v1.0.0 // indirect
  github.com/go-chi/chi/v5 v5.0.8 // indirect
@@ -111,9 +114,12 @@ require (
  github.com/golang/glog v1.1.0 // indirect
  github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
  github.com/golang/protobuf v1.5.3 // indirect
+ github.com/gomodule/redigo v1.8.2 // indirect
  github.com/google/btree v1.1.2 // indirect
  github.com/google/go-cmp v0.5.9 // indirect
  github.com/google/uuid v1.3.0 // indirect
+ github.com/gorilla/handlers v1.5.1 // indirect
+ github.com/gorilla/mux v1.8.0 // indirect
  github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.1 // indirect
  github.com/hashicorp/errwrap v1.1.0 // indirect
  github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect
@@ -143,7 +149,7 @@ require (
  github.com/jsimonetti/rtnetlink v1.1.2-0.20220408201609-d380b505068b // indirect
  github.com/karrick/godirwalk v1.16.1 // indirect
  github.com/kevinburke/ssh_config v1.2.0 // indirect
- github.com/klauspost/compress v1.16.3 // indirect
+ github.com/klauspost/compress v1.16.5 // indirect
  github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a // indirect
  github.com/lib/pq v1.10.6 // indirect
  github.com/mattn/go-colorable v0.1.13 // indirect
@@ -173,7 +179,7 @@ require (
  github.com/morikuni/aec v1.0.0 // indirect
  github.com/open-policy-agent/opa v0.51.0 // indirect
  github.com/opencontainers/go-digest v1.0.0 // indirect
- github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
+ github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
  github.com/opencontainers/runc v1.1.5 // indirect
  github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
  github.com/opencontainers/selinux v1.11.0 // indirect
@@ -202,7 +208,7 @@ require (
  github.com/tonistiigi/fsutil v0.0.0-20230105215944-fb433841cbfa // indirect
  github.com/u-root/uio v0.0.0-20221213070652-c3537552635f // indirect
  github.com/valyala/fasthttp v1.48.0 // indirect
- github.com/vbatts/tar-split v0.11.2 // indirect
+ github.com/vbatts/tar-split v0.11.3 // indirect
  github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
  github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
  github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect