new overlaybd build support

Signed-off-by: liulanzheng <[email protected]>

README.md

@@ -1,3 +1,56 @@
+# Buildkit for Overlaybd
+
+[Overlaybd](https://github.com/containerd/overlaybd) is a novel layering block-level image format, which is design for container, secure container and applicable to virtual machine. And it is an open-source implementation of paper [DADI: Block-Level Image Service for Agile and Elastic Application Deployment. USENIX ATC'20"](https://www.usenix.org/conference/atc20/presentation/li-huiba).
+
+## Build Overlaybd Images
+
+Before building overlaybd images, ensure that `overlaybd-tcmu` and `overlaybd-snapshotter` are active by referring to the [QUICKSTART](https://github.com/containerd/accelerated-container-image/blob/main/docs/QUICKSTART.md#install) guide.
+
+To use buildkit to build overlaybd images, you should specify `--oci-worker-snapshotter=overlaybd` and `--oci-worker-proxy-snapshotter-path=/run/overlaybd-snapshotter/overlaybd.sock` when start buildkitd:
+
+```bash
+buildkitd  --oci-worker-snapshotter=overlaybd  --oci-worker-proxy-snapshotter-path=/run/overlaybd-snapshotter/overlaybd.sock
+```
+
+After that, you can build a new overlaybd image from an existing overlaybd image. It is essential to include `--oci-mediatypes=true` and `--compression=uncompressed` while running buildctl:
+
+```bash
+buildctl build ... \
+--output type=image,name=<new image name>,push=true,oci-mediatypes=true,compression=uncompressed
+```
+
+This version is compatible with standard OCIv1 image builds. When building from an OCIv1 image, the output is an OCIv1 image, and when building from an OverlayBD image, the output is an OverlayBD image.
+
+
+## Limitation
+The FROM and the output image must be in the same registry.
+Accelerator layer is not supported yet.
+Multi-fs support is not implemented, only ext4 is supported as default fs type.
+
+
+## Performance
+
+In our test case Dockerfile, we used a 5GB OCI image (and corresponding overlaybd format), wrote some new layers of identical size, and recorded the time cost of image pull (as **pull** in the table below), building all lines in Dockerfile (as **build**), and exporting to image and pushing (as **push**).
+
+OCI:
+
+| **size per layer** | **layers** | **pull** | **build** | **push** | **total**  |
+| -------- | ---- | ---- | ----- | ---- | ---- |
+| 4GB      | 1    | 105.7| 23.5  | 219.4| 348.6|
+| 1GB      | 4    | 88.5 | 34.0  | 123.8| 246.3|
+| 256MB    | 10   | 92.1 | 20.7  | 63.6 | 176.4|
+
+Overlaybd:
+
+| **size per layer** | **layers** | **pull** | **build** | **push** | **total**  |
+| -------- | ---- | ---- | ----- | ---- | ---- |
+| 4GB      | 1    | 0.9  | 21.5  | 166.2| 188.6|
+| 1GB      | 4    | 0.9	 | 24.9	 | 72.9 | 98.7 |
+| 256MB    | 10   | 0.7  | 18.4  | 48.9 | 68.0 |
+
+
+
+
 [![asciicinema example](https://asciinema.org/a/gPEIEo1NzmDTUu2bEPsUboqmU.png)](https://asciinema.org/a/gPEIEo1NzmDTUu2bEPsUboqmU)
 
 # BuildKit <!-- omit in toc -->
@@ -401,7 +454,7 @@ BuildKit supports the following cache exporters:
 * `gha`: export to GitHub Actions cache
 
 In most case you want to use the `inline` cache exporter.
-However, note that the `inline` cache exporter only supports `min` cache mode. 
+However, note that the `inline` cache exporter only supports `min` cache mode.
 To enable `max` cache mode, push the image and the cache separately by using `registry` cache exporter.
 
 `inline` and `registry` exporters both store the cache in the registry. For importing the cache, `type=registry` is sufficient for both, as specifying the cache format is not necessary.
@@ -419,7 +472,7 @@ Note that the inline cache is not imported unless [`--import-cache type=registry
 
 Inline cache embeds cache metadata into the image config. The layers in the image will be left untouched compared to the image with no cache information.
 
-:information_source: Docker-integrated BuildKit (`DOCKER_BUILDKIT=1 docker build`) and `docker buildx`requires 
+:information_source: Docker-integrated BuildKit (`DOCKER_BUILDKIT=1 docker build`) and `docker buildx`requires
 `--build-arg BUILDKIT_INLINE_CACHE=1` to be specified to enable the `inline` cache exporter.
 However, the standalone `buildctl` does NOT require `--opt build-arg:BUILDKIT_INLINE_CACHE=1` and the build-arg is simply ignored.
 

cache/blobs.go

@@ -1,11 +1,17 @@
 package cache
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"os"
+	"path"
 	"strconv"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
+	obdcmd "github.com/containerd/accelerated-container-image/pkg/utils"
+	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/diff/walking"
 	"github.com/containerd/containerd/labels"
@@ -96,8 +102,6 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 				}
 
 				compressorFunc, finalize := comp.Type.Compress(ctx, comp)
-				mediaType := comp.Type.MediaType()
-
 				var lowerRef *immutableRef
 				switch sr.kind() {
 				case Diff:
@@ -169,6 +173,20 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 						enableOverlay = false
 					}
 				}
+				mediaType := comp.Type.MediaType()
+				if sr.cm.Snapshotter.Name() == "overlaybd" {
+					snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID())
+					if err != nil {
+						return struct{}{}, errors.Wrapf(err, "failed to Stat overlaybd")
+					}
+					if bdPath := snStat.Labels[obdlabel.LocalOverlayBDPath]; bdPath != "" {
+						if err := commitOverlayBD(ctx, sr, &desc); err != nil {
+							return struct{}{}, err
+						}
+						mediaType = desc.MediaType
+						enableOverlay = false
+					}
+				}
 				if enableOverlay {
 					computed, ok, err := sr.tryComputeOverlayBlob(ctx, lower, upper, mediaType, sr.ID(), compressorFunc)
 					if !ok || err != nil {
@@ -463,3 +481,50 @@ func ensureCompression(ctx context.Context, ref *immutableRef, comp compression.
 	})
 	return err
 }
+
+func commitOverlayBD(ctx context.Context, sr *immutableRef, desc *ocispecs.Descriptor) error {
+	snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID())
+	if err != nil {
+		return errors.Wrapf(err, "failed to Stat overlaybd")
+	}
+	bdPath := snStat.Labels[obdlabel.LocalOverlayBDPath]
+	if bdPath == "" {
+		return errors.New("missing overlaybd path label")
+	}
+	dir := path.Dir(bdPath)
+	commitPath := path.Join(dir, "overlaybd.commit")
+	err = obdcmd.Commit(ctx, dir, dir, true, "-t", "-z")
+	if err != nil {
+		return errors.Wrapf(err, "failed to overlaybd-commit")
+	}
+	cw, err := sr.cm.ContentStore.Writer(ctx, content.WithRef(sr.ID()))
+	if err != nil {
+		return errors.Wrapf(err, "failed to open writer")
+	}
+	fi, err := os.Open(commitPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open overlaybd commit file")
+	}
+	sz, err := io.Copy(cw, bufio.NewReader(fi))
+	if err != nil {
+		return errors.Wrapf(err, "failed to do io.Copy()")
+	}
+	dgst := cw.Digest()
+	labels := map[string]string{
+		labels.LabelUncompressed:     dgst.String(),
+		obdlabel.OverlayBDBlobDigest: dgst.String(),
+		obdlabel.OverlayBDBlobSize:   fmt.Sprintf("%d", sz),
+	}
+	err = cw.Commit(ctx, sz, dgst, content.WithLabels(labels))
+	if err != nil {
+		return errors.Wrapf(err, "failed to do cw.Commit")
+	}
+	desc.Digest = dgst
+	desc.Size = sz
+	desc.MediaType = ocispecs.MediaTypeImageLayer
+	desc.Annotations = map[string]string{
+		obdlabel.OverlayBDBlobDigest: string(desc.Digest),
+		obdlabel.OverlayBDBlobSize:   fmt.Sprintf("%d", desc.Size),
+	}
+	return nil
+}

cache/manager.go

@@ -8,13 +8,15 @@ import (
 	"sync"
 	"time"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/filters"
 	"github.com/containerd/containerd/gc"
 	"github.com/containerd/containerd/labels"
 	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/cache/metadata"
 	"github.com/moby/buildkit/client"
@@ -619,6 +621,10 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, sess session.Gr
 		}); rerr != nil {
 			return nil, rerr
 		}
+	} else if cm.Snapshotter.Name() == "overlaybd" && parent != nil {
+		// Snapshotter will create a R/W block device directly as rootfs with this label
+		rwLabels := map[string]string{obdlabel.SupportReadWriteMode: "dev"}
+		err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID, snapshots.WithLabels(rwLabels))
 	} else {
 		err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID)
 	}

cache/refs.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
+
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
@@ -40,7 +42,7 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-var additionalAnnotations = append(compression.EStargzAnnotations, labels.LabelUncompressed)
+var additionalAnnotations = append(append(compression.EStargzAnnotations, obdlabel.OverlayBDAnnotations...), labels.LabelUncompressed)
 
 // Ref is a reference to cacheable objects.
 type Ref interface {
@@ -418,7 +420,11 @@ func (cr *cacheRecord) mount(ctx context.Context, s session.Group) (_ snapshot.M
 		// Return the mount direct from View rather than setting it using the Mounts call below.
 		// The two are equivalent for containerd snapshotters but the moby snapshotter requires
 		// the use of the mountable returned by View in this case.
-		mnts, err := cr.cm.Snapshotter.View(ctx, mountSnapshotID, cr.getSnapshotID())
+		labels := make(map[string]string)
+		if cr.cm.Snapshotter.Name() == "overlaybd" {
+			labels["containerd.io/snapshot/overlaybd.writable"] = "dev"
+		}
+		mnts, err := cr.cm.Snapshotter.View(ctx, mountSnapshotID, cr.getSnapshotID(), snapshots.WithLabels(labels))
 		if err != nil && !errdefs.IsAlreadyExists(err) {
 			return nil, err
 		}
@@ -1015,6 +1021,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro
 			return err
 		}
 		return rerr
+	} else if sr.cm.Snapshotter.Name() == "overlaybd" {
+		if rerr = sr.prepareRemoteSnapshotsOverlaybdMode(ctx, s); rerr == nil {
+			return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
+		}
 	}
 
 	return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
@@ -1139,6 +1149,59 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s
 	})
 	return err
 }
+func (sr *immutableRef) prepareRemoteSnapshotsOverlaybdMode(ctx context.Context, s session.Group) error {
+	_, err := g.Do(ctx, sr.ID()+"-prepare-remote-snapshot", func(ctx context.Context) (_ struct{}, rerr error) {
+		dhs := sr.descHandlers
+		for _, r := range sr.layerChain() {
+			r := r
+			snapshotID := r.getSnapshotID()
+			if _, err := r.cm.Snapshotter.Stat(ctx, snapshotID); err == nil {
+				continue
+			}
+
+			dh := dhs[digest.Digest(r.getBlob())]
+			if dh == nil {
+				// We cannot prepare remote snapshots without descHandler.
+				return struct{}{}, nil
+			}
+
+			defaultLabels := snapshots.FilterInheritedLabels(dh.SnapshotLabels)
+			if defaultLabels == nil {
+				defaultLabels = make(map[string]string)
+			}
+			defaultLabels["containerd.io/snapshot.ref"] = snapshotID
+
+			// Prepare remote snapshots
+			var (
+				key  = fmt.Sprintf("tmp-%s %s", identity.NewID(), r.getChainID())
+				opts = []snapshots.Opt{
+					snapshots.WithLabels(defaultLabels),
+				}
+			)
+			parentID := ""
+			if r.layerParent != nil {
+				parentID = r.layerParent.getSnapshotID()
+			}
+			if err := r.cm.Snapshotter.Prepare(ctx, key, parentID, opts...); err != nil {
+				if errdefs.IsAlreadyExists(err) {
+					// Check if the targeting snapshot ID has been prepared as
+					// a remote snapshot in the snapshotter.
+					_, err := r.cm.Snapshotter.Stat(ctx, snapshotID)
+					if err == nil { // usable as remote snapshot without unlazying.
+						// Try the next layer as well.
+						continue
+					}
+				}
+			}
+
+			// This layer and all upper layers cannot be prepared without unlazying.
+			break
+		}
+
+		return struct{}{}, nil
+	})
+	return err
+}
 
 func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields []string, res map[string]string) {
 	res = make(map[string]string)
@@ -1306,7 +1369,12 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr
 
 	key := fmt.Sprintf("extract-%s %s", identity.NewID(), sr.getChainID())
 
-	err = sr.cm.Snapshotter.Prepare(ctx, key, parentID)
+	if sr.cm.Snapshotter.Name() == "overlaybd" {
+		err = sr.cm.Snapshotter.Prepare(ctx, key, parentID,
+			snapshots.WithLabels(map[string]string{"containerd.io/snapshot.ref": string(desc.Digest)}))
+	} else {
+		err = sr.cm.Snapshotter.Prepare(ctx, key, parentID)
+	}
 	if err != nil {
 		return err
 	}

cache/remotecache/registry/registry.go

@@ -166,6 +166,7 @@ func (dsl *withDistributionSourceLabel) SnapshotLabels(descs []ocispecs.Descript
 	for k, v := range estargz.SnapshotLabels(dsl.ref, descs, index) {
 		labels[k] = v
 	}
+	labels["containerd.io/snapshot/image-ref"] = dsl.ref
 	return labels
 }
 

go.mod

@@ -163,3 +163,5 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	kernel.org/pub/linux/libs/security/libcap/psx v1.2.67 // indirect
 )
+
+require github.com/containerd/accelerated-container-image v1.0.2

go.sum

@@ -280,6 +280,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4=
+github.com/containerd/accelerated-container-image v1.0.2 h1:4GmZg/8TrxAbTTpighuipeFPrjqH1ZKZgoa4bggMZVs=
+github.com/containerd/accelerated-container-image v1.0.2/go.mod h1:0/cMmA65Zervb+pO+sZvxvhqiO/pWoKdTf2zgbh59Zo=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=

source/containerimage/pull.go

@@ -153,6 +153,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach
 				for k, v := range estargz.SnapshotLabels(p.manifest.Ref, p.manifest.Descriptors, i) {
 					labels[k] = v
 				}
+				labels["containerd.io/snapshot/image-ref"] = p.manifest.Ref
 				p.descHandlers[desc.Digest] = &cache.DescHandler{
 					Provider:       p.manifest.Provider,
 					Progress:       progressController,