diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 35aa37c0..52b35f5a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -26,6 +26,11 @@ jobs: test-and-build-windows: runs-on: windows-2019 steps: + - name: Set git to use LF and symlinks + run: | + git config --global core.autocrlf false + git config --global core.eol lf + git config --global core.symlinks true - uses: actions/checkout@v4 - name: Set up go uses: actions/setup-go@v5 diff --git a/cnb_image.go b/cnb_image.go index d36b2f53..af8c78a8 100644 --- a/cnb_image.go +++ b/cnb_image.go @@ -3,6 +3,7 @@ package imgutil import ( "errors" "fmt" + "io" "strings" "time" @@ -18,36 +19,20 @@ import ( // The working image could be any v1.Image, // but in practice will start off as a pointer to a locallayout.v1ImageFacade (or similar). type CNBImageCore struct { - v1.Image // the working image - Store ImageStore // required - repoName string + v1.Image // the working image // optional + createdAt time.Time preferredMediaTypes MediaTypes preserveHistory bool previousImage v1.Image } -type ImageStore interface { - Contains(identifier string) bool - Delete(identifier string) error - Save(image IdentifiableV1Image, withName string, withAdditionalNames ...string) (string, error) - SaveFile(image IdentifiableV1Image, withName string) (string, error) - - DownloadLayersFor(identifier string) error - Layers() []v1.Layer -} - -type IdentifiableV1Image interface { - v1.Image - Identifier() (Identifier, error) -} - var _ v1.Image = &CNBImageCore{} // FIXME: mark deprecated methods as deprecated on the interface when other packages (remote, layout) expose a v1.Image -// Deprecated: Architecture +// TBD Deprecated: Architecture func (i *CNBImageCore) Architecture() (string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -56,7 +41,7 @@ func (i *CNBImageCore) Architecture() (string, error) { return configFile.Architecture, nil } -// Deprecated: CreatedAt +// TBD Deprecated: CreatedAt func (i *CNBImageCore) CreatedAt() (time.Time, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -65,7 +50,7 @@ func (i *CNBImageCore) CreatedAt() (time.Time, error) { return configFile.Created.Time, nil } -// Deprecated: Entrypoint +// TBD Deprecated: Entrypoint func (i *CNBImageCore) Entrypoint() ([]string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -96,25 +81,28 @@ func (i *CNBImageCore) GetAnnotateRefName() (string, error) { return manifest.Annotations["org.opencontainers.image.ref.name"], nil } -// Deprecated: History -func (i *CNBImageCore) History() ([]v1.History, error) { - configFile, err := getConfigFile(i.Image) +func (i *CNBImageCore) GetLayer(diffID string) (io.ReadCloser, error) { + hash, err := v1.NewHash(diffID) if err != nil { return nil, err } - return configFile.History, nil + layer, err := i.LayerByDiffID(hash) + if err != nil { + return nil, err + } + return layer.Uncompressed() } -func (i *CNBImageCore) Kind() string { - storeType := fmt.Sprintf("%T", i.Store) - parts := strings.Split(storeType, ".") - if len(parts) < 2 { - return storeType +// TBD Deprecated: History +func (i *CNBImageCore) History() ([]v1.History, error) { + configFile, err := getConfigFile(i.Image) + if err != nil { + return nil, err } - return strings.TrimPrefix(parts[0], "*") + return configFile.History, nil } -// Deprecated: Label +// TBD Deprecated: Label func (i *CNBImageCore) Label(key string) (string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -123,7 +111,7 @@ func (i *CNBImageCore) Label(key string) (string, error) { return configFile.Config.Labels[key], nil } -// Deprecated: Labels +// TBD Deprecated: Labels func (i *CNBImageCore) Labels() (map[string]string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -132,16 +120,12 @@ func (i *CNBImageCore) Labels() (map[string]string, error) { return configFile.Config.Labels, nil } -// Deprecated: ManifestSize +// TBD Deprecated: ManifestSize func (i *CNBImageCore) ManifestSize() (int64, error) { return i.Image.Size() } -func (i *CNBImageCore) Name() string { - return i.repoName -} - -// Deprecated: OS +// TBD Deprecated: OS func (i *CNBImageCore) OS() (string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -150,7 +134,7 @@ func (i *CNBImageCore) OS() (string, error) { return configFile.OS, nil } -// Deprecated: OSVersion +// TBD Deprecated: OSVersion func (i *CNBImageCore) OSVersion() (string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -165,7 +149,7 @@ func (i *CNBImageCore) TopLayer() (string, error) { return "", err } if len(layers) == 0 { - return "", fmt.Errorf("image %q has no layers", i.Name()) + return "", errors.New("image has no layers") } topLayer := layers[len(layers)-1] hex, err := topLayer.DiffID() @@ -185,7 +169,7 @@ func (i *CNBImageCore) Valid() bool { return err == nil } -// Deprecated: Variant +// TBD Deprecated: Variant func (i *CNBImageCore) Variant() (string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -194,7 +178,7 @@ func (i *CNBImageCore) Variant() (string, error) { return configFile.Variant, nil } -// Deprecated: WorkingDir +// TBD Deprecated: WorkingDir func (i *CNBImageCore) WorkingDir() (string, error) { configFile, err := getConfigFile(i.Image) if err != nil { @@ -208,6 +192,9 @@ func (i *CNBImageCore) AnnotateRefName(refName string) error { if err != nil { return err } + if manifest.Annotations == nil { + manifest.Annotations = make(map[string]string) + } manifest.Annotations["org.opencontainers.image.ref.name"] = refName mutated := mutate.Annotations(i.Image, manifest.Annotations) image, ok := mutated.(v1.Image) @@ -218,25 +205,21 @@ func (i *CNBImageCore) AnnotateRefName(refName string) error { return nil } -func (i *CNBImageCore) Rename(name string) { - i.repoName = name -} - -// Deprecated: SetArchitecture +// TBD Deprecated: SetArchitecture func (i *CNBImageCore) SetArchitecture(architecture string) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.Architecture = architecture }) } -// Deprecated: SetCmd +// TBD Deprecated: SetCmd func (i *CNBImageCore) SetCmd(cmd ...string) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.Config.Cmd = cmd }) } -// Deprecated: SetEntrypoint +// TBD Deprecated: SetEntrypoint func (i *CNBImageCore) SetEntrypoint(ep ...string) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.Config.Entrypoint = ep @@ -266,7 +249,7 @@ func (i *CNBImageCore) SetEnv(key, val string) error { }) } -// Deprecated: SetHistory +// TBD Deprecated: SetHistory func (i *CNBImageCore) SetHistory(histories []v1.History) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.History = histories @@ -288,21 +271,21 @@ func (i *CNBImageCore) SetOS(osVal string) error { }) } -// Deprecated: SetOSVersion +// TBD Deprecated: SetOSVersion func (i *CNBImageCore) SetOSVersion(osVersion string) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.OSVersion = osVersion }) } -// Deprecated: SetVariant +// TBD Deprecated: SetVariant func (i *CNBImageCore) SetVariant(variant string) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.Variant = variant }) } -// Deprecated: SetWorkingDir +// TBD Deprecated: SetWorkingDir func (i *CNBImageCore) SetWorkingDir(dir string) error { return i.MutateConfigFile(func(c *v1.ConfigFile) { c.Config.WorkingDir = dir @@ -322,6 +305,13 @@ func (i *CNBImageCore) AddLayerWithDiffID(path, _ string) error { } func (i *CNBImageCore) AddLayerWithDiffIDAndHistory(path, _ string, history v1.History) error { + // ensure existing history + if err := i.MutateConfigFile(func(c *v1.ConfigFile) { + c.History = NormalizedHistory(c.History, len(c.RootFS.DiffIDs)) + }); err != nil { + return err + } + layer, err := tarball.LayerFromFile(path) if err != nil { return err @@ -329,11 +319,8 @@ func (i *CNBImageCore) AddLayerWithDiffIDAndHistory(path, _ string, history v1.H if !i.preserveHistory { history = emptyHistory } - configFile, err := getConfigFile(i) - if err != nil { - return err - } - history.Created = configFile.Created + history.Created = v1.Time{Time: i.createdAt} + i.Image, err = mutate.Append( i.Image, mutate.Addendum{ @@ -346,9 +333,6 @@ func (i *CNBImageCore) AddLayerWithDiffIDAndHistory(path, _ string, history v1.H } func (i *CNBImageCore) Rebase(baseTopLayerDiffID string, withNewBase Image) error { - if i.Kind() != withNewBase.Kind() { - return fmt.Errorf("expected new base to be a %s image; got %s", i.Kind(), withNewBase.Kind()) - } newBase := withNewBase.UnderlyingImage() // FIXME: when all imgutil.Images are v1.Images, we can remove this part var err error i.Image, err = mutate.Rebase(i.Image, i.newV1ImageFacade(baseTopLayerDiffID), newBase) @@ -409,11 +393,11 @@ func (i *CNBImageCore) ReuseLayer(diffID string) error { } idx, err := getLayerIndex(diffID, i.previousImage) if err != nil { - return err + return fmt.Errorf("failed to get layer index: %w", err) } previousHistory, err := getHistory(idx, i.previousImage) if err != nil { - return err + return fmt.Errorf("failed to get history: %w", err) } return i.ReuseLayerWithHistory(diffID, previousHistory) } @@ -421,11 +405,11 @@ func (i *CNBImageCore) ReuseLayer(diffID string) error { func getLayerIndex(forDiffID string, fromImage v1.Image) (int, error) { layerHash, err := v1.NewHash(forDiffID) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to get layer hash: %w", err) } configFile, err := getConfigFile(fromImage) if err != nil { - return -1, err + return -1, fmt.Errorf("failed to get config file: %w", err) } for idx, configHash := range configFile.RootFS.DiffIDs { if layerHash.String() == configHash.String() { @@ -449,13 +433,15 @@ func getHistory(forIndex int, fromImage v1.Image) (v1.History, error) { func (i *CNBImageCore) ReuseLayerWithHistory(diffID string, history v1.History) error { layerHash, err := v1.NewHash(diffID) if err != nil { - return err + return fmt.Errorf("failed to get layer hash: %w", err) } layer, err := i.previousImage.LayerByDiffID(layerHash) if err != nil { - return err + return fmt.Errorf("failed to get layer by diffID: %w", err) } - if !i.preserveHistory { + if i.preserveHistory { + history.Created = v1.Time{Time: i.createdAt} + } else { history = emptyHistory } i.Image, err = mutate.Append( @@ -482,6 +468,34 @@ func (i *CNBImageCore) MutateConfigFile(withFunc func(c *v1.ConfigFile)) error { return err } +func (i *CNBImageCore) SetCreatedAtAndHistory() error { + var err error + // set created at + if err = i.MutateConfigFile(func(c *v1.ConfigFile) { + c.Created = v1.Time{Time: i.createdAt} + c.Container = "" + }); err != nil { + return err + } + // set history + if i.preserveHistory { + // set created at for each history + err = i.MutateConfigFile(func(c *v1.ConfigFile) { + for j := range c.History { + c.History[j].Created = v1.Time{Time: i.createdAt} + } + }) + } else { + // zero history + err = i.MutateConfigFile(func(c *v1.ConfigFile) { + for j := range c.History { + c.History[j] = v1.History{Created: v1.Time{Time: i.createdAt}} + } + }) + } + return err +} + func getConfigFile(image v1.Image) (*v1.ConfigFile, error) { configFile, err := image.ConfigFile() if err != nil { diff --git a/image.go b/image.go index d6feaef0..87bd82c3 100644 --- a/image.go +++ b/image.go @@ -7,9 +7,7 @@ import ( "time" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/types" ) type Image interface { @@ -19,7 +17,7 @@ type Image interface { CreatedAt() (time.Time, error) Entrypoint() ([]string, error) Env(key string) (string, error) - // Found tells whether the image exists in the repository by `Name()`. + // Found reports if image exists in the image store with `Name()`. Found() bool GetAnnotateRefName() (string, error) // GetLayer retrieves layer by diff id. Returns a reader of the uncompressed contents of the layer. @@ -87,136 +85,16 @@ type Platform struct { OSVersion string } -type MediaTypes int - -const ( - MissingTypes MediaTypes = iota - DefaultTypes - OCITypes - DockerTypes -) - -func (t MediaTypes) ManifestType() types.MediaType { - switch t { - case OCITypes: - return types.OCIManifestSchema1 - case DockerTypes: - return types.DockerManifestSchema2 - default: - return "" - } -} - -func (t MediaTypes) ConfigType() types.MediaType { - switch t { - case OCITypes: - return types.OCIConfigJSON - case DockerTypes: - return types.DockerConfigJSON - default: - return "" - } -} - -func (t MediaTypes) LayerType() types.MediaType { - switch t { - case OCITypes: - return types.OCILayer - case DockerTypes: - return types.DockerLayer - default: - return "" - } -} - -// OverrideMediaTypes mutates the provided v1.Image to use the desired media types -// in the image manifest and config files (including the layers referenced in the manifest) -func OverrideMediaTypes(image v1.Image, mediaTypes MediaTypes) (v1.Image, error) { - if mediaTypes == DefaultTypes || mediaTypes == MissingTypes { - // without media types option, default to original media types - return image, nil - } - - // manifest media type - retImage := mutate.MediaType(empty.Image, mediaTypes.ManifestType()) - - // update empty image with image config - config, err := image.ConfigFile() - if err != nil { - return nil, err - } - history := config.History - // zero out diff IDs and history, as these will be updated when we call `mutate.Append` - config.RootFS.DiffIDs = make([]v1.Hash, 0) - config.History = []v1.History{} - retImage, err = mutate.ConfigFile(retImage, config) - if err != nil { - return nil, err - } - - // config media type - retImage = mutate.ConfigMediaType(retImage, mediaTypes.ConfigType()) - - // layers media type - layers, err := image.Layers() - if err != nil { - return nil, err - } - additions := layersAddendum(layers, history, mediaTypes.LayerType()) - retImage, err = mutate.Append(retImage, additions...) - if err != nil { - return nil, err - } - - return retImage, nil -} - // OverrideHistoryIfNeeded zeroes out the history if the number of history entries doesn't match the number of layers. func OverrideHistoryIfNeeded(image v1.Image) (v1.Image, error) { - configFile, err := image.ConfigFile() - if err != nil || configFile == nil { - return nil, fmt.Errorf("getting image config: %w", err) + configFile, err := getConfigFile(image) + if err != nil { + return nil, err } configFile.History = NormalizedHistory(configFile.History, len(configFile.RootFS.DiffIDs)) return mutate.ConfigFile(image, configFile) } -func NormalizedHistory(history []v1.History, nLayers int) []v1.History { - if history == nil { - return make([]v1.History, nLayers) - } - // ensure we remove history for empty layers - var nHistory []v1.History - for _, h := range history { - if !h.EmptyLayer { - nHistory = append(nHistory, h) - } - } - if len(nHistory) == nLayers { - return nHistory - } - return make([]v1.History, nLayers) -} - -// layersAddendum creates an Addendum array with the given layers -// and the desired media type -func layersAddendum(layers []v1.Layer, history []v1.History, mediaType types.MediaType) []mutate.Addendum { - additions := make([]mutate.Addendum, 0) - if len(history) != len(layers) { - history = make([]v1.History, len(layers)) - } - for idx, layer := range layers { - additions = append(additions, mutate.Addendum{ - Layer: layer, - History: history[idx], - MediaType: mediaType, - }) - } - return additions -} - -var NormalizedDateTime = time.Date(1980, time.January, 1, 0, 0, 1, 0, time.UTC) - type SaveDiagnostic struct { ImageName string Cause error diff --git a/layout/layout.go b/layout/layout.go index 715fc805..6c3f5e33 100644 --- a/layout/layout.go +++ b/layout/layout.go @@ -1,18 +1,9 @@ package layout import ( - "bytes" - "fmt" - "io" "os" "path/filepath" - "strings" - "time" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/google/go-containerregistry/pkg/v1/types" "github.com/pkg/errors" "github.com/buildpacks/imgutil" @@ -21,78 +12,29 @@ import ( var _ imgutil.Image = (*Image)(nil) type Image struct { - v1.Image - path string - prevLayers []v1.Layer - prevHistory []v1.History - createdAt time.Time - refName string // holds org.opencontainers.image.ref.name value - requestedMediaTypes imgutil.MediaTypes - withHistory bool + *imgutil.CNBImageCore + repoPath string + saveWithoutLayers bool } -// getters - -func (i *Image) Architecture() (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - if cfg.Architecture == "" { - return "", fmt.Errorf("missing Architecture for image at path %q", i.path) - } - return cfg.Architecture, nil -} - -func (i *Image) CreatedAt() (time.Time, error) { - configFile, err := i.Image.ConfigFile() - if err != nil { - return time.Time{}, errors.Wrapf(err, "getting createdAt time for image at path %q", i.path) - } - return configFile.Created.UTC(), nil +func (i *Image) Kind() string { + return "layout" } -func (i *Image) Env(key string) (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - for _, envVar := range cfg.Config.Env { - parts := strings.Split(envVar, "=") - if parts[0] == key { - return parts[1], nil - } - } - return "", nil +func (i *Image) Name() string { + return i.repoPath } -func (i *Image) Entrypoint() ([]string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return nil, errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return nil, fmt.Errorf("missing config for image at path %q", i.path) - } - return cfg.Config.Entrypoint, nil +func (i *Image) Rename(name string) { + i.repoPath = name } -// Found tells whether the image exists in the repository by `Name()`. +// Found reports if image exists in the image store with `Name()`. func (i *Image) Found() bool { - return ImageExists(i.path) -} - -func (i *Image) Valid() bool { - return i.Found() + return imageExists(i.repoPath) } -func ImageExists(path string) bool { +func imageExists(path string) bool { if !pathExists(path) { return false } @@ -112,442 +54,17 @@ func pathExists(path string) bool { return false } -func (i *Image) GetAnnotateRefName() (string, error) { - return i.refName, nil -} - -// GetLayer retrieves layer by diff id. Returns a reader of the uncompressed contents of the layer. -// When the layers (notExistsLayer) came from a sparse image returns an empty reader -func (i *Image) GetLayer(sha string) (io.ReadCloser, error) { - layers, err := i.Image.Layers() - if err != nil { - return nil, err - } - - layer, _, err := findLayerWithSha(layers, sha) - if err != nil { - return nil, err - } - - return layer.Uncompressed() -} - -func (i *Image) History() ([]v1.History, error) { - configFile, err := i.ConfigFile() - if err != nil { - return nil, err - } - return configFile.History, nil -} - // Identifier // Each image's ID is given by the SHA256 hash of its configuration JSON. It is represented as a hexadecimal encoding of 256 bits, // e.g., sha256:a9561eb1b190625c9adb5a9513e72c4dedafc1cb2d4c5236c9a6957ec7dfd5a9. func (i *Image) Identifier() (imgutil.Identifier, error) { hash, err := i.Image.Digest() if err != nil { - return nil, errors.Wrapf(err, "getting identifier for image at path %q", i.path) - } - return newLayoutIdentifier(i.path, hash) -} - -func (i *Image) Kind() string { - return `layout` -} - -func (i *Image) Label(key string) (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", fmt.Errorf("getting config for image at path %q: %w", i.path, err) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - labels := cfg.Config.Labels - return labels[key], nil -} - -func (i *Image) Labels() (map[string]string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return nil, errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return nil, fmt.Errorf("missing config for image at path %q", i.path) - } - return cfg.Config.Labels, nil -} - -// Layers overrides v1.Image Layers(), because we allow sparse image in OCI layout, sometimes some blobs -// are missing. This method checks: -// If there is data, return the layer -// If there is no data, return a notExistsLayer -func (i *Image) Layers() ([]v1.Layer, error) { - layers, err := i.Image.Layers() - if err != nil { - return nil, err - } - - var retLayers []v1.Layer - for pos, layer := range layers { - if hasData(layer) { - retLayers = append(retLayers, layer) - } else { - cfg, err := i.Image.ConfigFile() - if err != nil { - return nil, err - } - diffID := cfg.RootFS.DiffIDs[pos] - retLayers = append(retLayers, ¬ExistsLayer{Layer: layer, diffID: diffID}) - } - } - return retLayers, nil -} - -func hasData(layer v1.Layer) bool { - _, err := layer.Compressed() - return err == nil -} - -type notExistsLayer struct { - v1.Layer - diffID v1.Hash -} - -func (l *notExistsLayer) Compressed() (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader([]byte{})), nil -} - -func (l *notExistsLayer) DiffID() (v1.Hash, error) { - return l.diffID, nil -} - -func (l *notExistsLayer) Uncompressed() (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader([]byte{})), nil -} - -func (i *Image) ManifestSize() (int64, error) { - return i.Image.Size() -} - -func (i *Image) Name() string { - return i.path -} - -func (i *Image) OS() (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - if cfg.OS == "" { - return "", fmt.Errorf("missing OS for image at path %q", i.path) - } - return cfg.OS, nil -} - -func (i *Image) OSVersion() (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - return cfg.OSVersion, nil -} - -func (i *Image) TopLayer() (string, error) { - all, err := i.Image.Layers() - if err != nil { - return "", err - } - if len(all) == 0 { - return "", fmt.Errorf("image at path %q has no layers", i.Name()) - } - topLayer := all[len(all)-1] - hex, err := topLayer.DiffID() - if err != nil { - return "", err - } - return hex.String(), nil -} - -func (i *Image) UnderlyingImage() v1.Image { - return i.Image -} - -func (i *Image) Variant() (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - return cfg.Variant, nil -} - -func (i *Image) WorkingDir() (string, error) { - cfg, err := i.Image.ConfigFile() - if err != nil { - return "", errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return "", fmt.Errorf("missing config for image at path %q", i.path) - } - return cfg.Config.WorkingDir, nil -} - -// setters - -func (i *Image) AnnotateRefName(refName string) error { - i.refName = refName - return nil -} - -func (i *Image) Rename(name string) { - i.path = name -} - -func (i *Image) SetArchitecture(architecture string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - configFile.Architecture = architecture - err = i.mutateConfigFile(i.Image, configFile) - return err -} - -func (i *Image) SetCmd(cmd ...string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - config := *configFile.Config.DeepCopy() - config.Cmd = cmd - err = i.mutateConfig(i.Image, config) - return err -} - -func (i *Image) SetEntrypoint(ep ...string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - config := *configFile.Config.DeepCopy() - config.Entrypoint = ep - err = i.mutateConfig(i.Image, config) - return err -} - -func (i *Image) SetEnv(key string, val string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - config := *configFile.Config.DeepCopy() - ignoreCase := configFile.OS == "windows" - for idx, e := range config.Env { - parts := strings.Split(e, "=") - foundKey := parts[0] - searchKey := key - if ignoreCase { - foundKey = strings.ToUpper(foundKey) - searchKey = strings.ToUpper(searchKey) - } - if foundKey == searchKey { - config.Env[idx] = fmt.Sprintf("%s=%s", key, val) - err = i.mutateConfig(i.Image, config) - return err - } - } - config.Env = append(config.Env, fmt.Sprintf("%s=%s", key, val)) - err = i.mutateConfig(i.Image, config) - return err -} - -func (i *Image) SetHistory(history []v1.History) error { - configFile, err := i.Image.ConfigFile() // TODO: check if we need to use DeepCopy - if err != nil { - return err - } - configFile.History = history - i.Image, err = mutate.ConfigFile(i.Image, configFile) - return err -} - -func (i *Image) SetLabel(key string, val string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - config := *configFile.Config.DeepCopy() - if config.Labels == nil { - config.Labels = map[string]string{} - } - config.Labels[key] = val - err = i.mutateConfig(i.Image, config) - if err != nil { - return errors.Wrapf(err, "set label key=%s value=%s", key, val) - } - return nil -} - -func (i *Image) SetOS(osVal string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - configFile.OS = osVal - err = i.mutateConfigFile(i.Image, configFile) - return err -} - -func (i *Image) SetOSVersion(osVersion string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - configFile.OSVersion = osVersion - err = i.mutateConfigFile(i.Image, configFile) - return err -} - -func (i *Image) SetVariant(variant string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err + return nil, errors.Wrapf(err, "getting identifier for image at path %q", i.repoPath) } - configFile.Variant = variant - err = i.mutateConfigFile(i.Image, configFile) - return err -} - -func (i *Image) SetWorkingDir(dir string) error { - configFile, err := i.Image.ConfigFile() - if err != nil { - return err - } - config := *configFile.Config.DeepCopy() - config.WorkingDir = dir - err = i.mutateConfig(i.Image, config) - return err -} - -// modifiers - -// AddLayer adds an uncompressed tarred layer to the image -func (i *Image) AddLayer(path string) error { - return i.AddLayerWithDiffIDAndHistory(path, "ignored", v1.History{}) -} - -func (i *Image) addLayer(layer v1.Layer, history v1.History) error { - image, err := mutate.Append( - i.Image, - layerAddendum(layer, history, i.requestedMediaTypes.LayerType()), - ) - if err != nil { - return errors.Wrap(err, "add layer") - } - return i.setUnderlyingImage(image) -} - -func layerAddendum(layer v1.Layer, history v1.History, mediaType types.MediaType) mutate.Addendum { - return mutate.Addendum{ - Layer: layer, - History: history, - MediaType: mediaType, - } -} - -func (i *Image) AddLayerWithDiffID(path, diffID string) error { - return i.AddLayerWithDiffIDAndHistory(path, "ignored", v1.History{}) -} - -func (i *Image) AddLayerWithDiffIDAndHistory(path, diffID string, history v1.History) error { - // add layer - layer, err := tarball.LayerFromFile(path) - if err != nil { - return err - } - return i.addLayer(layer, history) + return newLayoutIdentifier(i.repoPath, hash) } func (i *Image) Delete() error { - return os.RemoveAll(i.path) -} - -func (i *Image) Rebase(s string, image imgutil.Image) error { - return errors.New("not yet implemented") -} - -func (i *Image) RemoveLabel(key string) error { - cfg, err := i.Image.ConfigFile() - if err != nil { - return errors.Wrapf(err, "getting config file for image at path %q", i.path) - } - if cfg == nil { - return fmt.Errorf("missing config for image at path %q", i.path) - } - config := *cfg.Config.DeepCopy() - delete(config.Labels, key) - err = i.mutateConfig(i.Image, config) - return err -} - -func (i *Image) ReuseLayer(sha string) error { - layer, idx, err := findLayerWithSha(i.prevLayers, sha) - if err != nil { - return err - } - return i.addLayer(layer, i.prevHistory[idx]) -} - -func (i *Image) ReuseLayerWithHistory(sha string, history v1.History) error { - layer, _, err := findLayerWithSha(i.prevLayers, sha) - if err != nil { - return err - } - return i.addLayer(layer, history) -} - -// helpers - -func findLayerWithSha(layers []v1.Layer, diffID string) (v1.Layer, int, error) { - for idx, layer := range layers { - dID, err := layer.DiffID() - if err != nil { - return nil, idx, errors.Wrap(err, "get diff ID for previous image layer") - } - if diffID == dID.String() { - return layer, idx, nil - } - } - return nil, -1, fmt.Errorf("previous image did not have layer with diff id %q", diffID) -} - -// mutateConfig mutates the provided v1.Image to have the provided v1.Config, -// wraps the result into a layout.Image, -// and sets it as the underlying image for the receiving layout.Image (required for overriding methods like Layers()) -func (i *Image) mutateConfig(base v1.Image, config v1.Config) error { - image, err := mutate.Config(base, config) - if err != nil { - return err - } - return i.setUnderlyingImage(image) -} - -// mutateConfigFile mutates the provided v1.Image to have the provided v1.ConfigFile, -// wraps the result into a layout.Image, -// and sets it as the underlying image for the receiving layout.Image (required for overriding methods like Layers()) -func (i *Image) mutateConfigFile(base v1.Image, configFile *v1.ConfigFile) error { - image, err := mutate.ConfigFile(base, configFile) - if err != nil { - return err - } - return i.setUnderlyingImage(image) + return os.RemoveAll(i.repoPath) } diff --git a/layout/layout_test.go b/layout/layout_test.go index 14582624..973cb9a4 100644 --- a/layout/layout_test.go +++ b/layout/layout_test.go @@ -118,7 +118,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) { h.AssertEq(t, osVersion, "10.0.17763.316") _, err = img.TopLayer() - h.AssertError(t, err, "has no layers") + h.AssertNil(t, err) // Window images include a runnable base layer }) it("sets all platform required fields for linux", func() { @@ -255,6 +255,16 @@ func testImage(t *testing.T, when spec.G, it spec.S) { h.AssertError(t, err, "has no layers") }) }) + + when("existing config has extra fields", func() { + it("returns an unmodified digest", func() { + img, err := layout.NewImage(imagePath, layout.FromBaseImagePath(filepath.Join("testdata", "layout", "busybox-sparse"))) + h.AssertNil(t, err) + digest, err := img.Digest() + h.AssertNil(t, err) + h.AssertEq(t, digest.String(), "sha256:f75f3d1a317fc82c793d567de94fc8df2bece37acd5f2bd364a0d91a0d1f3dab") + }) + }) }) when("#WithMediaTypes", func() { @@ -273,6 +283,25 @@ func testImage(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, img.Save()) h.AssertDockerMediaTypes(t, img) // after saving }) + + when("using a sparse image", func() { + it("sets the requested media types", func() { + img, err := layout.NewImage( + imagePath, + layout.FromBaseImagePath(sparseBaseImagePath), + layout.WithMediaTypes(imgutil.OCITypes), + ) + h.AssertNil(t, err) + h.AssertOCIMediaTypes(t, img) // before saving + // add a random layer + path, diffID, _ := h.RandomLayer(t, tmpDir) + err = img.AddLayerWithDiffID(path, diffID) + h.AssertNil(t, err) + h.AssertOCIMediaTypes(t, img) // after adding a layer + h.AssertNil(t, img.Save()) + h.AssertOCIMediaTypes(t, img) // after saving + }) + }) }) when("#WithPreviousImage", func() { diff --git a/layout/new.go b/layout/new.go index f27df0e6..9d8586d1 100644 --- a/layout/new.go +++ b/layout/new.go @@ -4,224 +4,118 @@ import ( "fmt" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/pkg/errors" "github.com/buildpacks/imgutil" ) func NewImage(path string, ops ...ImageOption) (*Image, error) { - imageOpts := &options{} + options := &imgutil.ImageOptions{} for _, op := range ops { - if err := op(imageOpts); err != nil { - return nil, err - } + op(options) } - platform := defaultPlatform() - if (imageOpts.platform != imgutil.Platform{}) { - platform = imageOpts.platform - } + options.Platform = processDefaultPlatformOption(options.Platform) - image, err := emptyImage(platform) - if err != nil { - return nil, err - } + var err error - ri := &Image{ - Image: image, - path: path, - withHistory: imageOpts.withHistory, + if options.BaseImage == nil && options.BaseImageRepoName != "" { // options.BaseImage supersedes options.BaseImageRepoName + options.BaseImage, err = newImageFromPath(options.BaseImageRepoName, options.Platform) + if err != nil { + return nil, err + } } - - if imageOpts.prevImagePath != "" { - if err := processPreviousImageOption(ri, imageOpts.prevImagePath, platform); err != nil { + options.MediaTypes = imgutil.GetPreferredMediaTypes(*options) + if options.BaseImage != nil { + options.BaseImage, err = newImageFacadeFrom(options.BaseImage, options.MediaTypes) + if err != nil { return nil, err } } - hasBaseImage := imageOpts.baseImagePath != "" || imageOpts.baseImage != nil - if imageOpts.baseImagePath != "" { - if err := processBaseImageOption(ri, imageOpts.baseImagePath, platform); err != nil { + if options.PreviousImageRepoName != "" { + options.PreviousImage, err = newImageFromPath(options.PreviousImageRepoName, options.Platform) + if err != nil { return nil, err } - } else if imageOpts.baseImage != nil { - if err := ri.setUnderlyingImage(imageOpts.baseImage); err != nil { + } + if options.PreviousImage != nil { + options.PreviousImage, err = newImageFacadeFrom(options.PreviousImage, options.MediaTypes) + if err != nil { return nil, err } } - if imageOpts.createdAt.IsZero() { - ri.createdAt = imgutil.NormalizedDateTime - } else { - ri.createdAt = imageOpts.createdAt - } - - if imageOpts.mediaTypes != imgutil.MissingTypes { - ri.requestedMediaTypes = imageOpts.mediaTypes - } else if !hasBaseImage { - ri.requestedMediaTypes = imgutil.OCITypes - } - if err = ri.setUnderlyingImage(ri.Image); err != nil { // update media types + cnbImage, err := imgutil.NewCNBImage(*options) + if err != nil { return nil, err } - return ri, nil + return &Image{ + CNBImageCore: cnbImage, + repoPath: path, + saveWithoutLayers: options.WithoutLayers, + }, nil } -func defaultPlatform() imgutil.Platform { +func processDefaultPlatformOption(requestedPlatform imgutil.Platform) imgutil.Platform { + var emptyPlatform imgutil.Platform + if requestedPlatform != emptyPlatform { + return requestedPlatform + } return imgutil.Platform{ OS: "linux", Architecture: "amd64", } } -func emptyImage(platform imgutil.Platform) (v1.Image, error) { - cfg := &v1.ConfigFile{ - Architecture: platform.Architecture, - History: []v1.History{}, - OS: platform.OS, - OSVersion: platform.OSVersion, - RootFS: v1.RootFS{ - Type: "layers", - DiffIDs: []v1.Hash{}, - }, +// newImageFromPath creates a layout image from the given path. +// * If an image index for multiple platforms exists, it will try to select the image according to the platform provided. +// * If the image does not exist, then nothing is returned. +func newImageFromPath(path string, withPlatform imgutil.Platform) (v1.Image, error) { + if !imageExists(path) { + return nil, nil } - image := mutate.MediaType(empty.Image, types.OCIManifestSchema1) - image = mutate.ConfigMediaType(image, types.OCIConfigJSON) - return mutate.ConfigFile(image, cfg) -} -func processPreviousImageOption(ri *Image, prevImagePath string, platform imgutil.Platform) error { - prevImage, err := newV1Image(prevImagePath, platform, ri.withHistory) + layoutPath, err := FromPath(path) if err != nil { - return err + return nil, fmt.Errorf("failed to load layout from path: %w", err) } - - prevLayers, err := prevImage.Layers() + index, err := layoutPath.ImageIndex() if err != nil { - return errors.Wrapf(err, "getting layers for previous image with path %q", prevImagePath) + return nil, fmt.Errorf("failed to load index: %w", err) } - - ri.prevLayers = prevLayers - configFile, err := prevImage.ConfigFile() + image, err := imageFromIndex(index, withPlatform) if err != nil { - return err - } - ri.prevHistory = configFile.History - - return nil -} - -// newV1Image creates a layout image from the given path. -// - If a ImageIndex for multiples platforms exists, then it will try to select the image -// according to the platform provided -// - If the image does not exist, then an empty image is returned -func newV1Image(path string, platform imgutil.Platform, withHistory bool) (v1.Image, error) { - var ( - image v1.Image - layout Path - err error - ) - - if ImageExists(path) { - layout, err = FromPath(path) - if err != nil { - return nil, fmt.Errorf("loading layout from path new: %w", err) - } - - index, err := layout.ImageIndex() - if err != nil { - return nil, fmt.Errorf("reading index: %w", err) - } - - image, err = imageFromIndex(index, platform) - if err != nil { - return nil, fmt.Errorf("getting image from index: %w", err) - } - } else { - image, err = emptyImage(platform) - if err != nil { - return nil, fmt.Errorf("initializing empty image: %w", err) - } - } - - if withHistory { - if image, err = imgutil.OverrideHistoryIfNeeded(image); err != nil { - return nil, fmt.Errorf("overriding history: %w", err) - } + return nil, fmt.Errorf("failed to load image from index: %w", err) } - - return &Image{ - Image: image, - path: path, - }, nil + return image, nil } // imageFromIndex creates a v1.Image from the given Image Index, selecting the image manifest // that matches the given OS and architecture. func imageFromIndex(index v1.ImageIndex, platform imgutil.Platform) (v1.Image, error) { - indexManifest, err := index.IndexManifest() + manifestList, err := index.IndexManifest() if err != nil { return nil, err } - - if len(indexManifest.Manifests) == 0 { - return nil, errors.New("no underlyingImage indexManifest found") + if len(manifestList.Manifests) == 0 { + return nil, fmt.Errorf("failed to find manifest at index") } - manifest := indexManifest.Manifests[0] - if len(indexManifest.Manifests) > 1 { - // Find based on platform (os/arch) - for _, m := range indexManifest.Manifests { - if m.Platform.OS == platform.OS && m.Platform.Architecture == platform.OS { + // find manifest for platform + var manifest v1.Descriptor + if len(manifestList.Manifests) == 1 { + manifest = manifestList.Manifests[0] + } else { + for _, m := range manifestList.Manifests { + if m.Platform.OS == platform.OS && + m.Platform.Architecture == platform.Architecture { manifest = m break } } - return nil, fmt.Errorf("manifest matching platform %v not found", platform) - } - - image, err := index.Image(manifest.Digest) - if err != nil { - return nil, err + return nil, fmt.Errorf("failed to find manifest matching platform %v", platform) } - return image, nil -} - -func processBaseImageOption(ri *Image, baseImagePath string, platform imgutil.Platform) error { - baseImage, err := newV1Image(baseImagePath, platform, ri.withHistory) - if err != nil { - return err - } - - return ri.setUnderlyingImage(baseImage) -} - -// setUnderlyingImage wraps the provided v1.Image into a layout.Image and sets it as the underlying image for the receiving layout.Image -func (i *Image) setUnderlyingImage(base v1.Image) error { - manifest, err := base.Manifest() - if err != nil { - return err - } - if i.requestedMediaTypesMatch(manifest) { - i.Image = &Image{Image: base} - return nil - } - // provided v1.Image media types differ from requested, override them - newBase, err := imgutil.OverrideMediaTypes(base, i.requestedMediaTypes) - if err != nil { - return err - } - i.Image = &Image{Image: newBase} - return nil -} - -// requestedMediaTypesMatch returns true if the manifest and config file use the requested media types -func (i *Image) requestedMediaTypesMatch(manifest *v1.Manifest) bool { - return manifest.MediaType == i.requestedMediaTypes.ManifestType() && - manifest.Config.MediaType == i.requestedMediaTypes.ConfigType() + return index.Image(manifest.Digest) } diff --git a/layout/options.go b/layout/options.go index 23b49e10..1204943f 100644 --- a/layout/options.go +++ b/layout/options.go @@ -8,79 +8,75 @@ import ( "github.com/buildpacks/imgutil" ) -type ImageOption func(*options) error +type ImageOption func(*imgutil.ImageOptions) -type options struct { - platform imgutil.Platform - baseImage v1.Image - baseImagePath string - prevImagePath string - withHistory bool - createdAt time.Time - mediaTypes imgutil.MediaTypes +// FromBaseImage loads the provided image as the manifest, config, and layers for the working image. +// If the image is not found, it does nothing. +func FromBaseImage(image v1.Image) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.BaseImage = image + } } -// FromBaseImage loads the given image as the config and layers for the new image. -// Ignored if image is not found. -func FromBaseImage(base v1.Image) ImageOption { - return func(i *options) error { - i.baseImage = base - return nil +// FromBaseImagePath (layout only) loads the image at the provided path as the manifest, config, and layers for the working image. +// If the image is not found, it does nothing. +func FromBaseImagePath(name string) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.BaseImageRepoName = name } } -// FromBaseImagePath (layout only) loads an existing image as the config and layers for the new underlyingImage. -// Ignored if underlyingImage is not found. -func FromBaseImagePath(path string) ImageOption { - return func(i *options) error { - i.baseImagePath = path - return nil +// WithCreatedAt lets a caller set the "created at" timestamp for the working image when saved. +// If not provided, the default is imgutil.NormalizedDateTime. +func WithCreatedAt(t time.Time) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.CreatedAt = t } } -// WithCreatedAt lets a caller set the created at timestamp for the image. -// Defaults for a new image is imgutil.NormalizedDateTime -func WithCreatedAt(createdAt time.Time) ImageOption { - return func(i *options) error { - i.createdAt = createdAt - return nil +// WithConfig lets a caller provided a `config` object for the working image. +func WithConfig(c *v1.Config) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.Config = c } } -// WithDefaultPlatform provides Architecture/OS/OSVersion defaults for the new image. -// Defaults for a new image are ignored when FromBaseImage returns an image. -// FromBaseImage and WithPreviousImage will use the platform to choose an image from a manifest list. -func WithDefaultPlatform(platform imgutil.Platform) ImageOption { - return func(i *options) error { - i.platform = platform - return nil +// WithDefaultPlatform provides the default Architecture/OS/OSVersion if no base image is provided, +// or if the provided image inputs (base and previous) are manifest lists. +func WithDefaultPlatform(p imgutil.Platform) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.Platform = p } } // WithHistory if provided will configure the image to preserve history when saved // (including any history from the base image if valid). -func WithHistory() ImageOption { - return func(opts *options) error { - opts.withHistory = true - return nil +func WithHistory() func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.PreserveHistory = true } } -// WithMediaTypes lets a caller set the desired media types for the image manifest and config files, -// including the layers referenced in the manifest, to be either OCI media types or Docker media types. -func WithMediaTypes(requested imgutil.MediaTypes) ImageOption { - return func(i *options) error { - i.mediaTypes = requested - return nil +// WithMediaTypes lets a caller set the desired media types for the manifest and config (including layers referenced in the manifest) +// to be either OCI media types or Docker media types. +func WithMediaTypes(m imgutil.MediaTypes) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.MediaTypes = m } } -// WithPreviousImage loads an existing image as a source for reusable layers. +// WithPreviousImage loads an existing image as the source for reusable layers. // Use with ReuseLayer(). -// Ignored if underlyingImage is not found. -func WithPreviousImage(path string) ImageOption { - return func(i *options) error { - i.prevImagePath = path - return nil +// If the image is not found, it does nothing. +func WithPreviousImage(name string) func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.PreviousImageRepoName = name + } +} + +// WithoutLayersWhenSaved (layout only) if provided will cause the image to be written without layers in the `blobs` directory. +func WithoutLayersWhenSaved() func(*imgutil.ImageOptions) { + return func(o *imgutil.ImageOptions) { + o.WithoutLayers = true } } diff --git a/layout/save.go b/layout/save.go index d0edb141..bc6bfca8 100644 --- a/layout/save.go +++ b/layout/save.go @@ -1,12 +1,7 @@ package layout import ( - "fmt" - - v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/pkg/errors" "github.com/buildpacks/imgutil" ) @@ -17,56 +12,34 @@ func (i *Image) Save(additionalNames ...string) error { // SaveAs ignores the image `Name()` method and saves the image according to name & additional names provided to this method func (i *Image) SaveAs(name string, additionalNames ...string) error { - err := i.mutateCreatedAt(i.Image, v1.Time{Time: i.createdAt}) - if err != nil { - return errors.Wrap(err, "set creation time") - } - - if i.Image, err = imgutil.OverrideHistoryIfNeeded(i.Image); err != nil { - return fmt.Errorf("override history: %w", err) + if err := i.SetCreatedAtAndHistory(); err != nil { + return err } - - cfg, err := i.Image.ConfigFile() + refName, err := i.GetAnnotateRefName() if err != nil { - return errors.Wrap(err, "get image config") - } - cfg = cfg.DeepCopy() - - created := v1.Time{Time: i.createdAt} - if i.withHistory { - // set created - for j := range cfg.History { - cfg.History[j].Created = created - } - } else { - // zero history, set created - for j := range cfg.History { - cfg.History[j] = v1.History{Created: created} - } + return err } - cfg.DockerVersion = "" - cfg.Container = "" - err = i.mutateConfigFile(i.Image, cfg) - if err != nil { - return errors.Wrap(err, "zeroing history") + ops := []AppendOption{WithAnnotations(ImageRefAnnotation(refName))} + if i.saveWithoutLayers { + ops = append(ops, WithoutLayers()) } - var diagnostics []imgutil.SaveDiagnostic - annotations := ImageRefAnnotation(i.refName) - pathsToSave := append([]string{name}, additionalNames...) + var ( + pathsToSave = append([]string{name}, additionalNames...) + diagnostics []imgutil.SaveDiagnostic + ) for _, path := range pathsToSave { - // initialize image path - path, err := Write(path, empty.Index) + layoutPath, err := initEmptyIndexAt(path) if err != nil { return err } - - err = path.AppendImage(i.Image, WithAnnotations(annotations)) - if err != nil { + if err = layoutPath.AppendImage( + i.Image, + ops..., + ); err != nil { diagnostics = append(diagnostics, imgutil.SaveDiagnostic{ImageName: i.Name(), Cause: err}) } } - if len(diagnostics) > 0 { return imgutil.SaveError{Errors: diagnostics} } @@ -74,12 +47,6 @@ func (i *Image) SaveAs(name string, additionalNames ...string) error { return nil } -// mutateCreatedAt mutates the provided v1.Image to have the provided v1.Time and wraps the result -// into a layout.Image (requires for override methods like Layers() -func (i *Image) mutateCreatedAt(base v1.Image, created v1.Time) error { // FIXME: this function doesn't need arguments; we should also probably do this mutation at the time of image instantiation instead of at the point of saving - image, err := mutate.CreatedAt(i.Image, v1.Time{Time: i.createdAt}) - if err != nil { - return err - } - return i.setUnderlyingImage(image) +func initEmptyIndexAt(path string) (Path, error) { + return Write(path, empty.Index) } diff --git a/layout/sparse/new.go b/layout/sparse/new.go index 64944a6c..89ddfe0b 100644 --- a/layout/sparse/new.go +++ b/layout/sparse/new.go @@ -7,15 +7,14 @@ import ( ) // NewImage returns a new Image saved on disk that can be modified -func NewImage(path string, from v1.Image, ops ...layout.ImageOption) (*Image, error) { - allOps := append([]layout.ImageOption{layout.FromBaseImage(from)}, ops...) - img, err := layout.NewImage(path, allOps...) +func NewImage(path string, from v1.Image, ops ...layout.ImageOption) (*layout.Image, error) { + ops = append([]layout.ImageOption{ + layout.FromBaseImage(from), + layout.WithoutLayersWhenSaved(), + }, ops...) + img, err := layout.NewImage(path, ops...) if err != nil { return nil, err } - - image := &Image{ - Image: *img, - } - return image, nil + return img, nil } diff --git a/layout/sparse/options.go b/layout/sparse/options.go deleted file mode 100644 index d4cb7959..00000000 --- a/layout/sparse/options.go +++ /dev/null @@ -1 +0,0 @@ -package sparse diff --git a/layout/sparse/save.go b/layout/sparse/save.go deleted file mode 100644 index f083a759..00000000 --- a/layout/sparse/save.go +++ /dev/null @@ -1,38 +0,0 @@ -package sparse - -import ( - "github.com/google/go-containerregistry/pkg/v1/empty" - - "github.com/buildpacks/imgutil" - "github.com/buildpacks/imgutil/layout" -) - -func (i *Image) Save(additionalNames ...string) error { - return i.SaveAs(i.Name(), additionalNames...) -} - -func (i *Image) SaveAs(name string, additionalNames ...string) error { - var diagnostics []imgutil.SaveDiagnostic - - refName, _ := i.Image.GetAnnotateRefName() - annotations := layout.ImageRefAnnotation(refName) - - pathsToSave := append([]string{name}, additionalNames...) - for _, path := range pathsToSave { - layoutPath, err := layout.Write(path, empty.Index) - if err != nil { - return err - } - - err = layoutPath.AppendImage(i, layout.WithoutLayers(), layout.WithAnnotations(annotations)) - if err != nil { - diagnostics = append(diagnostics, imgutil.SaveDiagnostic{ImageName: name, Cause: err}) - } - } - - if len(diagnostics) > 0 { - return imgutil.SaveError{Errors: diagnostics} - } - - return nil -} diff --git a/layout/sparse/sparse.go b/layout/sparse/sparse.go deleted file mode 100644 index aac6f37d..00000000 --- a/layout/sparse/sparse.go +++ /dev/null @@ -1,15 +0,0 @@ -package sparse - -import ( - "github.com/buildpacks/imgutil/layout" - - "github.com/buildpacks/imgutil" -) - -var _ imgutil.Image = (*Image)(nil) - -// Image is a struct created to override the Save() method of a layout image, -// so that when the image is saved to disk, it does not include any layers in the `blobs` directory. -type Image struct { - layout.Image -} diff --git a/layout/sparse/sparse_test.go b/layout/sparse/sparse_test.go index a1b73914..147a164e 100644 --- a/layout/sparse/sparse_test.go +++ b/layout/sparse/sparse_test.go @@ -141,9 +141,6 @@ func testImage(t *testing.T, when spec.G, it spec.S) { image, err := sparse.NewImage(imagePath, testImage) h.AssertNil(t, err) - err = image.Save() - h.AssertNil(t, err) - expectedDigest, err := testImage.Digest() h.AssertNil(t, err) diff --git a/layout/v1_facade.go b/layout/v1_facade.go new file mode 100644 index 00000000..8a7fa530 --- /dev/null +++ b/layout/v1_facade.go @@ -0,0 +1,156 @@ +package layout + +import ( + "bytes" + "fmt" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/buildpacks/imgutil" +) + +type v1ImageFacade struct { + v1.Image + diffIDMap map[v1.Hash]v1.Layer + digestMap map[v1.Hash]v1.Layer +} + +func newImageFacadeFrom(original v1.Image, withMediaTypes imgutil.MediaTypes) (v1.Image, error) { + configFile, err := original.ConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to get config: %w", err) + } + manifestFile, err := original.Manifest() + if err != nil { + return nil, fmt.Errorf("failed to get manifest: %w", err) + } + originalLayers, err := original.Layers() + if err != nil { + return nil, fmt.Errorf("failed to get layers: %w", err) + } + + ensureLayers := func(idx int, layer v1.Layer) (v1.Layer, error) { + return newLayerOrFacadeFrom(*configFile, *manifestFile, idx, layer) + } + // first, ensure media types + image, mutated, err := imgutil.EnsureMediaTypesAndLayers(original, withMediaTypes, ensureLayers) // if no media types are requested, this does nothing + if err != nil { + return nil, fmt.Errorf("failed to ensure media types: %w", err) + } + // then, ensure layers + if mutated { + // layers are wrapped in a facade, it is possible to call layer.Compressed or layer.Uncompressed without error + return image, nil + } + // we didn't mutate the image (possibly to preserve the digest), we must wrap the image in a facade + facade := &v1ImageFacade{ + Image: original, + diffIDMap: make(map[v1.Hash]v1.Layer), + digestMap: make(map[v1.Hash]v1.Layer), + } + for idx, l := range originalLayers { + layer, err := newLayerOrFacadeFrom(*configFile, *manifestFile, idx, l) + if err != nil { + return nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, err + } + facade.diffIDMap[diffID] = layer + digest, err := layer.Digest() + if err != nil { + return nil, err + } + facade.digestMap[digest] = layer + } + + return facade, nil +} + +func (i *v1ImageFacade) Layers() ([]v1.Layer, error) { + var layers []v1.Layer + configFile, err := i.ConfigFile() + if err != nil { + return nil, err + } + if configFile == nil { + return nil, nil + } + for _, diffID := range configFile.RootFS.DiffIDs { + l, err := i.LayerByDiffID(diffID) + if err != nil { + return nil, err + } + layers = append(layers, l) + } + return layers, nil +} + +func (i *v1ImageFacade) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.diffIDMap[h]; ok { + return layer, nil + } + return nil, fmt.Errorf("failed to find layer with diffID %s", h) // shouldn't get here +} + +func (i *v1ImageFacade) LayerByDigest(h v1.Hash) (v1.Layer, error) { + if layer, ok := i.digestMap[h]; ok { + return layer, nil + } + return nil, fmt.Errorf("failed to find layer with digest %s", h) // shouldn't get here +} + +type v1LayerFacade struct { + v1.Layer + diffID v1.Hash + digest v1.Hash + size int64 +} + +func (l *v1LayerFacade) Compressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte{})), nil +} + +func (l *v1LayerFacade) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +func (l *v1LayerFacade) Digest() (v1.Hash, error) { + return l.digest, nil +} + +func (l *v1LayerFacade) Uncompressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte{})), nil +} + +func (l *v1LayerFacade) Size() (int64, error) { + return l.size, nil +} + +func newLayerOrFacadeFrom(configFile v1.ConfigFile, manifestFile v1.Manifest, layerIndex int, originalLayer v1.Layer) (v1.Layer, error) { + if hasData(originalLayer) { + return originalLayer, nil + } + if layerIndex > len(configFile.RootFS.DiffIDs) { + return nil, fmt.Errorf("failed to find layer for index %d in config file", layerIndex) + } + if layerIndex > (len(manifestFile.Layers)) { + return nil, fmt.Errorf("failed to find layer for index %d in manifest file", layerIndex) + } + return &v1LayerFacade{ + Layer: originalLayer, + diffID: configFile.RootFS.DiffIDs[layerIndex], + digest: manifestFile.Layers[layerIndex].Digest, + size: manifestFile.Layers[layerIndex].Size, + }, nil +} + +func hasData(layer v1.Layer) bool { + if rc, err := layer.Compressed(); err == nil { + defer rc.Close() + return true + } + return false +} diff --git a/locallayout/image.go b/locallayout/image.go index 99a27919..f48c434b 100644 --- a/locallayout/image.go +++ b/locallayout/image.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "strings" - "sync" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -15,12 +14,23 @@ import ( // Image wraps an imgutil.CNBImageCore and implements the methods needed to complete the imgutil.Image interface. type Image struct { *imgutil.CNBImageCore + repoName string + store *Store lastIdentifier string daemonOS string - downloadOnce *sync.Once } -var _ imgutil.Image = &Image{} +func (i *Image) Kind() string { + return "locallayout" +} + +func (i *Image) Name() string { + return i.repoName +} + +func (i *Image) Rename(name string) { + i.repoName = name +} func (i *Image) Found() bool { return i.lastIdentifier != "" @@ -67,11 +77,7 @@ func (i *Image) GetLayer(diffID string) (io.ReadCloser, error) { } func (i *Image) ensureLayers() error { - var err error - i.downloadOnce.Do(func() { - err = i.Store.DownloadLayersFor(i.lastIdentifier) - }) - if err != nil { + if err := i.store.downloadLayersFor(i.lastIdentifier); err != nil { return fmt.Errorf("fetching base layers: %w", err) } return nil @@ -92,21 +98,27 @@ func (i *Image) Rebase(baseTopLayerDiffID string, withNewBase imgutil.Image) err } func (i *Image) Save(additionalNames ...string) error { - var err error - i.lastIdentifier, err = i.Store.Save(i, i.Name(), additionalNames...) + err := i.SetCreatedAtAndHistory() + if err != nil { + return err + } + i.lastIdentifier, err = i.store.Save(i, i.Name(), additionalNames...) return err } func (i *Image) SaveAs(name string, additionalNames ...string) error { - var err error - i.lastIdentifier, err = i.Store.Save(i, name, additionalNames...) + err := i.SetCreatedAtAndHistory() + if err != nil { + return err + } + i.lastIdentifier, err = i.store.Save(i, name, additionalNames...) return err } func (i *Image) SaveFile() (string, error) { - return i.Store.SaveFile(i, i.Name()) + return i.store.SaveFile(i, i.Name()) } func (i *Image) Delete() error { - return i.Store.Delete(i.lastIdentifier) + return i.store.Delete(i.lastIdentifier) } diff --git a/locallayout/image_test.go b/locallayout/image_test.go index 40ac66df..61f0d848 100644 --- a/locallayout/image_test.go +++ b/locallayout/image_test.go @@ -27,7 +27,7 @@ const someSHA = "sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6cc var localTestRegistry *h.DockerRegistry -func TestLocal(t *testing.T) { +func TestLocalLayout(t *testing.T) { localTestRegistry = h.NewDockerRegistry() localTestRegistry.Start(t) defer localTestRegistry.Stop(t) @@ -709,10 +709,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) { }) }) - when.Pend("#CreatedAt", func() { - // Previously, we only zeroed CreatedAt at the point of save. - // Now, we zero CreatedAt at the point of instantiation. - // If this behavior change is acceptable, we can remove this test. + when("#CreatedAt", func() { it("returns the containers created at time", func() { img, err := local.NewImage(newTestImageName(), dockerClient, local.FromBaseImage(runnableBaseImageName)) h.AssertNil(t, err) diff --git a/locallayout/new.go b/locallayout/new.go index a6eb4057..1ca41033 100644 --- a/locallayout/new.go +++ b/locallayout/new.go @@ -8,6 +8,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/image" "github.com/docker/docker/client" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/buildpacks/imgutil" ) @@ -26,35 +27,41 @@ func NewImage(repoName string, dockerClient DockerClient, ops ...func(*imgutil.I return nil, err } - options.PreviousImage, err = processPreviousImageOption(options.PreviousImageRepoName, dockerClient) + processPrevious, err := processImageOption(options.PreviousImageRepoName, dockerClient, true) if err != nil { return nil, err } + if processPrevious.image != nil { + options.PreviousImage = processPrevious.image + } var ( baseIdentifier string - store imgutil.ImageStore = &Store{dockerClient: dockerClient} + store *Store ) - baseImage, err := processBaseImageOption(options.BaseImageRepoName, dockerClient) + processBase, err := processImageOption(options.BaseImageRepoName, dockerClient, false) if err != nil { return nil, err } - if baseImage != nil { - options.BaseImage = baseImage - baseIdentifier = baseImage.identifier - store = baseImage.store + if processBase.image != nil { + options.BaseImage = processBase.image + baseIdentifier = processBase.identifier + store = processBase.layerStore + } else { + store = &Store{dockerClient: dockerClient, downloadOnce: &sync.Once{}} } - cnbImage, err := imgutil.NewCNBImage(repoName, store, *options) + cnbImage, err := imgutil.NewCNBImage(*options) if err != nil { return nil, err } return &Image{ CNBImageCore: cnbImage, + repoName: repoName, + store: store, lastIdentifier: baseIdentifier, daemonOS: options.Platform.OS, - downloadOnce: &sync.Once{}, }, nil } @@ -84,32 +91,33 @@ func defaultPlatform(dockerClient DockerClient) (imgutil.Platform, error) { }, nil } -func processPreviousImageOption(repoName string, dockerClient DockerClient) (*v1ImageFacade, error) { - if repoName == "" { - return nil, nil - } - inspect, history, err := getInspectAndHistory(repoName, dockerClient) - if err != nil { - return nil, err - } - if inspect == nil { - return nil, nil - } - return newV1ImageFacadeFromInspect(*inspect, history, dockerClient, true) +type imageResult struct { + image v1.Image + identifier string + layerStore *Store } -func processBaseImageOption(repoName string, dockerClient DockerClient) (*v1ImageFacade, error) { +func processImageOption(repoName string, dockerClient DockerClient, downloadLayersOnAccess bool) (imageResult, error) { if repoName == "" { - return nil, nil + return imageResult{}, nil } inspect, history, err := getInspectAndHistory(repoName, dockerClient) if err != nil { - return nil, err + return imageResult{}, err } if inspect == nil { - return nil, nil + return imageResult{}, nil } - return newV1ImageFacadeFromInspect(*inspect, history, dockerClient, false) + layerStore := &Store{dockerClient: dockerClient, downloadOnce: &sync.Once{}} + image, err := newV1ImageFacadeFromInspect(*inspect, history, layerStore, downloadLayersOnAccess) + if err != nil { + return imageResult{}, err + } + return imageResult{ + image: image, + identifier: inspect.ID, + layerStore: layerStore, + }, nil } func getInspectAndHistory(repoName string, dockerClient DockerClient) (*types.ImageInspect, []image.HistoryResponseItem, error) { diff --git a/locallayout/store.go b/locallayout/store.go index f001f6ff..08e98ed0 100644 --- a/locallayout/store.go +++ b/locallayout/store.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/image" @@ -17,6 +18,7 @@ import ( "github.com/docker/docker/pkg/jsonmessage" registryName "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" "golang.org/x/sync/errgroup" "github.com/buildpacks/imgutil" @@ -29,6 +31,7 @@ type Store struct { // required dockerClient DockerClient // optional + downloadOnce *sync.Once onDiskLayers []v1.Layer } @@ -44,8 +47,6 @@ type DockerClient interface { ServerVersion(ctx context.Context) (types.Version, error) } -var _ imgutil.ImageStore = &Store{} - // images func (s *Store) Contains(identifier string) bool { @@ -65,17 +66,13 @@ func (s *Store) Delete(identifier string) error { return err } -func (s *Store) Save(image imgutil.IdentifiableV1Image, withName string, withAdditionalNames ...string) (string, error) { +func (s *Store) Save(image *Image, withName string, withAdditionalNames ...string) (string, error) { withName = tryNormalizing(withName) // save inspect, err := s.doSave(image, withName) if err != nil { - identifier, err := image.Identifier() - if err != nil { - return "", err - } - if err = s.DownloadLayersFor(identifier.String()); err != nil { + if err = image.ensureLayers(); err != nil { return "", err } inspect, err = s.doSave(image, withName) @@ -223,7 +220,7 @@ func (s *Store) addLayerToTar(tw *tar.Writer, layer v1.Layer) (string, error) { } withName := fmt.Sprintf("/%s.tar", layerDiffID.String()) - uncompressedSize, err := getLayerSize(layer) + uncompressedSize, err := getLayerSize(layer) // FIXME: this degrades performance compared to `local` package if err != nil { return "", err } @@ -292,7 +289,7 @@ func ensureReaderClosed(r io.ReadCloser) error { return err } -func (s *Store) SaveFile(image imgutil.IdentifiableV1Image, withName string) (string, error) { +func (s *Store) SaveFile(image *Image, withName string) (string, error) { withName = tryNormalizing(withName) f, err := os.CreateTemp("", "imgutil.local.image.export.*.tar") @@ -310,11 +307,7 @@ func (s *Store) SaveFile(image imgutil.IdentifiableV1Image, withName string) (st // (1) WithPreviousImage(), or (2) FromBaseImage(). // The former is only relevant if ReuseLayers() has been called which takes care of resolving them. // The latter case needs to be handled explicitly. - identifier, err := image.Identifier() - if err != nil { - return "", err - } - if err = s.DownloadLayersFor(identifier.String()); err != nil { + if err = image.ensureLayers(); err != nil { return "", err } @@ -347,40 +340,39 @@ func (s *Store) SaveFile(image imgutil.IdentifiableV1Image, withName string) (st // layers -func (s *Store) DownloadLayersFor(identifier string) error { - layers, err := downloadLayersFor(identifier, s.dockerClient) - if err != nil { - return err - } - s.onDiskLayers = append(s.onDiskLayers, layers...) - return nil +func (s *Store) downloadLayersFor(identifier string) error { + var err error + s.downloadOnce.Do(func() { + err = s.doDownloadLayersFor(identifier) + }) + return err } -func downloadLayersFor(identifier string, dockerClient DockerClient) ([]v1.Layer, error) { +func (s *Store) doDownloadLayersFor(identifier string) error { if identifier == "" { - return nil, nil + return nil } ctx := context.Background() - imageReader, err := dockerClient.ImageSave(ctx, []string{identifier}) + imageReader, err := s.dockerClient.ImageSave(ctx, []string{identifier}) if err != nil { - return nil, fmt.Errorf("saving base image with ID %q from the docker daemon: %w", identifier, err) + return fmt.Errorf("saving base image with ID %q from the docker daemon: %w", identifier, err) } defer ensureReaderClosed(imageReader) tmpDir, err := os.MkdirTemp("", "imgutil.local.image.") if err != nil { - return nil, fmt.Errorf("failed to create temp dir: %w", err) + return fmt.Errorf("failed to create temp dir: %w", err) } err = untar(imageReader, tmpDir) if err != nil { - return nil, err + return err } mf, err := os.Open(filepath.Clean(filepath.Join(tmpDir, "manifest.json"))) if err != nil { - return nil, err + return err } defer mf.Close() @@ -389,15 +381,15 @@ func downloadLayersFor(identifier string, dockerClient DockerClient) ([]v1.Layer Layers []string } if err := json.NewDecoder(mf).Decode(&manifest); err != nil { - return nil, err + return err } if len(manifest) != 1 { - return nil, fmt.Errorf("manifest.json had unexpected number of entries: %d", len(manifest)) + return fmt.Errorf("manifest.json had unexpected number of entries: %d", len(manifest)) } cfg, err := os.Open(filepath.Clean(filepath.Join(tmpDir, manifest[0].Config))) if err != nil { - return nil, err + return err } defer cfg.Close() var configFile struct { @@ -406,22 +398,17 @@ func downloadLayersFor(identifier string, dockerClient DockerClient) ([]v1.Layer } `json:"rootfs"` } if err = json.NewDecoder(cfg).Decode(&configFile); err != nil { - return nil, err + return err } - layers := make([]v1.Layer, len(configFile.RootFS.DiffIDs)) - for idx, diffID := range configFile.RootFS.DiffIDs { - var h v1.Hash - h, err = v1.NewHash(diffID) + for idx := range configFile.RootFS.DiffIDs { + layer, err := tarball.LayerFromFile(filepath.Join(tmpDir, manifest[0].Layers[idx])) if err != nil { - return nil, err - } - layers[idx] = &v1LayerFacade{ - diffID: h, - optionalLayerPath: filepath.Join(tmpDir, manifest[0].Layers[idx]), + return err } + s.onDiskLayers = append(s.onDiskLayers, layer) } - return layers, nil + return nil } func untar(r io.Reader, dest string) error { @@ -488,6 +475,23 @@ func cleanPath(dest, header string) (string, error) { return "", fmt.Errorf("bad filepath: %s", header) } -func (s *Store) Layers() []v1.Layer { - return s.onDiskLayers +func (s *Store) LayerByDiffID(h v1.Hash) (v1.Layer, error) { + layer := findLayer(h, s.onDiskLayers) + if layer == nil { + return nil, fmt.Errorf("failed to find layer with diff ID %q", h.String()) + } + return layer, nil +} + +func findLayer(withHash v1.Hash, inLayers []v1.Layer) v1.Layer { + for _, layer := range inLayers { + layerHash, err := layer.DiffID() + if err != nil { + continue + } + if layerHash.String() == withHash.String() { + return layer + } + } + return nil } diff --git a/locallayout/v1_facade.go b/locallayout/v1_facade.go index af8fbb61..35418e29 100644 --- a/locallayout/v1_facade.go +++ b/locallayout/v1_facade.go @@ -4,8 +4,6 @@ import ( "bytes" "fmt" "io" - "os" - "sync" "time" "github.com/docker/docker/api/types" @@ -19,68 +17,12 @@ import ( "github.com/buildpacks/imgutil" ) -// v1ImageFacade wraps a v1.Image constructed from the output of `docker inspect`. +// newV1ImageFacadeFromInspect returns a v1.Image constructed from the output of `docker inspect`. // It is used to provide a v1.Image implementation for previous images and base images. -// The v1ImageFacade is never modified, but it may become the underlying v1.Image for imgutil.CNBImageCore images. -// A v1ImageFacade will try to return layer data if the layers exist on disk, -// otherwise it will return empty layer data. -// By storing a pointer to the image store, users can update the store to force a v1ImageFacade to return layer data. -type v1ImageFacade struct { - v1.Image - emptyLayers []v1.Layer - - // for downloading layers from the daemon as needed - store imgutil.ImageStore - downloadLayersOnAccess bool // set to true to downloading ALL the image layers from the daemon when LayerByDiffID is called - downloadOnce *sync.Once - identifier string -} - -var _ v1.Image = &v1ImageFacade{} - -func (i *v1ImageFacade) LayerByDiffID(h v1.Hash) (v1.Layer, error) { - if layer := findLayer(h, i.store.Layers()); layer != nil { - return layer, nil - } - if i.downloadLayersOnAccess { - if err := i.ensureLayers(); err != nil { - return nil, err - } - if layer := findLayer(h, i.store.Layers()); layer != nil { - return layer, nil - } - } - if layer := findLayer(h, i.emptyLayers); layer != nil { - return layer, nil - } - return nil, fmt.Errorf("failed to find layer with diff ID %q", h.String()) -} - -func (i *v1ImageFacade) ensureLayers() error { - var err error - i.downloadOnce.Do(func() { - err = i.store.DownloadLayersFor(i.identifier) - }) - if err != nil { - return fmt.Errorf("fetching base layers: %w", err) - } - return nil -} - -func findLayer(withHash v1.Hash, inLayers []v1.Layer) v1.Layer { - for _, layer := range inLayers { - layerHash, err := layer.DiffID() - if err != nil { - continue - } - if layerHash.String() == withHash.String() { - return layer - } - } - return nil -} - -func newV1ImageFacadeFromInspect(dockerInspect types.ImageInspect, history []image.HistoryResponseItem, dockerClient DockerClient, downloadLayersOnAccess bool) (*v1ImageFacade, error) { +// The facade is never modified, but it may become the underlying v1.Image for imgutil.CNBImageCore images. +// The underlying layers will return data if they are contained in the store. +// By storing a pointer to the image store, callers can update the store to force the layers to return data. +func newV1ImageFacadeFromInspect(dockerInspect types.ImageInspect, history []image.HistoryResponseItem, withStore *Store, downloadLayersOnAccess bool) (v1.Image, error) { rootFS, err := toV1RootFS(dockerInspect.RootFS) if err != nil { return nil, err @@ -98,18 +40,61 @@ func newV1ImageFacadeFromInspect(dockerInspect types.ImageInspect, history []ima OSVersion: dockerInspect.OsVersion, Variant: dockerInspect.Variant, } - img, err := mutate.ConfigFile(empty.Image, configFile) + layersToSet := newEmptyLayerListFrom(configFile, dockerInspect.ID, withStore, downloadLayersOnAccess) + return imageFrom(layersToSet, configFile, imgutil.DockerTypes) +} + +func imageFrom(layers []v1.Layer, configFile *v1.ConfigFile, requestedTypes imgutil.MediaTypes) (v1.Image, error) { + // (1) construct a new image with the right manifest media type + manifestType := requestedTypes.ManifestType() + retImage := mutate.MediaType(empty.Image, manifestType) + + // (2) set config media type + configType := requestedTypes.ConfigType() + // zero out history and diff IDs, as these will be updated when we call `mutate.Append` to add the layers + beforeHistory := imgutil.NormalizedHistory(configFile.History, len(configFile.RootFS.DiffIDs)) + configFile.History = []v1.History{} + configFile.RootFS.DiffIDs = make([]v1.Hash, 0) + // set config + var err error + retImage, err = mutate.ConfigFile(retImage, configFile) if err != nil { return nil, err } - return &v1ImageFacade{ - Image: img, - emptyLayers: newEmptyLayerListFrom(configFile), - store: &Store{dockerClient: dockerClient}, - downloadLayersOnAccess: downloadLayersOnAccess, - downloadOnce: &sync.Once{}, - identifier: dockerInspect.ID, - }, nil + retImage = mutate.ConfigMediaType(retImage, configType) + // (3) set layers with the right media type + additions := layersAddendum(layers, beforeHistory, requestedTypes.LayerType()) + if err != nil { + return nil, err + } + retImage, err = mutate.Append(retImage, additions...) + if err != nil { + return nil, err + } + afterLayers, err := retImage.Layers() + if err != nil { + return nil, err + } + if len(afterLayers) != len(layers) { + return nil, fmt.Errorf("found %d layers for image; expected %d", len(afterLayers), len(layers)) + } + return retImage, nil +} + +func layersAddendum(layers []v1.Layer, history []v1.History, requestedType v1types.MediaType) []mutate.Addendum { + addendums := make([]mutate.Addendum, 0) + if len(history) != len(layers) { + history = make([]v1.History, len(layers)) + } + for idx, layer := range layers { + layerType := requestedType + addendums = append(addendums, mutate.Addendum{ + Layer: layer, + History: history[idx], + MediaType: layerType, + }) + } + return addendums } func toV1RootFS(dockerRootFS types.RootFS) (v1.RootFS, error) { @@ -197,58 +182,79 @@ func toV1Config(dockerCfg *container.Config) v1.Config { var _ v1.Layer = &v1LayerFacade{} type v1LayerFacade struct { - diffID v1.Hash - optionalLayerPath string + diffID v1.Hash + store *Store + // for downloading layers from the daemon as needed + downloadOnAccess bool + imageIdentifier string } -func newEmptyLayerListFrom(configFile *v1.ConfigFile) []v1.Layer { +func newEmptyLayerListFrom(configFile *v1.ConfigFile, withImageIdentifier string, withStore *Store, downloadOnAccess bool) []v1.Layer { layers := make([]v1.Layer, len(configFile.RootFS.DiffIDs)) for idx, diffID := range configFile.RootFS.DiffIDs { layers[idx] = &v1LayerFacade{ - diffID: diffID, + diffID: diffID, + store: withStore, + downloadOnAccess: downloadOnAccess, + imageIdentifier: withImageIdentifier, } } return layers } -func (l v1LayerFacade) Digest() (v1.Hash, error) { - return l.diffID, nil +func (l v1LayerFacade) Compressed() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader([]byte{})), nil } func (l v1LayerFacade) DiffID() (v1.Hash, error) { return l.diffID, nil } -func (l v1LayerFacade) Compressed() (io.ReadCloser, error) { - if l.optionalLayerPath != "" { - f, err := os.Open(l.optionalLayerPath) - if err != nil { - return nil, err - } - return f, nil - } - return io.NopCloser(bytes.NewReader([]byte{})), nil +func (l v1LayerFacade) Digest() (v1.Hash, error) { + return v1.Hash{}, nil } func (l v1LayerFacade) Uncompressed() (io.ReadCloser, error) { - if l.optionalLayerPath != "" { - f, err := os.Open(l.optionalLayerPath) - if err != nil { - return nil, err - } - return f, nil + layer, err := l.store.LayerByDiffID(l.diffID) + if err == nil { + return layer.Uncompressed() } - return io.NopCloser(bytes.NewReader([]byte{})), nil + if !l.downloadOnAccess { + return io.NopCloser(bytes.NewReader([]byte{})), nil + } + if err = l.store.downloadLayersFor(l.imageIdentifier); err != nil { + return nil, err + } + layer, err = l.store.LayerByDiffID(l.diffID) + if err != nil { + return io.NopCloser(bytes.NewReader([]byte{})), nil + } + return layer.Uncompressed() } // Size returns a sentinel value indicating if the layer has data. func (l v1LayerFacade) Size() (int64, error) { - if l.optionalLayerPath != "" { - return 1, nil + layer, err := l.store.LayerByDiffID(l.diffID) + if err == nil { + return layer.Size() + } + if !l.downloadOnAccess { + return -1, nil + } + if err = l.store.downloadLayersFor(l.imageIdentifier); err != nil { + return -1, err } - return -1, nil + layer, err = l.store.LayerByDiffID(l.diffID) + if err != nil { + return -1, nil + } + return layer.Size() } func (l v1LayerFacade) MediaType() (v1types.MediaType, error) { - return v1types.OCILayer, nil + layer, err := l.store.LayerByDiffID(l.diffID) + if err != nil { + return v1types.OCILayer, nil + } + return layer.MediaType() } diff --git a/new.go b/new.go index 7a7c0d69..5dfae9eb 100644 --- a/new.go +++ b/new.go @@ -6,62 +6,40 @@ import ( "fmt" "io" "os" + "time" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/buildpacks/imgutil/layer" ) -func NewCNBImage(repoName string, store ImageStore, options ImageOptions) (*CNBImageCore, error) { +func NewCNBImage(options ImageOptions) (*CNBImageCore, error) { image := &CNBImageCore{ - Image: options.BaseImage, - Store: store, - // required - repoName: repoName, - // optional - preferredMediaTypes: options.MediaTypes, + Image: options.BaseImage, // the working image + createdAt: getCreatedAt(options), + preferredMediaTypes: GetPreferredMediaTypes(options), preserveHistory: options.PreserveHistory, previousImage: options.PreviousImage, } + // ensure base image var err error if image.Image == nil { - image.Image, err = emptyV1(options.Platform) + image.Image, err = emptyV1(options.Platform, image.preferredMediaTypes) if err != nil { return nil, err } } - if image.Image, err = OverrideMediaTypes(image.Image, options.MediaTypes); err != nil { - return nil, err - } + // ensure windows if err = prepareNewWindowsImageIfNeeded(image); err != nil { return nil, err } - createdAt := NormalizedDateTime - if !options.CreatedAt.IsZero() { - createdAt = options.CreatedAt - } - if err = image.MutateConfigFile(func(c *v1.ConfigFile) { - c.Created = v1.Time{Time: createdAt} - c.Container = "" - }); err != nil { - return nil, err - } - - if !options.PreserveHistory { - if err = image.MutateConfigFile(func(c *v1.ConfigFile) { - for j := range c.History { - c.History[j] = v1.History{Created: v1.Time{Time: createdAt}} - } - }); err != nil { - return nil, err - } - } - + // set config if requested if options.Config != nil { if err = image.MutateConfigFile(func(c *v1.ConfigFile) { c.Config = *options.Config @@ -73,7 +51,69 @@ func NewCNBImage(repoName string, store ImageStore, options ImageOptions) (*CNBI return image, nil } -func emptyV1(withPlatform Platform) (v1.Image, error) { +func getCreatedAt(options ImageOptions) time.Time { + if !options.CreatedAt.IsZero() { + return options.CreatedAt + } + return NormalizedDateTime +} + +var NormalizedDateTime = time.Date(1980, time.January, 1, 0, 0, 1, 0, time.UTC) + +func GetPreferredMediaTypes(options ImageOptions) MediaTypes { + if options.MediaTypes != MissingTypes { + return options.MediaTypes + } + if options.MediaTypes == MissingTypes && + options.BaseImage == nil { + return OCITypes + } + return DefaultTypes +} + +type MediaTypes int + +const ( + MissingTypes MediaTypes = iota + DefaultTypes + OCITypes + DockerTypes +) + +func (t MediaTypes) ManifestType() types.MediaType { + switch t { + case OCITypes: + return types.OCIManifestSchema1 + case DockerTypes: + return types.DockerManifestSchema2 + default: + return "" + } +} + +func (t MediaTypes) ConfigType() types.MediaType { + switch t { + case OCITypes: + return types.OCIConfigJSON + case DockerTypes: + return types.DockerConfigJSON + default: + return "" + } +} + +func (t MediaTypes) LayerType() types.MediaType { + switch t { + case OCITypes: + return types.OCILayer + case DockerTypes: + return types.DockerLayer + default: + return "" + } +} + +func emptyV1(withPlatform Platform, withMediaTypes MediaTypes) (v1.Image, error) { configFile := &v1.ConfigFile{ Architecture: withPlatform.Architecture, History: []v1.History{}, @@ -84,7 +124,137 @@ func emptyV1(withPlatform Platform) (v1.Image, error) { DiffIDs: []v1.Hash{}, }, } - return mutate.ConfigFile(empty.Image, configFile) + image, err := mutate.ConfigFile(empty.Image, configFile) + if err != nil { + return nil, err + } + image, _, err = EnsureMediaTypesAndLayers(image, withMediaTypes, PreserveLayers) + return image, err +} + +func PreserveLayers(_ int, layer v1.Layer) (v1.Layer, error) { + return layer, nil +} + +// EnsureMediaTypesAndLayers replaces the provided image with a new image that has the desired media types. +// It does this by constructing a manifest and config from the provided image, +// and adding the layers from the provided image to the new image with the right media type. +// If requested types are missing or default, it does nothing. +// While adding the layers, each layer can be additionally mutated by providing a "mutate layer" function. +func EnsureMediaTypesAndLayers(image v1.Image, requestedTypes MediaTypes, mutateLayer func(idx int, layer v1.Layer) (v1.Layer, error)) (v1.Image, bool, error) { + if requestedTypes == MissingTypes || requestedTypes == DefaultTypes { + return image, false, nil + } + // (1) get data from the original image + // manifest + beforeManifest, err := image.Manifest() + if err != nil { + return nil, false, fmt.Errorf("failed to get manifest: %w", err) + } + // config + beforeConfig, err := image.ConfigFile() + if err != nil { + return nil, false, fmt.Errorf("failed to get config: %w", err) + } + // layers + beforeLayers, err := image.Layers() + if err != nil { + return nil, false, fmt.Errorf("failed to get layers: %w", err) + } + var layersToAdd []v1.Layer + for idx, l := range beforeLayers { + layer, err := mutateLayer(idx, l) + if err != nil { + return nil, false, fmt.Errorf("failed to mutate layer: %w", err) + } + layersToAdd = append(layersToAdd, layer) + } + + // (2) construct a new image manifest with the right media type + manifestType := requestedTypes.ManifestType() + if manifestType == "" { + manifestType = beforeManifest.MediaType + } + retImage := mutate.MediaType(empty.Image, manifestType) + + // (3) set config with the right media type + configType := requestedTypes.ConfigType() + if configType == "" { + configType = beforeManifest.Config.MediaType + } + // zero out diff IDs and history, these will be added back when we append the layers + beforeHistory := beforeConfig.History + beforeConfig.History = []v1.History{} + beforeConfig.RootFS.DiffIDs = []v1.Hash{} + retImage, err = mutate.ConfigFile(retImage, beforeConfig) + if err != nil { + return nil, false, fmt.Errorf("failed to set config: %w", err) + } + retImage = mutate.ConfigMediaType(retImage, configType) + + // (4) set layers with the right media type + additions := layersAddendum(layersToAdd, beforeHistory, requestedTypes.LayerType()) + if err != nil { + return nil, false, err + } + retImage, err = mutate.Append(retImage, additions...) + if err != nil { + return nil, false, fmt.Errorf("failed to append layers: %w", err) + } + + // (5) force compute + afterLayers, err := retImage.Layers() + if err != nil { + return nil, false, fmt.Errorf("failed to get layers: %w", err) + } + if len(afterLayers) != len(beforeLayers) { + return nil, false, fmt.Errorf("expected %d layers; got %d", len(beforeLayers), len(afterLayers)) + } + + return retImage, true, nil +} + +// layersAddendum creates an Addendum array with the given layers +// and the desired media type +func layersAddendum(layers []v1.Layer, history []v1.History, requestedType types.MediaType) []mutate.Addendum { + addendums := make([]mutate.Addendum, 0) + history = NormalizedHistory(history, len(layers)) + if len(history) != len(layers) { + history = make([]v1.History, len(layers)) + } + var err error + for idx, l := range layers { + layerType := requestedType + if requestedType == "" { + // try to get a non-empty media type + if layerType, err = l.MediaType(); err != nil { + layerType = "" + } + } + addendums = append(addendums, mutate.Addendum{ + Layer: l, + History: history[idx], + MediaType: layerType, + }) + } + return addendums +} + +func NormalizedHistory(history []v1.History, nLayers int) []v1.History { + if history == nil { + return make([]v1.History, nLayers) + } + // ensure we remove history for empty layers + var normalizedHistory []v1.History + for _, h := range history { + if !h.EmptyLayer { + normalizedHistory = append(normalizedHistory, h) + } + } + if len(normalizedHistory) == nLayers { + return normalizedHistory + } + return make([]v1.History, nLayers) } func prepareNewWindowsImageIfNeeded(image *CNBImageCore) error { @@ -92,6 +262,7 @@ func prepareNewWindowsImageIfNeeded(image *CNBImageCore) error { if err != nil { return err } + // only append base layer to empty image if !(configFile.OS == "windows") || len(configFile.RootFS.DiffIDs) > 0 { return nil diff --git a/options.go b/options.go index ed058a61..061e33a3 100644 --- a/options.go +++ b/options.go @@ -8,12 +8,14 @@ import ( type ImageOptions struct { BaseImageRepoName string + PreviousImageRepoName string Config *v1.Config CreatedAt time.Time + MediaTypes MediaTypes Platform Platform + PreserveDigest bool PreserveHistory bool - PreviousImageRepoName string - MediaTypes MediaTypes + WithoutLayers bool // only relevant for layout images // These options are specified in each implementation's image constructor BaseImage v1.Image diff --git a/remote/new.go b/remote/new.go index 72aedd3e..a392de57 100644 --- a/remote/new.go +++ b/remote/new.go @@ -292,7 +292,7 @@ func (i *Image) setUnderlyingImage(base v1.Image) error { return nil } // provided v1.Image media types differ from requested, override them - newBase, err := imgutil.OverrideMediaTypes(base, i.requestedMediaTypes) + newBase, _, err := imgutil.EnsureMediaTypesAndLayers(base, i.requestedMediaTypes, imgutil.PreserveLayers) if err != nil { return err } diff --git a/testhelpers/testhelpers.go b/testhelpers/testhelpers.go index cb1abc40..288bae5a 100644 --- a/testhelpers/testhelpers.go +++ b/testhelpers/testhelpers.go @@ -22,20 +22,18 @@ import ( "testing" "time" - "github.com/google/go-containerregistry/pkg/v1/types" - - "github.com/buildpacks/imgutil/layer" - dockertypes "github.com/docker/docker/api/types" dockercli "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" - "github.com/google/go-cmp/cmp" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/pkg/errors" + + "github.com/buildpacks/imgutil/layer" ) func RandString(n int) string {