Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fast-load images from remote pull service #277

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fast-load images from remote pull service #277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 31 additions & 225 deletions pkg/load/load.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"fmt"
"runtime"
"strings"
"time"

depotbuild "github.com/depot/cli/pkg/buildx/build"
depotprogress "github.com/depot/cli/pkg/progress"
Expand All @@ -33,40 +32,44 @@ func DepotFastLoad(ctx context.Context, dockerapi docker.APIClient, resp []depot
nodeRes := chooseNodeResponse(buildRes.NodeResponses)
pullOpt := pullOpts[buildRes.Name]

architecture := nodeRes.Node.DriverOpts["platform"]
manifest, config, err := decodeNodeResponse(architecture, nodeRes)
if err != nil {
return err
digest := nodeRes.SolveResponse.ExporterResponse[exptypes.ExporterImageDigestKey]
if v, ok := nodeRes.SolveResponse.ExporterResponse[exptypes.ExporterImageConfigDigestKey]; ok {
digest = v
}
if digest == "" {
return errors.New("missing image digest")
}
proxyOpts := &ProxyConfig{
RawManifest: manifest,
RawConfig: config,
Addr: nodeRes.Node.DriverOpts["addr"],
CACert: []byte(nodeRes.Node.DriverOpts["caCert"]),
Key: []byte(nodeRes.Node.DriverOpts["key"]),
Cert: []byte(nodeRes.Node.DriverOpts["cert"]),

info := struct {
Address string `json:"address"`
Cert string `json:"cert"`
Key string `json:"key"`
CaCert string `json:"caCert"`
}{
Address: nodeRes.Node.DriverOpts["addr"],
Cert: base64.StdEncoding.EncodeToString([]byte(nodeRes.Node.DriverOpts["cert"])),
Key: base64.StdEncoding.EncodeToString([]byte(nodeRes.Node.DriverOpts["key"])),
CaCert: base64.StdEncoding.EncodeToString([]byte(nodeRes.Node.DriverOpts["caCert"])),
}

// Start the depot registry proxy.
var registry *RegistryProxy
err = progress.Wrap("preparing to load", pw.Write, func(logger progress.SubLogger) error {
registry, err = NewRegistryProxy(ctx, proxyOpts, dockerapi)
if err != nil {
err = logger.Wrap(fmt.Sprintf("[registry] unable to start: %s", err), func() error { return err })
}
return err
})
username := "x-info"
passwordBytes, err := json.Marshal(info)
if err != nil {
return err
return fmt.Errorf("failed to marshal info: %w", err)
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
registry.Close(ctx)
cancel()
}()
password := string(passwordBytes)
serverAddress := "depot-pull.fly.dev" // TODO: move this to the API

pullOpt.Username = &username
pullOpt.Password = &password
pullOpt.ServerAddress = &serverAddress

randomImageName := RandImageName()
tag := "manifest"
imageToPull := fmt.Sprintf("%s/%s:%s@%s", serverAddress, randomImageName, tag, digest)

// Pull the image and relabel it with the user specified tags.
err = PullImages(ctx, dockerapi, registry.ImageToPull, pullOpt, pw)
err = PullImages(ctx, dockerapi, imageToPull, pullOpt, pw)
if err != nil {
return fmt.Errorf("failed to pull image: %w", err)
}
Expand Down Expand Up @@ -94,44 +97,6 @@ func chooseNodeResponse(nodeResponses []depotbuild.DepotNodeResponse) depotbuild
// ImageExported is the solve response key added for `depot.export.image.version=2`.
const ImagesExported = "depot/images.exported"

func decodeNodeResponse(architecture string, nodeRes depotbuild.DepotNodeResponse) (rawManifest, rawConfig []byte, err error) {
if _, err := EncodedExportedImages(nodeRes.SolveResponse.ExporterResponse); err == nil {
return decodeNodeResponseV2(architecture, nodeRes)
}

// Needed until all depot builds and CLI versions are updated.
return decodeNodeResponseV1(architecture, nodeRes)
}

func decodeNodeResponseV2(architecture string, nodeRes depotbuild.DepotNodeResponse) (rawManifest, rawConfig []byte, err error) {
encodedExportedImages, err := EncodedExportedImages(nodeRes.SolveResponse.ExporterResponse)
if err != nil {
return nil, nil, err
}

exportedImages, _, imageConfigs, err := DecodeExportImages(encodedExportedImages)
if err != nil {
return nil, nil, err
}

idx, err := chooseBestImageManifestV2(architecture, imageConfigs)
if err != nil {
return nil, nil, err
}

return exportedImages[idx].Manifest, exportedImages[idx].Config, nil
}

// EncodedExportedImages returns the encoded exported images from the solve response.
// This uses the `depot.export.image.version=2` format.
func EncodedExportedImages(exporterResponse map[string]string) (string, error) {
encodedExportedImages, ok := exporterResponse[ImagesExported]
if !ok {
return "", errors.New("missing image export response")
}
return encodedExportedImages, nil
}

// RawExportedImage is the JSON-encoded image manifest and config used loading the image.
type RawExportedImage struct {
// JSON-encoded ocispecs.Manifest.
Expand Down Expand Up @@ -177,162 +142,3 @@ func DecodeExportImages(encodedExportedImages string) ([]RawExportedImage, []oci

return exportedImages, manifests, imageConfigs, nil
}

// We encode the image manifest and image config within the buildkitd Solve response
// because the content may be GCed by the time this load occurs.
func decodeNodeResponseV1(architecture string, nodeRes depotbuild.DepotNodeResponse) (rawManifest, rawConfig []byte, err error) {
encodedDesc, ok := nodeRes.SolveResponse.ExporterResponse[exptypes.ExporterImageDescriptorKey]
if !ok {
return nil, nil, errors.New("missing image descriptor")
}

jsonImageDesc, err := base64.StdEncoding.DecodeString(encodedDesc)
if err != nil {
return nil, nil, fmt.Errorf("invalid image descriptor: %w", err)
}

var imageDesc ocispecs.Descriptor
if err := json.Unmarshal(jsonImageDesc, &imageDesc); err != nil {
return nil, nil, fmt.Errorf("invalid image descriptor json: %w", err)
}

var imageManifest ocispecs.Descriptor = imageDesc
{
// These checks handle situations where the image does and does not have attestations.
// If there are no attestations, then the imageDesc contains the manifest and config.
// Otherwise the imageDesc's `depot.containerimage.index` will contain the manifest and config.

encodedIndex, ok := imageDesc.Annotations["depot.containerimage.index"]
if ok {
var index ocispecs.Index
if err := json.Unmarshal([]byte(encodedIndex), &index); err != nil {
return nil, nil, fmt.Errorf("invalid image index json: %w", err)
}

imageManifest, err = chooseBestImageManifest(architecture, &index)
if err != nil {
return nil, nil, err
}
}
}

m, ok := imageManifest.Annotations["depot.containerimage.manifest"]
if !ok {
return nil, nil, errors.New("missing image manifest")
}
rawManifest = []byte(m)

c, ok := imageManifest.Annotations["depot.containerimage.config"]
if !ok {
return nil, nil, errors.New("missing image config")
}
rawConfig = []byte(c)

// Decoding both the manifest and config to ensure they are valid.
var manifest ocispecs.Manifest
if err := json.Unmarshal(rawManifest, &manifest); err != nil {
return nil, nil, fmt.Errorf("invalid image manifest json: %w", err)
}

var image ocispecs.Image
if err := json.Unmarshal(rawConfig, &image); err != nil {
return nil, nil, fmt.Errorf("invalid image config json: %w", err)
}
return rawManifest, rawConfig, nil
}

type RegistryProxy struct {
// ImageToPull is the image that should be pulled.
ImageToPull string
// ProxyContainerID is the ID of the container that is proxying the registry.
// Make sure to remove this container when finished.
ProxyContainerID string

// Used to stop and remove the proxy container.
DockerAPI docker.APIClient
}

// NewRegistryProxy creates a registry proxy that can be used to pull images from
// buildkitd cache.
//
// This also handles docker for desktop issues that prevent the registry from being
// accessed directly because the proxy is accessible by the docker daemon.
// The proxy registry translates pull requests into requests to containerd via mTLS.
//
// The running server and proxy container will be cleaned-up when Close() is called.
func NewRegistryProxy(ctx context.Context, config *ProxyConfig, dockerapi docker.APIClient) (*RegistryProxy, error) {
proxyContainer, err := RunProxyImage(ctx, dockerapi, config)
if err != nil {
return nil, err
}

randomImageName := RandImageName()
// The tag is only for the UX during a pull. The first line will be "pulling manifest".
tag := "manifest"
// Docker is able to pull from the proxyPort on localhost. The proxy
// forwards registry requests to buildkitd via mTLS.
imageToPull := fmt.Sprintf("localhost:%s/%s:%s", proxyContainer.Port, randomImageName, tag)

registryProxy := &RegistryProxy{
ImageToPull: imageToPull,
ProxyContainerID: proxyContainer.ID,
DockerAPI: dockerapi,
}

return registryProxy, nil
}

// Close will stop and remove the registry proxy container if it was created.
func (l *RegistryProxy) Close(ctx context.Context) error {
return StopProxyContainer(ctx, l.DockerAPI, l.ProxyContainerID)
}

// Prefer architecture, otherwise, take first available index.
func chooseBestImageManifestV2(architecture string, imageConfigs []ocispecs.Image) (int, error) {
archIdx := map[string]int{}
for i, imageConfig := range imageConfigs {
if imageConfig.Architecture == "unknown" {
continue
}

archIdx[imageConfig.Architecture] = i
}

// Prefer the architecture of the depot CLI host, otherwise, take first available.
if idx, ok := archIdx[architecture]; ok {
return idx, nil
}

for _, idx := range archIdx {
return idx, nil
}

return 0, errors.New("no manifests found")
}

// Prefer architecture, otherwise, take first available.
func chooseBestImageManifest(architecture string, index *ocispecs.Index) (ocispecs.Descriptor, error) {
archDescriptors := map[string]ocispecs.Descriptor{}
for _, manifest := range index.Manifests {
if manifest.Platform == nil {
continue
}

if manifest.Platform.Architecture == "unknown" {
continue
}

archDescriptors[manifest.Platform.Architecture] = manifest
}

// Prefer the architecture of the depot CLI host, otherwise, take first available.
if descriptor, ok := archDescriptors[architecture]; ok {
return descriptor, nil
}

for _, descriptor := range archDescriptors {
return descriptor, nil
}

return ocispecs.Descriptor{}, errors.New("no manifests found")
}
Loading
Loading