introducing release_artefacts table for OCI based artefacts

backend/controller/artefacts/hybrid_registry.go

@@ -60,8 +60,12 @@ func (s *hybridRegistry) Download(ctx context.Context, digest sha256.SHA256) (io
 }
 
 func (s *hybridRegistry) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
-	// note: the container and database store currently use release_artefacts to associated
-	return s.container.GetReleaseArtefacts(ctx, releaseID)
+	if ras, err := s.dal.GetReleaseArtefacts(ctx, releaseID); err != nil {
+		return nil, fmt.Errorf("unable to get release artefacts from container store: %w", err)
+	} else if len(ras) > 0 {
+		return ras, nil
+	}
+	return s.dal.GetReleaseArtefacts(ctx, releaseID)
 }
 
 func (s *hybridRegistry) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error {

backend/controller/artefacts/internal/sql/queries.sql

@@ -26,4 +26,15 @@ WHERE deployment_id = $1;
 
 -- name: AssociateArtefactWithDeployment :exec
 INSERT INTO deployment_artefacts (deployment_id, artefact_id, executable, path)
-VALUES ((SELECT id FROM deployments WHERE key = @key::deployment_key), (SELECT id FROM artefacts WHERE digest = @digest::bytea), $3, $4);
+VALUES ((SELECT id FROM deployments WHERE key = @key::deployment_key), (SELECT id FROM artefacts WHERE digest = @digest::bytea), $3, $4);
+
+-- name: GetReleaseArtefacts :many
+-- Get all artefacts associated with the specified release_id
+SELECT created_at, digest, executable, path
+FROM release_artefacts
+WHERE release_id = $1;
+
+-- name: PublishReleaseArtefact :exec
+INSERT INTO release_artefacts(release_id, digest, executable, path)
+VALUES ((SELECT id FROM deployments WHERE key = @key::deployment_key), $2, $3, $4)
+    ON CONFLICT (release_id, digest) DO NOTHING;

backend/controller/artefacts/oci_registry.go

@@ -6,9 +6,9 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"github.com/TBD54566975/ftl/internal/slices"
 	"io"
 
-	"github.com/opencontainers/go-digest"
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content"
 	"oras.land/oras-go/v2/errdef"
@@ -23,14 +23,14 @@ import (
 )
 
 const (
-	ModuleBlobsPrefix = "ftl/modules/"
+	ModuleBlobsPrefix = "ftl/modules"
 )
 
 type ContainerConfig struct {
-	Registry       string `help:"OCI container registry host:port" env:"FTL_ARTEFACTS_REGISTRY"`
-	Username       string `help:"OCI container registry username" env:"FTL_ARTEFACTS_USER"`
-	Password       string `help:"OCI container registry password" env:"FTL_ARTEFACTS_PWD"`
-	AllowPlainHTTP bool   `help:"Allows OCI container requests to accept plain HTTP responses" env:"FTL_ARTEFACTS_ALLOW_HTTP"`
+	Registry          string `help:"OCI container registry host:port" default:"127.0.0.1:5001"  env:"FTL_ARTEFACTS_REGISTRY"`
+	RegistryUsername  string `help:"OCI container registry username" env:"FTL_ARTEFACTS_USER"`
+	RegistryPassword  string `help:"OCI container registry password" env:"FTL_ARTEFACTS_PWD"`
+	RegistryAllowHTTP bool   `help:"Allows OCI container requests to accept plain HTTP responses" default:"true" env:"FTL_ARTEFACTS_ALLOW_HTTP"`
 }
 
 type containerRegistry struct {
@@ -42,20 +42,6 @@ type containerRegistry struct {
 	db     sql.Querier
 }
 
-type ArtefactRepository struct {
-	ModuleDigest     sha256.SHA256
-	MediaType        string
-	ArtefactType     string
-	RepositoryDigest digest.Digest
-	Size             int64
-}
-
-type ArtefactBlobs struct {
-	Digest    sha256.SHA256
-	MediaType string
-	Size      int64
-}
-
 func newContainerRegistry(c ContainerConfig, conn libdal.Connection) *containerRegistry {
 	// Connect the registry targeting the specified container
 	repoFactory := func() (*remote.Repository, error) {
@@ -69,11 +55,11 @@ func newContainerRegistry(c ContainerConfig, conn libdal.Connection) *containerR
 			Client: retry.DefaultClient,
 			Cache:  auth.NewCache(),
 			Credential: auth.StaticCredential(c.Registry, auth.Credential{
-				Username: c.Username,
-				Password: c.Password,
+				Username: c.RegistryUsername,
+				Password: c.RegistryPassword,
 			}),
 		}
-		reg.PlainHTTP = c.AllowPlainHTTP
+		reg.PlainHTTP = c.RegistryAllowHTTP
 
 		return reg, nil
 	}
@@ -148,11 +134,30 @@ func (s *containerRegistry) Download(ctx context.Context, digest sha256.SHA256)
 }
 
 func (s *containerRegistry) GetReleaseArtefacts(ctx context.Context, releaseID int64) ([]ReleaseArtefact, error) {
-	return getDatabaseReleaseArtefacts(ctx, s.db, releaseID)
+	rows, err := s.db.GetReleaseArtefacts(ctx, releaseID)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get release artefacts: %w", libdal.TranslatePGError(err))
+	}
+	return slices.Map(rows, func(row sql.GetReleaseArtefactsRow) ReleaseArtefact {
+		return ReleaseArtefact{
+			Artefact:   ArtefactKey{Digest: sha256.FromBytes(row.Digest)},
+			Path:       row.Path,
+			Executable: row.Executable,
+		}
+	}), nil
 }
 
 func (s *containerRegistry) AddReleaseArtefact(ctx context.Context, key model.DeploymentKey, ra ReleaseArtefact) error {
-	return addReleaseArtefacts(ctx, s.db, key, ra)
+	params := sql.PublishReleaseArtefactParams{
+		Key:        key,
+		Digest:     ra.Artefact.Digest[:],
+		Executable: ra.Executable,
+		Path:       ra.Path,
+	}
+	if err := s.db.PublishReleaseArtefact(ctx, params); err != nil {
+		return libdal.TranslatePGError(err)
+	}
+	return nil
 }
 
 // createModuleRepositoryPathFromDigest creates the path to the repository, relative to the registries root

backend/controller/controller.go

@@ -109,7 +109,7 @@ type Config struct {
 	KMSURI                       *string             `help:"URI for KMS key e.g. with fake-kms:// or aws-kms://arn:aws:kms:ap-southeast-2:12345:key/0000-1111" env:"FTL_KMS_URI"`
 	MaxOpenDBConnections         int                 `help:"Maximum number of database connections." default:"20" env:"FTL_MAX_OPEN_DB_CONNECTIONS"`
 	MaxIdleDBConnections         int                 `help:"Maximum number of idle database connections." default:"20" env:"FTL_MAX_IDLE_DB_CONNECTIONS"`
-	ContainerConfig              artefacts.ContainerConfig
+	artefacts.ContainerConfig
 	CommonConfig
 }
 

backend/controller/sql/schema/20241030190730_create_release_artefacts.sql

@@ -0,0 +1,14 @@
+-- migrate:up
+
+-- release_artefacts stores references to OCI artefacts (compiled modules)
+CREATE TABLE release_artefacts (
+    release_id BIGINT      NOT NULL REFERENCES deployments (id) ON DELETE CASCADE,
+    digest     BYTEA       UNIQUE NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'),
+    executable BOOLEAN     NOT NULL,
+    -- Path relative to the module root.
+    path       TEXT        NOT NULL,
+
+    UNIQUE (release_id, digest)
+);
+-- migrate:down

frontend/cli/cmd_release.go

@@ -83,7 +83,7 @@ func createContainerService(release *releaseCmd) (artefacts.Service, error) {
 	conn.SetMaxOpenConns(release.MaxOpenDBConnections)
 
 	return artefacts.New(artefacts.ContainerConfig{
-		Registry:       release.Registry,
-		AllowPlainHTTP: true,
+		Registry:          release.Registry,
+		RegistryAllowHTTP: true,
 	}, conn), nil
 }