Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cascadeDelete option to the Image and Build objects #1742

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions api/openapi-spec/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -5591,6 +5591,9 @@
"cache": {
"$ref": "#/definitions/kpack.build.v1alpha2.BuildCacheConfig"
},
"cascadeDelete": {
"type": "boolean"
},
"cnbBindings": {
"type": "array",
"items": {
Expand Down Expand Up @@ -6724,6 +6727,9 @@
"cache": {
"$ref": "#/definitions/kpack.build.v1alpha2.ImageCacheConfig"
},
"cascadeDelete": {
"type": "boolean"
},
"cosign": {
"$ref": "#/definitions/kpack.build.v1alpha2.CosignConfig"
},
Expand Down Expand Up @@ -7340,6 +7346,9 @@
"url"
],
"properties": {
"auth": {
"type": "string"
},
"stripComponents": {
"type": "integer",
"format": "int64"
Expand Down
2 changes: 1 addition & 1 deletion cmd/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ func main() {
SystemServiceAccountName: cfg.SystemServiceAccount,
}

buildController := build.NewController(ctx, options, k8sClient, buildInformer, podInformer, metadataRetriever, buildpodGenerator, podProgressLogger, keychainFactory, &slsaAttester, secretFetcher, featureFlags)
buildController := build.NewController(ctx, options, k8sClient, buildInformer, podInformer, metadataRetriever, buildpodGenerator, podProgressLogger, keychainFactory, &slsaAttester, secretFetcher, featureFlags, &registry.Client{})
imageController := image.NewController(ctx, options, k8sClient, imageInformer, buildInformer, duckBuilderInformer, sourceResolverInformer, pvcInformer, cfg.EnablePriorityClasses)
sourceResolverController := sourceresolver.NewController(ctx, options, sourceResolverInformer, gitResolver, blobResolver, registryResolver)
builderController, builderResync := builder.NewController(ctx, options, builderInformer, builderCreator, keychainFactory, clusterStoreInformer, buildpackInformer, clusterBuildpackInformer, clusterStackInformer, secretFetcher)
Expand Down
4 changes: 3 additions & 1 deletion docs/build.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ spec:
git:
url: https://github.com/buildpack/sample-java-app.git
revision: main
cascadeDelete: true
activeDeadlineSeconds: 1800
env:
- name: "JAVA_BP_ENV"
Expand Down Expand Up @@ -72,10 +73,11 @@ spec:
- `env`: Optional list of build time environment variables.
- `defaultProcess`: The [default process type](https://buildpacks.io/docs/app-developer-guide/run-an-app/) for the built OCI image
- `projectDescriptorPath`: Path to the [project descriptor file](https://buildpacks.io/docs/reference/config/project-descriptor/) relative to source root dir or `subPath` if set. If unset, kpack will look for `project.toml` at the root dir or `subPath` if set.
- `cascadeDelete`: If set to `true`, produced image will garbage collected from the registry upon object deletion.
- `resources`: Optional configurable resource limits on `CPU` and `memory`.
- `tolerations`: Optional configurable pod spec tolerations
- `nodeSelector`: Optional configurable pod spec nodeSelector
- `affinity`: Optional configurabl pod spec affinity
- `affinity`: Optional configurable pod spec affinity

> Note: All fields on a build are immutable. Instead of updating a build, create a new one.

Expand Down
1 change: 1 addition & 0 deletions docs/image.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ The following defines the relevant fields of the `image` resource spec in more d
- `build`: Configuration that is passed to every image build. See [Build Configuration](#build-config) section below.
- `defaultProcess`: The [default process type](https://buildpacks.io/docs/app-developer-guide/run-an-app/) for the built OCI image
- `projectDescriptorPath`: Path to the [project descriptor file](https://buildpacks.io/docs/reference/config/project-descriptor/) relative to source root dir or `subPath` if set. If unset, kpack will look for `project.toml` at the root dir or `subPath` if set.
- `cascadeDelete`: If set to `true`, produced image will garbage collected from the registry upon `Build` objects deletion.
- `cosign`: Configuration for additional cosign image signing. See [Cosign Configuration](#cosign-config) section below.

### <a id='tags-config'></a> Configuring Tags
Expand Down
4 changes: 4 additions & 0 deletions pkg/apis/build/v1alpha2/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ func (b *Build) DefaultProcess() string {
return b.Spec.DefaultProcess
}

func (b *Build) CascadeDeleteImage() bool {
return b.Spec.CascadeDelete
}

var buildSteps = map[string]struct{}{
PrepareContainerName: {},
AnalyzeContainerName: {},
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/build/v1alpha2/build_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ type BuildSpec struct {
SchedulerName string `json:"schedulerName,omitempty"`
PriorityClassName string `json:"priorityClassName,omitempty"`
CreationTime string `json:"creationTime,omitempty"`
// +optional
CascadeDelete bool `json:"cascadeDelete,omitempty"`
}

func (bs *BuildSpec) RegistryCacheTag() string {
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/build/v1alpha2/image_builds.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ func (im *Image) Build(sourceResolver *SourceResolver, builder BuilderResource,
PriorityClassName: priorityClass,
ActiveDeadlineSeconds: im.BuildTimeout(),
CreationTime: im.Spec.creationTime(),
CascadeDelete: im.Spec.CascadeDelete,
},
}
}
Expand Down
16 changes: 9 additions & 7 deletions pkg/apis/build/v1alpha2/image_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ type ImageSpec struct {
DefaultProcess string `json:"defaultProcess,omitempty"`
// +listType
AdditionalTags []string `json:"additionalTags,omitempty"`
// +optional
CascadeDelete bool `json:"cascadeDelete,omitempty"`
}

// +k8s:openapi-gen=true
Expand All @@ -72,13 +74,13 @@ type ImageBuild struct {
Env []corev1.EnvVar `json:"env,omitempty"`
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
// +listType
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
Affinity *corev1.Affinity `json:"affinity,omitempty"`
RuntimeClassName *string `json:"runtimeClassName,omitempty"`
SchedulerName string `json:"schedulerName,omitempty"`
BuildTimeout *int64 `json:"buildTimeout,omitempty"`
CreationTime string `json:"creationTime,omitempty"`
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
Affinity *corev1.Affinity `json:"affinity,omitempty"`
RuntimeClassName *string `json:"runtimeClassName,omitempty"`
SchedulerName string `json:"schedulerName,omitempty"`
BuildTimeout *int64 `json:"buildTimeout,omitempty"`
CreationTime string `json:"creationTime,omitempty"`
}

// +k8s:openapi-gen=true
Expand Down
30 changes: 24 additions & 6 deletions pkg/openapi/openapi_generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

82 changes: 79 additions & 3 deletions pkg/reconciler/build/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"slices"

"github.com/google/go-containerregistry/pkg/authn"
ggcrv1 "github.com/google/go-containerregistry/pkg/v1"
Expand Down Expand Up @@ -41,6 +42,7 @@ const (
Kind = "Build"
k8sOSLabel = "kubernetes.io/os"
ReasonCompleted = "Completed"
BuildFinalizer = "builds.kpack.io/finalizer"
)

//go:generate counterfeiter . MetadataRetriever
Expand Down Expand Up @@ -69,6 +71,11 @@ type SecretFetcher interface {
SecretsForSystemServiceAccount(context.Context) ([]*corev1.Secret, error)
}

//go:generate counterfeiter . RegistryClient
type RegistryClient interface {
Delete(keychain authn.Keychain, repoName string) error
}

func NewController(
ctx context.Context, opt reconciler.Options, k8sClient k8sclient.Interface,
informer buildinformers.BuildInformer, podInformer corev1Informers.PodInformer,
Expand All @@ -78,6 +85,7 @@ func NewController(
attester SLSAAttester,
secretFetcher SecretFetcher,
featureFlags config.FeatureFlags,
registryClient RegistryClient,
) *controller.Impl {
c := &Reconciler{
Client: opt.Client,
Expand All @@ -91,6 +99,7 @@ func NewController(
Attester: attester,
SecretFetcher: secretFetcher,
FeatureFlags: featureFlags,
RegistryClient: registryClient,
}

logger := opt.Logger.With(
Expand Down Expand Up @@ -121,6 +130,7 @@ type Reconciler struct {
Attester SLSAAttester
SecretFetcher SecretFetcher
FeatureFlags config.FeatureFlags
RegistryClient RegistryClient
}

func (c *Reconciler) Reconcile(ctx context.Context, key string) error {
Expand All @@ -136,6 +146,14 @@ func (c *Reconciler) Reconcile(ctx context.Context, key string) error {
return err
}

if !build.DeletionTimestamp.IsZero() {
return c.finalize(ctx, build)
}

if err := c.setFinalizer(ctx, build); err != nil {
return err
}

build = build.DeepCopy()
build.SetDefaults(ctx)

Expand Down Expand Up @@ -352,6 +370,64 @@ func (c *Reconciler) conditionForPod(pod *corev1.Pod, stepsCompleted []string) c
}
}

func (c *Reconciler) finalize(ctx context.Context, build *buildapi.Build) error {
if !slices.Contains(build.GetFinalizers(), BuildFinalizer) {
return nil
}

if build.Finished() {
keychain, err := c.KeychainFactory.KeychainForSecretRef(ctx, registry.SecretRef{
ServiceAccount: build.Spec.ServiceAccountName,
Namespace: build.Namespace,
})
if err != nil {
return err
}

if err := c.RegistryClient.Delete(keychain, build.Status.LatestImage); err != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We would also need to delete all of the additional tags as well.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was under the assumption that a Build object produces exactly one image, while Image can produce multiple Build's. If each Build cleans up it's own image, would there be any leftovers? Or the assumption is incorrect?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A build can push multiple images (see additionalTags). There is also the signed images and image attestations but that might be too much to try to delete and we probably don’t want to delete the attestations

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to docs, the status.latestImage field contain the image's digest. It should be enough to delete just this (no necessity to delete all tags individually): https://github.com/opencontainers/distribution-spec/blob/main/spec.md#deleting-manifests.

Once deleted, a GET to /v2//manifests/ and any tag pointing to that digest will return a 404.

//logger.Printf(errors.Wrapf(err, "Could not delete image %q", build.Status.LatestImage))

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How should we handle the error in that case? It seems like error logging is not supposed to happen at the reconciliation stage.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can log and it will print to the controller logs

}
}

mergePatch := map[string]interface{}{
"metadata": map[string]interface{}{
"finalizers": slices.DeleteFunc(build.GetFinalizers(), func(f string) bool {
return f == BuildFinalizer
}),
"resourceVersion": build.ResourceVersion,
},
}

patch, err := json.Marshal(mergePatch)
if err != nil {
return err
}

_, err = c.Client.KpackV1alpha2().Builds(build.Namespace).Patch(ctx, build.Name, types.MergePatchType, patch, metav1.PatchOptions{})
return err
}

func (c *Reconciler) setFinalizer(ctx context.Context, build *buildapi.Build) error {
if slices.Contains(build.GetFinalizers(), BuildFinalizer) || !build.CascadeDeleteImage() {
return nil
}

mergePatch := map[string]interface{}{
"metadata": map[string]interface{}{
"finalizers": append(build.GetFinalizers(), BuildFinalizer),
"resourceVersion": build.ResourceVersion,
},
}

patch, err := json.Marshal(mergePatch)
if err != nil {
return err
}

_, err = c.Client.KpackV1alpha2().Builds(build.Namespace).Patch(ctx, build.Name, types.MergePatchType, patch, metav1.PatchOptions{})
return err
}

func stepStates(pod *corev1.Pod) []corev1.ContainerState {
states := make([]corev1.ContainerState, 0, len(buildapi.BuildSteps()))
for _, s := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
Expand Down Expand Up @@ -442,17 +518,17 @@ func (c *Reconciler) attestBuild(ctx context.Context, build *buildapi.Build, bui
signers[i] = s
}

buildId := slsa.UnsignedBuildID
buildID := slsa.UnsignedBuildID
if len(signers) > 0 {
buildId = slsa.SignedBuildID
buildID = slsa.SignedBuildID
}

deps, err := c.attestBuildDeps(ctx, build, pod, secrets)
if err != nil {
return "", fmt.Errorf("failed to gather build deps: %v", err)
}

statement, err := c.Attester.AttestBuild(build, buildMetadata, pod, keychain, buildId, deps...)
statement, err := c.Attester.AttestBuild(build, buildMetadata, pod, keychain, buildID, deps...)
if err != nil {
return "", fmt.Errorf("failed to generate statement: %v", err)
}
Expand Down
Loading
Loading