From 4ededc85973249cb574dd2494a7fa55a936d3d66 Mon Sep 17 00:00:00 2001 From: Ben Jackson Date: Sat, 2 Dec 2023 15:42:31 +1100 Subject: [PATCH] feat: clean up unused services and resources --- cmd/identify_ingress_test.go | 4 + cmd/identify_lagoonservices.go | 70 +++---- cmd/identify_lagoonservices_test.go | 262 +++++++++----------------- legacy/build-deploy-docker-compose.sh | 212 +++++++++++++++++++++ 4 files changed, 334 insertions(+), 214 deletions(-) diff --git a/cmd/identify_ingress_test.go b/cmd/identify_ingress_test.go index 0f3ea1e2..c6dc334d 100644 --- a/cmd/identify_ingress_test.go +++ b/cmd/identify_ingress_test.go @@ -383,6 +383,10 @@ func TestIdentifyRoute(t *testing.T) { if string(retJSON) != tt.wantJSON { t.Errorf("returned autogen %v doesn't match want %v", string(retJSON), tt.wantJSON) } + t.Cleanup(func() { + helpers.UnsetEnvVars(nil) + helpers.UnsetEnvVars(tt.args.BuildPodVariables) + }) }) } } diff --git a/cmd/identify_lagoonservices.go b/cmd/identify_lagoonservices.go index d17b8ad9..64ad3420 100644 --- a/cmd/identify_lagoonservices.go +++ b/cmd/identify_lagoonservices.go @@ -1,7 +1,6 @@ package cmd import ( - "encoding/base64" "encoding/json" "fmt" @@ -11,18 +10,9 @@ import ( ) type identifyServices struct { - Name string `json:"name"` - Type string `json:"type"` - Containers []containers `json:"containers,omitempty"` -} - -type containers struct { - Name string `json:"name"` - Ports []ports `json:"ports,omitempty"` -} - -type ports struct { - Port int32 `json:"port"` + Deployments []string `json:"deployments,omitempty"` + Volumes []string `json:"volumes,omitempty"` + Services []string `json:"services,omitempty"` } var lagoonServiceIdentify = &cobra.Command{ @@ -30,7 +20,7 @@ var lagoonServiceIdentify = &cobra.Command{ Aliases: []string{"ls"}, Short: "Identify the lagoon services for a Lagoon build", RunE: func(cmd *cobra.Command, args []string) error { - gen, err := generator.GenerateInput(*rootCmd, true) + gen, err := generator.GenerateInput(*rootCmd, false) if err != nil { return err } @@ -38,22 +28,17 @@ var lagoonServiceIdentify = &cobra.Command{ if err != nil { return fmt.Errorf("error reading images flag: %v", err) } - var imageRefs struct { - Images map[string]string `json:"images"` - } - imagesStr, err := base64.StdEncoding.DecodeString(images) + imageRefs, err := loadImagesFromFile(images) if err != nil { - return fmt.Errorf("error decoding images payload: %v", err) - } - if err := json.Unmarshal(imagesStr, &imageRefs); err != nil { - return fmt.Errorf("error unmarshalling images payload: %v", err) + return err } gen.ImageReferences = imageRefs.Images out, err := LagoonServiceTemplateIdentification(gen) if err != nil { return err } - fmt.Println(out) + b, _ := json.Marshal(out) + fmt.Println(string(b)) return nil }, } @@ -62,9 +47,9 @@ var lagoonServiceIdentify = &cobra.Command{ // about the services that lagoon will be deploying (this will be kubernetes `kind: deployment`, but lagoon calls them services ¯\_(ツ)_/¯) // this command can be used to identify services that are deployed by the build, so that services that may remain in the environment can be identified // and eventually removed -func LagoonServiceTemplateIdentification(g generator.GeneratorInput) ([]identifyServices, error) { +func LagoonServiceTemplateIdentification(g generator.GeneratorInput) (*identifyServices, error) { - lServices := []identifyServices{} + servicesData := identifyServices{} lagoonBuild, err := generator.NewGenerator( g, ) @@ -74,27 +59,26 @@ func LagoonServiceTemplateIdentification(g generator.GeneratorInput) ([]identify deployments, err := servicestemplates.GenerateDeploymentTemplate(*lagoonBuild.BuildValues) if err != nil { - return nil, fmt.Errorf("couldn't generate template: %v", err) + return nil, fmt.Errorf("couldn't identify deployments: %v", err) } for _, d := range deployments { - dcs := []containers{} - for _, dc := range d.Spec.Template.Spec.Containers { - dcp := []ports{} - for _, p := range dc.Ports { - dcp = append(dcp, ports{Port: p.ContainerPort}) - } - dcs = append(dcs, containers{ - Name: dc.Name, - Ports: dcp, - }) - } - lServices = append(lServices, identifyServices{ - Name: d.Name, - Type: d.ObjectMeta.Labels["lagoon.sh/service-type"], - Containers: dcs, - }) + servicesData.Deployments = append(servicesData.Deployments, d.ObjectMeta.Name) + } + pvcs, err := servicestemplates.GeneratePVCTemplate(*lagoonBuild.BuildValues) + if err != nil { + return nil, fmt.Errorf("couldn't identify volumes: %v", err) + } + for _, pvc := range pvcs { + servicesData.Volumes = append(servicesData.Volumes, pvc.ObjectMeta.Name) + } + services, err := servicestemplates.GenerateServiceTemplate(*lagoonBuild.BuildValues) + if err != nil { + return nil, fmt.Errorf("couldn't identify services: %v", err) + } + for _, service := range services { + servicesData.Services = append(servicesData.Services, service.ObjectMeta.Name) } - return lServices, nil + return &servicesData, nil } func init() { diff --git a/cmd/identify_lagoonservices_test.go b/cmd/identify_lagoonservices_test.go index 59521a60..e2303a58 100644 --- a/cmd/identify_lagoonservices_test.go +++ b/cmd/identify_lagoonservices_test.go @@ -1,10 +1,12 @@ package cmd import ( + "encoding/json" "os" "reflect" "testing" + "github.com/andreyvit/diff" "github.com/uselagoon/build-deploy-tool/internal/dbaasclient" "github.com/uselagoon/build-deploy-tool/internal/helpers" "github.com/uselagoon/build-deploy-tool/internal/lagoon" @@ -19,7 +21,7 @@ func TestIdentifyLagoonServices(t *testing.T) { name string description string args testdata.TestData - want []identifyServices + want *identifyServices }{ { name: "test1 basic deployment", @@ -33,20 +35,12 @@ func TestIdentifyLagoonServices(t *testing.T) { "node": "harbor.example/example-project/main/node@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", }, }, true), - want: []identifyServices{ - { - Name: "node", - Type: "basic", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 1234}, - {Port: 8191}, - {Port: 9001}, - }, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "node", + }, + Services: []string{ + "node", }, }, }, @@ -66,59 +60,20 @@ func TestIdentifyLagoonServices(t *testing.T) { "varnish": "harbor.example/example-project/main/varnish@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", }, }, true), - want: []identifyServices{ - { - Name: "cli", - Type: "cli-persistent", - Containers: []containers{ - { - Name: "cli", - Ports: []ports{}, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "cli", + "redis", + "varnish", + "nginx-php", }, - { - Name: "redis", - Type: "redis", - Containers: []containers{ - { - Name: "redis", - Ports: []ports{ - {Port: 6379}, - }, - }, - }, + Volumes: []string{ + "nginx-php", }, - { - Name: "varnish", - Type: "varnish", - Containers: []containers{ - { - Name: "varnish", - Ports: []ports{ - {Port: 8080}, - {Port: 6082}, - }, - }, - }, - }, - { - Name: "nginx-php", - Type: "nginx-php-persistent", - Containers: []containers{ - { - Name: "nginx", - Ports: []ports{ - {Port: 8080}, - }, - }, - { - Name: "php", - Ports: []ports{ - {Port: 9000}, - }, - }, - }, + Services: []string{ + "redis", + "varnish", + "nginx-php", }, }, }, @@ -145,59 +100,20 @@ func TestIdentifyLagoonServices(t *testing.T) { }, }, }, true), - want: []identifyServices{ - { - Name: "cli", - Type: "cli-persistent", - Containers: []containers{ - { - Name: "cli", - Ports: []ports{}, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "cli", + "redis", + "varnish", + "nginx-php", }, - { - Name: "redis", - Type: "redis", - Containers: []containers{ - { - Name: "redis", - Ports: []ports{ - {Port: 6379}, - }, - }, - }, + Volumes: []string{ + "nginx-php", }, - { - Name: "varnish", - Type: "varnish", - Containers: []containers{ - { - Name: "varnish", - Ports: []ports{ - {Port: 8080}, - {Port: 6082}, - }, - }, - }, - }, - { - Name: "nginx-php", - Type: "nginx-php-persistent", - Containers: []containers{ - { - Name: "nginx", - Ports: []ports{ - {Port: 8080}, - }, - }, - { - Name: "php", - Ports: []ports{ - {Port: 9000}, - }, - }, - }, + Services: []string{ + "redis", + "varnish", + "nginx-php", }, }, }, @@ -223,44 +139,19 @@ func TestIdentifyLagoonServices(t *testing.T) { }, }, }, true), - want: []identifyServices{ - { - Name: "lnd", - Type: "basic-persistent", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 8080}, - {Port: 10009}, - }, - }, - }, + want: &identifyServices{ + Deployments: []string{ + "lnd", + "thunderhub", + "tor", }, - { - Name: "thunderhub", - Type: "basic-persistent", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 3000}, - }, - }, - }, + Volumes: []string{ + "lnd", }, - { - Name: "tor", - Type: "basic", - Containers: []containers{ - { - Name: "basic", - Ports: []ports{ - {Port: 9050}, - {Port: 9051}, - }, - }, - }, + Services: []string{ + "lnd", + "thunderhub", + "tor", }, }, }, @@ -285,25 +176,52 @@ func TestIdentifyLagoonServices(t *testing.T) { }, }, }, true), - want: []identifyServices{ - { - Name: "lnd", - Type: "basic-persistent", - Containers: []containers{ - {Name: "basic", - Ports: []ports{ - {Port: 8080}, - {Port: 10009}, - }}, - }, + want: &identifyServices{ + Deployments: []string{ + "lnd", + "tor", + }, + Volumes: []string{ + "lnd", }, - { - Name: "tor", - Type: "worker-persistent", - Containers: []containers{ - {Name: "worker", - Ports: []ports{}}, + Services: []string{ + "lnd", + }, + }, + }, + + { + name: "test5-complex-custom-volumes", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + BuildType: "branch", + LagoonYAML: "internal/testdata/complex/lagoon.multiple-volumes.yml", + ImageReferences: map[string]string{ + "nginx": "harbor.example/example-project/main/nginx@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "php": "harbor.example/example-project/main/php@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "cli": "harbor.example/example-project/main/cli@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "nginx2": "harbor.example/example-project/main/nginx2@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "php2": "harbor.example/example-project/main/php2@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", + "mariadb": "harbor.example/example-project/main/mariadb@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8", }, + }, true), + want: &identifyServices{ + Deployments: []string{ + "cli", + "mariadb", + "nginx", + }, + Volumes: []string{ + "mariadb", + "nginx", + "custom-files", + }, + Services: []string{ + "mariadb", + "nginx", }, }, }, @@ -337,7 +255,9 @@ func TestIdentifyLagoonServices(t *testing.T) { t.Errorf("%v", err) } if !reflect.DeepEqual(out, tt.want) { - t.Errorf("returned output %v doesn't match want %v", out, tt.want) + r1, _ := json.MarshalIndent(out, "", " ") + s1, _ := json.MarshalIndent(tt.want, "", " ") + t.Errorf("LagoonServiceTemplateIdentification() = \n%v", diff.LineDiff(string(r1), string(s1))) } t.Cleanup(func() { helpers.UnsetEnvVars(nil) diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 9e11fe6a..e1000a39 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -1610,6 +1610,218 @@ done currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "deploymentApplyComplete" "Applying Deployments" "false" previousStepEnd=${currentStepEnd} + +############################################## +### CLEANUP services which have been removed from docker-compose.yaml +##############################################s + +# collect the current deployments, volumes, and services in the environment +CURRENT_DEPLOYMENTS=$(kubectl -n ${NAMESPACE} get deployments -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) +CURRENT_PVCS=$(kubectl -n ${NAMESPACE} get pvc -l "lagoon.sh/service-type" --no-headers | cut -d " " -f 1 | xargs) +CURRENT_SERVICES=$(kubectl -n ${NAMESPACE} get services -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) + +CURRENT_MARIADB_CONSUMERS=$(kubectl -n ${NAMESPACE} get mariadbconsumers -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) +CURRENT_POSTGRES_CONSUMERS=$(kubectl -n ${NAMESPACE} get postgresqlconsumers -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) +CURRENT_MONGODB_CONSUMERS=$(kubectl -n ${NAMESPACE} get mongodbconsumers -l "lagoon.sh/service-type" -l "lagoon.sh/service" --no-headers | cut -d " " -f 1 | xargs) + +# using the build-deploy-tool identify the deployments, volumes, and services that this build has created +LAGOON_DEPLOYMENTS_TO_JSON=$(build-deploy-tool identify lagoon-services --images /kubectl-build-deploy/images.yaml | jq -r ) + +echo "${LAGOON_DEPLOYMENTS_TO_JSON}" +MATCHED_MARIADB=false +DELETE_MARIADB=() +for EXIST_CONSUMERS in ${CURRENT_MARIADB_CONSUMERS}; do + for DBAAS_ENTRY in "${DBAAS[@]}" + do + IFS=':' read -ra DBAAS_ENTRY_SPLIT <<< "$DBAAS_ENTRY" + SERVICE_NAME=${DBAAS_ENTRY_SPLIT[0]} + if [ "${EXIST_CONSUMERS}" == "${SERVICE_NAME}" ]; then + MATCHED_MARIADB=true + continue + fi + done + if [ "${MATCHED_MARIADB}" != "true" ]; then + DELETE_MARIADB+=($EXIST_CONSUMERS) + fi + MATCHED_MARIADB=false +done + + +MATCHED_POSTGRES=false +DELETE_POSTGRES=() +for EXIST_CONSUMERS in ${CURRENT_POSTGRES_CONSUMERS}; do + for DBAAS_ENTRY in "${DBAAS[@]}" + do + IFS=':' read -ra DBAAS_ENTRY_SPLIT <<< "$DBAAS_ENTRY" + SERVICE_NAME=${DBAAS_ENTRY_SPLIT[0]} + if [ "${EXIST_CONSUMERS}" == "${SERVICE_NAME}" ]; then + MATCHED_POSTGRES=true + continue + fi + done + if [ "${MATCHED_POSTGRES}" != "true" ]; then + DELETE_POSTGRES+=($EXIST_CONSUMERS) + fi + MATCHED_POSTGRES=false +done + +MATCHED_MONGODB=false +DELETE_MONGODB=() +for EXIST_CONSUMERS in ${CURRENT_MONGODB_CONSUMERS}; do + for DBAAS_ENTRY in "${DBAAS[@]}" + do + IFS=':' read -ra DBAAS_ENTRY_SPLIT <<< "$DBAAS_ENTRY" + SERVICE_NAME=${DBAAS_ENTRY_SPLIT[0]} + if [ "${EXIST_CONSUMERS}" == "${SERVICE_NAME}" ]; then + MATCHED_MONGODB=true + continue + fi + done + if [ "${MATCHED_MONGODB}" != "true" ]; then + DELETE_MONGODB+=($EXIST_CONSUMERS) + fi + MATCHED_MONGODB=false +done + +# check the current deployments in the environment against what the build has created and mark anything that isnt in this build as needing removal +MATCHED_DEPLOYMENT=false +DELETE_DEPLOYMENT=() +for EXIST_DEPLOYMENT in ${CURRENT_DEPLOYMENTS}; do + for DEPLOYMENT in $(echo "$LAGOON_DEPLOYMENTS_TO_JSON" | jq -rc '.deployments[]?') + do + if [ "${EXIST_DEPLOYMENT}" == "${DEPLOYMENT}" ]; then + MATCHED_DEPLOYMENT=true + continue + fi + done + if [ "${MATCHED_DEPLOYMENT}" != "true" ]; then + DELETE_DEPLOYMENT+=($EXIST_DEPLOYMENT) + fi + MATCHED_DEPLOYMENT=false +done +# check the current volumes in the environment against what the build has created and mark anything that isnt in this build as needing removal +MATCHED_VOLUME=false +DELETE_VOLUME=() +for EXIST_PVC in ${CURRENT_PVCS}; do + for VOLUME in $(echo "$LAGOON_DEPLOYMENTS_TO_JSON" | jq -rc '.volumes[]?') + do + if [ "${EXIST_PVC}" == "${VOLUME}" ]; then + MATCHED_VOLUME=true + continue + fi + done + if [ "${MATCHED_VOLUME}" != "true" ]; then + DELETE_VOLUME+=($EXIST_PVC) + fi + MATCHED_VOLUME=false +done +# check the current services in the environment against what the build has created and mark anything that isnt in this build as needing removal +MATCHED_SERVICE=false +DELETE_SERVICE=() +for EXIST_SERVICE in ${CURRENT_SERVICES}; do + for SERVICE in $(echo "$LAGOON_DEPLOYMENTS_TO_JSON" | jq -rc '.services[]?') + do + if [ "${EXIST_SERVICE}" == "${SERVICE}" ]; then + MATCHED_SERVICE=true + continue + fi + done + if [ "${MATCHED_SERVICE}" != "true" ]; then + DELETE_SERVICE+=($EXIST_SERVICE) + fi + MATCHED_SERVICE=false +done + +if [[ ${#DELETE_DEPLOYMENT[@]} -ne 0 ]] || [[ ${#DELETE_SERVICE[@]} -ne 0 ]] || [[ ${#DELETE_VOLUME[@]} -ne 0 ]] || [[ ${#DELETE_MARIADB[@]} -ne 0 ]] || [[ ${#DELETE_POSTGRES[@]} -ne 0 ]] || [[ ${#DELETE_MONGODB[@]} -ne 0 ]]; then + # only show the service cleanup section if there is anything to actually clean up + beginBuildStep "Unused Service Cleanup" "unusedServiceCleanup" + ((++BUILD_WARNING_COUNT)) + echo ">> Lagoon detected services or volumes that have been removed from the docker-compose file" + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" != enabled ]; then + echo "> If you no longer need these services, you can instruct Lagoon to remove it from the environment by setting the following variable" + echo " 'LAGOON_FEATURE_FLAG_CLEANUP_REMOVED_LAGOON_SERVICES=enabled' as a GLOBAL scoped variable to this environment or project." + echo " Removing unused resources will result in the services and any data they had being deleted." + echo " Ensure your application is no longer configured to use these resources before removing them." + echo " If you're not sure, contact your support team with a link to this build." + else + echo "> The flag 'LAGOON_FEATURE_FLAG_CLEANUP_REMOVED_LAGOON_SERVICES' is enabled." + echo " Resources that were removed from the docker-compose file will now be removed from the environment." + echo " The services and any data they had will be deleted." + echo " You should remove this variable if you don't want services to be removed automatically in the future." + fi + echo "> Future releases of Lagoon may remove services automatically, you should ensure that your services are up always up to date if you see this warning." + for DD in ${DELETE_DEPLOYMENT[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + if kubectl -n ${NAMESPACE} get deployments ${DD} &> /dev/null; then + echo ">> Removing deployment ${DD}" + kubectl -n ${NAMESPACE} delete deployment ${DD} + fi + else + echo ">> Would remove deployment ${DD}" + fi + done + for DD in ${DELETE_SERVICE[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + if kubectl -n ${NAMESPACE} get services ${DD} &> /dev/null; then + echo ">> Removing service reference ${DD}" + kubectl -n ${NAMESPACE} delete service ${DD} + fi + else + echo ">> Would remove service ${DD}" + fi + done + for DD in ${DELETE_VOLUME[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + if kubectl -n ${NAMESPACE} get pvc ${DD} &> /dev/null; then + echo ">> Removing volume ${DD}" + kubectl -n ${NAMESPACE} delete pvc ${DD} + fi + else + echo ">> Would remove volume ${DD}" + fi + done + for DD in ${DELETE_MARIADB[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + if kubectl -n ${NAMESPACE} get mariadbconsumers ${DD} &> /dev/null; then + echo ">> Removing mariadb-dbaas ${DD}" + kubectl -n ${NAMESPACE} delete mariadbconsumer ${DD} + fi + else + echo ">> Would remove mariadb-dbaas ${DD}" + fi + done + for DD in ${DELETE_POSTGRES[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + if kubectl -n ${NAMESPACE} get postgresqlconsumers ${DD} &> /dev/null; then + echo ">> Removing postgres-dbaas ${DD}" + kubectl -n ${NAMESPACE} delete postgresqlconsumer ${DD} + fi + else + echo ">> Would remove postgres-dbaas ${DD}" + fi + done + for DD in ${DELETE_MONGODB[@]} + do + if [ "$(featureFlag CLEANUP_REMOVED_LAGOON_SERVICES)" = enabled ]; then + if kubectl -n ${NAMESPACE} get mongodbconsumers ${DD} &> /dev/null; then + echo ">> Removing mongodb-dbaas ${DD}" + kubectl -n ${NAMESPACE} delete mongodbconsumer ${DD} + fi + else + echo ">> Would remove mongodb-dbaas ${DD}" + fi + done + # finalize the service cleanup + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" + patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "unusedServiceCleanupComplete" "Unused Service Cleanup" "true" + previousStepEnd=${currentStepEnd} +fi + beginBuildStep "Cronjob Cleanup" "cleaningUpCronjobs" ##############################################