From 64b31fb90e286bbe7056b4d6c793d2a8421095b5 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy <41936996+vasireddy99@users.noreply.github.com> Date: Mon, 5 Feb 2024 09:38:09 -0800 Subject: [PATCH] Add Lambda Layer Version cleaner (#2611) * Initial commit to add layer cleaner * Print the layer ARNs for visibility and use ListLayersInput * Use Regex, type Pair for layer and version * Iterate on versions for that layer to delete every layer version * Modify variables usage * Remove extra fmt * Add comments for context * Fix fmt and comments * Add functionality to shouldDeleteLayer * Add regex as appropriate * Make it more generalized to match more * Remove NodeProxyAgentLayerand custom-config-layer form regex list --- tools/workflow/cleaner/lambda/clean.go | 117 ++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 11 deletions(-) diff --git a/tools/workflow/cleaner/lambda/clean.go b/tools/workflow/cleaner/lambda/clean.go index 443cf808f..9c2e2b9f1 100644 --- a/tools/workflow/cleaner/lambda/clean.go +++ b/tools/workflow/cleaner/lambda/clean.go @@ -1,6 +1,7 @@ package lambda import ( + "errors" "fmt" "log" "os" @@ -15,48 +16,114 @@ const Type = "lambda" var logger = log.New(os.Stdout, fmt.Sprintf("[%s] ", Type), log.LstdFlags) +type Layer struct { + LayerARN *string + Version *int64 +} + func Clean(sess *session.Session, expirationDate time.Time) error { - logger.Printf("Begin to clean Lambda functions") + return errors.Join(cleanFunctions(sess, expirationDate), cleanLayers(sess, expirationDate)) +} - lambdaclient := lambda.New(sess) +func cleanFunctions(sess *session.Session, expirationDate time.Time) error { + logger.Printf("Begin to clean Lambda functions") + lambdaClient := lambda.New(sess) var deleteFunctionIDs []*string var nextToken *string + for { - ListFunctionsInput := &lambda.ListFunctionsInput{Marker: nextToken} - ListFunctionsOutput, err := lambdaclient.ListFunctions(ListFunctionsInput) + listFunctionsInput := &lambda.ListFunctionsInput{Marker: nextToken} + listFunctionsOutput, err := lambdaClient.ListFunctions(listFunctionsInput) if err != nil { - return fmt.Errorf("unable to retrieve APIs: %w", err) + return fmt.Errorf("unable to retrieve functions: %w", err) } - for _, lf := range ListFunctionsOutput.Functions { + + for _, lf := range listFunctionsOutput.Functions { doesNameMatch, err := shouldDelete(lf) if err != nil { return fmt.Errorf("error during pattern match: %w", err) } created, err := time.Parse("2006-01-02T15:04:05Z0700", *lf.LastModified) if err != nil { - return fmt.Errorf("error parting last modified time: %w", err) + return fmt.Errorf("error parsing last modified time: %w", err) } if expirationDate.After(created) && doesNameMatch { logger.Printf("Try to delete function %s created-at %s", *lf.FunctionArn, *lf.LastModified) deleteFunctionIDs = append(deleteFunctionIDs, lf.FunctionArn) } } - if ListFunctionsOutput.NextMarker == nil { + if listFunctionsOutput.NextMarker == nil { break } - nextToken = ListFunctionsOutput.NextMarker + nextToken = listFunctionsOutput.NextMarker } if len(deleteFunctionIDs) < 1 { logger.Printf("No Lambda functions to delete") return nil } - for _, id := range deleteFunctionIDs { terminateFunctionInput := &lambda.DeleteFunctionInput{FunctionName: id} - if _, err := lambdaclient.DeleteFunction(terminateFunctionInput); err != nil { + if _, err := lambdaClient.DeleteFunction(terminateFunctionInput); err != nil { return fmt.Errorf("unable to delete function: %w", err) } + logger.Printf("Deleted %d Lambda functions", len(deleteFunctionIDs)) + } + + return nil +} + +func cleanLayers(sess *session.Session, expirationDate time.Time) error { + logger.Printf("Begin to clean Lambda layers") + lambdaClient := lambda.New(sess) + var deleteLayerVersions []Layer + var nextToken *string + + for { + listLayerVersionsInput := &lambda.ListLayersInput{ + Marker: nextToken, + } + + // Retrieve layer versions from Lambda service + listLayerVersionsOutput, err := lambdaClient.ListLayers(listLayerVersionsInput) + if err != nil { + return fmt.Errorf("unable to retrieve layer versions: %w", err) + } + + // Loop through retrieved layer versions + for _, layer := range listLayerVersionsOutput.Layers { + + if shouldDeleteLayer(layer, expirationDate) { + logger.Printf("Try to delete layer version %s created-at %s", *layer.LayerArn, *layer.LatestMatchingVersion.CreatedDate) + deleteLayerVersions = append(deleteLayerVersions, Layer{layer.LayerArn, layer.LatestMatchingVersion.Version}) + } + } + + if listLayerVersionsOutput.NextMarker == nil { + break + } + nextToken = listLayerVersionsOutput.NextMarker + } + + if len(deleteLayerVersions) < 1 { + logger.Printf("No Lambda layers to delete") + return nil + } + for _, id := range deleteLayerVersions { + for *id.Version > 0 { + // Prepare input for deleting a specific layer version + deleteLayerVersionInput := &lambda.DeleteLayerVersionInput{ + LayerName: id.LayerARN, + VersionNumber: id.Version, + } + if _, err := lambdaClient.DeleteLayerVersion(deleteLayerVersionInput); err != nil { + return fmt.Errorf("unable to delete layer version: %w", err) + } + // Decrement the version number for the next iteration + *id.Version-- + } + logger.Printf("Deleted %d Lambda layer versions", len(deleteLayerVersions)) } + return nil } @@ -79,3 +146,31 @@ func shouldDelete(lf *lambda.FunctionConfiguration) (bool, error) { } return false, nil } + +func shouldDeleteLayer(layerList *lambda.LayersListItem, expirationDate time.Time) bool { + layerARN := layerList.LayerArn + regexList := []string{ + ".*:layer:aws-otel-collector.*$", + ".*:layer:aws-otel-lambda-python.*$", + ".*:layer:aws-otel-java.*$", + ".*:layer:aws-otel-lambda-nodejs.*$", + ".*:layer:aws-otel-go-wrapper.*$", + ".*:layer:aws-observability.*$", + ".*:layer:aws-distro-for-opentelemetry.*$", + } + + for _, rx := range regexList { + matched, _ := regexp.MatchString(rx, *layerARN) + if matched { + created, err := time.Parse("2006-01-02T15:04:05Z0700", *layerList.LatestMatchingVersion.CreatedDate) + if err != nil { + logger.Printf("Error Parsing the created time for layer %s", err) + return false + } + if expirationDate.After(created) { + return true + } + } + } + return false +}