Add command to delete stale elastic indices

README.md

@@ -21,6 +21,24 @@ This will store the encrypted file at `keys/production/oauth.pem.asc`.
 cat oauth.pen.asc | bay kms decrypt > oauth.pem
 ```
 
+## Elastic Cloud
+Commands for querying and interacting with the Elastic Cloud API.
+
+#### Required inputs
+
+> [!CAUTION]
+> Variables are deployment specific - make sure the deployment you are targeting is not a production deployment.
+
+* `EC_DEPLOYMENT_API_KEY` (environment variable) - Generated from the deployments Kibana settings
+* `--deployment-id` (command line flag) - Found on the deployments Elastic Cloud 'manage' page. Can be set with `EC_DEPLOYMENT_CLOUD_ID` envvar.
+
+#### Usage
+`delete-stale` Delete indices that are greater than 30 days old
+
+```
+bay elastic-cloud delete-stale --deployment-id 'string'
+```
+
 # Installation
 
 ## Homebrew (OSX)

cmd/deployment/metadata.go

@@ -1,4 +1,4 @@
-package project_map
+package deployment
 
 import (
 	"encoding/json"

cmd/elastic-cloud/delete-stale.go

@@ -0,0 +1,207 @@
+package elastic_cloud
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dpc-sdp/bay-cli/internal/helpers"
+	elasticsearch "github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/esapi"
+	"github.com/manifoldco/promptui"
+	errors "github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	lagoon_client "github.com/uselagoon/machinery/api/lagoon/client"
+	"github.com/uselagoon/machinery/api/schema"
+)
+
+type IndexSettings struct {
+	IndexItem struct {
+		IndexDetail struct {
+			CreationDate string `json:"creation_date"`
+		} `json:"index"`
+	} `json:"settings"`
+}
+
+type Indices map[string]IndexSettings
+
+type AliasAttr struct {
+	IsHidden bool `json:"is_hidden"`
+}
+
+type Aliases struct {
+	Aliases map[string]AliasAttr `json:"aliases"`
+}
+
+func DeleteStaleIndices(c *cli.Context) error {
+	force := c.Bool("force")
+	apiKey := c.String("deployment-api-key")
+	cloudId := c.String("deployment-id")
+	age := c.Int64("age")
+	deleteList := make([]string, 0)
+
+	hashes, err := NewHashMap()
+	if err != nil {
+		return err
+	}
+
+	client, err := elasticsearch.NewClient(elasticsearch.Config{APIKey: apiKey, CloudID: cloudId})
+	if err != nil {
+		return err
+	}
+
+	settings, err := esapi.IndicesGetSettingsRequest{FilterPath: []string{"*.settings.index.creation_date"}}.Do(context.TODO(), client)
+
+	if err != nil {
+		return err
+	}
+
+	indicesList := Indices{}
+
+	if err := json.NewDecoder(settings.Body).Decode(&indicesList); err != nil {
+		return errors.Wrap(err, "Error parsing the response body")
+	} else {
+		for k, i := range indicesList {
+			if strings.Contains(k, "elasticsearch_index") {
+				a, err := esapi.IndicesGetAliasRequest{Index: []string{k}}.Do(context.TODO(), client)
+				if err != nil {
+					return err
+				}
+				aliasList := map[string]Aliases{}
+				if err := json.NewDecoder(a.Body).Decode(&aliasList); err != nil {
+					return errors.Wrap(err, "Error parsing the response body")
+				}
+				now := time.Now().UnixMilli()
+				created, err := strconv.ParseInt(i.IndexItem.IndexDetail.CreationDate, 10, 64)
+				if err != nil {
+					return err
+				}
+
+				diffInDays := (now - created) / (1000 * 60 * 60 * 24)
+
+				if diffInDays > age {
+					// Add helper function to compute hash from k.
+					hash := strings.Split(k, "--")[0]
+					project, err := hashes.LookupProjectFromHash(hash)
+					if err != nil {
+						fmt.Printf("Error looking up project for hash %+v\n", hash)
+					}
+
+					fmt.Printf("Project: %+v\n", project)
+
+					if len(aliasList[k].Aliases) > 0 {
+						for aliasName := range aliasList[k].Aliases {
+							fmt.Fprintf(c.App.Writer, "The index %s is %d days old but will not be deleted because it has an associated alias %s\n", k, diffInDays, aliasName)
+						}
+					} else {
+						fmt.Fprintf(c.App.Writer, "The index %s is %d days old and will be marked for deletion\n", k, diffInDays)
+						deleteList = append(deleteList, k)
+					}
+				}
+			}
+		}
+		if i := len(deleteList); i > 0 {
+			if force {
+				fmt.Fprint(c.App.Writer, "Deleting indices marked for deletion.")
+				statusCode, err := deleteIndices(client, deleteList, i)
+				if err != nil {
+					return errors.Wrap(err, "error deleting indices")
+				} else {
+					if statusCode == 200 {
+						fmt.Fprintf(c.App.Writer, "Deletion request failed. Status code %d", statusCode)
+					} else {
+						fmt.Fprintf(c.App.Writer, "%+v indices successfully deleted.", i)
+					}
+				}
+			} else {
+				prompt := promptui.Prompt{
+					Label:     "Delete indices",
+					IsConfirm: true,
+				}
+
+				prompt_result, _ := prompt.Run()
+
+				if prompt_result == "y" {
+					_, err := deleteIndices(client, deleteList, i)
+					if err != nil {
+						return err
+					}
+				} else {
+					fmt.Printf("Operation cancelled.\nThere are %d indices marked for deletion.\n", i)
+				}
+			}
+		} else {
+			fmt.Printf("No indices meet the criteria for deletion.")
+		}
+	}
+	return nil
+}
+
+func deleteIndices(client *elasticsearch.Client, deleteList []string, c int) (int, error) {
+	res, err := esapi.IndicesDeleteRequest{Index: deleteList}.Do(context.TODO(), client)
+	if err != nil {
+		return res.StatusCode, err
+	} else {
+		if res.StatusCode != 200 {
+			fmt.Printf("Deletion request failed. Status code %+v", res.StatusCode)
+			return res.StatusCode, errors.New("non 200 status code")
+		} else {
+			fmt.Printf("%+v indices successfully deleted.", c)
+			return res.StatusCode, nil
+		}
+	}
+}
+
+type HashMap struct {
+	Hashes map[string]string `json:"hashes"`
+}
+
+func (h *HashMap) LookupProjectFromHash(hash string) (string, error) {
+	if val, ok := h.Hashes[hash]; ok {
+		return val, nil
+	}
+	return "", errors.New("hash not found")
+
+}
+
+// Compute the lookup table of hash to project name.
+func NewHashMap() (HashMap, error) {
+	hashMap := HashMap{
+		Hashes: make(map[string]string),
+	}
+	client, err := helpers.NewLagoonClient(nil)
+	if err != nil {
+		return hashMap, err
+	}
+	projects, _ := getLagoonProjects(context.TODO(), client)
+
+	for _, project := range projects {
+		searchHash, _ := getLagoonProjectVar(context.TODO(), client, project.Name, "SEARCH_HASH")
+		hashMap.Hashes[searchHash] = project.Name
+	}
+	fmt.Printf("Hashmap: %+v\n", hashMap)
+	return hashMap, nil
+}
+
+// Lookup Lagoon projects
+func getLagoonProjects(ctx context.Context, client *lagoon_client.Client) ([]schema.ProjectMetadata, error) {
+	projects := make([]schema.ProjectMetadata, 0)
+
+	err := client.ProjectsByMetadata(ctx, "type", "tide", &projects)
+	return projects, err
+}
+
+// Lookup Lagoon projects
+func getLagoonProjectVar(ctx context.Context, client *lagoon_client.Client, projectName string, varName string) (string, error) {
+	vars := []schema.EnvKeyValue{}
+	err := client.GetEnvVariablesByProjectEnvironmentName(ctx, &schema.EnvVariableByProjectEnvironmentNameInput{Project: projectName}, &vars)
+	for _, v := range vars {
+		if v.Name == varName {
+			return strings.ToLower(v.Value), nil
+		}
+	}
+	return "", err
+}

go.mod

@@ -1,19 +1,22 @@
 module github.com/dpc-sdp/bay-cli
 
-go 1.21
+go 1.22
 
 toolchain go1.23.3
 
 require (
 	github.com/alexeyco/simpletable v1.0.0
 	github.com/aws/aws-sdk-go-v2/config v1.28.3
 	github.com/aws/aws-sdk-go-v2/service/kms v1.37.5
+	github.com/elastic/go-elasticsearch/v8 v8.16.0
 	github.com/go-git/go-git/v5 v5.12.0
+	github.com/manifoldco/promptui v0.9.0
 	github.com/pkg/errors v0.9.1
 	github.com/urfave/cli/v2 v2.27.5
 	github.com/uselagoon/machinery v0.0.31
 	golang.org/x/crypto v0.29.0
 	gopkg.in/yaml.v3 v3.0.1
+	sigs.k8s.io/controller-runtime v0.17.2
 )
 
 require (
@@ -32,12 +35,16 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 // indirect
 	github.com/aws/smithy-go v1.22.0 // indirect
+	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/guregu/null v4.0.0+incompatible // indirect
@@ -53,9 +60,12 @@ require (
 	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.27.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/tools v0.16.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )