From 6264463618c727f2fa2c2079d2ec3226aa42b583 Mon Sep 17 00:00:00 2001 From: Blaize Kaye Date: Tue, 7 Nov 2023 06:09:31 +1300 Subject: [PATCH] Writes to insights remote endpoint with in cluster gatherer command * Adds insights remote lib * Updates arguments * Removes local token * Small updates to Facts * Updates defaults to bring in line with insights-remote * Completing work on in cluster gathering * Removes extraneous exit * Fixes error handling in remote writer * Refactors token gathering * Removes test gatherer --------- Co-authored-by: Chris --- .github/workflows/go.yml | 2 +- cmd/gatherInCluster.go | 110 ++++++++++++++++++++++++++++ gatherers/graphql.go | 3 +- gatherers/insights-remote-writer.go | 62 ++++++++++++++++ gatherers/main.go | 2 +- go.mod | 2 +- go.work | 6 ++ insightRemoteLib/go.mod | 3 + insightRemoteLib/remote.go | 21 ++++++ 9 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 cmd/gatherInCluster.go create mode 100644 gatherers/insights-remote-writer.go create mode 100644 go.work create mode 100644 insightRemoteLib/go.mod create mode 100644 insightRemoteLib/remote.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 5d4b8fb..e22a58b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v2 with: - go-version: ^1.13 + go-version: ^1.18 - name: Check out code into the Go module directory uses: actions/checkout@v2 diff --git a/cmd/gatherInCluster.go b/cmd/gatherInCluster.go new file mode 100644 index 0000000..14bc47a --- /dev/null +++ b/cmd/gatherInCluster.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "encoding/json" + "log" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/uselagoon/lagoon-facts-app/gatherers" +) + +var tokenValue string +var tokenFile string +var insightsRemoteEndpoint string + +// gatherCmd represents the gather command +var gatherInClusterCmd = &cobra.Command{ + Use: "gather-in-cluster", + Short: "Running this command will invoke the registered gatherers in cluster", + Long: `Running all the registered gatherers will inspect the system and write FACT data back to the Lagoon insights system via insights-remote`, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + //get the basic env vars + if argStatic && argDynamic { + log.Fatalf("Cannot use both 'static' and 'dynamic' only gatherers - exiting") + } + }, + Run: func(cmd *cobra.Command, args []string) { + + if tokenValue == "" { + if tokenFile == "" { + log.Fatal("Either a token or a token file needs to be passed as an argument") + } + var err error + tokenValue, err = getTokenFromFile(tokenFile) + if err != nil { + log.Fatalf("Unable to load token: %v - %v", tokenFile, err.Error()) + } + } + + //set gatherer type to be static by default + gathererTypeArg := gatherers.GATHERER_TYPE_STATIC + if argDynamic { + gathererTypeArg = gatherers.GATHERER_TYPE_DYNAMIC + } + + //run the gatherers... + gathererSlice := gatherers.GetGatherers() + + var facts []gatherers.GatheredFact + + for _, e := range gathererSlice { + if e.GetGathererCmdType() == gathererTypeArg { + if e.AppliesToEnvironment() { + gatheredFacts, err := e.GatherFacts() + if err != nil { + log.Println(err.Error()) + continue + } + for _, f := range gatheredFacts { + if verbose := viper.Get("verbose"); verbose == true { + log.Printf("Registering %s", f.Name) + } + } + facts = append(facts, gatheredFacts...) + } + } + } + + if !dryRun { + err := gatherers.WriteFactsToInsightsRemote(tokenValue, facts) + if err != nil { + log.Println(err.Error()) + } + } + + if dryRun { + if facts != nil { + log.Println("---- Dry run ----") + log.Printf("Would post the follow facts to '%s:%s'", projectName, environmentName) + s, _ := json.MarshalIndent(facts, "", "\t") + log.Println(string(s)) + } + } + }, +} + +func getTokenFromFile(tokenFile string) (string, error) { + _, err := os.Stat(tokenFile) + if err != nil { + return "", err + } + + ba, err := os.ReadFile(tokenFile) + if err != nil { + return "", err + } + return string(ba), nil +} + +//var GatherCommand = gatherCmd + +func init() { + gatherInClusterCmd.PersistentFlags().StringVarP(&tokenValue, "token", "t", "", "The Lagoon insights remote token") + gatherInClusterCmd.PersistentFlags().StringVarP(&tokenFile, "token-file", "", "/var/run/secrets/lagoon/dynamic/insights-token/INSIGHTS_TOKEN", "Read the Lagoon insights remote token from a file") + gatherInClusterCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "d", false, "run gathers and print to screen without running write methods") + gatherInClusterCmd.PersistentFlags().StringVar(&insightsRemoteEndpoint, "insights-remote-endpoint", "http://lagoon-remote-insights-remote.lagoon.svc/facts", "The Lagoon insights remote endpoint") + viper.BindPFlag("insights-remote-endpoint", gatherInClusterCmd.PersistentFlags().Lookup("insights-remote-endpoint")) + rootCmd.AddCommand(gatherInClusterCmd) +} diff --git a/gatherers/graphql.go b/gatherers/graphql.go index 63f48f1..040f910 100644 --- a/gatherers/graphql.go +++ b/gatherers/graphql.go @@ -3,11 +3,10 @@ package gatherers import ( "context" "fmt" - "log" - "github.com/machinebox/graphql" "github.com/uselagoon/lagoon-facts-app/utils" "golang.org/x/oauth2" + "log" ) const lagoonAPIEndpoint = "https://api.lagoon.amazeeio.cloud/graphql" diff --git a/gatherers/insights-remote-writer.go b/gatherers/insights-remote-writer.go new file mode 100644 index 0000000..d95775a --- /dev/null +++ b/gatherers/insights-remote-writer.go @@ -0,0 +1,62 @@ +package gatherers + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/spf13/viper" + insightRemoteLib "github.com/uselagoon/insights-remote-lib" + "io/ioutil" + "log" + "net/http" +) + +func WriteFactsToInsightsRemote(token string, facts []GatheredFact) error { + + insightsRemoteFacts := insightRemoteLib.Facts{ + Facts: []insightRemoteLib.Fact{}, + } + + for _, fact := range facts { + f := insightRemoteLib.Fact{ + Name: fact.Name, + Value: fact.Value, + Source: fact.Source, + Description: fact.Description, + Category: string(fact.Category), + } + + insightsRemoteFacts.Facts = append(insightsRemoteFacts.Facts, f) + } + + bodyString, err := json.Marshal(insightsRemoteFacts) + if err != nil { + log.Fatal(err.Error()) + } + + fmt.Printf("Sending %v fact(s) to insights core\n", len(facts)) + + serviceEndpoint := viper.GetString("insights-remote-endpoint") + req, _ := http.NewRequest(http.MethodPost, serviceEndpoint, bytes.NewBuffer(bodyString)) + req.Header.Set("Authorization", token) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + response, err := client.Do(req) + + if err != nil { + return err + } + + if response.StatusCode != 200 { + bodyData, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Fatal(err.Error()) + } + + log.Fatalf("There was an error sending the facts to '%s': %v- %v \n", serviceEndpoint, response.StatusCode, string(bodyData)) + } + + defer response.Body.Close() + + return err +} diff --git a/gatherers/main.go b/gatherers/main.go index 3fc23fd..b191d1a 100644 --- a/gatherers/main.go +++ b/gatherers/main.go @@ -11,4 +11,4 @@ func LoadYamlConfig(yamlFilePath string) ([]byte, error) { return []byte{}, err } return data, nil -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 19c21cc..d19c4a9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/uselagoon/lagoon-facts-app -go 1.13 +go 1.18 require ( github.com/machinebox/graphql v0.2.2 diff --git a/go.work b/go.work new file mode 100644 index 0000000..08851dc --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.18 + +use ( + . + insightRemoteLib +) \ No newline at end of file diff --git a/insightRemoteLib/go.mod b/insightRemoteLib/go.mod new file mode 100644 index 0000000..ff637c3 --- /dev/null +++ b/insightRemoteLib/go.mod @@ -0,0 +1,3 @@ +module github.com/uselagoon/insights-remote-lib + +go 1.18 diff --git a/insightRemoteLib/remote.go b/insightRemoteLib/remote.go new file mode 100644 index 0000000..dbe992f --- /dev/null +++ b/insightRemoteLib/remote.go @@ -0,0 +1,21 @@ +package insightRemoteLib + +type Fact struct { + EnvironmentId string `json:"environment"` + ProjectName string `json:"projectName"` + EnvironmentName string `json:"environmentName"` + Name string `json:"name"` + Value string `json:"value"` + Source string `json:"source"` + Description string `json:"description"` + Type string `json:"type"` + Category string `json:"category"` + Service string `json:"service"` +} + +type Facts struct { + EnvironmentId int `json:"environment"` + ProjectName string `json:"projectName"` + EnvironmentName string `json:"environmentName"` + Facts []Fact `json:"facts"` +}