From 63b867745dd88306f30a6193c8e898e1e5702c75 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Thu, 16 Nov 2023 13:16:30 +1300 Subject: [PATCH] Introduces direct problems processing and logic fixes --- internal/handler/insightsParserFilter.go | 2 +- internal/handler/main.go | 7 +++ internal/handler/main_test.go | 6 +- internal/handler/messaging.go | 28 +++++++-- internal/handler/processing.go | 60 ++++++++++++++++++- .../testassets/directProblemsPayload.json | 22 +++++++ .../testassets/directProblemsPayload2.json | 20 +++++++ internal/lagoonclient/problems.go | 2 +- 8 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 internal/handler/testassets/directProblemsPayload.json create mode 100644 internal/handler/testassets/directProblemsPayload2.json diff --git a/internal/handler/insightsParserFilter.go b/internal/handler/insightsParserFilter.go index aeda938..fb5baba 100644 --- a/internal/handler/insightsParserFilter.go +++ b/internal/handler/insightsParserFilter.go @@ -71,7 +71,7 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC fmt.Println("trivy is alive") } if isAlive { - err = SbomToProblems(apiClient, h.TrivyServerEndpoint, "/tmp/", environment.Id, "insights-handler", *bom) + err = SbomToProblems(apiClient, h.TrivyServerEndpoint, "/tmp/", environment.Id, resource.Service, *bom) } if err != nil { return nil, "", err diff --git a/internal/handler/main.go b/internal/handler/main.go index 1062cc2..5633326 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -96,6 +96,13 @@ type DirectFacts struct { InsightsType string `json:"insightsType"` } +type DirectProblems struct { + EnvironmentId int `json:"environment"` + ProjectName string `json:"projectName"` + EnvironmentName string `json:"environmentName"` + Problems []lagoonclient.LagoonProblem `json:"problems"` + Type string `json:"type"` +} type InsightsData struct { InputType string InputPayload PayloadType diff --git a/internal/handler/main_test.go b/internal/handler/main_test.go index b33bfbf..7d0c7de 100644 --- a/internal/handler/main_test.go +++ b/internal/handler/main_test.go @@ -122,13 +122,13 @@ func Test_processDirectFacts(t *testing.T) { fmt.Println(string(message.Body())) - got := processItemsDirectly(tt.args.message, tt.args.h) + got := processFactsDirectly(tt.args.message, tt.args.h) if (err != nil) != tt.wantErr { - t.Errorf("processItemsDirectly() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("processFactsDirectly() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("processItemsDirectly() got = %v, want %v", got, tt.want) + t.Errorf("processFactsDirectly() got = %v, want %v", got, tt.want) } }) } diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index 6281397..1c7f643 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -39,7 +39,6 @@ func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts func (h *Messaging) processMessageQueue(message mq.Message) { var insights InsightsData var resource ResourceDestination - acknowledgeMessage := func(message mq.Message) func() { return func() { // Ack to remove from queue @@ -61,13 +60,32 @@ func (h *Messaging) processMessageQueue(message mq.Message) { }(message) incoming := &InsightsMessage{} - json.Unmarshal(message.Body(), incoming) + err := json.Unmarshal(message.Body(), incoming) + + if err != nil { + fmt.Printf(err.Error()) + acknowledgeMessage() + return + } // if we have direct problems or facts, we process them differently - skipping all // the extra processing below. - if incoming.Type == "direct.facts" || incoming.Type == "direct.problems" { - resp := processItemsDirectly(message, h) - log.Println(resp) + if incoming.Type == "direct.facts" { + resp := processFactsDirectly(message, h) + if h.EnableDebug { + log.Println(resp) + } + acknowledgeMessage() + return + } + + if incoming.Type == "direct.problems" { + resp, _ := processProblemsDirectly(message, h) + if h.EnableDebug { + for _, d := range resp { + log.Println(d) + } + } acknowledgeMessage() return } diff --git a/internal/handler/processing.go b/internal/handler/processing.go index 7e0609c..16f5d1a 100644 --- a/internal/handler/processing.go +++ b/internal/handler/processing.go @@ -13,12 +13,12 @@ import ( // processing.go contains the functions that actually process the incoming messages -func processItemsDirectly(message mq.Message, h *Messaging) string { +func processFactsDirectly(message mq.Message, h *Messaging) string { var directFacts DirectFacts json.Unmarshal(message.Body(), &directFacts) err := json.Unmarshal(message.Body(), &directFacts) if err != nil { - log.Println("Error unmarshaling JSON:", err) + log.Println("Error unmarshaling JSON:", err.Error()) return "exciting, unable to process direct facts" } @@ -63,7 +63,7 @@ func processItemsDirectly(message mq.Message, h *Messaging) string { if err != nil { log.Println(err) } - log.Printf("Deleted facts on '%v:%v' for source %v", directFacts.ProjectName, directFacts.EnvironmentName, s) + log.Printf("Deleted facts on '%v:%v' for source %v\n", directFacts.ProjectName, directFacts.EnvironmentName, s) } facts, err := lagoonclient.AddFacts(context.TODO(), apiClient, processedFacts) @@ -73,3 +73,57 @@ func processItemsDirectly(message mq.Message, h *Messaging) string { return facts } + +func processProblemsDirectly(message mq.Message, h *Messaging) ([]string, error) { + var directProblems DirectProblems + json.Unmarshal(message.Body(), &directProblems) + log.Println(directProblems) + err := json.Unmarshal(message.Body(), &directProblems) + if err != nil { + log.Println("Error unmarshaling JSON:", err) + return []string{}, err + } + + if h.EnableDebug { + log.Print("[DEBUG] problems", directProblems) + } + + apiClient := graphql.NewClient(h.LagoonAPI.Endpoint, &http.Client{Transport: &authedTransport{wrapped: http.DefaultTransport, h: h}}) + + // serviceSource just gives us simple structure to do the deletions + type serviceSource struct { + Source string + Service string + } + problemSources := map[string]serviceSource{} + + for i, problem := range directProblems.Problems { + + // We want to ensure that the incoming problems aren't malformed or trying to do anything dodgy with env ids + + if problem.Environment != directProblems.EnvironmentId { + directProblems.Problems[i].Environment = directProblems.EnvironmentId + } + + problemSources[problem.Service+problem.Source] = serviceSource{ + Source: problem.Source, + Service: problem.Service, + } + } + + for _, s := range problemSources { + _, err := lagoonclient.DeleteProblemsFromSource(context.TODO(), apiClient, directProblems.EnvironmentId, s.Service, s.Source) + if err != nil { + log.Println(err) //This could potentially mess up the state if we've already deleted source info, might + return []string{}, err + } + log.Printf("Deleted Problems on '%v:%v' for source %v\n", directProblems.ProjectName, directProblems.EnvironmentName, s) + } + + resptext, err := lagoonclient.AddProblems(context.TODO(), apiClient, directProblems.Problems) + if err != nil { + log.Println(err) + } + + return resptext, nil +} diff --git a/internal/handler/testassets/directProblemsPayload.json b/internal/handler/testassets/directProblemsPayload.json new file mode 100644 index 0000000..c1b040f --- /dev/null +++ b/internal/handler/testassets/directProblemsPayload.json @@ -0,0 +1,22 @@ +{ + "projectName": "test6-drupal-example-simple", + "environment": 1236, + "environmentName": "test1copy", + "problems": [ + { + "environment": 3, + "service": "cli", + "identifier": "testidentifier", + "version": "1.0.0", + "fixedVersion": "1.0.1", + "source": "insights:problems:cli", + "data": "Nothing to write home about", + "severityScore": 0.1, + "severity": "CRITICAL", + "description": "Test description" + + } + ], + "type": "direct.problems", + "source": "insights:problems:cli" +} diff --git a/internal/handler/testassets/directProblemsPayload2.json b/internal/handler/testassets/directProblemsPayload2.json new file mode 100644 index 0000000..1624347 --- /dev/null +++ b/internal/handler/testassets/directProblemsPayload2.json @@ -0,0 +1,20 @@ +{ + "environment": 1236, + "projectName": "test6-drupal-example-simple", + "environmentName": "test1copy", + "problems": [ + { + "environment": 1236, + "identifier": "1234", + "version": "1.0.1", + "fixedVersion": "1.0.2", + "source": "test", + "service": "cli", + "data": "{}", + "severity": "CRITICAL", + "severityScore": 0.5, + "description": "This is a test 1" + } + ], + "type": "direct.problems" +} diff --git a/internal/lagoonclient/problems.go b/internal/lagoonclient/problems.go index 3815fc1..56fe84f 100644 --- a/internal/lagoonclient/problems.go +++ b/internal/lagoonclient/problems.go @@ -59,7 +59,7 @@ func AddProblems(ctx context.Context, client graphql.Client, problems []LagoonPr } func DeleteProblemsFromSource(ctx context.Context, client graphql.Client, environmentID int, service string, source string) (string, error) { - resp, err := deleteProblemsFromSource(ctx, client, environmentID, service, source) + resp, err := deleteProblemsFromSource(ctx, client, environmentID, source, service) if err != nil { return "", err }