Merge pull request #30 from uselagoon/feature/trivy_integration

Trivy integration

.gitignore

@@ -1,3 +1,4 @@
 .idea
 .env
-internal/handler/testassets/bin/*
+internal/handler/testassets/bin/*
+

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.18-alpine3.15 as builder
+FROM golang:1.20.8-alpine3.18 as builder
 
 COPY . /go/src/github.com/uselagoon/lagoon/services/insights-handler/
 WORKDIR /go/src/github.com/uselagoon/lagoon/services/insights-handler/

Makefile

@@ -0,0 +1,10 @@
+.PHONY: gettrivy
+gettrivy:
+	mkdir -p internal/handler/testassets/bin/trivy/ && wget -O - https://github.com/aquasecurity/trivy/releases/download/v0.45.0/trivy_0.45.0_Linux-64bit.tar.gz | tar -zxvf - -C internal/handler/testassets/bin/trivy/
+
+
+.PHONY: runlocal
+runlocal:
+	go run main.go --problems-from-sbom=true --rabbitmq-username=guest  --rabbitmq-password=guest --lagoon-api-host=http://localhost:8888/graphql   --jwt-token-signing-key=secret   --access-key-id=minio     --secret-access-key=minio123 --disable-s3-upload=true
+
+

README.md

@@ -12,7 +12,7 @@ and push to the api and s3 bucket.
     go run main.go \
         -rabbitmq-username guest \
         -rabbitmq-password guest \
-        -lagoon-api-host http://localhost:7070/graphql \
+        -lagoon-api-host http://localhost:8888/graphql \
         --jwt-token-signing-key secret  \
         --access-key-id minio \
         --secret-access-key minio123

internal/handler/insightsParserFilter.go

@@ -61,6 +61,23 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC
 	}
 	source := fmt.Sprintf("insights:sbom:%s", resource.Service)
 
+	// we process the SBOM here
+
+	if h.ProblemsFromSBOM == true {
+		isAlive, err := IsTrivyServerIsAlive(h.TrivyServerEndpoint)
+		if err != nil {
+			return nil, "", fmt.Errorf("trivy server not alive: %v", err.Error())
+		} else {
+			fmt.Println("trivy is alive")
+		}
+		if isAlive {
+			err = SbomToProblems(apiClient, h.TrivyServerEndpoint, "/tmp/", environment.Id, "insights-handler", *bom)
+		}
+		if err != nil {
+			return nil, "", err
+		}
+	}
+
 	// Process SBOM into facts
 	facts := processFactsFromSBOM(bom.Components, environment.Id, source)
 
@@ -74,6 +91,7 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC
 	}
 
 	log.Printf("Successfully decoded SBOM of image %s with %s, found %d for '%s:%s'", bom.Metadata.Component.Name, (*bom.Metadata.Tools)[0].Name, len(*bom.Components), resource.Project, resource.Environment)
+
 	return facts, source, nil
 }
 

internal/handler/main.go

@@ -202,6 +202,7 @@ type ResourceDestination struct {
 // Consumer handles consuming messages sent to the queue that this action handler is connected to and processes them accordingly
 func (h *Messaging) Consumer() {
 	var messageQueue mq.MQ
+
 	// if no mq is found when the goroutine starts, retry a few times before exiting
 	// default is 10 retry with 30 second delay = 5 minutes
 	err := try.Do(func(attempt int) (bool, error) {
@@ -266,16 +267,16 @@ func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource Resource
 	apiClient := h.getApiClient()
 
 	if resource.Project == "" && resource.Environment == "" {
-		log.Println("no resource definition labels could be found in payload (i.e. lagoon.sh/project or lagoon.sh/environment)")
+		return fmt.Errorf("no resource definition labels could be found in payload (i.e. lagoon.sh/project or lagoon.sh/environment)")
 	}
 
-	if insights.InputPayload == Payload {
+	if insights.InputPayload == Payload && insights.LagoonType == Facts {
 		for _, p := range incoming.Payload {
 			parserFilterLoopForPayloads(insights, p, h, apiClient, resource)
 		}
 	}
 
-	if insights.InputPayload == BinaryPayload {
+	if insights.InputPayload == BinaryPayload && insights.LagoonType == Facts {
 		for _, p := range incoming.BinaryPayload {
 			parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource)
 		}
@@ -286,30 +287,13 @@ func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource Resource
 
 func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) {
 	for _, filter := range parserFilters {
-
-		if insights.LagoonType == Facts { // This should be more or less trivially true
-
-			result, source, err := filter(h, insights, p, apiClient, resource)
-			if err != nil {
-				log.Println(fmt.Errorf(err.Error()))
-			}
 
-			for _, r := range result {
-				if fact, ok := r.(LagoonFact); ok {
-					// Handle single fact
-					err = h.sendFactsToLagoonAPI([]LagoonFact{fact}, apiClient, resource, source)
-					if err != nil {
-						fmt.Println(err)
-					}
-				} else if facts, ok := r.([]LagoonFact); ok {
-					// Handle slice of facts
-					h.sendFactsToLagoonAPI(facts, apiClient, resource, source)
-				} else {
-					// Unexpected type returned from filter()
-					log.Printf("unexpected type returned from filter(): %T\n", r)
-				}
-			}
+		result, source, err := filter(h, insights, p, apiClient, resource)
+		if err != nil {
+			log.Println(fmt.Errorf(err.Error()))
 		}
+
+		processResultset(result, err, h, apiClient, resource, source)
 	}
 }
 
@@ -318,32 +302,35 @@ func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messa
 		var result []interface{}
 		var source string
 
-		if insights.LagoonType == Facts { // This should be more or less trivially true
-			json, err := json.Marshal(p)
-			if err != nil {
-				log.Println(fmt.Errorf(err.Error()))
-			}
+		json, err := json.Marshal(p)
+		if err != nil {
+			log.Println(fmt.Errorf(err.Error()))
+		}
 
-			result, source, err = filter(h, insights, fmt.Sprintf("%s", json), apiClient, resource)
-			if err != nil {
-				log.Println(fmt.Errorf(err.Error()))
-			}
+		result, source, err = filter(h, insights, fmt.Sprintf("%s", json), apiClient, resource)
+		if err != nil {
+			log.Println(fmt.Errorf(err.Error()))
+		}
 
-			for _, r := range result {
-				if fact, ok := r.(LagoonFact); ok {
-					// Handle single fact
-					err = h.sendFactsToLagoonAPI([]LagoonFact{fact}, apiClient, resource, source)
-					if err != nil {
-						fmt.Println(err)
-					}
-				} else if facts, ok := r.([]LagoonFact); ok {
-					// Handle slice of facts
-					h.sendFactsToLagoonAPI(facts, apiClient, resource, source)
-				} else {
-					// Unexpected type returned from filter()
-					log.Printf("unexpected type returned from filter(): %T\n", r)
-				}
+		processResultset(result, err, h, apiClient, resource, source)
+	}
+}
+
+// processResultset will send results as facts to the lagoon api after processing via a parser filter
+func processResultset(result []interface{}, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) {
+	for _, r := range result {
+		if fact, ok := r.(LagoonFact); ok {
+			// Handle single fact
+			err = h.sendFactsToLagoonAPI([]LagoonFact{fact}, apiClient, resource, source)
+			if err != nil {
+				fmt.Println(err)
 			}
+		} else if facts, ok := r.([]LagoonFact); ok {
+			// Handle slice of facts
+			h.sendFactsToLagoonAPI(facts, apiClient, resource, source)
+		} else {
+			// Unexpected type returned from filter()
+			log.Printf("unexpected type returned from filter(): %T\n", r)
 		}
 	}
 }
@@ -394,11 +381,11 @@ func determineResourceFromLagoonAPI(apiClient graphql.Client, resource ResourceD
 	// Get project data (we need the project ID to be able to utilise the environmentByName query)
 	project, err := lagoonclient.GetProjectByName(context.TODO(), apiClient, resource.Project)
 	if err != nil {
-		return lagoonclient.Project{}, lagoonclient.Environment{}, fmt.Errorf("error: unable to determine resource destination (does %s:%s exist?)", resource.Project, resource.Environment)
+		return lagoonclient.Project{}, lagoonclient.Environment{}, fmt.Errorf("error: unable to determine resource destination (does %s:%s exist?): %v", resource.Project, resource.Environment, err.Error())
 	}
 
 	if project.Id == 0 || project.Name == "" {
-		return lagoonclient.Project{}, lagoonclient.Environment{}, fmt.Errorf("error: unable to determine resource destination (does %s:%s exist?)", resource.Project, resource.Environment)
+		return lagoonclient.Project{}, lagoonclient.Environment{}, fmt.Errorf("error: unable to determine resource destination (does %s:%s exist?): %v", resource.Project, resource.Environment, err.Error())
 	}
 
 	environment, err := lagoonclient.GetEnvironmentFromName(context.TODO(), apiClient, resource.Environment, project.Id)

internal/handler/messaging.go

@@ -17,17 +17,21 @@ type Messaging struct {
 	ConnectionAttempts      int
 	ConnectionRetryInterval int
 	EnableDebug             bool
+	ProblemsFromSBOM        bool
+	TrivyServerEndpoint     string
 }
 
 // NewMessaging returns a messaging with config
-func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts int, startupInterval int, enableDebug bool) *Messaging {
+func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts int, startupInterval int, enableDebug bool, problemsFromSBOM bool, trivyServerEndpoint string) *Messaging {
 	return &Messaging{
 		Config:                  config,
 		LagoonAPI:               lagoonAPI,
 		S3Config:                s3,
 		ConnectionAttempts:      startupAttempts,
 		ConnectionRetryInterval: startupInterval,
 		EnableDebug:             enableDebug,
+		ProblemsFromSBOM:        problemsFromSBOM,
+		TrivyServerEndpoint:     trivyServerEndpoint,
 	}
 }
 
@@ -36,12 +40,23 @@ func (h *Messaging) processMessageQueue(message mq.Message) {
 	var insights InsightsData
 	var resource ResourceDestination
 
-	// set up defer to ack the message after we're done processing
-	defer func(message mq.Message) {
-		// Ack to remove from queue
-		err := message.Ack(false)
-		if err != nil {
-			fmt.Printf("Failed to acknowledge message: %s\n", err.Error())
+	acknowledgeMessage := func(message mq.Message) func() {
+		return func() {
+			// Ack to remove from queue
+			err := message.Ack(false)
+			if err != nil {
+				fmt.Printf("Failed to acknowledge message: %s\n", err.Error())
+			}
+		}
+	}(message)
+
+	rejectMessage := func(message mq.Message) func(bool) {
+		return func(requeue bool) {
+			// Ack to remove from queue
+			err := message.Reject(requeue)
+			if err != nil {
+				fmt.Printf("Failed to requect message: %s\n", err.Error())
+			}
 		}
 	}(message)
 
@@ -53,6 +68,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) {
 	if incoming.Type == "direct.facts" || incoming.Type == "direct.problems" {
 		resp := processItemsDirectly(message, h)
 		log.Println(resp)
+		acknowledgeMessage()
 		return
 	}
 
@@ -120,10 +136,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) {
 		if h.EnableDebug {
 			log.Printf("[DEBUG] no payload was found")
 		}
-		err := message.Reject(false)
-		if err != nil {
-			fmt.Printf("Unable to reject payload: %s\n", err.Error())
-		}
+		rejectMessage(false)
 		return
 	}
 	if len(incoming.Payload) != 0 {
@@ -145,6 +158,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) {
 			err := h.sendToLagoonS3(incoming, insights, resource)
 			if err != nil {
 				log.Printf("Unable to send to S3: %s", err.Error())
+				// TODO: do we reque here? Reject
 			}
 		}
 	}
@@ -158,9 +172,13 @@ func (h *Messaging) processMessageQueue(message mq.Message) {
 			log.Println("only 'sbom', 'direct', 'raw', and 'image' types are currently supported for api processing")
 		} else {
 			err := h.sendToLagoonAPI(incoming, resource, insights)
+
 			if err != nil {
 				log.Printf("Unable to send to the api: %s", err.Error())
+				rejectMessage(false)
+				return
 			}
 		}
 	}
+	acknowledgeMessage()
 }