Adding database

docker-compose.yml

@@ -0,0 +1,10 @@
+version: '2'
+
+# For testing purposes
+services:
+  postgres-16:
+    image: uselagoon/postgres-16:latest
+    labels:
+      lagoon.type: postgres
+    ports:
+      - '5433:5432'

go.mod

@@ -6,25 +6,47 @@ require (
 	github.com/Khan/genqlient v0.7.0
 	github.com/cheshir/go-mq v1.0.2
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/gin-gonic/gin v1.9.1
 	github.com/joho/godotenv v1.5.1
 	github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2
 	github.com/stretchr/testify v1.9.0
+	gorm.io/driver/postgres v1.5.9
+	gorm.io/driver/sqlite v1.5.4
+	gorm.io/gorm v1.25.10
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/NeowayLabs/wabbit v0.0.0-20200409220312-12e68ab5b0c6 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/containerd/containerd v1.7.13 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.5.5 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v1.14.18 // indirect
 	github.com/minio/md5-simd v1.1.0 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -34,17 +56,22 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
 	github.com/pborman/uuid v1.2.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/rs/xid v1.2.1 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.11 // indirect
+	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )
 

internal/handler/main.go

@@ -386,6 +386,11 @@ func (h *Messaging) SendResultsetToLagoon(result []LagoonFact, resource Resource
 	return nil
 }
 
+func (h *Messaging) sendFactsToDB(facts []LagoonFact, resource ResourceDestination, source string) error {
+
+	return nil
+}
+
 func (h *Messaging) sendFactsToLagoonAPI(facts []LagoonFact, apiClient graphql.Client, resource ResourceDestination, source string) error {
 
 	slog.Debug("Matched facts",

internal/handler/messaging.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/cheshir/go-mq"
+	"gorm.io/gorm"
 	"log/slog"
 	"sort"
 	"strconv"
@@ -19,10 +20,12 @@ type Messaging struct {
 	EnableDebug             bool
 	ProblemsFromSBOM        bool
 	TrivyServerEndpoint     string
+	DBConnection            *gorm.DB
 }
 
 // NewMessaging returns a messaging with config
-func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts int, startupInterval int, enableDebug bool, problemsFromSBOM bool, trivyServerEndpoint string) *Messaging {
+func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts int, startupInterval int, enableDebug bool, problemsFromSBOM bool, trivyServerEndpoint string, db *gorm.DB) *Messaging {
+
 	return &Messaging{
 		Config:                  config,
 		LagoonAPI:               lagoonAPI,
@@ -32,6 +35,7 @@ func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts
 		EnableDebug:             enableDebug,
 		ProblemsFromSBOM:        problemsFromSBOM,
 		TrivyServerEndpoint:     trivyServerEndpoint,
+		DBConnection:            db,
 	}
 }
 

internal/handler/processing.go

@@ -6,6 +6,7 @@ import (
 	"github.com/Khan/genqlient/graphql"
 	"github.com/cheshir/go-mq"
 	"github.com/uselagoon/lagoon/services/insights-handler/internal/lagoonclient"
+	"github.com/uselagoon/lagoon/services/insights-handler/internal/service"
 	"log/slog"
 	"net/http"
 	"strconv"
@@ -30,10 +31,6 @@ func processFactsDirectly(message mq.Message, h *Messaging) string {
 		return "exciting, unable to process direct facts"
 	}
 
-	//if h.EnableDebug {
-	//	log.Print("[DEBUG] facts", directFacts)
-	//}
-	//
 	slog.Debug("Facts info", "data", directFacts)
 
 	apiClient := graphql.NewClient(h.LagoonAPI.Endpoint, &http.Client{Transport: &authedTransport{wrapped: http.DefaultTransport, h: h}})
@@ -73,7 +70,31 @@ func processFactsDirectly(message mq.Message, h *Messaging) string {
 				"Error", err,
 			)
 		}
-		//log.Printf("Deleted facts on '%v:%v' for source %v\n", directFacts.ProjectName, directFacts.EnvironmentName, s)
+
+		// Now we do the same in the DB
+		if h.DBConnection != nil {
+			n, err := directFacts.EnvironmentId.Int64()
+			if err != nil {
+				slog.Error("Unable to convert json.Number to int64",
+					"EnvironmentId", directFacts.EnvironmentId,
+					"ProjectName", directFacts.ProjectName,
+					"EnvironmentName", directFacts.EnvironmentName,
+					"Source", s,
+					"Error", err,
+				)
+			}
+			if _, err := service.DeleteFacts(h.DBConnection, int(n), s); err != nil {
+				slog.Error("Unable to delete facts from DB",
+					"EnvironmentId", directFacts.EnvironmentId,
+					"ProjectName", directFacts.ProjectName,
+					"EnvironmentName", directFacts.EnvironmentName,
+					"Source", s,
+					"Error", err,
+				)
+			}
+
+		}
+
 		slog.Info("Deleted facts",
 			"EnvironmentId", directFacts.EnvironmentId,
 			"ProjectName", directFacts.ProjectName,
@@ -85,7 +106,30 @@ func processFactsDirectly(message mq.Message, h *Messaging) string {
 	facts, err := lagoonclient.AddFacts(context.TODO(), apiClient, processedFacts)
 	if err != nil {
 		//log.Println(err)
-		slog.Error("Issue adding facts", "Error", err.Error())
+		slog.Error("Issue adding facts to API", "Error", err.Error())
+	}
+
+	// Now add facts to DB
+	if h.DBConnection != nil {
+		lagoonFacts := []lagoonclient.Fact{}
+		for _, e := range processedFacts {
+			lf := lagoonclient.Fact{
+				Environment: environmentId,
+				Name:        e.Name,
+				Value:       e.Value,
+				Source:      e.Source,
+				Description: e.Description,
+				KeyFact:     e.KeyFact,
+				Type:        e.Type,
+				Category:    e.Category,
+			}
+			lagoonFacts = append(lagoonFacts, lf)
+		}
+
+		err = service.CreateFacts(h.DBConnection, &lagoonFacts)
+		if err != nil {
+			slog.Error("Issue adding facts to DB", "Error", err.Error())
+		}
 	}
 
 	return facts

internal/handler/testassets/directFactsPayload.json

@@ -12,7 +12,8 @@
       "type": "TEXT",
       "category": "Framework",
       "service": "cli",
-      "keyFact": true
+      "keyFact": true,
+      "source": "somethingwicked"
     },
     {
       "projectName": "lagoon-demo-org",
@@ -23,7 +24,8 @@
       "type": "TEXT",
       "category": "Programming language",
       "service": "cli",
-      "keyFact": false
+      "keyFact": false,
+      "source": "somethingwicked"
     }
   ],
   "type": "direct.facts",