From b13a4df0ed6c59843c9057ff0b96558461863e6f Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Thu, 30 Nov 2023 13:08:59 +0545 Subject: [PATCH] feat: embedded db (#309) * feat: Add support for embedded db * feat: support embedded database in Helm chart * fix: improve docker file * fix: templates/deployment.yaml * feat: ability to create postgresql stateful set * fix: calling db.MustInit twice on operator mode --- .dockerignore | 15 +++++++ Makefile | 2 +- Dockerfile => build/Dockerfile | 14 ++++-- chart/Chart.yaml | 2 +- chart/templates/deployment.yaml | 39 +++++++++++++++-- chart/templates/postgres.yaml | 22 ++++------ chart/values.yaml | 49 +++++++++++++++------ cmd/operator.go | 9 ++-- cmd/run.go | 7 +-- cmd/server.go | 25 ++++++++--- db/init.go | 68 +++++++++++++++++++++++++++--- main.go | 19 ++++++++- scrapers/runscrapers_suite_test.go | 3 +- 13 files changed, 215 insertions(+), 59 deletions(-) create mode 100644 .dockerignore rename Dockerfile => build/Dockerfile (65%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ce2ac009 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.bin/ +.config-db/ +.github/ +.vscode/ + +build + +chart/ +CONTRIBUTING.md +SECURITY.md +README.md +PROJECT + +cover.out +.releaserc diff --git a/Makefile b/Makefile index 64f36078..f322061f 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GOBIN=$(shell go env GOBIN) endif docker: - docker build . -t ${IMG} + docker build . -f build/Dockerfile -t ${IMG} # Push the docker image docker-push: diff --git a/Dockerfile b/build/Dockerfile similarity index 65% rename from Dockerfile rename to build/Dockerfile index 65444a5a..88ddf68b 100644 --- a/Dockerfile +++ b/build/Dockerfile @@ -1,13 +1,13 @@ FROM golang:1.20@sha256:bc5f0b5e43282627279fe5262ae275fecb3d2eae3b33977a7fd200c7a760d6f1 as builder WORKDIR /app -COPY ./ ./ ARG VERSION + COPY go.mod /app/go.mod COPY go.sum /app/go.sum RUN go mod download -WORKDIR /app -RUN go version + +COPY ./ ./ RUN make build FROM ubuntu:jammy@sha256:0bced47fffa3361afa981854fcabcd4577cd43cebbb808cea2b1f33a3dd7f508 @@ -16,5 +16,11 @@ WORKDIR /app COPY --from=builder /app/.bin/config-db /app COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +RUN mkdir /opt/database && groupadd --gid 1000 catalog && \ + useradd catalog --uid 1000 -g catalog -m -d /var/lib/catalog && \ + chown -R 1000:1000 /opt/database && chown -R 1000:1000 /app + +USER catalog:catalog + RUN /app/config-db go-offline -ENTRYPOINT ["/app/config-db"] +ENTRYPOINT ["/app/config-db"] \ No newline at end of file diff --git a/chart/Chart.yaml b/chart/Chart.yaml index c9a29d46..5ef95d0c 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -4,6 +4,6 @@ description: A Helm chart for config-db type: application -version: 0.2.0 +version: 0.3.0 appVersion: "0.0.5" diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index cb656ac6..eef5b954 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -1,5 +1,11 @@ +{{$embeddedDB := and (eq .Values.db.external.enabled false) (eq .Values.db.embedded.persist true) }} +--- apiVersion: apps/v1 +{{- if $embeddedDB}} +kind: StatefulSet +{{- else }} kind: Deployment +{{- end }} metadata: name: {{ include "config-db.name" . }} labels: {{- include "config-db.labels" . | nindent 4 }} @@ -7,11 +13,29 @@ spec: replicas: {{ .Values.replicas }} selector: matchLabels: {{- include "config-db.selectorLabels" . | nindent 6 }} + {{- if $embeddedDB }} + serviceName: {{ include "config-db.name" . }} + volumeClaimTemplates: + - metadata: + name: config-db-embedded-database + labels: + {{- include "config-db.labels" . | nindent 10 }} + spec: + {{- if not (eq .Values.db.embedded.storageClass "") }} + storageClassName: {{ .Values.db.embedded.storageClass }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.db.embedded.storage }} + {{- end }} template: metadata: labels: {{- include "config-db.selectorLabels" . | nindent 8 }} spec: serviceAccountName: {{ template "config-db.serviceAccountName" . }} + securityContext: + fsGroup: 1000 containers: - name: {{ include "config-db.name" . }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" @@ -42,7 +66,7 @@ spec: - --disable-postgrest={{ .Values.disablePostgrest }} - --change-retention-days={{ .Values.configChangeRetentionDays }} - --analysis-retention-days={{ .Values.configAnalysisRetentionDays }} - {{- if .Values.db.enabled}} + {{- if .Values.db.runMigrations}} - --db-migrations {{- end}} {{- if .Values.upstream.enabled}} @@ -52,10 +76,14 @@ spec: {{- end}} env: - name: DB_URL + {{- if .Values.db.external.enabled}} valueFrom: secretKeyRef: - name: {{ .Values.db.secretKeyRef.name }} - key: {{ .Values.db.secretKeyRef.key }} + name: {{ .Values.db.external.secretKeyRef.name }} + key: {{ .Values.db.external.secretKeyRef.key }} + {{- else}} + value: "embedded:///opt/database" + {{- end}} - name: NAMESPACE value: {{ .Values.namespace | default .Release.Namespace }} {{- if .Values.upstream.enabled}} @@ -66,6 +94,11 @@ spec: {{- end}} resources: {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + {{- if $embeddedDB}} + - name: config-db-embedded-database + mountPath: "/opt/database" + {{- end }} {{- with .Values.extra }} {{- toYaml . | nindent 6 }} {{- end }} diff --git a/chart/templates/postgres.yaml b/chart/templates/postgres.yaml index c274372e..668fe2a3 100644 --- a/chart/templates/postgres.yaml +++ b/chart/templates/postgres.yaml @@ -1,5 +1,4 @@ -{{- if eq .Values.db.enabled true }} - +{{- if .Values.db.external.create }} --- # PostgreSQL StatefulSet apiVersion: apps/v1 @@ -25,18 +24,18 @@ spec: mountPath: /data envFrom: - secretRef: - name: {{ .Values.db.secretKeyRef.name }} + name: {{ .Values.db.external.secretKeyRef.name }} volumeClaimTemplates: - metadata: name: postgresql spec: accessModes: ["ReadWriteOnce"] - {{- if not (eq .Values.db.storageClass "") }} - storageClassName: {{ .Values.db.storageClass }} + {{- if ne .Values.db.external.storageClass "" }} + storageClassName: {{ .Values.db.external.storageClass }} {{- end }} resources: requests: - storage: {{ .Values.db.storage }} + storage: {{ .Values.db.external.storage }} --- # PostgreSQL StatefulSet Service apiVersion: v1 @@ -50,11 +49,10 @@ spec: - port: 5432 targetPort: 5432 --- -{{- if .Values.db.secretKeyRef.create }} apiVersion: v1 kind: Secret metadata: - name: {{ .Values.db.secretKeyRef.name }} + name: {{ .Values.db.external.secretKeyRef.name }} annotations: "helm.sh/resource-policy": "keep" type: Opaque @@ -66,16 +64,12 @@ stringData: {{- $dbname := (( get $secretData "POSTGRES_DB" ) | b64dec ) | default "config_db" }} {{- $host := print (include "config-db.name" .) "-postgresql." .Release.Namespace ".svc.cluster.local:5432" }} {{- $url := print "postgresql://" $user ":" $password "@" $host }} - {{- $configDbUrl := ( get $secretData .Values.db.secretKeyRef.key ) | default ( print $url "/config_db?sslmode=disable" ) }} - + {{- $configDbUrl := ( get $secretData .Values.db.external.secretKeyRef.key ) | default ( print $url "/config_db?sslmode=disable" ) }} POSTGRES_USER: {{ $user | quote }} POSTGRES_PASSWORD: {{ $password | quote }} POSTGRES_HOST: {{ $host | quote }} POSTGRES_DB: {{ $dbname | quote }} - {{ .Values.db.secretKeyRef.key }}: {{ $configDbUrl | quote }} - -{{- end }} - + {{ .Values.db.external.secretKeyRef.key }}: {{ $configDbUrl | quote }} --- {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 59b99ad8..15abd60c 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -4,16 +4,16 @@ replicas: 1 # Use this only if you want to replace the default that is .Chart.Name as the name of all the objects. -nameOverride: "" +nameOverride: '' # Set to true if you want to disable the postgrest service -disablePostgrest: false +disablePostgrest: false image: repository: docker.io/flanksource/config-db pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "latest" + tag: 'latest' configChangeRetentionDays: 60 configAnalysisRetentionDays: 60 @@ -21,17 +21,38 @@ configAnalysisRetentionDays: 60 # a list of configmaps to load scrape rules from, the configmap should have a single entry called "config.yaml" scrapeRuleConfigMaps: - config-db-rules + db: - # Setting this to true will create a postgres stateful set for config-db to connect to. - enabled: true - secretKeyRef: - create: true - # The name of the secret to look for. - name: config-db-postgresql - # This is the key that we look for in the secret. - key: DB_URL - storageClass: "" - storage: 20Gi + runMigrations: true + embedded: + # If the database is embedded, setting this to true will persist the contents of the database + # through a persistent volume + persist: true + storageClass: '' + storage: 20Gi + external: + # Setting enabled to true will use an external postgres DB. + # You can either use the embedded db or an external db. + # If both is enabled, then embedded db will take precedence. + enabled: false + # Setting create:true will create + # - a postgres stateful set + # - the secret & + # - the service to expose the postgres stateful set. + # By default, the generated secret will use 'postgres' as the username and a randomly generated password. + # If you need to set a custom username and password, you can populate a secret named 'postgres-connection' before install + # with POSTGRES_USER and POSTGRES_PASSWORD + # + # If create:false, a prexisting secret containing the URI to an existing postgres database must be provided + # The URI must be in the format 'postgresql://"$user":"$password"@"$host"/"$database"' + create: false + secretKeyRef: + # The name of the secret to look for. + name: config-db-postgresql + # This is the key that we look for in the secret. + key: DB_URL + storageClass: '' + storage: 20Gi ingress: enabled: false @@ -55,7 +76,7 @@ resources: serviceAccount: create: true - name: "" + name: '' annotations: {} upstream: diff --git a/cmd/operator.go b/cmd/operator.go index dc4b9633..d5b38118 100644 --- a/cmd/operator.go +++ b/cmd/operator.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "fmt" "os" @@ -39,8 +38,10 @@ func init() { } func run(cmd *cobra.Command, args []string) { - db.MustInit() - api.DefaultContext = api.NewScrapeContext(context.Background(), db.DefaultDB(), db.Pool) + ctx := cmd.Context() + + db.MustInit(ctx) + api.DefaultContext = api.NewScrapeContext(ctx, db.DefaultDB(), db.Pool) zapLogger := logger.GetZapLogger() if zapLogger == nil { @@ -64,7 +65,7 @@ func run(cmd *cobra.Command, args []string) { utilruntime.Must(configsv1.AddToScheme(scheme)) // Start the server - go serve(args) + go serve(ctx, args) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/cmd/run.go b/cmd/run.go index 68500f3b..d4e86396 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "encoding/json" "fmt" "os" @@ -23,6 +22,8 @@ var Run = &cobra.Command{ Use: "run ", Short: "Run scrapers and return", Run: func(cmd *cobra.Command, configFiles []string) { + ctx := cmd.Context() + logger.Infof("Scraping %v", configFiles) scraperConfigs, err := v1.ParseConfigs(configFiles...) if err != nil { @@ -30,8 +31,8 @@ var Run = &cobra.Command{ } if db.ConnectionString != "" { - db.MustInit() - api.DefaultContext = api.NewScrapeContext(context.Background(), db.DefaultDB(), db.Pool) + db.MustInit(ctx) + api.DefaultContext = api.NewScrapeContext(ctx, db.DefaultDB(), db.Pool) } if db.ConnectionString == "" && outputDir == "" { diff --git a/cmd/server.go b/cmd/server.go index 1bf6af90..31cb7565 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -23,13 +23,15 @@ import ( var Serve = &cobra.Command{ Use: "serve", Run: func(cmd *cobra.Command, args []string) { - serve(args) + ctx := cmd.Context() + + db.MustInit(ctx) + serve(ctx, args) }, } -func serve(configFiles []string) { - db.MustInit() - api.DefaultContext = api.NewScrapeContext(context.Background(), db.DefaultDB(), db.Pool) +func serve(ctx context.Context, configFiles []string) { + api.DefaultContext = api.NewScrapeContext(ctx, db.DefaultDB(), db.Pool) e := echo.New() // PostgREST needs to know how it is exposed to create the correct links @@ -60,8 +62,19 @@ func serve(configFiles []string) { go jobs.ScheduleJobs() - if err := e.Start(fmt.Sprintf(":%d", httpPort)); err != nil { - e.Logger.Fatal(err) + go func() { + if err := e.Start(fmt.Sprintf(":%d", httpPort)); err != nil { + e.Logger.Fatal(err) + } + }() + + <-ctx.Done() + if err := db.StopEmbeddedPGServer(); err != nil { + logger.Errorf("failed to stop server: %v", err) + } + + if err := e.Shutdown(ctx); err != nil { + logger.Errorf("failed to shutdown echo HTTP server: %v", err) } } diff --git a/db/init.go b/db/init.go index 2d0925a4..b810b729 100644 --- a/db/init.go +++ b/db/init.go @@ -3,7 +3,12 @@ package db import ( "context" "database/sql" + "fmt" + "os" + "path" + "strings" + embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/flanksource/commons/logger" "github.com/flanksource/duty" "github.com/flanksource/duty/migrate" @@ -20,11 +25,15 @@ var ( HTTPEndpoint = "http://localhost:8080/db" db *gorm.DB runMigrations = false + + EmbeddedPGServer *embeddedpostgres.EmbeddedPostgres + EmbeddedPGPort = uint32(6432) + EmbeddedPGDB = "catalog" ) // Flags ... func Flags(flags *pflag.FlagSet) { - flags.StringVar(&ConnectionString, "db", "DB_URL", "Connection string for the postgres database") + flags.StringVar(&ConnectionString, "db", "DB_URL", "Connection string for the postgres database. Use embedded:// to use the embedded database") flags.StringVar(&Schema, "db-schema", "public", "") flags.StringVar(&LogLevel, "db-log-level", "warn", "") flags.BoolVar(&runMigrations, "db-migrations", false, "Run database migrations") @@ -34,27 +43,58 @@ func Flags(flags *pflag.FlagSet) { var Pool *pgxpool.Pool // MustInit initializes the database or fatally exits -func MustInit() { - if err := Init(ConnectionString); err != nil { +func MustInit(ctx context.Context) { + if err := Init(ctx, ConnectionString); err != nil { logger.Fatalf("Failed to initialize db: %v", err.Error()) } } +func embeddedDB(database string, port uint32) (string, error) { + embeddedPath := strings.TrimSuffix(strings.TrimPrefix(ConnectionString, "embedded://"), "/") + if err := os.Chmod(embeddedPath, 0750); err != nil { + logger.Errorf("failed to chmod %s: %v", embeddedPath, err) + } + + logger.Infof("Starting embedded postgres server at %s", embeddedPath) + + EmbeddedPGServer = embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig(). + Port(port). + DataPath(path.Join(embeddedPath, "data")). + RuntimePath(path.Join(embeddedPath, "runtime")). + BinariesPath(path.Join(embeddedPath, "bin")). + Version(embeddedpostgres.V14). + Username("postgres").Password("postgres"). + Database(database)) + + if err := EmbeddedPGServer.Start(); err != nil { + return "", fmt.Errorf("error starting embedded postgres: %w", err) + } + + return fmt.Sprintf("postgres://postgres:postgres@localhost:%d/%s?sslmode=disable", port, database), nil +} + // Init ... -func Init(connection string) error { +func Init(ctx context.Context, connection string) error { var err error + + if strings.HasPrefix(ConnectionString, "embedded://") { + if connection, err = embeddedDB(EmbeddedPGDB, EmbeddedPGPort); err != nil { + return fmt.Errorf("failed to setup embedded postgres: %w", err) + } + } + Pool, err = duty.NewPgxPool(connection) if err != nil { return err } - conn, err := Pool.Acquire(context.Background()) + conn, err := Pool.Acquire(ctx) if err != nil { return err } defer conn.Release() - if err := conn.Ping(context.Background()); err != nil { + if err := conn.Ping(ctx); err != nil { return err } @@ -92,3 +132,19 @@ func Ping() error { func DefaultDB() *gorm.DB { return db } + +func StopEmbeddedPGServer() error { + if EmbeddedPGServer == nil { + return nil + } + + logger.Infof("Stopping embedded postgres database server") + err := EmbeddedPGServer.Stop() + if err != nil { + return err + } + + EmbeddedPGServer = nil + logger.Infof("Stoped database server") + return nil +} diff --git a/main.go b/main.go index 23fe5ceb..13de0a2c 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,29 @@ package main import ( + "context" "os" + "os/signal" "github.com/flanksource/config-db/cmd" ) func main() { - - if err := cmd.Root.Execute(); err != nil { + if err := cmd.Root.ExecuteContext(newCancelableContext()); err != nil { os.Exit(1) } } + +func newCancelableContext() context.Context { + doneCh := make(chan os.Signal, 1) + signal.Notify(doneCh, os.Interrupt) + + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + <-doneCh + cancel() + }() + + return ctx +} diff --git a/scrapers/runscrapers_suite_test.go b/scrapers/runscrapers_suite_test.go index d4c724f7..cb6b91d5 100644 --- a/scrapers/runscrapers_suite_test.go +++ b/scrapers/runscrapers_suite_test.go @@ -1,6 +1,7 @@ package scrapers import ( + "context" "os" "path/filepath" "testing" @@ -49,7 +50,7 @@ var _ = BeforeSuite(func() { if _, err := duty.NewDB(pgUrl); err != nil { Fail(err.Error()) } - if err := db.Init(pgUrl); err != nil { + if err := db.Init(context.Background(), pgUrl); err != nil { Fail(err.Error()) }