chore: use koanf instead of viper to allow more possibility on config…

…uration (#160) **Describe the pull request** This pull request involves switching from the Viper library to the Koanf library in a Go application. The primary motivation behind this change is to enhance the application's configuration possibilities. Viper, although a comprehensive solution for configuration in Go applications, has some limitations, particularly its case-insensitive behavior and a large number of hardcoded dependencies, which result in a bigger binary size compared to Koanf. For instance, a simple program using Viper is significantly larger (about 313%) than its Koanf equivalent **Checklist** - [x] I have linked the relative issue to this pull request - [x] I have made the modifications or added tests related to my PR - [x] I have added/updated the documentation for my RP - [x] I put my PR in Ready for Review only when all the checklist is checked **Additional context** This switch to Koanf aims to reduce the application's binary size and dependency footprint while maintaining or enhancing configuration capabilities. The choice between Viper and Koanf largely depends on the specific needs and constraints of the application in question.
42atomys · Nov 19, 2023 · 828953d · 828953d
1 parent 7837602
commit 828953d
Show file tree

Hide file tree

Showing 21 changed files with 395 additions and 424 deletions.
diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml
@@ -5,7 +5,7 @@ services:
     volumes:
       - ..:/workspace:cached
     environment:
-      DEBUG: "true"
+      WH_DEBUG: "true"
       POSTGRES_DB: postgres
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres

diff --git a/.github/workflows/k6.yaml b/.github/workflows/k6.yaml
@@ -31,7 +31,7 @@ jobs:
     - name: Start application and run K6
       continue-on-error: true
       run: |
-        go run main.go serve --config tests/webhooks.tests.yml >/dev/null 2>&1 &
+        go run main.go serve --config tests/webhooks.tests.yaml >/dev/null 2>&1 &
         until $(curl --output /dev/null --silent --head --fail http://localhost:8080/metrics); do
           printf '.'
           sleep 1

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -6,7 +6,7 @@ on:
     - main
     paths:
     - '**/*.go'
-    - '**/*.yml'
+    - '**/*.yaml'
   workflow_dispatch:
 permissions:
   contents: read
@@ -19,6 +19,7 @@ jobs:
       matrix:
         goVersion: [ '1.18', '1.19', '1.20' ]
     env:
+      WH_DEBUG: 'true'
       REDIS_HOST: '127.0.0.1'
       REDIS_PORT: '6379'
       REDIS_PASSWORD: ''
@@ -62,9 +63,7 @@ jobs:
       with:
         version: latest
     - name: Run Unit tests
-      run: |
-        go test ./... -coverprofile coverage.out -covermode count
-        go tool cover -func coverage.out
+      run: make test-units
     - name: Quality Gate - Test coverage shall be above threshold
       env:
           TESTCOVERAGE_THRESHOLD: 90
@@ -82,8 +81,7 @@ jobs:
         fi
     - uses: codecov/codecov-action@v3
     - name: Run Go Build
-      run: |
-        go build -o /tmp/applications-test-units
+      run: make build
   integration-tests:
     name: "Integration tests"
     runs-on: ubuntu-latest
@@ -92,6 +90,7 @@ jobs:
       matrix:
         goVersion: [ '1.18', '1.19', '1.20' ]
     env:
+      WH_DEBUG: 'true'
       REDIS_HOST: '127.0.0.1'
       REDIS_PORT: '6379'
       REDIS_PASSWORD: ''
@@ -102,14 +101,11 @@ jobs:
       uses: supercharge/[email protected]
       with:
         redis-version: 6
-    - name: Install k6
-      run: |
-        curl https://github.com/grafana/k6/releases/download/v0.45.0/k6-v0.45.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1
     - name: Run Integration tests
       run: |
-        go run main.go serve --config tests/integrations/webhooked_config.integration.yaml >/dev/null 2>&1 &
+        make run-integration >/dev/null 2>&1 &
         until $(curl --output /dev/null --silent --head --fail http://localhost:8080/metrics); do
           printf '.'
           sleep 1
         done
-        ./k6 run tests/integrations/scenarios.js
+        make test-integrations
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+bin
 
 # Test binary, built with `go test -c`
 *.test
@@ -12,8 +13,8 @@
 *.out
 
 # Configuration file
-config/*.yml
-!config/webhooks.example.yml
+config/*.yaml
+!config/webhooked.example.yaml
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Start webhooked",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "main.go",
+      "args": ["serve"]
+    }
+  ]
+}
diff --git a/Makefile b/Makefile
@@ -0,0 +1,46 @@
+
+test-payload:
+	curl -XPOST -H 'X-Hook-Secret:test' \
+		-d "{\"time\": \"$(date +"%Y-%m-%dT%H:%M:%S")\", \"content\": \"Hello World\"}" \
+		http://localhost:8080/v1alpha1/webhooks/example
+
+install-k6:
+	@if ! which k6 > /dev/null; then \
+		echo "Installing k6..." \
+		sudo gpg -k; \
+		sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69; \
+		echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list; \
+		sudo apt-get update; \
+		sudo apt-get install k6; \
+		echo "k6 installed successfully"; \
+	else \
+		echo "k6 is already installed"; \
+	fi
+
+build:
+	@echo "Building webhooked..."
+	@GOOS=linux GOARCH=amd64 go build -o ./bin/webhooked ./main.go
+
+tests: test-units test-integrations
+
+test-units:
+	@echo "Running unit tests..."
+	@export WH_DEBUG=true
+	@go test ./... -coverprofile coverage.out -covermode count
+	@go tool cover -func coverage.out
+
+run-integration: build
+	@./bin/webhooked --config ./tests/integrations/webhooked_config.integration.yaml serve
+
+test-integrations: install-k6
+	@echo "Running integration tests..."
+
+	@if ! pgrep -f "./bin/webhooked" > /dev/null; then \
+		echo "PID file not found. Please run 'make run-integration' in another terminal."; \
+		exit 1; \
+	fi
+
+	@echo "Running k6 tests..."
+	@k6 run ./tests/integrations/scenarios.js
+
+.PHONY: test-payload install-k6 build run-integration test-integration
diff --git a/README.md b/README.md
@@ -122,7 +122,7 @@ You can use the docker image [atomys/webhooked](https://hub.docker.com/r/atomys/
 
 ```sh
 # Basic launch instruction using the default configuration path
-docker run -it --rm -p 8080:8080 -v ${PWD}/myconfig.yaml:/config/webhooks.yaml atomys/webhooked:latest
+docker run -it --rm -p 8080:8080 -v ${PWD}/myconfig.yaml:/config/webhooked.yaml atomys/webhooked:latest
 # Use custom configuration file
 docker run -it --rm -p 8080:8080 -v ${PWD}/myconfig.yaml:/myconfig.yaml atomys/webhooked:latest serve --config /myconfig.yaml
 ```

diff --git a/cmd/root.go b/cmd/root.go
@@ -24,17 +24,16 @@ THE SOFTWARE.
 package cmd
 
 import (
-	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
-var cfgFile string
+// configFilePath represents the location of the configuration file
+var configFilePath string
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "webhooks",
-	Short: "webhooks is a simple program to receive webhooks from around the world and place them in a storage space of your choice.",
+	Use:   "webhooked",
+	Short: "webhooked is a simple program to receive webhooks and forward them to a destination",
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
@@ -44,30 +43,8 @@ func Execute() {
 }
 
 func init() {
-	cobra.OnInitialize(initConfig)
-
 	// Here you will define your flags and configuration settings.
 	// Cobra supports persistent flags, which, if defined here,
 	// will be global for your application.
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is config/webhooks.yaml)")
-}
-
-// initConfig reads in config file and ENV variables if set.
-func initConfig() {
-	if cfgFile != "" {
-		// Use config file from the flag.
-		viper.SetConfigFile(cfgFile)
-	} else {
-		viper.SetConfigName("webhooks")
-		viper.SetConfigType("yaml")
-		viper.AddConfigPath("./config")
-	}
-
-	viper.AutomaticEnv() // read in environment variables that match
-
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err != nil {
-		log.Error().Msgf("Error reading config file, %s", err.Error())
-	}
-	log.Info().Msgf("Using config file: %s", viper.ConfigFileUsed())
+	rootCmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", "config/webhooked.yaml", "config file (default is config/webhooked.yaml)")
 }
diff --git a/cmd/serve.go b/cmd/serve.go
@@ -1,7 +1,7 @@
 /*
 Package cmd : cobra package
 
-Copyright © 2022 42Stellar
+# Copyright © 2022 42Stellar
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -38,7 +38,7 @@ var (
 		Use:   "serve",
 		Short: "serve the http server",
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := config.Load(); err != nil {
+			if err := config.Load(configFilePath); err != nil {
 				log.Fatal().Err(err).Msg("invalid configuration")
 			}
 

diff --git a/config/webhooks.example.yml → config/webhooked.example.yaml b/config/webhooks.example.yml → config/webhooked.example.yaml
diff --git a/examples/kubernetes/deployment.yaml b/examples/kubernetes/deployment.yaml
@@ -9,7 +9,7 @@ kind: ConfigMap
 metadata:
   name: webhooked
 data:
-  webhooks.yaml: |
+  webhooked.yaml: |
     apiVersion: v1alpha1
     specs:
     - name: exampleHook
@@ -68,9 +68,9 @@ spec:
         - containerPort: 8080
           name: http
         volumeMounts:
-        - mountPath: /config/webhooks.yaml
+        - mountPath: /config/webhooked.yaml
           name: configuration
-          subPath: webhooks.yaml
+          subPath: webhooked.yaml
       volumes:
       - name: configuration
         configMap:

diff --git a/go.mod b/go.mod
@@ -6,12 +6,12 @@ require (
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/gorilla/mux v1.8.1
 	github.com/jmoiron/sqlx v1.3.5
+	github.com/knadh/koanf v1.5.0
 	github.com/lib/pq v1.10.9
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/rs/zerolog v1.31.0
 	github.com/spf13/cobra v1.8.0
-	github.com/spf13/viper v1.16.0
 	github.com/streadway/amqp v1.1.0
 	github.com/stretchr/testify v1.8.4
 )
@@ -23,25 +23,19 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cast v1.5.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
 	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )