Downloader: Add forwarding to HTTP endpoint (#442)

* started with forwarding support in downloader

* Add missing files.

* Add missing files.

* Raise needed Go version

* More Go version bumping.

* Fix forwarding

* Go 1.21+ needed

* Make terminating forwarder more robust.

* Better var naming

* Remove dead code. Improve commentary.

* Prepare validation status adjustment.

* Move validations to functions to make them executable in a loop.

* Introduce validation mode flag (strict, unsafe)

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.3
+          go-version: 1.21.0
 
       - name: Build
         run: go build -v ./cmd/...

.github/workflows/itest.yml

@@ -9,7 +9,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.3
+          go-version: 1.21.0
 
       - name: Set up Node.js
         uses: actions/setup-node@v3

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-            go-version: '^1.20.3'
+            go-version: '^1.21.0'
 
       - name: Build
         run: make dist

README.md

@@ -37,7 +37,7 @@ Download the binaries from the most recent release assets on Github.
 
 ### Build from sources
 
-- A recent version of **Go** (1.20+) should be installed. [Go installation](https://go.dev/doc/install)
+- A recent version of **Go** (1.21+) should be installed. [Go installation](https://go.dev/doc/install)
 
 - Clone the repository `git clone https://github.com/csaf-poc/csaf_distribution.git `
 

cmd/csaf_downloader/config.go

@@ -10,6 +10,7 @@ package main
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net/http"
 
 	"github.com/csaf-poc/csaf_distribution/v2/internal/certs"
@@ -19,8 +20,17 @@ import (
 )
 
 const (
-	defaultWorker = 2
-	defaultPreset = "mandatory"
+	defaultWorker         = 2
+	defaultPreset         = "mandatory"
+	defaultForwardQueue   = 5
+	defaultValidationMode = validationStrict
+)
+
+type validationMode string
+
+const (
+	validationStrict = validationMode("strict")
+	validationUnsafe = validationMode("unsafe")
 )
 
 type config struct {
@@ -32,6 +42,7 @@ type config struct {
 	ClientPassphrase     *string           `long:"client-passphrase" description:"Optional passphrase for the client cert (limited, experimental, see doc)" value-name:"PASSPHRASE" toml:"client_passphrase"`
 	Version              bool              `long:"version" description:"Display version of the binary" toml:"-"`
 	Verbose              bool              `long:"verbose" short:"v" description:"Verbose output" toml:"verbose"`
+	NoStore              bool              `long:"nostore" short:"n" description:"Do not store files" toml:"no_store"`
 	Rate                 *float64          `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"`
 	Worker               int               `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"`
 	Range                *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"`
@@ -43,6 +54,14 @@ type config struct {
 	RemoteValidatorCache   string   `long:"validatorcache" description:"FILE to cache remote validations" value-name:"FILE" toml:"validatorcache"`
 	RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validatorpreset"`
 
+	//lint:ignore SA5008 We are using choice twice: strict, unsafe.
+	ValidationMode validationMode `long:"validationmode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"`
+
+	ForwardURL      string      `long:"forwardurl" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"`
+	ForwardHeader   http.Header `long:"forwardheader" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"`
+	ForwardQueue    int         `long:"forwardqueue" description:"Maximal queue LENGTH before forwarder" value-name:"LENGTH" toml:"forward_queue"`
+	ForwardInsecure bool        `long:"forwardinsecure" description:"Do not check TLS certificates from forward endpoint" toml:"forward_insecure"`
+
 	Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"`
 
 	clientCerts   []tls.Certificate
@@ -66,6 +85,8 @@ func parseArgsConfig() ([]string, *config, error) {
 		SetDefaults: func(cfg *config) {
 			cfg.Worker = defaultWorker
 			cfg.RemoteValidatorPresets = []string{defaultPreset}
+			cfg.ValidationMode = defaultValidationMode
+			cfg.ForwardQueue = defaultForwardQueue
 		},
 		// Re-establish default values if not set.
 		EnsureDefaults: func(cfg *config) {
@@ -75,11 +96,27 @@ func parseArgsConfig() ([]string, *config, error) {
 			if cfg.RemoteValidatorPresets == nil {
 				cfg.RemoteValidatorPresets = []string{defaultPreset}
 			}
+			switch cfg.ValidationMode {
+			case validationStrict, validationUnsafe:
+			default:
+				cfg.ValidationMode = validationStrict
+			}
 		},
 	}
 	return p.Parse()
 }
 
+// UnmarshalText implements [encoding/text.TextUnmarshaler].
+func (vm *validationMode) UnmarshalText(text []byte) error {
+	switch m := validationMode(text); m {
+	case validationStrict, validationUnsafe:
+		*vm = m
+	default:
+		return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe"`, m)
+	}
+	return nil
+}
+
 // ignoreFile returns true if the given URL should not be downloaded.
 func (cfg *config) ignoreURL(u string) bool {
 	return cfg.ignorePattern.Matches(u)

cmd/csaf_downloader/downloader.go

@@ -42,6 +42,7 @@ type downloader struct {
 	keys      *crypto.KeyRing
 	eval      *util.PathEval
 	validator csaf.RemoteValidator
+	forwarder *forwarder
 	mkdirMu   sync.Mutex
 }
 
@@ -424,18 +425,26 @@ nextAdvisory:
 		}
 
 		// Compare the checksums.
-		if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
-			log.Printf("SHA256 checksum of %s does not match.\n", file.URL())
-			continue
+		s256Check := func() error {
+			if s256 != nil && !bytes.Equal(s256.Sum(nil), remoteSHA256) {
+				return fmt.Errorf("SHA256 checksum of %s does not match", file.URL())
+			}
+			return nil
 		}
 
-		if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
-			log.Printf("SHA512 checksum of %s does not match.\n", file.URL())
-			continue
+		s512Check := func() error {
+			if s512 != nil && !bytes.Equal(s512.Sum(nil), remoteSHA512) {
+				return fmt.Errorf("SHA512 checksum of %s does not match", file.URL())
+			}
+			return nil
 		}
 
-		// Only check signature if we have loaded keys.
-		if d.keys != nil {
+		// Validate OpenPGG signature.
+		keysCheck := func() error {
+			// Only check signature if we have loaded keys.
+			if d.keys == nil {
+				return nil
+			}
 			var sign *crypto.PGPSignature
 			sign, signData, err = loadSignature(client, file.SignURL())
 			if err != nil {
@@ -446,38 +455,83 @@ nextAdvisory:
 			}
 			if sign != nil {
 				if err := d.checkSignature(data.Bytes(), sign); err != nil {
-					log.Printf("Cannot verify signature for %s: %v\n", file.URL(), err)
 					if !d.cfg.IgnoreSignatureCheck {
-						continue
+						return fmt.Errorf("cannot verify signature for %s: %v", file.URL(), err)
 					}
 				}
 			}
+			return nil
 		}
 
 		// Validate against CSAF schema.
-		if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
-			d.logValidationIssues(file.URL(), errors, err)
-			continue
+		schemaCheck := func() error {
+			if errors, err := csaf.ValidateCSAF(doc); err != nil || len(errors) > 0 {
+				d.logValidationIssues(file.URL(), errors, err)
+				return fmt.Errorf("schema validation for %q failed", file.URL())
+			}
+			return nil
 		}
 
-		if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil {
-			log.Printf("Ignoring %s: %s.\n", file.URL(), err)
-			continue
+		// Validate if filename is conforming.
+		filenameCheck := func() error {
+			if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil {
+				return fmt.Errorf("filename not conforming %s: %s", file.URL(), err)
+			}
+			return nil
 		}
 
-		// Validate against remote validator
-		if d.validator != nil {
+		// Validate against remote validator.
+		remoteValidatorCheck := func() error {
+			if d.validator == nil {
+				return nil
+			}
 			rvr, err := d.validator.Validate(doc)
 			if err != nil {
 				errorCh <- fmt.Errorf(
 					"calling remote validator on %q failed: %w",
 					file.URL(), err)
-				continue
+				return nil
 			}
 			if !rvr.Valid {
-				log.Printf("Remote validation of %q failed\n", file.URL())
+				return fmt.Errorf("remote validation of %q failed", file.URL())
+			}
+			return nil
+		}
+
+		// Run all the validations.
+		valStatus := notValidatedValidationStatus
+		for _, check := range []func() error{
+			s256Check,
+			s512Check,
+			keysCheck,
+			schemaCheck,
+			filenameCheck,
+			remoteValidatorCheck,
+		} {
+			if err := check(); err != nil {
+				// TODO: Improve logging.
+				log.Printf("check failed: %v\n", err)
+				valStatus.update(invalidValidationStatus)
+				if d.cfg.ValidationMode == validationStrict {
+					continue nextAdvisory
+				}
 			}
 		}
+		valStatus.update(validValidationStatus)
+
+		// Send to forwarder
+		if d.forwarder != nil {
+			d.forwarder.forward(
+				filename, data.String(),
+				valStatus,
+				string(s256Data),
+				string(s512Data))
+		}
+
+		if d.cfg.NoStore {
+			// Do not write locally.
+			continue
+		}
 
 		if err := d.eval.Extract(`$.document.tracking.initial_release_date`, dateExtract, false, doc); err != nil {
 			log.Printf("Cannot extract initial_release_date from advisory '%s'\n", file.URL())