From 1d9969162f8beaf6632b43d8c6f5204a19674227 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Sun, 30 Jul 2023 22:37:32 +0200 Subject: [PATCH 1/7] Fix version config and make aggreator use new command line parser. --- cmd/csaf_aggregator/config.go | 53 +++++++++++++++++------------------ cmd/csaf_aggregator/main.go | 29 +++---------------- cmd/csaf_checker/config.go | 3 +- cmd/csaf_downloader/config.go | 3 +- docs/csaf_aggregator.md | 9 +++--- 5 files changed, 38 insertions(+), 59 deletions(-) diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index c8a807d3..da91c2f1 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -18,15 +18,14 @@ import ( "sync" "time" - "github.com/BurntSushi/toml" "github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/csaf-poc/csaf_distribution/v2/csaf" + "github.com/csaf-poc/csaf_distribution/v2/internal/options" "github.com/csaf-poc/csaf_distribution/v2/util" "golang.org/x/time/rate" ) const ( - defaultConfigPath = "aggregator.toml" defaultWorkers = 10 defaultFolder = "/var/www" defaultWeb = "/var/www/html" @@ -74,7 +73,8 @@ type config struct { LockFile *string `toml:"lock_file"` // Interim performs an interim scan. - Interim bool `toml:"interim"` + Interim bool `short:"i" long:"interim" description:"Perform an interim scan" toml:"interim"` + Version bool `long:"version" description:"Display version of the binary" toml:"-"` // InterimYears is numbers numbers of years to look back // for interim advisories. Less/equal zero means forever. @@ -90,11 +90,34 @@ type config struct { // 'update_interval'. UpdateInterval *string `toml:"update_interval"` + Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` + keyMu sync.Mutex key *crypto.Key keyErr error } +// configPaths are the potential file locations of the config file. +var configPaths = []string{ + // TODO: Make symmetric to checker and downloader. + "aggregator.toml", +} + +// parseArgsConfig parse the command arguments and loads configuration +// from a configuration file. +func parseArgsConfig() ([]string, *config, error) { + p := options.Parser[config]{ + DefaultConfigLocations: configPaths, + ConfigLocation: func(cfg *config) string { + return cfg.Config + }, + HasVersion: func(cfg *config) bool { return cfg.Version }, + // Establish default values if not set. + EnsureDefaults: (*config).setDefaults, + } + return p.Parse() +} + // tooOldForInterims returns a function that tells if a given // time is too old for the configured interims interval. func (c *config) tooOldForInterims() func(time.Time) bool { @@ -299,27 +322,3 @@ func (c *config) check() error { return c.checkMirror() } - -func loadConfig(path string) (*config, error) { - if path == "" { - path = defaultConfigPath - } - - var cfg config - md, err := toml.DecodeFile(path, &cfg) - if err != nil { - return nil, err - } - - if undecoded := md.Undecoded(); len(undecoded) != 0 { - return nil, fmt.Errorf("could not parse %q from config.toml", undecoded) - } - - cfg.setDefaults() - - if err := cfg.check(); err != nil { - return nil, err - } - - return &cfg, nil -} diff --git a/cmd/csaf_aggregator/main.go b/cmd/csaf_aggregator/main.go index 9af87441..d6261927 100644 --- a/cmd/csaf_aggregator/main.go +++ b/cmd/csaf_aggregator/main.go @@ -15,17 +15,11 @@ import ( "os" "path/filepath" - "github.com/csaf-poc/csaf_distribution/v2/util" + "github.com/csaf-poc/csaf_distribution/v2/internal/options" "github.com/gofrs/flock" "github.com/jessevdk/go-flags" ) -type options struct { - Config string `short:"c" long:"config" description:"File name of the configuration file" value-name:"CFG-FILE" default:"aggregator.toml"` - Version bool `long:"version" description:"Display version of the binary"` - Interim bool `short:"i" long:"interim" description:"Perform an interim scan"` -} - func errCheck(err error) { if err != nil { if flags.WroteHelp(err) { @@ -60,24 +54,9 @@ func lock(lockFile *string, fn func() error) error { } func main() { - opts := new(options) - - _, err := flags.Parse(opts) - errCheck(err) - - if opts.Version { - fmt.Println(util.SemVersion) - return - } - - interim := opts.Interim - - cfg, err := loadConfig(opts.Config) - errCheck(err) - - if interim { - cfg.Interim = true - } + _, cfg, err := parseArgsConfig() + options.ErrorCheck(err) + options.ErrorCheck(cfg.check()) p := processor{cfg: cfg} errCheck(lock(cfg.LockFile, p.process)) diff --git a/cmd/csaf_checker/config.go b/cmd/csaf_checker/config.go index b8148e53..a2581d9e 100644 --- a/cmd/csaf_checker/config.go +++ b/cmd/csaf_checker/config.go @@ -77,7 +77,8 @@ func parseArgsConfig() ([]string, *config, error) { ConfigLocation: func(cfg *config) string { return cfg.Config }, - Usage: "[OPTIONS] domain...", + Usage: "[OPTIONS] domain...", + HasVersion: func(cfg *config) bool { return cfg.Version }, SetDefaults: func(cfg *config) { cfg.Format = defaultFormat cfg.RemoteValidatorPresets = []string{defaultPreset} diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 07e9de64..fdf30996 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -53,7 +53,8 @@ func parseArgsConfig() ([]string, *config, error) { ConfigLocation: func(cfg *config) string { return cfg.Config }, - Usage: "[OPTIONS] domain...", + Usage: "[OPTIONS] domain...", + HasVersion: func(cfg *config) bool { return cfg.Version }, SetDefaults: func(cfg *config) { cfg.Worker = defaultWorker cfg.RemoteValidatorPresets = []string{defaultPreset} diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index 64491f8b..edbe8e81 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -6,13 +6,12 @@ csaf_aggregator [OPTIONS] Application Options: - -c, --config=CFG-FILE File name of the configuration file (default: - aggregator.toml) - --version Display version of the binary - -i, --interim Perform an interim scan + -i, --interim Perform an interim scan + --version Display version of the binary + -c, --config=TOML-FILE Path to config TOML file Help Options: - -h, --help Show this help message + -h, --help Show this help message ``` Usage example for a single run, to test if the config is good: From 383b0ca77b384e51d83a08ea012ceaea4de495d0 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 31 Jul 2023 17:19:38 +0200 Subject: [PATCH 2/7] Add an option to downloader to store advisories into a given folder. --- cmd/csaf_downloader/config.go | 1 + cmd/csaf_downloader/downloader.go | 9 ++++++++- docs/csaf_downloader.md | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index fdf30996..f8adb6c1 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -29,6 +29,7 @@ type config struct { Rate *float64 `long:"rate" short:"r" description:"The average upper limit of https operations per second (defaults to unlimited)" toml:"rate"` Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"` Range *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"` + Folder string `long:"folder" short:"f" description:"Download into a given FOLDER" value-name:"FOLDER" toml:"folder"` ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index cabe2fe3..8c5312bc 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -474,8 +474,15 @@ nextAdvisory: initialReleaseDate = initialReleaseDate.UTC() // Write advisory to file + newDir := path.Join(d.directory, lower) + + // Do we have a configured destination folder? + if d.cfg.Folder != "" { + newDir = path.Join(newDir, d.cfg.Folder) + } else { + newDir = path.Join(newDir, strconv.Itoa(initialReleaseDate.Year())) + } - newDir := path.Join(d.directory, lower, strconv.Itoa(initialReleaseDate.Year())) if newDir != lastDir { if err := d.mkdirAll(newDir, 0755); err != nil { errorCh <- err diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index 8874df13..bdab206b 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -15,6 +15,7 @@ Application Options: -r, --rate= The average upper limit of https operations per second (defaults to unlimited) -w, --worker=NUM NUMber of concurrent downloads (default: 2) -t, --timerange=RANGE RANGE of time from which advisories to download + -f, --folder=FOLDER Download into a given FOLDER -H, --header= One or more extra HTTP header fields --validator=URL URL to validate documents remotely --validatorcache=FILE FILE to cache remote validations @@ -86,3 +87,7 @@ into a given intervall. There are three possible notations: spans an interval from 1st January 2019 to the 1st January of 2024. All interval boundaries are inclusive. + +if the `folder` option is given the advisories are stored in this folder. +If it is omitted (by default) the advisories are stored in folder name by the +year they are from. From 28641761113da147a8b2a3f3a0cdb65c6f4d7d4f Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Aug 2023 01:46:58 +0200 Subject: [PATCH 3/7] Add ignore patterns to downloader. --- cmd/csaf_downloader/config.go | 39 +++++++++++++++++++++++++------ cmd/csaf_downloader/downloader.go | 7 ++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index f8adb6c1..417befce 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -9,7 +9,9 @@ package main import ( + "fmt" "net/http" + "regexp" "github.com/csaf-poc/csaf_distribution/v2/internal/models" "github.com/csaf-poc/csaf_distribution/v2/internal/options" @@ -30,6 +32,7 @@ type config struct { Worker int `long:"worker" short:"w" description:"NUMber of concurrent downloads" value-name:"NUM" toml:"worker"` Range *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"` Folder string `long:"folder" short:"f" description:"Download into a given FOLDER" value-name:"FOLDER" toml:"folder"` + IgnorePattern []string `long:"ignorepattern" short:"i" description:"Dont download files if there URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignorepattern"` ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` @@ -38,6 +41,8 @@ type config struct { RemoteValidatorPresets []string `long:"validatorpreset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validatorpreset"` Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` + + ignorePattern []*regexp.Regexp } // configPaths are the potential file locations of the config file. @@ -51,11 +56,9 @@ var configPaths = []string{ func parseArgsConfig() ([]string, *config, error) { p := options.Parser[config]{ DefaultConfigLocations: configPaths, - ConfigLocation: func(cfg *config) string { - return cfg.Config - }, - Usage: "[OPTIONS] domain...", - HasVersion: func(cfg *config) bool { return cfg.Version }, + ConfigLocation: func(cfg *config) string { return cfg.Config }, + Usage: "[OPTIONS] domain...", + HasVersion: func(cfg *config) bool { return cfg.Version }, SetDefaults: func(cfg *config) { cfg.Worker = defaultWorker cfg.RemoteValidatorPresets = []string{defaultPreset} @@ -73,8 +76,30 @@ func parseArgsConfig() ([]string, *config, error) { return p.Parse() } +// ignoreFile returns true if the given URL should not be downloaded. +func (cfg *config) ignoreURL(u string) bool { + for _, expr := range cfg.ignorePattern { + if expr.MatchString(u) { + return true + } + } + return false +} + +// compileIgnorePatterns compiles the configure patterns to be ignored. +func (cfg *config) compileIgnorePatterns() error { + cfg.ignorePattern = make([]*regexp.Regexp, 0, len(cfg.IgnorePattern)) + for _, pattern := range cfg.IgnorePattern { + expr, err := regexp.Compile(pattern) + if err != nil { + return fmt.Errorf("invalid ignorepattern: %w", err) + } + cfg.ignorePattern = append(cfg.ignorePattern, expr) + } + return nil +} + // prepare prepares internal state of a loaded configuration. func (cfg *config) prepare() error { - // TODO: Implement me! - return nil + return cfg.compileIgnorePatterns() } diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 8c5312bc..0ae3b958 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -347,6 +347,13 @@ nextAdvisory: continue } + if d.cfg.ignoreURL(file.URL()) { + if d.cfg.Verbose { + log.Printf("Igoring %q.\n", file.URL()) + } + continue + } + resp, err := client.Get(file.URL()) if err != nil { log.Printf("WARN: cannot get '%s': %v\n", file.URL(), err) From 8aa31984df1754c2bdf1d40c469ac7bb9f2853fc Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Aug 2023 09:54:43 +0200 Subject: [PATCH 4/7] Add docs and fix output. --- cmd/csaf_downloader/config.go | 4 ++++ cmd/csaf_downloader/downloader.go | 2 +- docs/csaf_downloader.md | 15 +++++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index 417befce..d8dbee11 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -10,6 +10,7 @@ package main import ( "fmt" + "log" "net/http" "regexp" @@ -94,6 +95,9 @@ func (cfg *config) compileIgnorePatterns() error { if err != nil { return fmt.Errorf("invalid ignorepattern: %w", err) } + if cfg.Verbose { + log.Printf("ignore advisories containing pattern %q\n", expr) + } cfg.ignorePattern = append(cfg.ignorePattern, expr) } return nil diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index 0ae3b958..8d6fd805 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -349,7 +349,7 @@ nextAdvisory: if d.cfg.ignoreURL(file.URL()) { if d.cfg.Verbose { - log.Printf("Igoring %q.\n", file.URL()) + log.Printf("Ignoring %q.\n", file.URL()) } continue } diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index bdab206b..57b6fd11 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -16,6 +16,7 @@ Application Options: -w, --worker=NUM NUMber of concurrent downloads (default: 2) -t, --timerange=RANGE RANGE of time from which advisories to download -f, --folder=FOLDER Download into a given FOLDER + -i, --ignorepattern=PATTERN Dont download files if there URLs match any of the given PATTERNs -H, --header= One or more extra HTTP header fields --validator=URL URL to validate documents remotely --validatorcache=FILE FILE to cache remote validations @@ -52,11 +53,13 @@ ignoresigcheck = false verbose = false # rate # set to unlimited worker = 2 +# timerange # not set by default +# folder # not set by default +# ignorepattern # not set by default # header # not set by default # validator # not set by default # validatorcache # not set by default validatorpreset = ["mandatory"] -# timerange # not set by default ``` The `timerange` parameter enables downloading advisories which last changes falls @@ -88,6 +91,14 @@ into a given intervall. There are three possible notations: All interval boundaries are inclusive. -if the `folder` option is given the advisories are stored in this folder. +If the `folder` option is given the advisories are stored in this folder. If it is omitted (by default) the advisories are stored in folder name by the year they are from. + +You can ignore certain advisories while downloading by specifying a list +of regular expressions to match their URLs by using the `ignorepattern` option. +E.g. `-i='.*white.*' -i='*.red.*'` will ignore files wich URLs contain the sub strings **white** or **red**. +In the config file this has to be noted as: +``` +ignorepattern = [".*white.*", ".*red.*"] +``` From 97304ab38ed0d2806cda71fe3a44deba59fa64c7 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Aug 2023 16:43:31 +0200 Subject: [PATCH 5/7] Typo fix. --- docs/csaf_downloader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index 57b6fd11..6008e26d 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -97,7 +97,7 @@ year they are from. You can ignore certain advisories while downloading by specifying a list of regular expressions to match their URLs by using the `ignorepattern` option. -E.g. `-i='.*white.*' -i='*.red.*'` will ignore files wich URLs contain the sub strings **white** or **red**. +E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain the sub strings **white** or **red**. In the config file this has to be noted as: ``` ignorepattern = [".*white.*", ".*red.*"] From 5a4e5607cb2c4adb9fb71345792b23348b25b1ae Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 1 Aug 2023 17:05:49 +0200 Subject: [PATCH 6/7] Factored out matching lists of regular expressions. --- cmd/csaf_downloader/config.go | 27 ++++++---------------- internal/filter/filter.go | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 internal/filter/filter.go diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index d8dbee11..2c18ebec 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -9,11 +9,9 @@ package main import ( - "fmt" - "log" "net/http" - "regexp" + "github.com/csaf-poc/csaf_distribution/v2/internal/filter" "github.com/csaf-poc/csaf_distribution/v2/internal/models" "github.com/csaf-poc/csaf_distribution/v2/internal/options" ) @@ -43,7 +41,7 @@ type config struct { Config string `short:"c" long:"config" description:"Path to config TOML file" value-name:"TOML-FILE" toml:"-"` - ignorePattern []*regexp.Regexp + ignorePattern filter.PatternMatcher } // configPaths are the potential file locations of the config file. @@ -79,27 +77,16 @@ func parseArgsConfig() ([]string, *config, error) { // ignoreFile returns true if the given URL should not be downloaded. func (cfg *config) ignoreURL(u string) bool { - for _, expr := range cfg.ignorePattern { - if expr.MatchString(u) { - return true - } - } - return false + return cfg.ignorePattern.Matches(u) } // compileIgnorePatterns compiles the configure patterns to be ignored. func (cfg *config) compileIgnorePatterns() error { - cfg.ignorePattern = make([]*regexp.Regexp, 0, len(cfg.IgnorePattern)) - for _, pattern := range cfg.IgnorePattern { - expr, err := regexp.Compile(pattern) - if err != nil { - return fmt.Errorf("invalid ignorepattern: %w", err) - } - if cfg.Verbose { - log.Printf("ignore advisories containing pattern %q\n", expr) - } - cfg.ignorePattern = append(cfg.ignorePattern, expr) + pm, err := filter.NewPatternMatcher(cfg.IgnorePattern) + if err != nil { + return err } + cfg.ignorePattern = pm return nil } diff --git a/internal/filter/filter.go b/internal/filter/filter.go new file mode 100644 index 00000000..bdc6afb3 --- /dev/null +++ b/internal/filter/filter.go @@ -0,0 +1,42 @@ +// This file is Free Software under the MIT License +// without warranty, see README.md and LICENSES/MIT.txt for details. +// +// SPDX-License-Identifier: MIT +// +// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) +// Software-Engineering: 2023 Intevation GmbH + +// Package filter implements helps to filter advisories. +package filter + +import ( + "fmt" + "regexp" +) + +// PatternMatcher is a list of regular expressions. +type PatternMatcher []*regexp.Regexp + +// NewPatternMatcher compiles a new list of regular expression from +// a given list of strings. +func NewPatternMatcher(patterns []string) (PatternMatcher, error) { + pm := make(PatternMatcher, 0, len(patterns)) + for _, pattern := range patterns { + expr, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("invalid ignore pattern: %w", err) + } + pm = append(pm, expr) + } + return pm, nil +} + +// Matches returns true if the given string matches any of the expressions. +func (pm PatternMatcher) Matches(s string) bool { + for _, expr := range pm { + if expr.MatchString(s) { + return true + } + } + return false +} From 083ccc10f1003e3995daf5fcb7ad8f992bd28543 Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer Date: Wed, 16 Aug 2023 09:03:34 +0200 Subject: [PATCH 7/7] fmt --- cmd/csaf_downloader/config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/csaf_downloader/config.go b/cmd/csaf_downloader/config.go index be9b56fa..8a9bf68a 100644 --- a/cmd/csaf_downloader/config.go +++ b/cmd/csaf_downloader/config.go @@ -33,8 +33,6 @@ type config struct { Folder string `long:"folder" short:"f" description:"Download into a given subFOLDER" value-name:"FOLDER" toml:"folder"` IgnorePattern []string `long:"ignorepattern" short:"i" description:"Dont download files if there URLs match any of the given PATTERNs" value-name:"PATTERN" toml:"ignorepattern"` - - ExtraHeader http.Header `long:"header" short:"H" description:"One or more extra HTTP header fields" toml:"header"` RemoteValidator string `long:"validator" description:"URL to validate documents remotely" value-name:"URL" toml:"validator"`