From d49049c3afef7db26675f44c58cae4b5ed0145e7 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 17 Aug 2023 16:54:11 +0200 Subject: [PATCH 1/3] Aggregator: Add time range filtering --- cmd/csaf_aggregator/config.go | 19 +++++++++++++++++++ cmd/csaf_aggregator/mirror.go | 3 +++ docs/csaf_aggregator.md | 2 ++ docs/examples/aggregator.toml | 2 ++ 4 files changed, 26 insertions(+) diff --git a/cmd/csaf_aggregator/config.go b/cmd/csaf_aggregator/config.go index 8e79021c..eefcc3b2 100644 --- a/cmd/csaf_aggregator/config.go +++ b/cmd/csaf_aggregator/config.go @@ -22,6 +22,7 @@ import ( "github.com/csaf-poc/csaf_distribution/v2/csaf" "github.com/csaf-poc/csaf_distribution/v2/internal/certs" "github.com/csaf-poc/csaf_distribution/v2/internal/filter" + "github.com/csaf-poc/csaf_distribution/v2/internal/models" "github.com/csaf-poc/csaf_distribution/v2/internal/options" "github.com/csaf-poc/csaf_distribution/v2/util" "golang.org/x/time/rate" @@ -61,6 +62,8 @@ type provider struct { ClientKey *string `toml:"client_key"` ClientPassphrase *string `toml:"client_passphrase"` + Range *models.TimeRange `toml:"timerange"` + clientCerts []tls.Certificate ignorePattern filter.PatternMatcher } @@ -88,6 +91,8 @@ type config struct { ClientKey *string `toml:"client_key"` ClientPassphrase *string `toml:"client_passphrase"` + Range *models.TimeRange `long:"timerange" short:"t" description:"RANGE of time from which advisories to download" value-name:"RANGE" toml:"timerange"` + // LockFile tries to lock to a given file. LockFile *string `toml:"lock_file"` @@ -156,6 +161,20 @@ func (c *config) tooOldForInterims() func(time.Time) bool { return func(t time.Time) bool { return t.Before(from) } } +// ageAccept returns a function which checks if a given time +// is in the accepted download interval of the provider or +// the global config. +func (p *provider) ageAccept(c *config) func(time.Time) bool { + switch { + case p.Range != nil: + return p.Range.Contains + case c.Range != nil: + return c.Range.Contains + default: + return nil + } +} + // ignoreFile returns true if the given URL should not be downloaded. func (p *provider) ignoreURL(u string, c *config) bool { return p.ignorePattern.Matches(u) || c.ignorePattern.Matches(u) diff --git a/cmd/csaf_aggregator/mirror.go b/cmd/csaf_aggregator/mirror.go index 64ef18a4..13b0d1b4 100644 --- a/cmd/csaf_aggregator/mirror.go +++ b/cmd/csaf_aggregator/mirror.go @@ -78,6 +78,8 @@ func (w *worker) mirrorInternal() (*csaf.AggregatorCSAFProvider, error) { w.metadataProvider, base) + afp.AgeAccept = w.provider.ageAccept(w.processor.cfg) + if err := afp.Process(w.mirrorFiles); err != nil { return nil, err } @@ -494,6 +496,7 @@ func (w *worker) mirrorFiles(tlpLabel csaf.TLPLabel, files []csaf.AdvisoryFile) yearDirs := make(map[int]string) for _, file := range files { + u, err := url.Parse(file.URL()) if err != nil { log.Printf("error: %s\n", err) diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index c94dd90f..e1e60d7e 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -6,6 +6,7 @@ csaf_aggregator [OPTIONS] Application Options: + -t, --timerange=RANGE RANGE of time from which advisories to download -i, --interim Perform an interim scan --version Display version of the binary -c, --config=TOML-FILE Path to config TOML file @@ -99,6 +100,7 @@ client_cert // path to client certificate to access access-protected client_key // path to client key to access access-protected advisories client_passphrase // client passphrase to access access-protected advisories header // adds extra HTTP header fields to the client +timerange // Accepted time range of advisories to handle. See checker doc for details. ``` Next we have two TOML _tables_: diff --git a/docs/examples/aggregator.toml b/docs/examples/aggregator.toml index 4abbba3a..3595c44a 100644 --- a/docs/examples/aggregator.toml +++ b/docs/examples/aggregator.toml @@ -10,6 +10,7 @@ insecure = true #interim_years = #passphrase = #write_indices = false +#timerange = # specification requires at least two providers (default), # to override for testing, enable: @@ -31,6 +32,7 @@ insecure = true create_service_document = true # rate = 1.5 # insecure = true +# timerange = [[providers]] name = "local-dev-provider2" From 0b914f7e7ae62927b80219a7ea0f72b282b0e405 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Thu, 17 Aug 2023 17:02:10 +0200 Subject: [PATCH 2/3] Document regular expression syntax used for filtering URLs. (#433) * Document regular expression syntax used for filtering URLs. * Typo: describes -> described * Forget to add aggregator doc --------- Co-authored-by: JanHoefelmeyer --- docs/csaf_aggregator.md | 2 +- docs/csaf_checker.md | 4 +++- docs/csaf_downloader.md | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/csaf_aggregator.md b/docs/csaf_aggregator.md index e1e60d7e..9b92dd72 100644 --- a/docs/csaf_aggregator.md +++ b/docs/csaf_aggregator.md @@ -95,7 +95,7 @@ lock_file // path to lockfile, to stop other instances if one is n interim_years // limiting the years for which interim documents are searched (default 0) verbose // print more diagnostic output, e.g. https requests (default false) allow_single_provider // debugging option (default false) -ignorepattern // patterns of advisory URLs to be ignored +ignorepattern // patterns of advisory URLs to be ignored (see checker doc for details) client_cert // path to client certificate to access access-protected advisories client_key // path to client key to access access-protected advisories client_passphrase // client passphrase to access access-protected advisories diff --git a/docs/csaf_checker.md b/docs/csaf_checker.md index 6dc103ba..f555c696 100644 --- a/docs/csaf_checker.md +++ b/docs/csaf_checker.md @@ -100,7 +100,7 @@ There are following variants: All interval boundaries are inclusive. You can ignore certain advisories while checking by specifying a list -of regular expressions to match their URLs by using the `ignorepattern` +of regular expressions[^1] to match their URLs by using the `ignorepattern` option. E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain the sub strings **white** or **red**. @@ -119,3 +119,5 @@ If a provider hosts one or more advisories with a TLP level of AMBER or RED, the To check these advisories, authorization can be given via custom headers or certificates. The authorization method chosen needs to grant access to all advisories, as otherwise the checker will be unable to check the advisories it doesn't have permission for, falsifying the result. + +[^1]: Accepted syntax is described [here](https://github.com/google/re2/wiki/Syntax). diff --git a/docs/csaf_downloader.md b/docs/csaf_downloader.md index 2fb98d97..f2ed4192 100644 --- a/docs/csaf_downloader.md +++ b/docs/csaf_downloader.md @@ -103,7 +103,7 @@ of this name. Otherwise the advisories are each stored in a folder named by the year they are from. You can ignore certain advisories while downloading by specifying a list -of regular expressions to match their URLs by using the `ignorepattern` +of regular expressions[^1] to match their URLs by using the `ignorepattern` option. E.g. `-i='.*white.*' -i='*.red.*'` will ignore files which URLs contain @@ -112,3 +112,5 @@ In the config file this has to be noted as: ``` ignorepattern = [".*white.*", ".*red.*"] ``` + +[^1]: Accepted syntax is described [here](https://github.com/google/re2/wiki/Syntax). From bda7ade837b02a000166eada2baccde9affc8d57 Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer <107021473+JanHoefelmeyer@users.noreply.github.com> Date: Thu, 17 Aug 2023 20:17:15 +0200 Subject: [PATCH 3/3] Fix script scripts/prepareUbuntuInstanceForITests.sh for new go.dev API * Adapt finding the current go version to the new result of https://go.dev/VERSION\?m=text by using only the first line. Co-authored-by: JanHoefelmeyer --- docs/scripts/prepareUbuntuInstanceForITests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scripts/prepareUbuntuInstanceForITests.sh b/docs/scripts/prepareUbuntuInstanceForITests.sh index 35a17453..f99bc268 100755 --- a/docs/scripts/prepareUbuntuInstanceForITests.sh +++ b/docs/scripts/prepareUbuntuInstanceForITests.sh @@ -9,7 +9,7 @@ apt update apt install -y make bash curl gnupg sed tar git nginx fcgiwrap gnutls-bin # Install Go from binary distribution -latest_go="$(curl https://go.dev/VERSION\?m=text).linux-amd64.tar.gz" +latest_go="$(curl https://go.dev/VERSION\?m=text| head -1).linux-amd64.tar.gz" curl -O https://dl.google.com/go/$latest_go rm -rf /usr/local/go # be sure that we do not have an old installation tar -C /usr/local -xzf $latest_go