From a131b0fb4bc97592d8ac4d80280706359b2a6811 Mon Sep 17 00:00:00 2001 From: koplas <54645365+koplas@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:36:54 +0200 Subject: [PATCH] Improve SHA* marking --- cmd/csaf_checker/processor.go | 45 ++++++++++++++--- cmd/csaf_downloader/downloader.go | 34 ++++++++----- csaf/advisories.go | 83 +++++++++++++++---------------- 3 files changed, 99 insertions(+), 63 deletions(-) diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go index 451a315c..de42e184 100644 --- a/cmd/csaf_checker/processor.go +++ b/cmd/csaf_checker/processor.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "log" + "log/slog" "net/http" "net/url" "path/filepath" @@ -138,7 +139,7 @@ func (m *topicMessages) info(format string, args ...any) { m.add(InfoType, format, args...) } -// use signals that we going to use this topic. +// use signals that we're going to use this topic. func (m *topicMessages) use() { if *m == nil { *m = []Message{} @@ -164,7 +165,7 @@ func (m *topicMessages) hasErrors() bool { return false } -// newProcessor returns an initilaized processor. +// newProcessor returns an initialized processor. func newProcessor(cfg *config) (*processor, error) { var validator csaf.RemoteValidator @@ -594,10 +595,15 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) { var file csaf.AdvisoryFile - if sha256 != "" || sha512 != "" || sign != "" { - file = csaf.HashedAdvisoryFile{url, sha256, sha512, sign} - } else { - file = csaf.PlainAdvisoryFile(url) + switch { + case sha256 == "" && sha512 == "": + slog.Error("No hash listed on ROLIE feed", "file", url) + return + case sign == "": + slog.Error("No signature listed on ROLIE feed", "file", url) + return + default: + file = csaf.PlainAdvisoryFile{Path: url, SHA256: sha256, SHA512: sha512, Sign: sign} } files = append(files, file) @@ -888,7 +894,16 @@ func (p *processor) checkIndex(base string, mask whereType) error { p.badIntegrities.error("index.txt contains invalid URL %q in line %d", u, line) continue } - files = append(files, csaf.PlainAdvisoryFile(u)) + + SHA256 := p.checkURL(u + ".sha256") + SHA512 := p.checkURL(u + ".sha512") + sign := p.checkURL(u + ".asc") + files = append(files, csaf.PlainAdvisoryFile{ + Path: u, + SHA256: SHA256, + SHA512: SHA512, + Sign: sign, + }) } return files, scanner.Err() }() @@ -906,6 +921,15 @@ func (p *processor) checkIndex(base string, mask whereType) error { return p.integrity(files, base, mask, p.badIndices.add) } +// checkURL returns the URL if it is accessible. +func (p *processor) checkURL(url string) string { + _, err := p.client.Head(url) + if err != nil { + return url + } + return "" +} + // checkChanges fetches the "changes.csv" and calls the "checkTLS" method for HTTPs checks. // It extracts the file content, tests the column number and the validity of the time format // of the fields' values and if they are sorted properly. Then it passes the files to the @@ -970,9 +994,14 @@ func (p *processor) checkChanges(base string, mask whereType) error { continue } path := r[pathColumn] + + SHA256 := p.checkURL(path + ".sha256") + SHA512 := p.checkURL(path + ".sha512") + sign := p.checkURL(path + ".asc") + times, files = append(times, t), - append(files, csaf.PlainAdvisoryFile(path)) + append(files, csaf.PlainAdvisoryFile{Path: path, SHA256: SHA256, SHA512: SHA512, Sign: sign}) } return times, files, nil }() diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index badf0605..025ed65c 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -501,23 +501,31 @@ nextAdvisory: signData []byte ) - // Only hash when we have a remote counter part we can compare it with. - if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil { - slog.Warn("Cannot fetch SHA256", - "url", file.SHA256URL(), - "error", err) + if file.SHA256URL() == "" { + slog.Info("SHA256 not present", "file", file.URL()) } else { - s256 = sha256.New() - writers = append(writers, s256) + // Only hash when we have a remote counterpart we can compare it with. + if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil { + slog.Warn("Cannot fetch SHA256", + "url", file.SHA256URL(), + "error", err) + } else { + s256 = sha256.New() + writers = append(writers, s256) + } } - if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil { - slog.Warn("Cannot fetch SHA512", - "url", file.SHA512URL(), - "error", err) + if file.SHA512URL() == "" { + slog.Info("SHA512 not present", "file", file.URL()) } else { - s512 = sha512.New() - writers = append(writers, s512) + if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil { + slog.Warn("Cannot fetch SHA512", + "url", file.SHA512URL(), + "error", err) + } else { + s512 = sha512.New() + writers = append(writers, s512) + } } // Remember the data as we need to store it to file later. diff --git a/csaf/advisories.go b/csaf/advisories.go index 6f07648b..4aa7f52f 100644 --- a/csaf/advisories.go +++ b/csaf/advisories.go @@ -34,55 +34,30 @@ type AdvisoryFile interface { // PlainAdvisoryFile is a simple implementation of checkFile. // The hash and signature files are directly constructed by extending // the file name. -type PlainAdvisoryFile string +type PlainAdvisoryFile struct { + Path string + SHA256 string + SHA512 string + Sign string +} // URL returns the URL of this advisory. -func (paf PlainAdvisoryFile) URL() string { return string(paf) } +func (paf PlainAdvisoryFile) URL() string { return paf.Path } // SHA256URL returns the URL of SHA256 hash file of this advisory. -func (paf PlainAdvisoryFile) SHA256URL() string { return string(paf) + ".sha256" } +func (paf PlainAdvisoryFile) SHA256URL() string { return paf.SHA256 } // SHA512URL returns the URL of SHA512 hash file of this advisory. -func (paf PlainAdvisoryFile) SHA512URL() string { return string(paf) + ".sha512" } +func (paf PlainAdvisoryFile) SHA512URL() string { return paf.SHA512 } // SignURL returns the URL of signature file of this advisory. -func (paf PlainAdvisoryFile) SignURL() string { return string(paf) + ".asc" } +func (paf PlainAdvisoryFile) SignURL() string { return paf.Sign } // LogValue implements [slog.LogValuer] func (paf PlainAdvisoryFile) LogValue() slog.Value { return slog.GroupValue(slog.String("url", paf.URL())) } -// HashedAdvisoryFile is a more involed version of checkFile. -// Here each component can be given explicitly. -// If a component is not given it is constructed by -// extending the first component. -type HashedAdvisoryFile [4]string - -func (haf HashedAdvisoryFile) name(i int, ext string) string { - if haf[i] != "" { - return haf[i] - } - return haf[0] + ext -} - -// URL returns the URL of this advisory. -func (haf HashedAdvisoryFile) URL() string { return haf[0] } - -// SHA256URL returns the URL of SHA256 hash file of this advisory. -func (haf HashedAdvisoryFile) SHA256URL() string { return haf.name(1, ".sha256") } - -// SHA512URL returns the URL of SHA512 hash file of this advisory. -func (haf HashedAdvisoryFile) SHA512URL() string { return haf.name(2, ".sha512") } - -// SignURL returns the URL of signature file of this advisory. -func (haf HashedAdvisoryFile) SignURL() string { return haf.name(3, ".asc") } - -// LogValue implements [slog.LogValuer] -func (haf HashedAdvisoryFile) LogValue() slog.Value { - return slog.GroupValue(slog.String("url", haf.URL())) -} - // AdvisoryFileProcessor implements the extraction of // advisory file names from a given provider metadata. type AdvisoryFileProcessor struct { @@ -120,7 +95,7 @@ func empty(arr []string) bool { return true } -// Process extracts the adivisory filenames and passes them with +// Process extracts the advisory filenames and passes them with // the corresponding label to fn. func (afp *AdvisoryFileProcessor) Process( fn func(TLPLabel, []AdvisoryFile) error, @@ -201,6 +176,15 @@ func (afp *AdvisoryFileProcessor) Process( return nil } +// checkURL returns the URL if it is accessible. +func (afp *AdvisoryFileProcessor) checkURL(url string) string { + _, err := afp.client.Head(url) + if err != nil { + return url + } + return "" +} + // loadChanges loads baseURL/changes.csv and returns a list of files // prefixed by baseURL/. func (afp *AdvisoryFileProcessor) loadChanges( @@ -257,8 +241,19 @@ func (afp *AdvisoryFileProcessor) loadChanges( lg("%q contains an invalid URL %q in line %d", changesURL, path, line) continue } + + self := base.JoinPath(path).String() + sha256 := afp.checkURL(self + ".sha256") + sha512 := afp.checkURL(self + ".sha512") + sign := afp.checkURL(self + ".asc") + files = append(files, - PlainAdvisoryFile(base.JoinPath(path).String())) + PlainAdvisoryFile{ + Path: path, + SHA256: sha256, + SHA512: sha512, + Sign: sign, + }) } return files, nil } @@ -325,7 +320,6 @@ func (afp *AdvisoryFileProcessor) processROLIE( } rfeed.Entries(func(entry *Entry) { - // Filter if we have date checking. if afp.AgeAccept != nil { if t := time.Time(entry.Updated); !t.IsZero() && !afp.AgeAccept(t) { @@ -359,10 +353,15 @@ func (afp *AdvisoryFileProcessor) processROLIE( var file AdvisoryFile - if sha256 != "" || sha512 != "" || sign != "" { - file = HashedAdvisoryFile{self, sha256, sha512, sign} - } else { - file = PlainAdvisoryFile(self) + switch { + case sha256 == "" && sha512 == "": + slog.Error("No hash listed on ROLIE feed", "file", self) + return + case sign == "": + slog.Error("No signature listed on ROLIE feed", "file", self) + return + default: + file = PlainAdvisoryFile{self, sha256, sha512, sign} } files = append(files, file)