Skip to content

Commit

Permalink
Improve SHA* marking
Browse files Browse the repository at this point in the history
  • Loading branch information
@koplas
koplas committed Jul 25, 2024
1 parent 257c316 commit a131b0f
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 63 deletions.
45 changes: 37 additions & 8 deletions cmd/csaf_checker/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"io"
"log"
"log/slog"
"net/http"
"net/url"
"path/filepath"
Expand Down Expand Up @@ -138,7 +139,7 @@ func (m *topicMessages) info(format string, args ...any) {
m.add(InfoType, format, args...)
}

// use signals that we going to use this topic.
// use signals that we're going to use this topic.
func (m *topicMessages) use() {
if *m == nil {
*m = []Message{}
Expand All @@ -164,7 +165,7 @@ func (m *topicMessages) hasErrors() bool {
return false
}

// newProcessor returns an initilaized processor.
// newProcessor returns an initialized processor.
func newProcessor(cfg *config) (*processor, error) {

var validator csaf.RemoteValidator
Expand Down Expand Up @@ -594,10 +595,15 @@ func (p *processor) rolieFeedEntries(feed string) ([]csaf.AdvisoryFile, error) {

var file csaf.AdvisoryFile

if sha256 != "" || sha512 != "" || sign != "" {
file = csaf.HashedAdvisoryFile{url, sha256, sha512, sign}
} else {
file = csaf.PlainAdvisoryFile(url)
switch {
case sha256 == "" && sha512 == "":
slog.Error("No hash listed on ROLIE feed", "file", url)
return
case sign == "":
slog.Error("No signature listed on ROLIE feed", "file", url)
return
default:
file = csaf.PlainAdvisoryFile{Path: url, SHA256: sha256, SHA512: sha512, Sign: sign}
}

files = append(files, file)
Expand Down Expand Up @@ -888,7 +894,16 @@ func (p *processor) checkIndex(base string, mask whereType) error {
p.badIntegrities.error("index.txt contains invalid URL %q in line %d", u, line)
continue
}
files = append(files, csaf.PlainAdvisoryFile(u))

SHA256 := p.checkURL(u + ".sha256")
SHA512 := p.checkURL(u + ".sha512")
sign := p.checkURL(u + ".asc")
files = append(files, csaf.PlainAdvisoryFile{
Path: u,
SHA256: SHA256,
SHA512: SHA512,
Sign: sign,
})
}
return files, scanner.Err()
}()
Expand All @@ -906,6 +921,15 @@ func (p *processor) checkIndex(base string, mask whereType) error {
return p.integrity(files, base, mask, p.badIndices.add)
}

// checkURL returns the URL if it is accessible.
func (p *processor) checkURL(url string) string {
_, err := p.client.Head(url)
if err != nil {
return url
}
return ""
}

// checkChanges fetches the "changes.csv" and calls the "checkTLS" method for HTTPs checks.
// It extracts the file content, tests the column number and the validity of the time format
// of the fields' values and if they are sorted properly. Then it passes the files to the
Expand Down Expand Up @@ -970,9 +994,14 @@ func (p *processor) checkChanges(base string, mask whereType) error {
continue
}
path := r[pathColumn]

SHA256 := p.checkURL(path + ".sha256")
SHA512 := p.checkURL(path + ".sha512")
sign := p.checkURL(path + ".asc")

times, files =
append(times, t),
append(files, csaf.PlainAdvisoryFile(path))
append(files, csaf.PlainAdvisoryFile{Path: path, SHA256: SHA256, SHA512: SHA512, Sign: sign})
}
return times, files, nil
}()
Expand Down
34 changes: 21 additions & 13 deletions cmd/csaf_downloader/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,23 +501,31 @@ nextAdvisory:
signData []byte
)

// Only hash when we have a remote counter part we can compare it with.
if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
slog.Warn("Cannot fetch SHA256",
"url", file.SHA256URL(),
"error", err)
if file.SHA256URL() == "" {
slog.Info("SHA256 not present", "file", file.URL())
} else {
s256 = sha256.New()
writers = append(writers, s256)
// Only hash when we have a remote counterpart we can compare it with.
if remoteSHA256, s256Data, err = loadHash(client, file.SHA256URL()); err != nil {
slog.Warn("Cannot fetch SHA256",
"url", file.SHA256URL(),
"error", err)
} else {
s256 = sha256.New()
writers = append(writers, s256)
}
}

if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
slog.Warn("Cannot fetch SHA512",
"url", file.SHA512URL(),
"error", err)
if file.SHA512URL() == "" {
slog.Info("SHA512 not present", "file", file.URL())
} else {
s512 = sha512.New()
writers = append(writers, s512)
if remoteSHA512, s512Data, err = loadHash(client, file.SHA512URL()); err != nil {
slog.Warn("Cannot fetch SHA512",
"url", file.SHA512URL(),
"error", err)
} else {
s512 = sha512.New()
writers = append(writers, s512)
}
}

// Remember the data as we need to store it to file later.
Expand Down
83 changes: 41 additions & 42 deletions csaf/advisories.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,55 +34,30 @@ type AdvisoryFile interface {
// PlainAdvisoryFile is a simple implementation of checkFile.
// The hash and signature files are directly constructed by extending
// the file name.
type PlainAdvisoryFile string
type PlainAdvisoryFile struct {
Path string
SHA256 string
SHA512 string
Sign string
}

// URL returns the URL of this advisory.
func (paf PlainAdvisoryFile) URL() string { return string(paf) }
func (paf PlainAdvisoryFile) URL() string { return paf.Path }

// SHA256URL returns the URL of SHA256 hash file of this advisory.
func (paf PlainAdvisoryFile) SHA256URL() string { return string(paf) + ".sha256" }
func (paf PlainAdvisoryFile) SHA256URL() string { return paf.SHA256 }

// SHA512URL returns the URL of SHA512 hash file of this advisory.
func (paf PlainAdvisoryFile) SHA512URL() string { return string(paf) + ".sha512" }
func (paf PlainAdvisoryFile) SHA512URL() string { return paf.SHA512 }

// SignURL returns the URL of signature file of this advisory.
func (paf PlainAdvisoryFile) SignURL() string { return string(paf) + ".asc" }
func (paf PlainAdvisoryFile) SignURL() string { return paf.Sign }

// LogValue implements [slog.LogValuer]
func (paf PlainAdvisoryFile) LogValue() slog.Value {
return slog.GroupValue(slog.String("url", paf.URL()))
}

// HashedAdvisoryFile is a more involed version of checkFile.
// Here each component can be given explicitly.
// If a component is not given it is constructed by
// extending the first component.
type HashedAdvisoryFile [4]string

func (haf HashedAdvisoryFile) name(i int, ext string) string {
if haf[i] != "" {
return haf[i]
}
return haf[0] + ext
}

// URL returns the URL of this advisory.
func (haf HashedAdvisoryFile) URL() string { return haf[0] }

// SHA256URL returns the URL of SHA256 hash file of this advisory.
func (haf HashedAdvisoryFile) SHA256URL() string { return haf.name(1, ".sha256") }

// SHA512URL returns the URL of SHA512 hash file of this advisory.
func (haf HashedAdvisoryFile) SHA512URL() string { return haf.name(2, ".sha512") }

// SignURL returns the URL of signature file of this advisory.
func (haf HashedAdvisoryFile) SignURL() string { return haf.name(3, ".asc") }

// LogValue implements [slog.LogValuer]
func (haf HashedAdvisoryFile) LogValue() slog.Value {
return slog.GroupValue(slog.String("url", haf.URL()))
}

// AdvisoryFileProcessor implements the extraction of
// advisory file names from a given provider metadata.
type AdvisoryFileProcessor struct {
Expand Down Expand Up @@ -120,7 +95,7 @@ func empty(arr []string) bool {
return true
}

// Process extracts the adivisory filenames and passes them with
// Process extracts the advisory filenames and passes them with
// the corresponding label to fn.
func (afp *AdvisoryFileProcessor) Process(
fn func(TLPLabel, []AdvisoryFile) error,
Expand Down Expand Up @@ -201,6 +176,15 @@ func (afp *AdvisoryFileProcessor) Process(
return nil
}

// checkURL returns the URL if it is accessible.
func (afp *AdvisoryFileProcessor) checkURL(url string) string {
_, err := afp.client.Head(url)
if err != nil {
return url
}
return ""
}

// loadChanges loads baseURL/changes.csv and returns a list of files
// prefixed by baseURL/.
func (afp *AdvisoryFileProcessor) loadChanges(
Expand Down Expand Up @@ -257,8 +241,19 @@ func (afp *AdvisoryFileProcessor) loadChanges(
lg("%q contains an invalid URL %q in line %d", changesURL, path, line)
continue
}

self := base.JoinPath(path).String()
sha256 := afp.checkURL(self + ".sha256")
sha512 := afp.checkURL(self + ".sha512")
sign := afp.checkURL(self + ".asc")

files = append(files,
PlainAdvisoryFile(base.JoinPath(path).String()))
PlainAdvisoryFile{
Path: path,
SHA256: sha256,
SHA512: sha512,
Sign: sign,
})
}
return files, nil
}
Expand Down Expand Up @@ -325,7 +320,6 @@ func (afp *AdvisoryFileProcessor) processROLIE(
}

rfeed.Entries(func(entry *Entry) {

// Filter if we have date checking.
if afp.AgeAccept != nil {
if t := time.Time(entry.Updated); !t.IsZero() && !afp.AgeAccept(t) {
Expand Down Expand Up @@ -359,10 +353,15 @@ func (afp *AdvisoryFileProcessor) processROLIE(

var file AdvisoryFile

if sha256 != "" || sha512 != "" || sign != "" {
file = HashedAdvisoryFile{self, sha256, sha512, sign}
} else {
file = PlainAdvisoryFile(self)
switch {
case sha256 == "" && sha512 == "":
slog.Error("No hash listed on ROLIE feed", "file", self)
return
case sign == "":
slog.Error("No signature listed on ROLIE feed", "file", self)
return
default:
file = PlainAdvisoryFile{self, sha256, sha512, sign}
}

files = append(files, file)
Expand Down

0 comments on commit a131b0f

Please sign in to comment.