From 41a3ce7364c492ddd1c6274f1090b9b97f20dadc Mon Sep 17 00:00:00 2001 From: MaineK00n Date: Tue, 18 Jan 2022 20:36:16 +0900 Subject: [PATCH] feat(rocky): extract package by module --- go.mod | 1 + rocky/rocky.go | 222 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 211 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index bb72338a..8678bbc6 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/simplereach/timeutils v1.2.0 // indirect github.com/spf13/afero v1.7.0 github.com/stretchr/testify v1.7.0 + github.com/ulikunitz/xz v0.5.8 // indirect golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/vuln v0.0.0-20211215213114-5e054cb3e47e golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 diff --git a/rocky/rocky.go b/rocky/rocky.go index 8fd264d4..d1cd6b3b 100644 --- a/rocky/rocky.go +++ b/rocky/rocky.go @@ -1,6 +1,7 @@ package rocky import ( + "bufio" "bytes" "compress/gzip" "encoding/xml" @@ -14,7 +15,9 @@ import ( "github.com/aquasecurity/vuln-list-update/utils" "github.com/cheggaaa/pb/v3" + "github.com/ulikunitz/xz" "golang.org/x/xerrors" + "gopkg.in/yaml.v2" ) const ( @@ -50,7 +53,7 @@ type UpdateInfo struct { RLSAList []RLSA `xml:"update"` } -// RLSA has detailed data of RLSA +// RLSA has detailed data of Rocky Linux Security Advisory type RLSA struct { ID string `xml:"id" json:"id,omitempty"` Title string `xml:"title" json:"title,omitempty"` @@ -59,10 +62,26 @@ type RLSA struct { Severity string `xml:"severity" json:"severity,omitempty"` Description string `xml:"description" json:"description,omitempty"` Packages []Package `xml:"pkglist>collection>package" json:"packages,omitempty"` + PkgLists []PkgList `json:"pkglists,omitempty"` References []Reference `xml:"references>reference" json:"references,omitempty"` CveIDs []string `json:"cveids,omitempty"` } +// PkgList has modular package information +type PkgList struct { + Packages []Package `json:"packages,omitempty"` + Module Module `json:"module,omitempty"` +} + +// Module has module information +type Module struct { + Stream string `json:"stream,omitempty"` + Name string `json:"name,omitempty"` + Version int64 `json:"version,omitempty"` + Arch string `json:"arch,omitempty"` + Context string `json:"context,omitempty"` +} + // Date has time information type Date struct { Date string `xml:"date,attr" json:"date,omitempty"` @@ -86,6 +105,10 @@ type Package struct { Filename string `xml:"filename" json:"filename,omitempty"` } +func (p Package) String() string { + return fmt.Sprintf("%s-%s:%s-%s.%s", p.Name, p.Epoch, p.Version, p.Release, p.Arch) +} + type options struct { url string dir string @@ -173,16 +196,30 @@ func (c Config) update(release, repo, arch string) error { } rootPath := u.Path u.Path = path.Join(rootPath, "repodata/repomd.xml") - updateInfoPath, err := fetchUpdateInfoPath(u.String()) + updateInfoPath, modulesPath, err := c.fetchUpdateInfoPath(u.String()) if err != nil { return xerrors.Errorf("failed to fetch updateInfo path from repomd.xml: %w", err) } + + var modules map[string]ModuleInfo + if modulesPath != "" { + u.Path = path.Join(rootPath, modulesPath) + modules, err = c.fetchModules(u.String()) + if err != nil { + return xerrors.Errorf("failed to fetch modules info: %w", err) + } + } + u.Path = path.Join(rootPath, updateInfoPath) - uinfo, err := fetchUpdateInfo(u.String()) + uinfo, err := c.fetchUpdateInfo(u.String()) if err != nil { return xerrors.Errorf("failed to fetch updateInfo: %w", err) } + if err := extractModulesToUpdateInfo(uinfo, modules); err != nil { + return xerrors.Errorf("failed to extract modules to updateinfo: %w", err) + } + secErrata := map[string][]RLSA{} for _, rlsa := range uinfo.RLSAList { if !strings.HasPrefix(rlsa.ID, "RLSA-") { @@ -213,31 +250,33 @@ func (c Config) update(release, repo, arch string) error { return nil } -func fetchUpdateInfoPath(repomdURL string) (updateInfoPath string, err error) { - res, err := utils.FetchURL(repomdURL, "", retry) +func (c Config) fetchUpdateInfoPath(repomdURL string) (updateInfoPath, modulesPath string, err error) { + res, err := utils.FetchURL(repomdURL, "", c.retry) if err != nil { - return "", xerrors.Errorf("failed to fetch %s: %w", repomdURL, err) + return "", "", xerrors.Errorf("failed to fetch %s: %w", repomdURL, err) } var repoMd RepoMd if err := xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil { - return "", xerrors.Errorf("failed to decode repomd.xml: %w", err) + return "", "", xerrors.Errorf("failed to decode repomd.xml: %w", err) } for _, repo := range repoMd.RepoList { if repo.Type == "updateinfo" { updateInfoPath = repo.Location.Href - break + } + if repo.Type == "modules" { + modulesPath = repo.Location.Href } } if updateInfoPath == "" { - return "", xerrors.New("no updateinfo field in the repomd") + return "", "", xerrors.New("no updateinfo field in the repomd") } - return updateInfoPath, nil + return updateInfoPath, modulesPath, nil } -func fetchUpdateInfo(url string) (*UpdateInfo, error) { - res, err := utils.FetchURL(url, "", retry) +func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) { + res, err := utils.FetchURL(url, "", c.retry) if err != nil { return nil, xerrors.Errorf("failed to fetch updateInfo: %w", err) } @@ -262,3 +301,162 @@ func fetchUpdateInfo(url string) (*UpdateInfo, error) { } return &updateInfo, nil } + +func (c Config) fetchModules(url string) (map[string]ModuleInfo, error) { + res, err := utils.FetchURL(url, "", c.retry) + if err != nil { + return nil, xerrors.Errorf("failed to fetch modules: %w", err) + } + + r, err := xz.NewReader(bytes.NewBuffer(res)) + if err != nil { + return nil, xerrors.Errorf("failed to decompress modules: %w", err) + } + + modules := map[string]ModuleInfo{} + scanner := bufio.NewScanner(r) + var contents []string + for scanner.Scan() { + str := scanner.Text() + switch str { + case "---": + { + contents = []string{} + } + case "...": + { + var module ModuleInfo + if err := yaml.NewDecoder(strings.NewReader(strings.Join(contents, "\n"))).Decode(&module); err != nil { + return nil, xerrors.Errorf("failed to decode module info: %w", err) + } + if module.Version == 2 { + modules[module.String()] = module + } + } + default: + { + contents = append(contents, str) + } + } + } + + return modules, nil +} + +type ModuleInfo struct { + Version int `yaml:"version"` + Data struct { + Name string `yaml:"name"` + Stream string `yaml:"stream"` + Version int64 `yaml:"version"` + Context string `yaml:"context"` + Arch string `yaml:"arch"` + Artifacts struct { + Rpms []string `yaml:"rpms"` + } `yaml:"artifacts"` + } `yaml:"data"` +} + +func (m ModuleInfo) String() string { + return fmt.Sprintf("%s:%s:%d:%s:%s", m.Data.Name, m.Data.Stream, m.Data.Version, m.Data.Context, m.Data.Arch) +} + +func extractModulesToUpdateInfo(uinfo *UpdateInfo, modules map[string]ModuleInfo) error { + if modules == nil { + return nil + } + + // pkgToModuleStr: convert from package information to moduleStr + pkgToModuleStr := map[string]string{} + for modularStr, module := range modules { + for _, pkg := range module.Data.Artifacts.Rpms { + pkgToModuleStr[pkg] = modularStr + } + } + + for i, rlsa := range uinfo.RLSAList { + // moduleStrToPkgs: convert from moduleStr to the relevant pkgs of the module (moduleStr is "" if it is not a module package) + moduleStrToPkgs := map[string][]Package{} + for _, pkg := range rlsa.Packages { + moduleStr := pkgToModuleStr[pkg.String()] + moduleStrToPkgs[moduleStr] = append(moduleStrToPkgs[moduleStr], pkg) + } + + pkgLists := []PkgList{} + for modularStr, pkgs := range moduleStrToPkgs { + if modularStr == "" { + pkgLists = append(pkgLists, PkgList{ + Packages: pkgs, + }) + continue + } + + // list the packages related to the module + module := modules[modularStr] + pkgs := []Package{} + for _, pkg := range module.Data.Artifacts.Rpms { + name, ver, rel, epoch, arch, err := splitFileName(pkg) + if err != nil { + return xerrors.Errorf("failed to split rpm filename: %w", err) + } + pkgs = append(pkgs, Package{ + Name: name, + Epoch: epoch, + Version: ver, + Release: rel, + Arch: arch, + Filename: fmt.Sprintf("%s-%s-%s.%s.rpm", name, ver, rel, arch), + }) + } + + pkgLists = append(pkgLists, PkgList{ + Packages: pkgs, + Module: Module{ + Stream: module.Data.Stream, + Name: module.Data.Name, + Version: module.Data.Version, + Arch: module.Data.Arch, + Context: module.Data.Context, + }, + }) + } + + uinfo.RLSAList[i].PkgLists = pkgLists + uinfo.RLSAList[i].Packages = nil + } + return nil +} + +// splitFileName returns a name, version, release, epoch, arch +func splitFileName(filename string) (name, ver, rel, epoch, arch string, err error) { + filename = strings.TrimSuffix(filename, ".rpm") + + archIndex := strings.LastIndex(filename, ".") + if archIndex == -1 { + return "", "", "", "", "", xerrors.Errorf("failed to parse arch from filename: %s", filename) + } + arch = filename[archIndex+1:] + + relIndex := strings.LastIndex(filename[:archIndex], "-") + if relIndex == -1 { + return "", "", "", "", "", xerrors.Errorf("failed to parse release from filename: %s", filename) + } + rel = filename[relIndex+1 : archIndex] + + verIndex := strings.LastIndex(filename[:relIndex], "-") + if verIndex == -1 { + return "", "", "", "", "", xerrors.Errorf("failed to parse version from filename: %s", filename) + } + ver = filename[verIndex+1 : relIndex] + + epochIndex := strings.Index(ver, ":") + if epochIndex == -1 { + epoch = "0" + } else { + epoch = ver[:epochIndex] + ver = ver[epochIndex+1:] + } + + name = filename[:verIndex] + return name, ver, rel, epoch, arch, nil +}