Skip to content

Commit

Permalink
feat(rocky): extract package by module
Browse files Browse the repository at this point in the history
  • Loading branch information
@MaineK00n
MaineK00n committed Jan 18, 2022
1 parent a5eed63 commit 41a3ce7
Show file tree
Hide file tree
Showing 2 changed files with 211 additions and 12 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/simplereach/timeutils v1.2.0 // indirect
github.com/spf13/afero v1.7.0
github.com/stretchr/testify v1.7.0
github.com/ulikunitz/xz v0.5.8 // indirect
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
golang.org/x/vuln v0.0.0-20211215213114-5e054cb3e47e
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
Expand Down
222 changes: 210 additions & 12 deletions rocky/rocky.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package rocky

import (
"bufio"
"bytes"
"compress/gzip"
"encoding/xml"
Expand All @@ -14,7 +15,9 @@ import (

"github.com/aquasecurity/vuln-list-update/utils"
"github.com/cheggaaa/pb/v3"
"github.com/ulikunitz/xz"
"golang.org/x/xerrors"
"gopkg.in/yaml.v2"
)

const (
Expand Down Expand Up @@ -50,7 +53,7 @@ type UpdateInfo struct {
RLSAList []RLSA `xml:"update"`
}

// RLSA has detailed data of RLSA
// RLSA has detailed data of Rocky Linux Security Advisory
type RLSA struct {
ID string `xml:"id" json:"id,omitempty"`
Title string `xml:"title" json:"title,omitempty"`
Expand All @@ -59,10 +62,26 @@ type RLSA struct {
Severity string `xml:"severity" json:"severity,omitempty"`
Description string `xml:"description" json:"description,omitempty"`
Packages []Package `xml:"pkglist>collection>package" json:"packages,omitempty"`
PkgLists []PkgList `json:"pkglists,omitempty"`
References []Reference `xml:"references>reference" json:"references,omitempty"`
CveIDs []string `json:"cveids,omitempty"`
}

// PkgList has modular package information
type PkgList struct {
Packages []Package `json:"packages,omitempty"`
Module Module `json:"module,omitempty"`
}

// Module has module information
type Module struct {
Stream string `json:"stream,omitempty"`
Name string `json:"name,omitempty"`
Version int64 `json:"version,omitempty"`
Arch string `json:"arch,omitempty"`
Context string `json:"context,omitempty"`
}

// Date has time information
type Date struct {
Date string `xml:"date,attr" json:"date,omitempty"`
Expand All @@ -86,6 +105,10 @@ type Package struct {
Filename string `xml:"filename" json:"filename,omitempty"`
}

func (p Package) String() string {
return fmt.Sprintf("%s-%s:%s-%s.%s", p.Name, p.Epoch, p.Version, p.Release, p.Arch)
}

type options struct {
url string
dir string
Expand Down Expand Up @@ -173,16 +196,30 @@ func (c Config) update(release, repo, arch string) error {
}
rootPath := u.Path
u.Path = path.Join(rootPath, "repodata/repomd.xml")
updateInfoPath, err := fetchUpdateInfoPath(u.String())
updateInfoPath, modulesPath, err := c.fetchUpdateInfoPath(u.String())
if err != nil {
return xerrors.Errorf("failed to fetch updateInfo path from repomd.xml: %w", err)
}

var modules map[string]ModuleInfo
if modulesPath != "" {
u.Path = path.Join(rootPath, modulesPath)
modules, err = c.fetchModules(u.String())
if err != nil {
return xerrors.Errorf("failed to fetch modules info: %w", err)
}
}

u.Path = path.Join(rootPath, updateInfoPath)
uinfo, err := fetchUpdateInfo(u.String())
uinfo, err := c.fetchUpdateInfo(u.String())
if err != nil {
return xerrors.Errorf("failed to fetch updateInfo: %w", err)
}

if err := extractModulesToUpdateInfo(uinfo, modules); err != nil {
return xerrors.Errorf("failed to extract modules to updateinfo: %w", err)
}

secErrata := map[string][]RLSA{}
for _, rlsa := range uinfo.RLSAList {
if !strings.HasPrefix(rlsa.ID, "RLSA-") {
Expand Down Expand Up @@ -213,31 +250,33 @@ func (c Config) update(release, repo, arch string) error {
return nil
}

func fetchUpdateInfoPath(repomdURL string) (updateInfoPath string, err error) {
res, err := utils.FetchURL(repomdURL, "", retry)
func (c Config) fetchUpdateInfoPath(repomdURL string) (updateInfoPath, modulesPath string, err error) {
res, err := utils.FetchURL(repomdURL, "", c.retry)
if err != nil {
return "", xerrors.Errorf("failed to fetch %s: %w", repomdURL, err)
return "", "", xerrors.Errorf("failed to fetch %s: %w", repomdURL, err)
}

var repoMd RepoMd
if err := xml.NewDecoder(bytes.NewBuffer(res)).Decode(&repoMd); err != nil {
return "", xerrors.Errorf("failed to decode repomd.xml: %w", err)
return "", "", xerrors.Errorf("failed to decode repomd.xml: %w", err)
}

for _, repo := range repoMd.RepoList {
if repo.Type == "updateinfo" {
updateInfoPath = repo.Location.Href
break
}
if repo.Type == "modules" {
modulesPath = repo.Location.Href
}
}
if updateInfoPath == "" {
return "", xerrors.New("no updateinfo field in the repomd")
return "", "", xerrors.New("no updateinfo field in the repomd")
}
return updateInfoPath, nil
return updateInfoPath, modulesPath, nil
}

func fetchUpdateInfo(url string) (*UpdateInfo, error) {
res, err := utils.FetchURL(url, "", retry)
func (c Config) fetchUpdateInfo(url string) (*UpdateInfo, error) {
res, err := utils.FetchURL(url, "", c.retry)
if err != nil {
return nil, xerrors.Errorf("failed to fetch updateInfo: %w", err)
}
Expand All @@ -262,3 +301,162 @@ func fetchUpdateInfo(url string) (*UpdateInfo, error) {
}
return &updateInfo, nil
}

func (c Config) fetchModules(url string) (map[string]ModuleInfo, error) {
res, err := utils.FetchURL(url, "", c.retry)
if err != nil {
return nil, xerrors.Errorf("failed to fetch modules: %w", err)
}

r, err := xz.NewReader(bytes.NewBuffer(res))
if err != nil {
return nil, xerrors.Errorf("failed to decompress modules: %w", err)
}

modules := map[string]ModuleInfo{}
scanner := bufio.NewScanner(r)
var contents []string
for scanner.Scan() {
str := scanner.Text()
switch str {
case "---":
{
contents = []string{}
}
case "...":
{
var module ModuleInfo
if err := yaml.NewDecoder(strings.NewReader(strings.Join(contents, "\n"))).Decode(&module); err != nil {
return nil, xerrors.Errorf("failed to decode module info: %w", err)
}
if module.Version == 2 {
modules[module.String()] = module
}
}
default:
{
contents = append(contents, str)
}
}
}

return modules, nil
}

type ModuleInfo struct {
Version int `yaml:"version"`
Data struct {
Name string `yaml:"name"`
Stream string `yaml:"stream"`
Version int64 `yaml:"version"`
Context string `yaml:"context"`
Arch string `yaml:"arch"`
Artifacts struct {
Rpms []string `yaml:"rpms"`
} `yaml:"artifacts"`
} `yaml:"data"`
}

func (m ModuleInfo) String() string {
return fmt.Sprintf("%s:%s:%d:%s:%s", m.Data.Name, m.Data.Stream, m.Data.Version, m.Data.Context, m.Data.Arch)
}

func extractModulesToUpdateInfo(uinfo *UpdateInfo, modules map[string]ModuleInfo) error {
if modules == nil {
return nil
}

// pkgToModuleStr: convert from package information to moduleStr
pkgToModuleStr := map[string]string{}
for modularStr, module := range modules {
for _, pkg := range module.Data.Artifacts.Rpms {
pkgToModuleStr[pkg] = modularStr
}
}

for i, rlsa := range uinfo.RLSAList {
// moduleStrToPkgs: convert from moduleStr to the relevant pkgs of the module (moduleStr is "" if it is not a module package)
moduleStrToPkgs := map[string][]Package{}
for _, pkg := range rlsa.Packages {
moduleStr := pkgToModuleStr[pkg.String()]
moduleStrToPkgs[moduleStr] = append(moduleStrToPkgs[moduleStr], pkg)
}

pkgLists := []PkgList{}
for modularStr, pkgs := range moduleStrToPkgs {
if modularStr == "" {
pkgLists = append(pkgLists, PkgList{
Packages: pkgs,
})
continue
}

// list the packages related to the module
module := modules[modularStr]
pkgs := []Package{}
for _, pkg := range module.Data.Artifacts.Rpms {
name, ver, rel, epoch, arch, err := splitFileName(pkg)
if err != nil {
return xerrors.Errorf("failed to split rpm filename: %w", err)
}
pkgs = append(pkgs, Package{
Name: name,
Epoch: epoch,
Version: ver,
Release: rel,
Arch: arch,
Filename: fmt.Sprintf("%s-%s-%s.%s.rpm", name, ver, rel, arch),
})
}

pkgLists = append(pkgLists, PkgList{
Packages: pkgs,
Module: Module{
Stream: module.Data.Stream,
Name: module.Data.Name,
Version: module.Data.Version,
Arch: module.Data.Arch,
Context: module.Data.Context,
},
})
}

uinfo.RLSAList[i].PkgLists = pkgLists
uinfo.RLSAList[i].Packages = nil
}
return nil
}

// splitFileName returns a name, version, release, epoch, arch
func splitFileName(filename string) (name, ver, rel, epoch, arch string, err error) {
filename = strings.TrimSuffix(filename, ".rpm")

archIndex := strings.LastIndex(filename, ".")
if archIndex == -1 {
return "", "", "", "", "", xerrors.Errorf("failed to parse arch from filename: %s", filename)
}
arch = filename[archIndex+1:]

relIndex := strings.LastIndex(filename[:archIndex], "-")
if relIndex == -1 {
return "", "", "", "", "", xerrors.Errorf("failed to parse release from filename: %s", filename)
}
rel = filename[relIndex+1 : archIndex]

verIndex := strings.LastIndex(filename[:relIndex], "-")
if verIndex == -1 {
return "", "", "", "", "", xerrors.Errorf("failed to parse version from filename: %s", filename)
}
ver = filename[verIndex+1 : relIndex]

epochIndex := strings.Index(ver, ":")
if epochIndex == -1 {
epoch = "0"
} else {
epoch = ver[:epochIndex]
ver = ver[epochIndex+1:]
}

name = filename[:verIndex]
return name, ver, rel, epoch, arch, nil
}

0 comments on commit 41a3ce7

Please sign in to comment.