add vulnerability and blob stores

Signed-off-by: Alex Goodman <[email protected]>

go.mod

@@ -5,6 +5,7 @@ go 1.23.0
 require (
 	github.com/CycloneDX/cyclonedx-go v0.9.1
 	github.com/Masterminds/sprig/v3 v3.3.0
+	github.com/OneOfOne/xxhash v1.2.8
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/adrg/xdg v0.5.3
 	github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9
@@ -59,11 +60,10 @@ require (
 	github.com/wagoodman/go-progress v0.0.0-20230925121702-07e42b3cdba0
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/gorm v1.25.12
 )
 
-require gopkg.in/yaml.v3 v3.0.1
-
 require (
 	cloud.google.com/go v0.112.1 // indirect
 	cloud.google.com/go/compute v1.24.0 // indirect

grype/db/internal/gormadapter/open.go

@@ -3,6 +3,7 @@ package gormadapter
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/glebarez/sqlite"
 	"gorm.io/gorm"
@@ -77,10 +78,8 @@ func Open(path string, options ...Option) (*gorm.DB, error) {
 	cfg := newConfig(path, options)
 
 	if cfg.shouldTruncate() {
-		if _, err := os.Stat(path); err == nil {
-			if err := os.Remove(path); err != nil {
-				return nil, fmt.Errorf("unable to remove existing DB file: %w", err)
-			}
+		if err := prepareWritableDB(path); err != nil {
+			return nil, err
 		}
 	}
 
@@ -103,3 +102,18 @@ func Open(path string, options ...Option) (*gorm.DB, error) {
 
 	return dbObj, nil
 }
+
+func prepareWritableDB(path string) error {
+	if _, err := os.Stat(path); err == nil {
+		if err := os.Remove(path); err != nil {
+			return fmt.Errorf("unable to remove existing DB file: %w", err)
+		}
+	}
+
+	parent := filepath.Dir(path)
+	if err := os.MkdirAll(parent, 0700); err != nil {
+		return fmt.Errorf("unable to create parent directory %q for DB file: %w", parent, err)
+	}
+
+	return nil
+}

grype/db/internal/gormadapter/open_test.go

@@ -1,6 +1,8 @@
 package gormadapter
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -125,3 +127,41 @@ func TestConfigConnectionString(t *testing.T) {
 		})
 	}
 }
+
+func TestPrepareWritableDB(t *testing.T) {
+
+	t.Run("creates new directory and file when path does not exist", func(t *testing.T) {
+		tempDir := t.TempDir()
+		dbPath := filepath.Join(tempDir, "newdir", "test.db")
+
+		err := prepareWritableDB(dbPath)
+		require.NoError(t, err)
+
+		_, err = os.Stat(filepath.Dir(dbPath))
+		require.NoError(t, err)
+	})
+
+	t.Run("removes existing file at path", func(t *testing.T) {
+		tempDir := t.TempDir()
+		dbPath := filepath.Join(tempDir, "test.db")
+
+		_, err := os.Create(dbPath)
+		require.NoError(t, err)
+
+		_, err = os.Stat(dbPath)
+		require.NoError(t, err)
+
+		err = prepareWritableDB(dbPath)
+		require.NoError(t, err)
+
+		_, err = os.Stat(dbPath)
+		require.True(t, os.IsNotExist(err))
+	})
+
+	t.Run("returns error if unable to create parent directory", func(t *testing.T) {
+		invalidDir := filepath.Join("/root", "invalidDir", "test.db")
+		err := prepareWritableDB(invalidDir)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "unable to create parent directory")
+	})
+}

grype/db/v6/blob_store.go

@@ -0,0 +1,112 @@
+package v6
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"gorm.io/gorm"
+
+	"github.com/anchore/grype/internal/log"
+)
+
+type blobable interface {
+	getBlobValue() any
+	setBlobID(int64)
+}
+
+type blobStore struct {
+	db *gorm.DB
+}
+
+func newBlobStore(db *gorm.DB) *blobStore {
+	return &blobStore{
+		db: db,
+	}
+}
+
+func (s *blobStore) addBlobable(bs ...blobable) error {
+	for i := range bs {
+		b := bs[i]
+		v := b.getBlobValue()
+		if v == nil {
+			continue
+		}
+		bl := newBlob(v)
+
+		if err := s.addBlobs(bl); err != nil {
+			return err
+		}
+
+		b.setBlobID(bl.ID)
+	}
+	return nil
+}
+
+func (s *blobStore) addBlobs(blobs ...*Blob) error {
+	for i := range blobs {
+		v := blobs[i]
+		digest := v.computeDigest()
+
+		var blobDigest BlobDigest
+		err := s.db.Where("id = ?", digest).First(&blobDigest).Error
+		if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+			return fmt.Errorf("failed to get blob digest: %w", err)
+		}
+
+		if blobDigest.BlobID != 0 {
+			v.ID = blobDigest.BlobID
+			continue
+		}
+
+		if err := s.db.Create(v).Error; err != nil {
+			return fmt.Errorf("failed to create blob: %w", err)
+		}
+
+		blobDigest = BlobDigest{
+			ID:     digest,
+			BlobID: v.ID,
+		}
+		if err := s.db.Create(blobDigest).Error; err != nil {
+			return fmt.Errorf("failed to create blob digest: %w", err)
+		}
+	}
+	return nil
+}
+
+func (s *blobStore) getBlobValue(id int64) (string, error) {
+	var blob Blob
+	if err := s.db.First(&blob, id).Error; err != nil {
+		return "", err
+	}
+	return blob.Value, nil
+}
+
+func (s *blobStore) Close() error {
+	var count int64
+	if err := s.db.Model(&Blob{}).Count(&count).Error; err != nil {
+		return fmt.Errorf("failed to count blobs: %w", err)
+	}
+
+	log.WithFields("records", count).Trace("finalizing blobs")
+
+	if err := s.db.Exec("DROP TABLE blob_digests").Error; err != nil {
+		return fmt.Errorf("failed to drop blob digests: %w", err)
+	}
+	return nil
+}
+
+func newBlob(obj any) *Blob {
+	sb := strings.Builder{}
+	enc := json.NewEncoder(&sb)
+	enc.SetEscapeHTML(false)
+
+	if err := enc.Encode(obj); err != nil {
+		panic("could not marshal object to json")
+	}
+
+	return &Blob{
+		Value: sb.String(),
+	}
+}

grype/db/v6/blob_store_test.go

@@ -0,0 +1,61 @@
+package v6
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBlobWriter_AddBlobs(t *testing.T) {
+	db := setupTestDB(t)
+	writer := newBlobStore(db)
+
+	obj1 := map[string]string{"key": "value1"}
+	obj2 := map[string]string{"key": "value2"}
+
+	blob1 := newBlob(obj1)
+	blob2 := newBlob(obj2)
+	blob3 := newBlob(obj1) // same as blob1
+
+	err := writer.addBlobs(blob1, blob2, blob3)
+	require.NoError(t, err)
+
+	require.NotZero(t, blob1.ID)
+	require.Equal(t, blob1.ID, blob3.ID) // blob3 should have the same ID as blob1 (natural deduplication)
+
+	var result1 Blob
+	require.NoError(t, db.Where("id = ?", blob1.ID).First(&result1).Error)
+	assert.Equal(t, blob1.Value, result1.Value)
+
+	var result2 Blob
+	require.NoError(t, db.Where("id = ?", blob2.ID).First(&result2).Error)
+	assert.Equal(t, blob2.Value, result2.Value)
+}
+
+func TestBlobWriter_Close(t *testing.T) {
+	db := setupTestDB(t)
+	writer := newBlobStore(db)
+
+	obj := map[string]string{"key": "value"}
+	blob := newBlob(obj)
+	require.NoError(t, writer.addBlobs(blob))
+
+	// ensure the blob digest table is created
+	var blobDigest BlobDigest
+	require.NoError(t, db.First(&blobDigest).Error)
+	require.NotZero(t, blobDigest.ID)
+
+	err := writer.Close()
+	require.NoError(t, err)
+
+	// ensure the blob digest table is deleted
+	err = db.First(&blobDigest).Error
+	require.ErrorContains(t, err, "no such table: blob_digests")
+}
+
+func TestBlob_computeDigest(t *testing.T) {
+	assert.Equal(t, "xxh64:0e6882304e9adbd5", Blob{Value: "test content"}.computeDigest())
+
+	assert.Equal(t, "xxh64:ea0c19ae9fbd93b3", Blob{Value: "different content"}.computeDigest())
+}

grype/db/v6/blobs.go

@@ -0,0 +1,105 @@
+package v6
+
+import "time"
+
+// VulnerabilityStatus is meant to convey the current point in the lifecycle for a vulnerability record.
+// This is roughly based on CVE status, NVD status, and vendor-specific status values (see https://nvd.nist.gov/vuln/vulnerability-status)
+type VulnerabilityStatus string
+
+const (
+	// VulnerabilityNoStatus is the default status for a vulnerability record
+	VulnerabilityNoStatus VulnerabilityStatus = "?"
+
+	// VulnerabilityActive means that the information from the vulnerability record is actionable
+	VulnerabilityActive VulnerabilityStatus = "active" // empty also means active
+
+	// VulnerabilityAnalyzing means that the vulnerability record is being reviewed, it may or may not be actionable
+	VulnerabilityAnalyzing VulnerabilityStatus = "analyzing"
+
+	// VulnerabilityRejected means that data from the vulnerability record should not be acted upon
+	VulnerabilityRejected VulnerabilityStatus = "rejected"
+
+	// VulnerabilityDisputed means that the vulnerability record is in contention, it may or may not be actionable
+	VulnerabilityDisputed VulnerabilityStatus = "disputed"
+)
+
+// SeverityScheme represents how to interpret the string value for a vulnerability severity
+type SeverityScheme string
+
+const (
+	// SeveritySchemeCVSSV2 is the CVSS v2 severity scheme
+	SeveritySchemeCVSSV2 SeverityScheme = "CVSSv2"
+
+	// SeveritySchemeCVSSV3 is the CVSS v3 severity scheme
+	SeveritySchemeCVSSV3 SeverityScheme = "CVSSv3"
+
+	// SeveritySchemeCVSSV4 is the CVSS v4 severity scheme
+	SeveritySchemeCVSSV4 SeverityScheme = "CVSSv4"
+
+	// SeveritySchemeHML is a string severity scheme (High, Medium, Low)
+	SeveritySchemeHML SeverityScheme = "HML"
+
+	// SeveritySchemeCHMLN is a string severity scheme (Critical, High, Medium, Low, Negligible)
+	SeveritySchemeCHMLN SeverityScheme = "CHMLN"
+)
+
+// VulnerabilityBlob represents the core advisory record for a single known vulnerability from a specific provider.
+type VulnerabilityBlob struct {
+	// ID is the lowercase unique string identifier for the vulnerability relative to the provider
+	ID string `json:"id"`
+
+	// ProviderName of the Vunnel provider (or sub processor responsible for data records from a single specific source, e.g. "ubuntu")
+	ProviderName string `json:"provider"`
+
+	// Assigner is a list of names, email, or organizations who submitted the vulnerability
+	Assigner []string `json:"assigner,omitempty"`
+
+	// Status conveys the actionability of the current record
+	Status VulnerabilityStatus `json:"status"`
+
+	// Description of the vulnerability as provided by the source
+	Description string `json:"description"`
+
+	// PublishedDate is the date the vulnerability record was first published
+	PublishedDate *time.Time `json:"published,omitempty"`
+
+	// ModifiedDate is the date the vulnerability record was last modified
+	ModifiedDate *time.Time `json:"modified,omitempty"`
+
+	// WithdrawnDate is the date the vulnerability record was withdrawn
+	WithdrawnDate *time.Time `json:"withdrawn,omitempty"`
+
+	// References are URLs to external resources that provide more information about the vulnerability
+	References []Reference `json:"refs,omitempty"`
+
+	// Aliases is a list of IDs of the same vulnerability in other databases, in the form of the ID field. This allows one database to claim that its own entry describes the same vulnerability as one or more entries in other databases.
+	Aliases []string `json:"aliases,omitempty"`
+
+	// Severities is a list of severity indications (quantitative or qualitative) for the vulnerability
+	Severities []Severity `json:"severities,omitempty"`
+}
+
+// Reference represents a single external URL and string tags to use for organizational purposes
+type Reference struct {
+	// URL is the external resource
+	URL string `json:"url"`
+
+	// Tags is a free-form organizational field to convey additional information about the reference
+	Tags []string `json:"tags,omitempty"`
+}
+
+// Severity represents a single string severity record for a vulnerability record
+type Severity struct {
+	// Scheme describes the quantitative method used to determine the Score, such as "CVSS_V3". Alternatively this makes
+	// claim that Value is qualitative, for example "HML" (High, Medium, Low), CHMLN (critical-high-medium-low-negligible)
+	Scheme SeverityScheme `json:"scheme"`
+
+	// Value is the severity score (e.g. "7.5", "CVSS:4.0/AV:N/AC:L/AT:N/PR:H/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N",  or "high" )
+	Value string `json:"value"`
+
+	// Source is the name of the source of the severity score (e.g. "[email protected]" or "[email protected]")
+	Source string `json:"source"`
+
+	// Rank is a free-form organizational field to convey priority over other severities
+	Rank int `json:"rank"`
+}