test: validate CycloneDX with the JSON schema (#4956)

* test: validate CycloneDX with the JSON schema

* fix(sbom): move licenses to `name` field in Cyclonedx format (#4941)

* use license.Name instead of Expression

* update tests

* test: add uuid package

* test: compare UUID

---------

Co-authored-by: DmitriyLewen <[email protected]>

go.mod

@@ -91,6 +91,7 @@ require (
  github.com/testcontainers/testcontainers-go/modules/localstack v0.21.0
  github.com/tetratelabs/wazero v1.2.1
  github.com/twitchtv/twirp v8.1.2+incompatible
+ github.com/xeipuuv/gojsonschema v1.2.0
  github.com/xlab/treeprint v1.1.0
  go.etcd.io/bbolt v1.3.7
  go.uber.org/zap v1.24.0
@@ -344,7 +345,6 @@ require (
  github.com/xanzy/ssh-agent v0.3.3 // indirect
  github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
  github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
- github.com/xeipuuv/gojsonschema v1.2.0 // indirect
  github.com/yashtewari/glob-intersection v0.1.0 // indirect
  github.com/yuin/gopher-lua v1.1.0 // indirect
  github.com/zclconf/go-cty v1.10.0 // indirect

integration/client_server_test.go

@@ -4,23 +4,21 @@ package integration
 
 import (
  "context"
- "encoding/json"
  "fmt"
  "os"
  "path/filepath"
  "strings"
  "testing"
  "time"
 
- cdx "github.com/CycloneDX/cyclonedx-go"
  "github.com/docker/go-connections/nat"
- "github.com/samber/lo"
  "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/require"
  testcontainers "github.com/testcontainers/testcontainers-go"
 
  "github.com/aquasecurity/trivy/pkg/clock"
  "github.com/aquasecurity/trivy/pkg/report"
+ "github.com/aquasecurity/trivy/pkg/uuid"
 )
 
 type csArgs struct {
@@ -402,45 +400,34 @@ func TestClientServerWithFormat(t *testing.T) {
 }
 
 func TestClientServerWithCycloneDX(t *testing.T) {
- if *update {
- t.Skipf("This test doesn't use golden files")
- }
  tests := []struct {
- name string
- args csArgs
- wantComponentsCount int
- wantDependenciesCount int
+ name string
+ args csArgs
+ golden string
  }{
  {
  name: "fluentd with RubyGems with CycloneDX format",
  args: csArgs{
  Format: "cyclonedx",
  Input: "testdata/fixtures/images/fluentd-multiple-lockfiles.tar.gz",
  },
- wantComponentsCount: 161,
- wantDependenciesCount: 162,
+ golden: "testdata/fluentd-multiple-lockfiles.cdx.json.golden",
  },
  }
 
  addr, cacheDir := setup(t, setupOptions{})
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
- osArgs, outputFile := setupClient(t, tt.args, addr, cacheDir, "")
+ clock.SetFakeTime(t, time.Date(2020, 9, 10, 14, 20, 30, 5, time.UTC))
+ uuid.SetFakeUUID(t, "3ff14136-e09f-4df9-80ea-%012d")
+
+ osArgs, outputFile := setupClient(t, tt.args, addr, cacheDir, tt.golden)
 
  // Run Trivy client
  err := execute(osArgs)
  require.NoError(t, err)
 
- f, err := os.Open(outputFile)
- require.NoError(t, err)
- defer f.Close()
-
- var got cdx.BOM
- err = json.NewDecoder(f).Decode(&got)
- require.NoError(t, err)
-
- assert.EqualValues(t, tt.wantComponentsCount, len(lo.FromPtr(got.Components)))
- assert.EqualValues(t, tt.wantDependenciesCount, len(lo.FromPtr(got.Dependencies)))
+ compareCycloneDX(t, tt.golden, outputFile)
  })
  }
 }

integration/integration_test.go

@@ -11,18 +11,20 @@ import (
  "os"
  "path/filepath"
  "sort"
+ "strings"
  "testing"
  "time"
 
  cdx "github.com/CycloneDX/cyclonedx-go"
+ "github.com/samber/lo"
  spdxjson "github.com/spdx/tools-golang/json"
  "github.com/spdx/tools-golang/spdx"
  "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/require"
+ "github.com/xeipuuv/gojsonschema"
 
  "github.com/aquasecurity/trivy-db/pkg/db"
  "github.com/aquasecurity/trivy-db/pkg/metadata"
-
  "github.com/aquasecurity/trivy/pkg/commands"
  "github.com/aquasecurity/trivy/pkg/dbtest"
  "github.com/aquasecurity/trivy/pkg/types"
@@ -138,10 +140,7 @@ func readCycloneDX(t *testing.T, filePath string) *cdx.BOM {
  err = decoder.Decode(bom)
  require.NoError(t, err)
 
- // We don't compare values which change each time an SBOM is generated
- bom.Metadata.Timestamp = ""
- bom.Metadata.Component.BOMRef = ""
- bom.SerialNumber = ""
+ // Sort components
  if bom.Components != nil {
  sort.Slice(*bom.Components, func(i, j int) bool {
  return (*bom.Components)[i].Name < (*bom.Components)[j].Name
@@ -153,12 +152,6 @@ func readCycloneDX(t *testing.T, filePath string) *cdx.BOM {
  })
  }
  }
- if bom.Dependencies != nil {
- for j := range *bom.Dependencies {
- (*bom.Dependencies)[j].Ref = ""
- (*bom.Dependencies)[j].Dependencies = nil
- }
- }
 
  return bom
 }
@@ -212,6 +205,20 @@ func compareCycloneDX(t *testing.T, wantFile, gotFile string) {
  want := readCycloneDX(t, wantFile)
  got := readCycloneDX(t, gotFile)
  assert.Equal(t, want, got)
+
+ // Validate CycloneDX output against the JSON schema
+ schemaLoader := gojsonschema.NewReferenceLoader(got.JSONSchema)
+ documentLoader := gojsonschema.NewGoLoader(got)
+
+ result, err := gojsonschema.Validate(schemaLoader, documentLoader)
+ require.NoError(t, err)
+
+ if valid := result.Valid(); !valid {
+ errs := lo.Map(result.Errors(), func(err gojsonschema.ResultError, _ int) string {
+ return err.String()
+ })
+ assert.True(t, valid, strings.Join(errs, "\n"))
+ }
 }
 
 func compareSpdxJson(t *testing.T, wantFile, gotFile string) {

integration/repo_test.go

@@ -7,12 +7,15 @@ import (
  "path/filepath"
  "strings"
  "testing"
+ "time"
 
  "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/require"
 
+ "github.com/aquasecurity/trivy/pkg/clock"
  ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
  "github.com/aquasecurity/trivy/pkg/types"
+ "github.com/aquasecurity/trivy/pkg/uuid"
 )
 
 // TestRepository tests `trivy repo` with the local code repositories
@@ -31,7 +34,7 @@ func TestRepository(t *testing.T) {
  skipFiles []string
  skipDirs []string
  command string
- format string
+ format types.Format
  includeDevDeps bool
  }
  tests := []struct {
@@ -367,7 +370,7 @@ func TestRepository(t *testing.T) {
  command = tt.args.command
  }
 
- format := "json"
+ format := types.FormatJSON
  if tt.args.format != "" {
  format = tt.args.format
  }
@@ -380,7 +383,7 @@ func TestRepository(t *testing.T) {
  "--skip-db-update",
  "--skip-policy-update",
  "--format",
- format,
+ string(format),
  "--offline-scan",
  }
 
@@ -458,17 +461,20 @@ func TestRepository(t *testing.T) {
  osArgs = append(osArgs, "--output", outputFile)
  osArgs = append(osArgs, tt.args.input)
 
+ clock.SetFakeTime(t, time.Date(2020, 9, 10, 14, 20, 30, 5, time.UTC))
+ uuid.SetFakeUUID(t, "3ff14136-e09f-4df9-80ea-%012d")
+
  // Run "trivy repo"
  err := execute(osArgs)
  require.NoError(t, err)
 
  // Compare want and got
  switch format {
- case "cyclonedx":
+ case types.FormatCycloneDX:
  compareCycloneDX(t, tt.golden, outputFile)
- case "spdx-json":
+ case types.FormatSPDXJSON:
  compareSpdxJson(t, tt.golden, outputFile)
- case "json":
+ case types.FormatJSON:
  compareReports(t, tt.golden, outputFile, tt.override)
  default:
  require.Fail(t, "invalid format", "format: %s", format)

integration/sbom_test.go

@@ -56,7 +56,7 @@ func TestSBOM(t *testing.T) {
  format: "json",
  artifactType: "cyclonedx",
  },
- golden: "testdata/fluentd-multiple-lockfiles-cyclonedx.json.golden",
+ golden: "testdata/fluentd-multiple-lockfiles.json.golden",
  },
  {
  name: "centos7 in in-toto attestation",

integration/testdata/conda-cyclonedx.json.golden

@@ -2,10 +2,10 @@
  "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
  "bomFormat": "CycloneDX",
  "specVersion": "1.5",
- "serialNumber": "urn:uuid:e7d2faf4-1d5f-4cd7-a792-8b9b5f6fe2d7",
+ "serialNumber": "urn:uuid:3ff14136-e09f-4df9-80ea-000000000001",
  "version": 1,
  "metadata": {
- "timestamp": "2023-08-04T05:57:22+00:00",
+ "timestamp": "2020-09-10T14:20:30+00:00",
  "tools": [
  {
  "vendor": "aquasecurity",
@@ -14,7 +14,7 @@
  }
  ],
  "component": {
- "bom-ref": "a80bd6fc-91e4-4e42-9941-eafc2423d031",
+ "bom-ref": "3ff14136-e09f-4df9-80ea-000000000002",
  "type": "application",
  "name": "testdata/fixtures/repo/conda",
  "properties": [
@@ -77,7 +77,7 @@
  ],
  "dependencies": [
  {
- "ref": "a80bd6fc-91e4-4e42-9941-eafc2423d031",
+ "ref": "3ff14136-e09f-4df9-80ea-000000000002",
  "dependsOn": [
  "pkg:conda/[email protected]?file_path=miniconda3%2Fenvs%2Ftestenv%2Fconda-meta%2Fopenssl-1.1.1q-h7f8727e_0.json",
  "pkg:conda/[email protected]?file_path=miniconda3%2Fenvs%2Ftestenv%2Fconda-meta%2Fpip-22.2.2-py38h06a4308_0.json"