Adding functionality for dirhash in cli (#436)

* chore: Adding dirhas-glob argument.
* chore: Update command docs
* chore: Add dirhash examples to tutorial
---------

Signed-off-by: Matthias Glastra <[email protected]>

cmd/run.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/gobwas/glob"
 	witness "github.com/in-toto/go-witness"
 	"github.com/in-toto/go-witness/archivista"
 	"github.com/in-toto/go-witness/attestation"
@@ -127,11 +128,18 @@ func runRun(ctx context.Context, ro options.RunOptions, args []string, signers .
 		roHashes = append(roHashes, cryptoutil.DigestValue{Hash: hash, GitOID: false})
 	}
 
+	for _, dirHashGlobItem := range ro.DirHashGlobs {
+		_, err := glob.Compile(dirHashGlobItem)
+		if err != nil {
+			return fmt.Errorf("failed to compile glob: %v", err)	
+		}
+	}
+
 	results, err := witness.RunWithExports(
 		ro.StepName,
 		witness.RunWithSigners(signers...),
 		witness.RunWithAttestors(attestors),
-		witness.RunWithAttestationOpts(attestation.WithWorkingDir(ro.WorkingDir), attestation.WithHashes(roHashes)),
+		witness.RunWithAttestationOpts(attestation.WithWorkingDir(ro.WorkingDir), attestation.WithHashes(roHashes), attestation.WithDirHashGlob(ro.DirHashGlobs)),
 		witness.RunWithTimestampers(timestampers...),
 	)
 	if err != nil {

cmd/verify.go

@@ -162,6 +162,15 @@ func runVerify(ctx context.Context, vo options.VerifyOptions, verifiers ...crypt
 	}
 
 	subjects := []cryptoutil.DigestSet{}
+	if len(vo.ArtifactDirectoryPath) > 0 {
+		artifactDigestSet, err := cryptoutil.CalculateDigestSetFromDir(vo.ArtifactDirectoryPath, []cryptoutil.DigestValue{{Hash: crypto.SHA256, GitOID: false}})
+		if err != nil {
+			return fmt.Errorf("failed to calculate dir digest: %w", err)
+		}
+
+		subjects = append(subjects, artifactDigestSet)
+	}
+
 	if len(vo.ArtifactFilePath) > 0 {
 		artifactDigestSet, err := cryptoutil.CalculateDigestSetFromFile(vo.ArtifactFilePath, []cryptoutil.DigestValue{{Hash: crypto.SHA256, GitOID: false}})
 		if err != nil {

cmd/verify_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/in-toto/go-witness/attestation/commandrun"
 	"github.com/in-toto/go-witness/cryptoutil"
 	"github.com/in-toto/go-witness/dsse"
+	"github.com/in-toto/go-witness/log"
 	"github.com/in-toto/go-witness/policy"
 	"github.com/in-toto/go-witness/signer"
 	"github.com/in-toto/go-witness/signer/file"
@@ -208,6 +209,9 @@ func TestRunVerifyCA(t *testing.T) {
 }
 
 func TestRunVerifyKeyPair(t *testing.T) {
+	logger := newLogger()
+	log.SetLogger(logger)
+
 	policy, funcPriv := makepolicyRSAPub(t)
 	signedPolicy, pub := signPolicyRSA(t, policy)
 	workingDir := t.TempDir()

docs/commands.md

@@ -50,6 +50,7 @@ witness run [cmd] [flags]
       --attestor-product-include-glob string          Pattern to use when recording products. Files that match this pattern will be included as subjects on the attestation. (default "*")
       --attestor-sbom-export                          Export the SBOM predicate in its own attestation
       --attestor-slsa-export                          Export the SLSA provenance predicate in its own attestation
+      --dirhash-glob strings                          Dirhash glob can be used to collapse material and product hashes on matching directory matches.
       --enable-archivista                             Use Archivista to store or retrieve attestations
       --hashes strings                                Hashes selected for digest calculation. Defaults to SHA256 (default [sha256])
   -h, --help                                          help for run
@@ -178,8 +179,9 @@ witness verify [flags]
 
 ```
       --archivista-server string                            URL of the Archivista server to store or retrieve attestations (default "https://archivista.testifysec.io")
-  -f, --artifactfile string                                 Path to the artifact to verify
+  -f, --artifactfile string                                 Path to the artifact subject to verify
   -a, --attestations strings                                Attestation files to test against the policy
+      --directory-path string                               Path to the directory subject to verify
       --enable-archivista                                   Use Archivista to store or retrieve attestations
   -h, --help                                                help for verify
   -p, --policy string                                       Path to the policy to verify

docs/tutorials/getting-started.md

@@ -1,6 +1,7 @@
 # Getting Started
 
 ## Intro
+
 This quick tutorial will walk you through a simple example of how Witness can be used. To complete it
 successfully, you will need the following:
 
@@ -16,12 +17,14 @@ You will also of course need to have witness installed, which can be achieved by
 ### 1. Create a Keypair
 
 ><span class="tip-text">💡 Tip: Witness supports keyless signing with [SPIRE](https://spiffe.io/)!</span>
-```
+
+```shell
 openssl genpkey -algorithm ed25519 -outform PEM -out testkey.pem
 openssl pkey -in testkey.pem -pubout > testpub.pem
 ```
 
 ### 2. Create a Witness Configuration
+
 ><span class="tip-text">💡 Tip: Witness supports creating attestations for a wide variety of services,
 > including Github Actions </span>
 
@@ -30,7 +33,7 @@ openssl pkey -in testkey.pem -pubout > testpub.pem
 - `witness help` will show all configuration options
 - command-line arguments overrides configuration file values.
 
-```
+```yaml
 ## .witness.yaml
 
 run:
@@ -44,25 +47,29 @@ verify:
 ```
 
 ### 3. Record attestations for a build step
+
 ><span class="tip-text">💡 Tip: You can upload the recorded attestations to an [Archivista](https://github.com/in-toto/archivista) server by using the `--enable-archivista` flag!</span>
+
 - The `-a {attestor}` flag allows you to define which attestors run
 - ex. `-a maven -a gcp -a gitlab` would be used for a maven build running on a GitLab runner on GCP.
 - Witness has a set of attestors that are always run. You can see them in the output of the `witness attestors list` command.
 - Defining step names is important, these will be used in the policy.
 - This should happen as a part of a CI step
 
-```
+```shell
 witness run --step build -o test-att.json -a slsa --attestor-slsa-export -- go build -o=testapp .
 ```
 
+><span class="tip-text">💡 Tip: When you run a step with many files as the product of that step, like node_modules, it could be beneficial to collapse the result into a hash of the directory content. You can use `--dirhash-glob <glob-pattern>` to match the directory or use it multiple times to use different glob patterns. E.g. `--dirhash-glob node_modules/*`</span>
+
 ><span class="tip-text">💡 Tip: The `-a slsa` option allows to generate the [SLSA Provenace](https://slsa.dev/spec/v1.0/provenance) predicate in the attestation. The `--attestor-slsa-export` option allows to write the Provenance in a dedicated file. This is a mandatory requirement for SLSA Level 1</span>
 
 ### 4. View the attestation data in the signed DSSE Envelope
 
 - This data can be stored and retrieved from Archivista
 - This is the data that is evaluated against the Rego policy
 
-```
+```shell
 cat test-att.json | jq -r .payload | base64 -d | jq
 ```
 
@@ -76,7 +83,7 @@ Look [here](/docs/concepts/policy.md) for full documentation on Witness Policies
 > - Witness will require all attestations to succeed
 > - Witness will evaluate the rego policy against the JSON object in the corresponding attestor
 
-```
+```json
 ## policy.json
 
 {
@@ -116,7 +123,7 @@ Look [here](/docs/concepts/policy.md) for full documentation on Witness Policies
 
 ### 6. Replace the variables in the policy
 
-```
+```shell
 id=`sha256sum testpub.pem | awk '{print $1}'` && sed -i "s/{{PUBLIC_KEY_ID}}/$id/g" policy.json
 pubb64=`cat testpub.pem | base64 -w 0` && sed -i "s/{{B64_PUBLIC_KEY}}/$pubb64/g" policy.json
 ```
@@ -125,19 +132,28 @@ pubb64=`cat testpub.pem | base64 -w 0` && sed -i "s/{{B64_PUBLIC_KEY}}/$pubb64/g
 
 Keep this key safe, its owner will control the policy gates.
 
-```
+```shell
 witness sign -f policy.json --signer-file-key-path testkey.pem --outfile policy-signed.json
 ```
 
 ### 8. Verify the Binary Meets Policy Requirements
 
-This process works across air-gap as long as you have the signed policy file, correct binary, and public key or certificate authority corresponding to the private 
-key that signed the policy.
-```
+This process works across air-gap as long as you have the signed policy file, correct binary, and public key or certificate authority corresponding to the private key that signed the policy.
+
+```shell
 witness verify -f testapp -a test-att.json -p policy-signed.json -k testpub.pem
 ```
+
+If you want to verify a directory as a subject you can use the following.
+
+```shell
+witness verify --directory-path node_modules/example -a test-att.json -p policy-signed.json -k testpub.pem
+```
+
 ### 9. Profit
+
 `witness verify` will return a `non-zero` exit and reason in the case of failure, but hopefully you should have gotten sweet sweet silence with a `0` exit status, victory! If not, try again and if that fails please [file an issue](https://github.com/in-toto/witness/issues/new/choose)!
 
 ## What's Next?
+
 If you enjoyed this intro to Witness, you might benefit from taking things a step further by learning about [Witness Policies](./artifact-policy.md).

options/run.go

@@ -28,6 +28,7 @@ type RunOptions struct {
 	ArchivistaOptions        ArchivistaOptions
 	WorkingDir               string
 	Attestations             []string
+	DirHashGlobs             []string
 	Hashes                   []string
 	OutFilePath              string
 	StepName                 string
@@ -51,6 +52,7 @@ func (ro *RunOptions) AddFlags(cmd *cobra.Command) {
 	ro.ArchivistaOptions.AddFlags(cmd)
 	cmd.Flags().StringVarP(&ro.WorkingDir, "workingdir", "d", "", "Directory from which commands will run")
 	cmd.Flags().StringSliceVarP(&ro.Attestations, "attestations", "a", DefaultAttestors, "Attestations to record ('product' and 'material' are always recorded)")
+	cmd.Flags().StringSliceVar(&ro.DirHashGlobs, "dirhash-glob", []string{}, "Dirhash glob can be used to collapse material and product hashes on matching directory matches.")
 	cmd.Flags().StringSliceVar(&ro.Hashes, "hashes", []string{"sha256"}, "Hashes selected for digest calculation. Defaults to SHA256")
 	cmd.Flags().StringVarP(&ro.OutFilePath, "outfile", "o", "", "File to write signed data to")
 	cmd.Flags().StringVarP(&ro.StepName, "step", "s", "", "Name of the step being run")

options/verify.go

@@ -27,6 +27,7 @@ type VerifyOptions struct {
 	AttestationFilePaths       []string
 	PolicyFilePath             string
 	ArtifactFilePath           string
+	ArtifactDirectoryPath      string
 	AdditionalSubjects         []string
 	PolicyFulcioCertExtensions certificate.Extensions
 	PolicyCARootPaths          []string
@@ -63,7 +64,8 @@ func (vo *VerifyOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&vo.KeyPath, "publickey", "k", "", "Path to the policy signer's public key")
 	cmd.Flags().StringSliceVarP(&vo.AttestationFilePaths, "attestations", "a", []string{}, "Attestation files to test against the policy")
 	cmd.Flags().StringVarP(&vo.PolicyFilePath, "policy", "p", "", "Path to the policy to verify")
-	cmd.Flags().StringVarP(&vo.ArtifactFilePath, "artifactfile", "f", "", "Path to the artifact to verify")
+	cmd.Flags().StringVarP(&vo.ArtifactFilePath, "artifactfile", "f", "", "Path to the artifact subject to verify")
+	cmd.Flags().StringVarP(&vo.ArtifactDirectoryPath, "directory-path", "", "", "Path to the directory subject to verify")
 	cmd.Flags().StringSliceVarP(&vo.AdditionalSubjects, "subjects", "s", []string{}, "Additional subjects to lookup attestations")
 	cmd.Flags().StringSliceVarP(&vo.PolicyCARootPaths, "policy-ca-roots", "", []string{}, "Paths to CA root certificates to use for verifying a policy signed with x.509")
 	cmd.Flags().StringSliceVarP(&vo.PolicyCAIntermediatePaths, "policy-ca-intermediates", "", []string{}, "Paths to CA intermediate certificates to use for verifying a policy signed with x.509")