Kbs key release rework

src/cloud-api-adaptor/test/e2e/assessment_helpers.go

@@ -472,6 +472,7 @@ func AssessPodTestCommands(ctx context.Context, client klient.Client, pod *v1.Po
 		return "Failed to list pod", err
 	}
 	for _, testCommand := range testCommands {
+		log.Tracef("Running test command: %v", testCommand)
 		var stdout, stderr bytes.Buffer
 		for _, podItem := range podlist.Items {
 			if podItem.ObjectMeta.Name == pod.Name {
@@ -480,24 +481,24 @@ func AssessPodTestCommands(ctx context.Context, client klient.Client, pod *v1.Po
 				if err := client.Resources(pod.Namespace).ExecInPod(ctx, pod.Namespace, pod.Name, testCommand.ContainerName, testCommand.Command, &stdout, &stderr); err != nil {
 					if testCommand.TestErrorFn != nil {
 						if !testCommand.TestErrorFn(err) {
-							return err.Error(), fmt.Errorf("Command %v running in container %s produced unexpected output on error: %s", testCommand.Command, testCommand.ContainerName, err.Error())
+							return err.Error(), fmt.Errorf("command %v running in container %s produced unexpected output on error: %s, stderr: %s", testCommand.Command, testCommand.ContainerName, err.Error(), stderr.String())
 						}
 					} else {
-						return err.Error(), err
+						return err.Error(), fmt.Errorf("command %v running in container %s produced unexpected output on error: %s, stderr: %s", testCommand.Command, testCommand.ContainerName, err.Error(), stderr.String())
 					}
 				} else if testCommand.TestErrorFn != nil {
 					return "", fmt.Errorf("We expected an error from Pod %s, but it was not found", pod.Name)
 				}
 				if testCommand.TestCommandStderrFn != nil {
 					if !testCommand.TestCommandStderrFn(stderr) {
-						return stderr.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stderr: %s", testCommand.Command, testCommand.ContainerName, stderr.String())
+						return stderr.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stderr: %s, stdout: %s", testCommand.Command, testCommand.ContainerName, stderr.String(), stdout.String())
 					} else {
 						return stderr.String(), nil
 					}
 				}
 				if testCommand.TestCommandStdoutFn != nil {
 					if !testCommand.TestCommandStdoutFn(stdout) {
-						return stdout.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stdout: %s", testCommand.Command, testCommand.ContainerName, stdout.String())
+						return stdout.String(), fmt.Errorf("Command %v running in container %s produced unexpected output on stdout: %s, stderr: %s", testCommand.Command, testCommand.ContainerName, stdout.String(), stderr.String())
 					} else {
 						return stdout.String(), nil
 					}

src/cloud-api-adaptor/test/e2e/assessment_runner.go

@@ -393,11 +393,9 @@ func (tc *TestCase) Run() {
 
 				if tc.podState == v1.PodRunning {
 					if len(tc.testCommands) > 0 {
-						if len(tc.testCommands) > 0 {
-							logString, err := AssessPodTestCommands(ctx, client, tc.pod, tc.testCommands)
-							if err != nil {
-								t.Errorf("AssessPodTestCommands failed, with output: %s and error: %v", logString, err)
-							}
+						logString, err := AssessPodTestCommands(ctx, client, tc.pod, tc.testCommands)
+						if err != nil {
+							t.Errorf("AssessPodTestCommands failed, with output: %s and error: %v", logString, err)
 						}
 					}
 

src/cloud-api-adaptor/test/e2e/azure_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/azure"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
 )
 
 func TestDeletePodAzure(t *testing.T) {
@@ -131,7 +132,13 @@ func TestKbsKeyRelease(t *testing.T) {
 	}
 	t.Parallel()
 	kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint()
-	DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint)
+	testSecret := envconf.RandomName("coco-pp-e2e-secret", 25)
+	resourcePath := "caa/workload_key/test_key.bin"
+	err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret))
+	if err != nil {
+		t.Fatalf("SetSecret failed with: %v", err)
+	}
+	DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
 }
 
 func TestRemoteAttestation(t *testing.T) {
@@ -142,7 +149,11 @@ func TestRemoteAttestation(t *testing.T) {
 	} else if keyBrokerService == nil {
 		t.Skip("Skipping because KBS config is missing")
 	} else {
-		kbsEndpoint, _ = keyBrokerService.GetCachedKbsEndpoint()
+		var err error
+		kbsEndpoint, err = keyBrokerService.GetCachedKbsEndpoint()
+		if err != nil {
+			t.Fatalf("GetCachedKbsEndpoint failed with: %v", err)
+		}
 	}
 	DoTestRemoteAttestation(t, testEnv, assert, kbsEndpoint)
 }
@@ -152,8 +163,11 @@ func TestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T) {
 		t.Skip("Skipping kbs related test as Trustee Operator is not deployed")
 	}
 	t.Parallel()
-	kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint()
-	DoTestTrusteeOperatorKeyReleaseForSpecificKey(t, testEnv, assert, kbsEndpoint)
+	kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint()
+	if err != nil {
+		t.Fatalf("GetCachedKbsEndpoint failed with: %v", err)
+	}
+	DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, "default/kbsres1/key1", "res1val1")
 }
 
 func TestAzureImageDecryption(t *testing.T) {

src/cloud-api-adaptor/test/e2e/common.go

@@ -29,7 +29,6 @@ import (
 
 const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180
 const DEFAULT_AUTH_SECRET = "auth-json-secret-default"
-const KBS_SECRET = "reponame/workload_key/key.bin"
 
 var testInitdata string = `algorithm = "sha384"
 version = "0.1.0"

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -547,35 +547,34 @@ func DoTestImageDecryption(t *testing.T, e env.Environment, assert CloudAssert,
 	NewTestCase(t, e, "TestImageDecryption", assert, "Encrypted image layers have been decrypted").WithPod(pod).WithDeleteAssertion(&duration).Run()
 }
 
-func DoTestSealedSecret(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) {
+func DoTestSealedSecret(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string, resourcePath, expectedSecret string) {
 	key := "MY_SECRET"
-	value := CreateSealedSecretValue("kbs:///" + KBS_SECRET)
+	value := CreateSealedSecretValue("kbs:///" + resourcePath)
 	podName := "sealed-secret"
 	imageName := getBusyboxTestImage(t)
 	env := []v1.EnvVar{{Name: key, Value: value}}
 	cmd := []string{"watch", "-n", "120", "-t", "--", "printenv MY_SECRET"}
 
 	pod := NewPod(E2eNamespace, podName, podName, imageName, WithEnvironmentVariables(env), WithInitdata(kbsEndpoint), WithCommand(cmd))
 
-	expectedPodLogString := "This is my"
-	NewTestCase(t, e, "TestSealedSecret", assert, "Unsealed secret has been set to ENV").WithPod(pod).WithExpectedPodLogString(expectedPodLogString).Run()
+	NewTestCase(t, e, "TestSealedSecret", assert, "Unsealed secret has been set to ENV").WithPod(pod).WithExpectedPodLogString(expectedSecret).Run()
 }
 
 // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs
 // as test cases might be run in parallel
-func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) {
+func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint, resourcePath, expectedSecret string) {
 	t.Log("Do test kbs key release")
 	pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "kbs-key-release", kbsEndpoint).GetPodOrFatal(t)
 	testCommands := []TestCommand{
 		{
-			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + KBS_SECRET},
+			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + resourcePath},
 			ContainerName: pod.Spec.Containers[0].Name,
 			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
-				if strings.Contains(stdout.String(), "This is my cluster name") {
-					t.Logf("Success to get key.bin: %s", stdout.String())
+				if strings.Contains(stdout.String(), expectedSecret) {
+					t.Logf("Success to get secret key: %s", stdout.String())
 					return true
 				} else {
-					t.Errorf("Failed to access key.bin: %s", stdout.String())
+					t.Errorf("Failed to access secret key: %s", stdout.String())
 					return false
 				}
 			},
@@ -587,12 +586,12 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kb
 
 // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs
 // as test cases might be run in parallel
-func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) {
+func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint, resourcePath, expectedSecret string) {
 	t.Log("Do test kbs key release failure case")
 	pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "kbs-failure", kbsEndpoint).GetPodOrFatal(t)
 	testCommands := []TestCommand{
 		{
-			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + KBS_SECRET},
+			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/" + resourcePath},
 			ContainerName: pod.Spec.Containers[0].Name,
 			TestErrorFn: func(err error) bool {
 				if strings.Contains(err.Error(), "command terminated with exit code 1") {
@@ -603,11 +602,11 @@ func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert Cloud
 				}
 			},
 			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
-				if strings.Contains(stdout.String(), "This is my cluster name") {
-					t.Errorf("FAIL as successed to get key.bin: %s", stdout.String())
+				if strings.Contains(stdout.String(), expectedSecret) {
+					t.Errorf("FAIL as succeed to get secret key: %s", stdout.String())
 					return false
 				} else {
-					t.Logf("PASS as failed to access key.bin: %s", stdout.String())
+					t.Logf("PASS as failed to access secret key: %s", stdout.String())
 					return true
 				}
 			},
@@ -617,29 +616,6 @@ func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert Cloud
 	NewTestCase(t, e, "DoTestKbsKeyReleaseForFailure", assert, "Kbs key release is failed").WithPod(pod).WithTestCommands(testCommands).Run()
 }
 
-// Test to check for specific key value from Trustee Operator Deployment
-func DoTestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) {
-	t.Log("Do test Trustee operator key release for specific key")
-	pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "op-key-release", kbsEndpoint).GetPodOrFatal(t)
-	testCommands := []TestCommand{
-		{
-			Command:       []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/default/kbsres1/key1"},
-			ContainerName: pod.Spec.Containers[0].Name,
-			TestCommandStdoutFn: func(stdout bytes.Buffer) bool {
-				if strings.Contains(stdout.String(), "res1val1") {
-					t.Logf("Success to get key %s", stdout.String())
-					return true
-				} else {
-					t.Errorf("Failed to access key: %s", stdout.String())
-					return false
-				}
-			},
-		},
-	}
-
-	NewTestCase(t, e, "KbsKeyReleasePod", assert, "Kbs key release from Trustee Operator is successful").WithPod(pod).WithTestCommands(testCommands).Run()
-}
-
 func DoTestRestrictivePolicyBlocksExec(t *testing.T, e env.Environment, assert CloudAssert) {
 	allowAllExceptExecPolicyFilePath := "fixtures/policies/allow-all-except-exec-process.rego"
 	podName := "policy-exec-rejected"

src/cloud-api-adaptor/test/e2e/docker_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/docker"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
 )
 
 func TestDockerCreateSimplePod(t *testing.T) {
@@ -102,14 +103,28 @@ func TestDockerKbsKeyRelease(t *testing.T) {
 	if !isTestWithKbs() {
 		t.Skip("Skipping kbs related test as kbs is not deployed")
 	}
-	keyBrokerService.SetSampleSecretKey()
-	keyBrokerService.EnableKbsCustomizedResourcePolicy("deny_all.rego")
-	kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint()
+	testSecret := envconf.RandomName("coco-pp-e2e-secret", 25)
+	resourcePath := "caa/workload_key/test_key.bin"
+	err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret))
+	if err != nil {
+		t.Fatalf("SetSecret failed with: %v", err)
+	}
+	err = keyBrokerService.EnableKbsCustomizedResourcePolicy("deny_all.rego")
+	if err != nil {
+		t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err)
+	}
+	kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint()
+	if err != nil {
+		t.Fatalf("GetCachedKbsEndpoint failed with: %v", err)
+	}
 	assert := DockerAssert{}
 	t.Parallel()
-	DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint)
-	keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
-	DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint)
+	DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
+	err = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
+	if err != nil {
+		t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err)
+	}
+	DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
 }
 
 func TestDockerCreatePeerPodWithAuthenticatedImageWithoutCredentials(t *testing.T) {

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
 )
 
 func TestLibvirtCreateSimplePod(t *testing.T) {
@@ -137,37 +138,76 @@ func TestLibvirtSealedSecret(t *testing.T) {
 	if !isTestWithKbs() {
 		t.Skip("Skipping kbs related test as kbs is not deployed")
 	}
-	_ = keyBrokerService.SetSampleSecretKey()
-	_ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
-	_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego")
-	kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint()
+
+	testSecret := envconf.RandomName("coco-pp-e2e-secret", 25)
+	resourcePath := "caa/workload_key/test_key.bin"
+	err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret))
+	if err != nil {
+		t.Fatalf("SetSecret failed with: %v", err)
+	}
+	err = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
+	if err != nil {
+		t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err)
+	}
+	err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego")
+	if err != nil {
+		t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err)
+	}
+	kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint()
+	if err != nil {
+		t.Fatalf("GetCachedKbsEndpoint failed with: %v", err)
+	}
 	assert := LibvirtAssert{}
-	DoTestSealedSecret(t, testEnv, assert, kbsEndpoint)
+	DoTestSealedSecret(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
 }
 
 func TestLibvirtKbsKeyRelease(t *testing.T) {
 	if !isTestWithKbs() {
 		t.Skip("Skipping kbs related test as kbs is not deployed")
 	}
-	_ = keyBrokerService.SetSampleSecretKey()
-	_ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
-	_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego")
-	kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint()
+
+	testSecret := envconf.RandomName("coco-pp-e2e-secret", 25)
+	resourcePath := "caa/workload_key/test_key.bin"
+	err := keyBrokerService.SetSecret(resourcePath, []byte(testSecret))
+	if err != nil {
+		t.Fatalf("SetSecret failed with: %v", err)
+	}
+	err = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego")
+	if err != nil {
+		t.Fatalf("EnableKbsCustomizedResourcePolicy failed with: %v", err)
+	}
+	err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego")
+	if err != nil {
+		t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err)
+	}
+	kbsEndpoint, err := keyBrokerService.GetCachedKbsEndpoint()
+	if err != nil {
+		t.Fatalf("GetCachedKbsEndpoint failed with: %v", err)
+	}
 	assert := LibvirtAssert{}
 	t.Parallel()
-	DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint)
+	DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
 	if isTestWithKbsIBMSE() {
 		t.Log("KBS with ibmse cases")
 		// the allow_*_.rego file is created by follow document
 		// https://github.com/confidential-containers/trustee/blob/main/deps/verifier/src/se/README.md#set-attestation-policy
-		_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego")
-		DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint)
-		_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego")
-		DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint)
+		err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego")
+		if err != nil {
+			t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err)
+		}
+		DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
+		err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego")
+		if err != nil {
+			t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err)
+		}
+		DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
 	} else {
 		t.Log("KBS normal cases")
-		_ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego")
-		DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint)
+		err = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego")
+		if err != nil {
+			t.Fatalf("EnableKbsCustomizedAttestationPolicy failed with: %v", err)
+		}
+		DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint, resourcePath, testSecret)
 	}
 }