Allow secrets get permissions for intents operator role, to support r…

…eading DB credentials from k8s secrets (#218)
otterize · Jun 23, 2024 · 667e964 · 667e964
1 parent 641a917
commit 667e964
Show file tree

Hide file tree

Showing 7 changed files with 200 additions and 55 deletions.
diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml
@@ -8,9 +8,24 @@ on:
       - ready_for_review
       - labeled
   workflow_call:
+    inputs:
+      github_ref:
+        required: false
+        type: string
+      gcr-registry:
+        required: false
+        type: string
+      intents-operator-tag:
+        required: false
+        type: string
+      credentials-operator-tag:
+        required: false
+        type: string
     secrets:
       AZURE_CREDENTIALS:
         required: true
+      B64_GCLOUD_SERVICE_ACCOUNT_JSON:
+        required: false
 
 jobs:
   test-chart-deployment:
@@ -19,6 +34,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          # explicitly checkout helm-charts repository since this is a reusable workflow that's called from other repositories
+          repository: 'otterize/helm-charts'
+          ref: ${{ inputs.github_ref }}
 
       - name: Set up Helm
         uses: azure/[email protected]
@@ -34,29 +53,68 @@ jobs:
           kubectl wait pods -n kube-system -l k8s-app=calico-node --for condition=Ready --timeout=90s
           kubectl wait pods -n kube-system -l k8s-app=calico-kube-controllers --for condition=Ready --timeout=90s
 
+      - name: Login to GCR
+        if: "${{ inputs.gcr-registry != '' }}"
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ inputs.gcr-registry }}
+          username: _json_key_base64
+          password: ${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON}}
+
+      - name: Load intents-operator docker image from GCR
+        if: "${{ inputs.gcr-registry != '' && inputs.intents-operator-tag != ''}}"
+        run: |-
+          docker pull ${{ inputs.gcr-registry }}/intents-operator:${{ inputs.intents-operator-tag }}
+          minikube image load ${{ inputs.gcr-registry }}/intents-operator:${{ inputs.intents-operator-tag }}
+
+      - name: Load credentials-operator docker image from GCR
+        if: "${{ inputs.gcr-registry != '' && inputs.credentials-operator-tag != ''}}"
+        run: |-
+          docker pull ${{ inputs.gcr-registry }}/credentials-operator:${{ inputs.credentials-operator-tag }}
+          minikube image load ${{ inputs.gcr-registry }}/credentials-operator:${{ inputs.credentials-operator-tag }}
+
       - name: Deploy Otterize
         run: |-
           helm dep up ./otterize-kubernetes
           # schema validation using kubectl dry run
+          OPERATOR_FLAGS=""
+          if [ -n "${{ inputs.intents-operator-tag }}" ]; then
+            OPERATOR_FLAGS="$OPERATOR_FLAGS --set-string intentsOperator.operator.repository=${{ inputs.gcr-registry }} --set-string intentsOperator.operator.image=intents-operator --set-string intentsOperator.operator.tag=${{ inputs.intents-operator-tag }} --set-string intentsOperator.operator.pullPolicy=Never"
+          fi
+          if [ -n "${{ inputs.credentials-operator-tag }}" ]; then
+              OPERATOR_FLAGS="$OPERATOR_FLAGS --set-string credentialsOperator.operator.repository=${{ inputs.gcr-registry }} --set-string credentialsOperator.operator.image=credentials-operator --set-string credentialsOperator.operator.tag=${{ inputs.credentials-operator-tag }} --set-string credentialsOperator.operator.pullPolicy=Never"
+          fi
+          TELEMETRY_FLAG="--set global.telemetry.enabled=false"
+          
           kubectl create namespace otterize-system # required for dry-run
-          helm template otterize ./otterize-kubernetes -n otterize-system --set global.telemetry.enabled=false | kubectl apply --dry-run=server -f -
+          helm template otterize ./otterize-kubernetes -n otterize-system $OPERATOR_FLAGS $TELEMETRY_FLAG | kubectl apply --dry-run=server -f -
           kubectl delete namespace otterize-system # clean up
           
           # installation
-          helm install otterize ./otterize-kubernetes -n otterize-system --wait --create-namespace --set global.telemetry.enabled=false
+          helm install otterize ./otterize-kubernetes -n otterize-system --wait --create-namespace $OPERATOR_FLAGS $TELEMETRY_FLAG
 
   test-database-integrations:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          # explicitly checkout helm-charts repository since this is a reusable workflow that's called from other repositories
+          repository: 'otterize/helm-charts'
+          ref: ${{ inputs.github_ref }}
 
       - name: Start minikube
         uses: medyagh/setup-minikube@master
 
       - name: Set up Helm
         uses: azure/[email protected]
 
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.22.1
+          cache-dependency-path: tests/go.sum
+
       - name: Set up gotestfmt
         uses: GoTestTools/gotestfmt-action@v2
         with:
@@ -66,21 +124,61 @@ jobs:
       - name: Helm dependency update
         run: helm dep up ./otterize-kubernetes
 
+      - name: Login to GCR
+        if: "${{ inputs.gcr-registry != '' }}"
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ inputs.gcr-registry }}
+          username: _json_key_base64
+          password: ${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON}}
+
+      - name: Load intents-operator docker image from GCR
+        if: "${{ inputs.gcr-registry != '' && inputs.intents-operator-tag != ''}}"
+        run: |-
+          docker pull ${{ inputs.gcr-registry }}/intents-operator:${{ inputs.intents-operator-tag }}
+          minikube image load ${{ inputs.gcr-registry }}/intents-operator:${{ inputs.intents-operator-tag }}
+
+      - name: Load credentials-operator docker image from GCR
+        if: "${{ inputs.gcr-registry != '' && inputs.credentials-operator-tag != ''}}"
+        run: |-
+          docker pull ${{ inputs.gcr-registry }}/credentials-operator:${{ inputs.credentials-operator-tag }}
+          minikube image load ${{ inputs.gcr-registry }}/credentials-operator:${{ inputs.credentials-operator-tag }}
+
       - name: Run E2E tests - database integrations
         run: |
           cd tests
+          if [ -n "${{ inputs.intents-operator-tag }}" ]; then
+            export INTENTS_OPERATOR_REPOSITORY=${{ inputs.gcr-registry }}
+            export INTENTS_OPERATOR_TAG=${{ inputs.intents-operator-tag }}
+            export INTENTS_OPERATOR_IMAGE=intents-operator
+          fi
+          if [ -n "${{ inputs.credentials-operator-tag }}" ]; then
+            export CREDENTIALS_OPERATOR_REPOSITORY=${{ inputs.gcr-registry }}
+            export CREDENTIALS_OPERATOR_TAG=${{ inputs.credentials-operator-tag }}
+            export CREDENTIALS_OPERATOR_IMAGE=credentials-operator
+          fi
           go test -v -json ./databases/... | tee gotest.log | gotestfmt
 
   test-azure-integration:
-    if: contains(github.event.pull_request.labels.*.name, 'run-azure-e2e-tests') || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')))
+    if: contains(github.event.pull_request.labels.*.name, 'run-azure-e2e-tests') || (github.event_name == 'push' && github.repository == 'otterize/helm-charts' && startsWith(github.ref, 'refs/tags/'))
     timeout-minutes: 5
     runs-on: ubuntu-latest
     concurrency:
       group: azure-e2e-tests # do not allow concurrent runs of this job
       cancel-in-progress: false
     steps:
+      - name: Fail on custom registry
+        if: "${{ inputs.gcr-registry != '' }}"
+        run: |
+          echo "This job does not support custom docker registry"
+          exit 1
+
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          # explicitly checkout helm-charts repository since this is a reusable workflow that's called from other repositories
+          repository: 'otterize/helm-charts'
+          ref: ${{ inputs.github_ref }}
 
       - name: Log in with Azure
         uses: azure/login@v2
@@ -128,4 +226,13 @@ jobs:
       - name: Run E2E tests - azure integrations
         run: |
           cd tests
-          go test -v -json ./azureiam/... | tee gotest.log | gotestfmt
+          go test -v -json ./azureiam/... | tee gotest.log | gotestfmt
+
+  e2e-test:
+    needs:
+      - test-chart-deployment
+      - test-database-integrations
+    runs-on: ubuntu-latest
+    steps:
+      - run: |-
+          echo Success! This step is only here to depend on the tests.
diff --git a/intents-operator/templates/intents-operator-manager-clusterrole.yaml b/intents-operator/templates/intents-operator-manager-clusterrole.yaml
@@ -113,6 +113,12 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
 - apiGroups:
   - iam.cnrm.cloud.google.com
   resources:
@@ -269,4 +275,4 @@ rules:
     - get
     - list
     - watch
-{{ end }}
+{{ end }}
diff --git a/tests/azureiam/azureiam_test.go b/tests/azureiam/azureiam_test.go
@@ -129,22 +129,16 @@ func (s *AzureIAMTestSuite) initAzureAgent() {
 }
 
 func (s *AzureIAMTestSuite) installOtterizeForAzureIAM() {
-	values := map[string]any{
-		"global": map[string]any{
-			"azure": map[string]any{
-				"enabled":                true,
-				"subscriptionID":         s.conf.SubscriptionID,
-				"resourceGroup":          s.conf.ResourceGroup,
-				"aksClusterName":         s.conf.AKSClusterName,
-				"userAssignedIdentityID": s.conf.OtterizeOperatorUserAssignedIdentityClientID,
-			},
-			"deployment": map[string]any{
-				"networkMapper": false,
-			},
-			"telemetry": map[string]interface{}{
-				"enabled": false,
-			},
-		},
+	values := s.GetDefaultHelmChartValues()
+	if _, ok := values["global"]; !ok {
+		values["global"] = map[string]any{}
+	}
+	values["global"].(map[string]any)["azure"] = map[string]any{
+		"enabled":                true,
+		"subscriptionID":         s.conf.SubscriptionID,
+		"resourceGroup":          s.conf.ResourceGroup,
+		"aksClusterName":         s.conf.AKSClusterName,
+		"userAssignedIdentityID": s.conf.OtterizeOperatorUserAssignedIdentityClientID,
 	}
 
 	s.InstallOtterizeHelmChart(values)

diff --git a/tests/base_suite.go b/tests/base_suite.go
@@ -89,6 +89,59 @@ func (s *BaseSuite) SetupSuite() {
 	})
 }
 
+func (s *BaseSuite) GetDefaultHelmChartValues() map[string]any {
+	defaultValues := map[string]any{
+		"global": map[string]any{
+			"deployment": map[string]any{
+				"networkMapper": false,
+			},
+			"telemetry": map[string]any{
+				"enabled": false,
+			},
+		},
+		"intentsOperator": map[string]any{
+			"debug": true,
+		},
+		"credentialsOperator": map[string]any{
+			"debug": true,
+		},
+	}
+
+	intentsOperatorRepository := os.Getenv("INTENTS_OPERATOR_REPOSITORY")
+	intentsOperatorImage := os.Getenv("INTENTS_OPERATOR_IMAGE")
+	intentsOperatorTag := os.Getenv("INTENTS_OPERATOR_TAG")
+
+	if intentsOperatorTag != "" {
+		if _, ok := defaultValues["intentsOperator"]; !ok {
+			defaultValues["intentsOperator"] = map[string]any{}
+		}
+		defaultValues["intentsOperator"].(map[string]any)["operator"] = map[string]any{
+			"tag":        intentsOperatorTag,
+			"image":      intentsOperatorImage,
+			"repository": intentsOperatorRepository,
+			"pullPolicy": "Never",
+		}
+	}
+
+	credentialsOperatorRepository := os.Getenv("CREDENTIALS_OPERATOR_REPOSITORY")
+	credentialsOperatorImage := os.Getenv("CREDENTIALS_OPERATOR_IMAGE")
+	credentialsOperatorTag := os.Getenv("CREDENTIALS_OPERATOR_TAG")
+
+	if credentialsOperatorTag != "" {
+		if _, ok := defaultValues["credentialsOperator"]; !ok {
+			defaultValues["credentialsOperator"] = map[string]any{}
+		}
+		defaultValues["credentialsOperator"].(map[string]any)["operator"] = map[string]any{
+			"tag":        credentialsOperatorTag,
+			"image":      credentialsOperatorImage,
+			"repository": credentialsOperatorRepository,
+			"pullPolicy": "Never",
+		}
+	}
+
+	return defaultValues
+}
+
 func (s *BaseSuite) InstallOtterizeHelmChart(values map[string]any) {
 	// Load Chart.yaml
 	chart, err := loader.Load(OtterizeKubernetesChartPath)

diff --git a/tests/databases/postgres/postgres_test.go b/tests/databases/postgres/postgres_test.go
@@ -39,7 +39,7 @@ type PostgresTestSuite struct {
 func (s *PostgresTestSuite) SetupSuite() {
 	s.BaseSuite.SetupSuite()
 
-	s.installOtterizeNetworkMapperDisabled()
+	s.InstallOtterizeHelmChart(s.GetDefaultHelmChartValues())
 
 	s.PGServerConfClient = s.DynamicClient.Resource(schema.GroupVersionResource{
 		Group:    "k8s.otterize.com",
@@ -54,20 +54,6 @@ func (s *PostgresTestSuite) TearDownSuite() {
 	s.UninstallOtterizeHelmChart(ctx)
 }
 
-func (s *PostgresTestSuite) installOtterizeNetworkMapperDisabled() {
-	values := map[string]interface{}{
-		"global": map[string]interface{}{
-			"deployment": map[string]interface{}{
-				"networkMapper": false,
-			},
-			"telemetry": map[string]interface{}{
-				"enabled": false,
-			},
-		},
-	}
-	s.InstallOtterizeHelmChart(values)
-}
-
 func (s *PostgresTestSuite) SetupTest() {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Minute))
 	defer cancel()
@@ -87,8 +73,6 @@ func (s *PostgresTestSuite) SetupTest() {
 
 	// Get client pod name
 	s.clientPod = s.FindPodByLabel(ctx, s.testNamespaceName, "app=psql-client")
-
-	s.applyPGServerConf(ctx)
 }
 
 func (s *PostgresTestSuite) TearDownTest() {
@@ -274,7 +258,7 @@ func (s *PostgresTestSuite) deployDatabaseClient(ctx context.Context) {
 	s.WaitForDeploymentAvailability(ctx, clientDeployment.Namespace, clientDeployment.Name)
 }
 
-func (s *PostgresTestSuite) applyPGServerConf(ctx context.Context) {
+func (s *PostgresTestSuite) applyPGServerConfWithInlinePassword(ctx context.Context) {
 	pgServerConf := v1alpha3.PostgreSQLServerConfig{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "PostgreSQLServerConfig",
@@ -329,6 +313,8 @@ func (s *PostgresTestSuite) TestWorkloadFailsToAccessDatabase() {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
 	defer cancel()
 
+	s.applyPGServerConfWithInlinePassword(ctx)
+
 	logrus.Info("Validating client pod fails to access the database")
 	s.ReadPodLogsUntilSubstring(ctx, s.clientPod, "password authentication failed")
 }
@@ -337,6 +323,8 @@ func (s *PostgresTestSuite) TestAddSelectAndInsertPermissionsForDB() {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
 	defer cancel()
 
+	s.applyPGServerConfWithInlinePassword(ctx)
+
 	s.ReadPodLogsUntilSubstring(ctx, s.clientPod, "password authentication failed")
 
 	s.applyIntents(ctx, []v1alpha3.DatabaseOperation{v1alpha3.DatabaseOperationInsert, v1alpha3.DatabaseOperationSelect})
@@ -350,6 +338,8 @@ func (s *PostgresTestSuite) TestInsertPermissionWithoutSelect() {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
 	defer cancel()
 
+	s.applyPGServerConfWithInlinePassword(ctx)
+
 	s.ReadPodLogsUntilSubstring(ctx, s.clientPod, "password authentication failed")
 
 	s.applyIntents(ctx, []v1alpha3.DatabaseOperation{v1alpha3.DatabaseOperationInsert})
@@ -363,6 +353,8 @@ func (s *PostgresTestSuite) TestSelectPermissionWithoutInsert() {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
 	defer cancel()
 
+	s.applyPGServerConfWithInlinePassword(ctx)
+
 	s.ReadPodLogsUntilSubstring(ctx, s.clientPod, "password authentication failed")
 
 	s.applyIntents(ctx, []v1alpha3.DatabaseOperation{v1alpha3.DatabaseOperationSelect})