ci(s3): fetch connector creds from s3 for added security (#3323)

juspay · Jan 16, 2024 · eaa8791 · eaa8791
1 parent 8678f8d
commit eaa8791
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 26 deletions.
diff --git a/.github/secrets/connector_auth.toml.gpg b/.github/secrets/connector_auth.toml.gpg
diff --git a/.github/workflows/connector-ui-sanity-tests.yml b/.github/workflows/connector-ui-sanity-tests.yml
@@ -84,22 +84,31 @@ jobs:
         if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }}
         uses: actions/checkout@v4
 
-      - name: Decrypt connector auth file
+      - name: Download Encrypted TOML from S3 and Decrypt
         if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }}
         env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CONNECTOR_CREDS_AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.CONNECTOR_CREDS_AWS_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CONNECTOR_CREDS_AWS_SECRET_ACCESS_KEY }}
           CONNECTOR_AUTH_PASSPHRASE: ${{ secrets.CONNECTOR_AUTH_PASSPHRASE }}
+          CONNECTOR_CREDS_S3_BUCKET_URI: ${{ secrets.CONNECTOR_CREDS_S3_BUCKET_URI}}
+          DESTINATION_FILE_NAME: "connector_auth.toml.gpg"
+          S3_SOURCE_FILE_NAME: "cf05a6ab-525e-4888-98b3-3b4a443b87c0.toml.gpg"
         shell: bash
-        run: ./scripts/decrypt_connector_auth.sh
+        run: |
+          mkdir -p ${HOME}/target/secrets ${HOME}/target/test
+          aws s3 cp "${CONNECTOR_CREDS_S3_BUCKET_URI}/${S3_SOURCE_FILE_NAME}" "${HOME}/target/secrets/${DESTINATION_FILE_NAME}"
+          gpg --quiet --batch --yes --decrypt --passphrase="${CONNECTOR_AUTH_PASSPHRASE}" --output "${HOME}/target/test/connector_auth.toml" "${HOME}/target/secrets/${DESTINATION_FILE_NAME}"
 
       - name: Set connector auth file path in env
         if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }}
         shell: bash
-        run: echo "CONNECTOR_AUTH_FILE_PATH=$HOME/target/test/connector_auth.toml" >> $GITHUB_ENV
+        run: echo "CONNECTOR_AUTH_FILE_PATH=${HOME}/target/test/connector_auth.toml" >> $GITHUB_ENV
 
       - name: Set connector tests file path in env
         if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }}
         shell: bash
-        run: echo "CONNECTOR_TESTS_FILE_PATH=$HOME/target/test/connector_tests.json" >> $GITHUB_ENV
+        run: echo "CONNECTOR_TESTS_FILE_PATH=${HOME}/target/test/connector_tests.json" >> $GITHUB_ENV
 
       - name: Set ignore_browser_profile usage in env
         if: ${{ (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name) }}
@@ -154,9 +163,9 @@ jobs:
           failed_connectors=()
 
           for i in $(echo "$INPUT" | tr "," "\n"); do
-              echo $i
+              echo "${i}"
               if ! cargo test --package test_utils --test connectors -- "${i}_ui::" --test-threads=1; then
-                  failed_connectors+=("$i")
+                  failed_connectors+=("${i}")
               fi
           done
 

diff --git a/.github/workflows/postman-collection-runner.yml b/.github/workflows/postman-collection-runner.yml
@@ -52,27 +52,37 @@ jobs:
       - name: Repository checkout
         uses: actions/checkout@v4
 
-      - name: Decrypt connector auth file
+      - name: Download Encrypted TOML from S3 and Decrypt
         if: ${{ ((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)) || (github.event_name == 'merge_group')}}
         env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CONNECTOR_CREDS_AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.CONNECTOR_CREDS_AWS_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CONNECTOR_CREDS_AWS_SECRET_ACCESS_KEY }}
           CONNECTOR_AUTH_PASSPHRASE: ${{ secrets.CONNECTOR_AUTH_PASSPHRASE }}
+          CONNECTOR_CREDS_S3_BUCKET_URI: ${{ secrets.CONNECTOR_CREDS_S3_BUCKET_URI}}
+          DESTINATION_FILE_NAME: "connector_auth.toml.gpg"
+          S3_SOURCE_FILE_NAME: "cf05a6ab-525e-4888-98b3-3b4a443b87c0.toml.gpg"
         shell: bash
-        run: ./scripts/decrypt_connector_auth.sh
+        run: |
+          mkdir -p ${HOME}/target/secrets ${HOME}/target/test
+
+          aws s3 cp "${CONNECTOR_CREDS_S3_BUCKET_URI}/${S3_SOURCE_FILE_NAME}" "${HOME}/target/secrets/${DESTINATION_FILE_NAME}"
+          gpg --quiet --batch --yes --decrypt --passphrase="${CONNECTOR_AUTH_PASSPHRASE}" --output "${HOME}/target/test/connector_auth.toml" "${HOME}/target/secrets/${DESTINATION_FILE_NAME}"
 
       - name: Set paths in env
         if: ${{ ((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)) || (github.event_name == 'merge_group')}}
         id: config_path
         shell: bash
         run: |
-          echo "CONNECTOR_AUTH_FILE_PATH=$HOME/target/test/connector_auth.toml" >> $GITHUB_ENV
+          echo "CONNECTOR_AUTH_FILE_PATH=${HOME}/target/test/connector_auth.toml" >> $GITHUB_ENV
 
       - name: Fetch keys
         if: ${{ ((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)) || (github.event_name == 'merge_group')}}
         env:
           TOML_PATH: "./config/development.toml"
         run: |
-          LOCAL_ADMIN_API_KEY=$(yq '.secrets.admin_api_key' $TOML_PATH)
-          echo "ADMIN_API_KEY=$LOCAL_ADMIN_API_KEY" >> $GITHUB_ENV
+          LOCAL_ADMIN_API_KEY=$(yq '.secrets.admin_api_key' ${TOML_PATH})
+          echo "ADMIN_API_KEY=${LOCAL_ADMIN_API_KEY}" >> $GITHUB_ENV
 
       - name: Install Rust
         if: ${{ ((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)) || (github.event_name == 'merge_group')}}
@@ -118,7 +128,7 @@ jobs:
           while ! nc -z localhost 8080; do
             if [ $COUNT -gt 12 ]; then # Wait for up to 2 minutes (12 * 10 seconds)
               echo "Server did not start within a reasonable time. Exiting."
-              kill $SERVER_PID
+              kill ${SERVER_PID}
               exit 1
             else
               COUNT=$((COUNT+1))
@@ -141,10 +151,10 @@ jobs:
           export PATH=${NEWMAN_PATH}:${PATH}
           failed_connectors=()
 
-          for i in $(echo "$CONNECTORS" | tr "," "\n"); do
-            echo $i
-            if ! cargo run --bin test_utils -- --connector-name="$i" --base-url="$BASE_URL" --admin-api-key="$ADMIN_API_KEY"; then
-              failed_connectors+=("$i")
+          for i in $(echo "${CONNECTORS}" | tr "," "\n"); do
+            echo "${i}"
+            if ! cargo run --bin test_utils -- --connector-name="${i}" --base-url="${BASE_URL}" --admin-api-key="${ADMIN_API_KEY}"; then
+              failed_connectors+=("${i}")
             fi
           done
 

diff --git a/scripts/decrypt_connector_auth.sh b/scripts/decrypt_connector_auth.sh