Integrate with localstack for testing AWS deployments (#145)

* Setup integration testing directories for AWS deployments * Add local.tfvars for modular and remote-state integration testing * Add verify_stack.sh scripts for AWS modular and remote state integration tests * Add __init__.py in tests/integration * Create GitHub Actions workflow for AWS integration testing * Add docker-compose.localstack.yml for LocalStack service emulation * Run format and lint scripts * Add README.md for integration tests documentation * Address typo in README * Add removal of localstack_providers_override.tf in GitHub Actions cleanup step * Revise README for enhanced Docker setup instructions and overall documentation clarity * Modified CI workflow to fix path issues, emulate tflocal, and ensure proper cleanup * Add _override.tf for LocalStack integration in tests * Improve verify_stack.sh scripts with debugging outputs * Integrate localstack-aws-integratin-test job into main CI workflow * Make aws-remote-state/verify_stack.sh executable * Add workflow dispatch trigger to workflow file * Fix path in cleanup step to correctly delete _override.tf for aws-remote-state * Update README to reflect workflow and testing approach modifications * Remove debug line displaying absolute YAML path from workflow and README * Update .github/workflows/aws-integration-test.yml Co-authored-by: Alex Strick van Linschoten <[email protected]> * Update tests/integration/aws-modular/local.tfvars Co-authored-by: Alex Strick van Linschoten <[email protected]> * Update tests/integration/aws-remote-state/local.tfvars Co-authored-by: Alex Strick van Linschoten <[email protected]> * Update tests/integration/README.md Co-authored-by: Alex Strick van Linschoten <[email protected]> * Remove cleanup steps from AWS Modular workflow Following @strickvl's suggestion, removed the terraform destroy and related cleanup commands. These steps are unnecessary for our ephemeral instances and do not impact the repo's state. Co-authored-by: Alex Strick van Linschoten <[email protected]> * Update README to reflect changes to terraform version and removal of cleanup steps --------- Co-authored-by: Alex Strick van Linschoten <[email protected]>
zenml-io · Feb 19, 2024 · 4cc2502 · 4cc2502
1 parent baccb96
commit 4cc2502
Show file tree

Hide file tree

Showing 9 changed files with 577 additions and 0 deletions.
diff --git a/.github/workflows/aws-integration-test.yml b/.github/workflows/aws-integration-test.yml
@@ -0,0 +1,103 @@
+permissions:
+  contents: read
+name: LocalStack AWS Integration Test
+on:
+  workflow_call:
+  workflow_dispatch:
+jobs:
+  aws_modular_integration_test:
+    name: aws_modular_integration_test
+    runs-on: ubuntu-latest
+    services:
+      setup-localstack-service:
+        image: localstack/localstack
+        ports:
+          - '4566:4566'
+        env:
+          SERVICES: 's3,iam,sts'
+          DEFAULT_REGION: eu-north-1
+          FORCE_NONINTERACTIVE: 1
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+    steps:
+      - name: Checkout Repository
+        uses: actions/[email protected]
+
+      - name: Setup Terraform
+        uses: hashicorp/[email protected]
+        with:
+          terraform_version: 1.6.0
+
+      - name: Copy Override File
+        run: |
+          cp tests/integration/_override.tf src/mlstacks/terraform/aws-modular/_override.tf
+
+      - name: Apply Terraform Configuration
+        run: |
+          export TF_CLI_ARGS_apply="-compact-warnings"
+          terraform init -backend-config="path=./terraform.tfstate"
+          terraform validate
+          terraform apply -auto-approve -var-file="../../../../tests/integration/aws-modular/local.tfvars"
+        working-directory: src/mlstacks/terraform/aws-modular
+
+      - name: Refresh Terraform State
+        run: terraform refresh
+        working-directory: src/mlstacks/terraform/aws-modular
+
+      - name: Output Stack YAML Path
+        id: set_output
+        run: |
+          OUTPUT=$(terraform-bin output -raw stack-yaml-path)
+          echo "stack_yaml_path=$OUTPUT" >> $GITHUB_OUTPUT
+        working-directory: src/mlstacks/terraform/aws-modular
+        env:
+          terraform_wrapper: false
+
+      - name: Run Tests to Verify Resource Provisioning
+        run: |
+          STACK_YAML_PATH="${{ steps.set_output.outputs.stack_yaml_path }}"
+          ABSOLUTE_PATH="${GITHUB_WORKSPACE}/src/mlstacks/terraform/aws-modular/${STACK_YAML_PATH}"
+          ../../../../tests/integration/aws-modular/verify_stack.sh "$ABSOLUTE_PATH"
+        working-directory: src/mlstacks/terraform/aws-modular
+
+  aws_remote_state_integration_test:
+    name: aws_remote_state_integration_test
+    runs-on: ubuntu-latest
+    services:
+      setup-localstack-service:
+        image: localstack/localstack
+        ports:
+          - '4566:4566'
+        env:
+          SERVICES: 's3,dynamodb,iam,sts'
+          DEFAULT_REGION: eu-north-1
+          FORCE_NONINTERACTIVE: 1
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+    steps:
+      - name: Checkout Repository
+        uses: actions/[email protected]
+
+      - name: Setup Terraform
+        uses: hashicorp/[email protected]
+        with:
+          terraform_version: 1.6.0
+
+      - name: Copy Override File
+        run: |
+          cp tests/integration/_override.tf src/mlstacks/terraform/aws-remote-state/_override.tf
+
+      - name: Apply Terraform Configuration for aws-remote-state
+        run: |
+          export TF_CLI_ARGS_apply="-compact-warnings"
+          cd src/mlstacks/terraform/aws-remote-state
+          terraform init -backend-config="path=./terraform.tfstate"
+          terraform validate
+          terraform apply -auto-approve -var-file="../../../../tests/integration/aws-remote-state/local.tfvars"
+
+      - name: Run Tests to Verify Resource Provisioning
+        run: ./tests/integration/aws-remote-state/verify_stack.sh
+        env:
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+          AWS_DEFAULT_REGION: eu-north-1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -111,6 +111,10 @@ jobs:
           files: "."
           config: ./.typos.toml
 
+  localstack-aws-integration-test:
+    uses: ./.github/workflows/aws-integration-test.yml
+    secrets: inherit
+
   aws_test:
     name: aws_test
     runs-on: ubuntu-latest