Merge pull request #265 from aws-samples/integration_test

Integration test

.github/workflows/integration.yml

@@ -1,13 +1,11 @@
 name: Integration Tests
+
 on:
+  # we use pull_request_target to run the CI also for forks
+  pull_request_target:
+    types: [opened, reopened, synchronize, labeled]
   push:
-    branches:
-      - "main"
-      - "*.latest"
-      - "releases/*"
-    paths-ignore:
-      - "**.MD"
-      - "**.md"
+    branches: [main]
 
 permissions:
   id-token: write
@@ -22,8 +20,10 @@ defaults:
     shell: bash
 
 jobs:
-  integ:
-    name: integration test / python ${{ matrix.python-version }}
+  # workflow that is invoked when for PRs with labels 'enable-functional-tests'
+  functional-tests-pr:
+    name: Functional Tests - PR / python ${{ matrix.python-version }}
+    if: contains(github.event.pull_request.labels.*.name, 'enable-functional-tests')
 
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -87,3 +87,71 @@ jobs:
         with:
           name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
           path: unit_results.csv
+
+  # workflow that is invoked when a push to main happens
+  functional-tests-main:
+    name: Functional Tests - main / python ${{ matrix.python-version }}
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"] # Use single version to avoid resource conflicts in an AWS account
+
+    env:
+      TOXENV: "integration"
+      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
+      DBT_AWS_ACCOUNT: ${{ secrets.DBT_AWS_ACCOUNT }}
+      DBT_GLUE_ROLE_ARN: ${{ secrets.DBT_GLUE_ROLE_ARN }}
+      DBT_GLUE_REGION: ${{ secrets.DBT_GLUE_REGION }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Generate session name
+        id: session
+        run: |
+          repo="${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}"
+          echo "name=${repo}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "${GITHUB_OUTPUT}"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-session-name: ${{ steps.session.outputs.name }}
+          role-to-assume: arn:aws:iam::${{ secrets.DBT_AWS_ACCOUNT }}:role/dbt-glue
+          aws-region: ${{ secrets.DBT_GLUE_REGION }}
+          mask-aws-account-id: true
+
+      - name: Run tox
+        run: |
+          export DBT_S3_LOCATION=${{ secrets.DBT_S3_LOCATION }}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${{ matrix.python-version }}
+          tox
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: unit_results.csv

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 skipsdist = True
-envlist = unit, flake8, integration-spark-thrift
+envlist = unit, integration
 
 [testenv:{unit}]
 allowlist_externals =
@@ -16,7 +16,7 @@ deps =
 [testenv:{integration}]
 allowlist_externals =
     /bin/bash
-commands = /bin/bash -c '{envpython} -m pytest -v {posargs}'
+commands = /bin/bash -c '{envpython} -m pytest -v {posargs} tests/functional/adapter/'
 passenv =
     DBT_*
     PYTEST_ADDOPTS