From 4fe15d330f28a69cd7bcfa496d6fe8825714764b Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Fri, 17 Nov 2023 08:16:21 -0600 Subject: [PATCH] refactor: move all test logic to standalone actions --- .github/actions/tests/pre-commit/action.yml | 30 ++++++ .github/actions/tests/python/action.yml | 56 +++++++++++ .github/actions/tests/reactjs/action.yml | 25 +++++ .github/actions/tests/terraform/action.yml | 40 ++++++++ .github/workflows/pullRequestController.yml | 103 +++++--------------- 5 files changed, 174 insertions(+), 80 deletions(-) create mode 100644 .github/actions/tests/pre-commit/action.yml create mode 100644 .github/actions/tests/python/action.yml create mode 100644 .github/actions/tests/reactjs/action.yml create mode 100644 .github/actions/tests/terraform/action.yml diff --git a/.github/actions/tests/pre-commit/action.yml b/.github/actions/tests/pre-commit/action.yml new file mode 100644 index 00000000..c15328f9 --- /dev/null +++ b/.github/actions/tests/pre-commit/action.yml @@ -0,0 +1,30 @@ +#------------------------------------------------------------------------------ +# Run pre-commit +#------------------------------------------------------------------------------ +name: Test pre-commit +branding: + icon: 'git-pull-request' + color: 'orange' +inputs: + python-version: + description: 'The version of Python to use, such as 3.11.0' + required: true + type: string + +runs: + using: "composite" + steps: + - name: Checkout code + id: checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + # see: https://pre-commit.ci/lite.html + - name: pre-commit ci + id: pre-commit-ci + if: always() + uses: pre-commit-ci/lite-action@v1.0.1 diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml new file mode 100644 index 00000000..60768c97 --- /dev/null +++ b/.github/actions/tests/python/action.yml @@ -0,0 +1,56 @@ +#------------------------------------------------------------------------------ +# Run Python unit tests +#------------------------------------------------------------------------------ +name: Test Python +branding: + icon: 'git-pull-request' + color: 'orange' +inputs: + python-version: + description: 'The version of Python to use, such as 3.11.0' + required: true + type: string +env: + REQUIREMENTS_PATH: 'api/terraform/python/layer_genai/requirements.txt' + +runs: + using: "composite" + steps: + - name: Checkout code + id: checkout + uses: actions/checkout@v4 + + - name: Set up Python + if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} }}} + + - name: Install dependencies + if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + run: | + python -m pip install --upgrade pip + pip install -r ./requirements.txt + pip install -r ${{ env.REQUIREMENTS_PATH }} + cp -R ./api/terraform/python/layer_genai/openai_utils venv/lib/python${{ inputs.python-version }}/site-packages/ + + - name: Create .env + if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + run: | + touch ./.env + echo "OPENAI_API_ORGANIZATION=${{ secrets.OPENAI_API_ORGANIZATION }}" >> ./.env + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ./.env + echo "PINECONE_API_KEY=${{ secrets.PINECONE_API_KEY }}" >> ./.env + echo "PINECONE_ENVIRONMENT=${{ secrets.PINECONE_ENVIRONMENT }}" >> ./.env + + - name: Test lambda_openai_v2 + if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + run: | + cd ./api/terraform/python/lambda_openai_v2 + pytest -v -s tests/ + + - name: Test lambda_langchain + if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + run: | + cd ./api/terraform/python/lambda_langchain + pytest -v -s tests/ diff --git a/.github/actions/tests/reactjs/action.yml b/.github/actions/tests/reactjs/action.yml new file mode 100644 index 00000000..a715b062 --- /dev/null +++ b/.github/actions/tests/reactjs/action.yml @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------------ +# Run ReactJS unit tests +#------------------------------------------------------------------------------ +name: Test ReactJS +branding: + icon: 'git-pull-request' + color: 'orange' + +runs: + using: "composite" + steps: + - name: Checkout code + id: checkout + uses: actions/checkout@v4 + + - name: Check for ReactJS files changed + id: file_changes + run: | + echo "::set-output name=reactjs_files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^client/' || true)" + + - name: Run ReactJS Tests + if: steps.file_changes.outputs.reactjs_files_changed != '' + id: reactjs_tests + run: | + echo "Test scaffolding for ReactJS" diff --git a/.github/actions/tests/terraform/action.yml b/.github/actions/tests/terraform/action.yml new file mode 100644 index 00000000..69d7d2d9 --- /dev/null +++ b/.github/actions/tests/terraform/action.yml @@ -0,0 +1,40 @@ +#------------------------------------------------------------------------------ +# Run Terraform tests +#------------------------------------------------------------------------------ +name: Test Terraform +branding: + icon: 'git-pull-request' + color: 'orange' + +runs: + using: "composite" + steps: + - name: Checkout code + id: checkout + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: steps.file_changes.outputs.terraform_files_changed != '' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Terraform Init + if: steps.file_changes.outputs.terraform_files_changed != '' + run: | + cd api/terraform + terraform init + + - name: Terraform Validate + if: steps.file_changes.outputs.terraform_files_changed != '' + run: | + cd api/terraform + terraform validate + + - name: Terraform Format + if: steps.file_changes.outputs.terraform_files_changed != '' + run: | + cd api/terraform + terraform fmt -check diff --git a/.github/workflows/pullRequestController.yml b/.github/workflows/pullRequestController.yml index ac29467d..0d560cc5 100644 --- a/.github/workflows/pullRequestController.yml +++ b/.github/workflows/pullRequestController.yml @@ -58,6 +58,9 @@ permissions: pages: write id-token: write +env: + python-version: "3.11" + jobs: check_for_pending_release: name: test-semantic-release @@ -103,53 +106,23 @@ jobs: python_tests: needs: check_for_pending_release runs-on: ubuntu-latest - env: - REQUIREMENTS_PATH: 'api/terraform/python/layer_genai/requirements.txt' steps: - name: Checkout code id: checkout uses: actions/checkout@v4 - - name: Check for Python files changed + - name: Check for changed files id: file_changes run: | - echo "::set-output name=py_files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '\.py$' || true)" - echo "::set-output name=requirements_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^${{ env.REQUIREMENTS_PATH }}$' || true)" + echo "::set-output name=files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '\.py$' || true)" + echo "::set-output name=requirements_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep 'requirements.txt$' || true)" - - name: Set up Python - if: steps.file_changes.outputs.py_files_changed != '' || steps.file_changes.outputs.requirements_changed != '' - uses: actions/setup-python@v4 + - name: Run Python tests + if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + uses: ./.github/actions/tests/python with: - python-version: 3.11 - - - name: Install dependencies - if: steps.file_changes.outputs.py_files_changed != '' || steps.file_changes.outputs.requirements_changed != '' - run: | - python -m pip install --upgrade pip - pip install -r ./requirements.txt - cp -R ./api/terraform/python/layer_genai/openai_utils /opt/hostedtoolcache/Python/3.11.6/x64/lib/python3.11/site-packages/ - - - name: Create .env - if: steps.file_changes.outputs.py_files_changed != '' || steps.file_changes.outputs.requirements_changed != '' - run: | - touch ./.env - echo "OPENAI_API_ORGANIZATION=${OPENAI_API_ORGANIZATION}" >> ./.env - echo "OPENAI_API_KEY=${OPENAI_API_KEY}" >> ./.env - echo "PINECONE_API_KEY=${PINECONE_API_KEY}" >> ./.env - echo "PINECONE_ENVIRONMENT=${PINECONE_ENVIRONMENT}" >> ./.env - - - name: Test lambda_openai_v2 - if: steps.file_changes.outputs.py_files_changed != '' || steps.file_changes.outputs.requirements_changed != '' - run: | - cd ./api/terraform/python/lambda_openai_v2 - pytest -v -s tests/ - - - name: Test lambda_langchain - if: steps.file_changes.outputs.py_files_changed != '' || steps.file_changes.outputs.requirements_changed != '' - run: | - cd ./api/terraform/python/lambda_langchain - pytest -v -s tests/ + python-version: "${{ env.python-version}}" terraform_tests: needs: check_for_pending_release @@ -159,36 +132,14 @@ jobs: id: checkout uses: actions/checkout@v4 - - name: Check for Terraform files changed + - name: Check for changed files id: file_changes run: | - echo "::set-output name=terraform_files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '\.tf$' || true)" + echo "::set-output name=files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '\.tf$' || true)" - - name: Configure AWS credentials - if: steps.file_changes.outputs.terraform_files_changed != '' - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Terraform Init - if: steps.file_changes.outputs.terraform_files_changed != '' - run: | - cd api/terraform - terraform init - - - name: Terraform Validate - if: steps.file_changes.outputs.terraform_files_changed != '' - run: | - cd api/terraform - terraform validate - - - name: Terraform Format - if: steps.file_changes.outputs.terraform_files_changed != '' - run: | - cd api/terraform - terraform fmt -check + - name: Run Terraform tests + if: steps.file_changes.outputs.files_changed != '' + uses: ./.github/actions/tests/reactjs reactjs_tests: needs: check_for_pending_release @@ -198,16 +149,14 @@ jobs: id: checkout uses: actions/checkout@v4 - - name: Check for ReactJS files changed + - name: Check for changed files id: file_changes run: | - echo "::set-output name=reactjs_files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^client/' || true)" + echo "::set-output name=files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} -- './client/' || true)" - - name: Run ReactJS Tests - if: steps.file_changes.outputs.reactjs_files_changed != '' - id: reactjs_tests - run: | - echo "Test scaffolding for ReactJS" + - name: Run ReactJS tests + if: steps.file_changes.outputs.files_changed != '' + uses: ./.github/actions/tests/reactjs pre_commit_tests: needs: check_for_pending_release @@ -217,13 +166,7 @@ jobs: id: checkout uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 + - name: Run pre-commit tests + uses: ./.github/actions/tests/pre-commit with: - python-version: 3.11 - - # see: https://pre-commit.ci/lite.html - - name: pre-commit ci - id: pre-commit-ci - if: always() - uses: pre-commit-ci/lite-action@v1.0.1 + python-version: "${{ env.python-version}}"