From 3cb6d41d8f7397f11e861ec094fe962726a30c67 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Fri, 17 Nov 2023 08:25:15 -0600 Subject: [PATCH 1/4] chore: validate that all required GitHub Secrets are set --- .github/actions/tests/python/action.yml | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml index 60768c97..0cf0cc25 100644 --- a/.github/actions/tests/python/action.yml +++ b/.github/actions/tests/python/action.yml @@ -20,6 +20,34 @@ runs: id: checkout uses: actions/checkout@v4 + - name: Verify OPENAI_API_ORGANIZATION + run: | + if [[ -z "${{ secrets.OPENAI_API_ORGANIZATION }}" ]]; then + echo "OPENAI_API_ORGANIZATION is not set" >&2 + exit 1 + fi + + - name: Verify OPENAI_API_KEY + run: | + if [[ -z "${{ secrets.OPENAI_API_KEY }}" ]]; then + echo "OPENAI_API_KEY is not set" >&2 + exit 1 + fi + + - name: Verify PINECONE_API_KEY + run: | + if [[ -z "${{ secrets.PINECONE_API_KEY }}" ]]; then + echo "PINECONE_API_KEY is not set" >&2 + exit 1 + fi + + - name: Verify PINECONE_ENVIRONMENT + run: | + if [[ -z "${{ secrets.PINECONE_ENVIRONMENT }}" ]]; then + echo "PINECONE_ENVIRONMENT is not set" >&2 + exit 1 + fi + - name: Set up Python if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' uses: actions/setup-python@v4 From 56feb8c92c44a20a63cb826cc940b6d4771dbf30 Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Fri, 17 Nov 2023 09:14:53 -0600 Subject: [PATCH 2/4] chore: remove file change checks. add shell type to steps --- .github/actions/tests/python/action.yml | 13 ++++++++----- .github/actions/tests/reactjs/action.yml | 7 +------ .github/actions/tests/terraform/action.yml | 7 +++---- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml index 0cf0cc25..85a073c6 100644 --- a/.github/actions/tests/python/action.yml +++ b/.github/actions/tests/python/action.yml @@ -21,6 +21,7 @@ runs: uses: actions/checkout@v4 - name: Verify OPENAI_API_ORGANIZATION + shell: bash run: | if [[ -z "${{ secrets.OPENAI_API_ORGANIZATION }}" ]]; then echo "OPENAI_API_ORGANIZATION is not set" >&2 @@ -28,6 +29,7 @@ runs: fi - name: Verify OPENAI_API_KEY + shell: bash run: | if [[ -z "${{ secrets.OPENAI_API_KEY }}" ]]; then echo "OPENAI_API_KEY is not set" >&2 @@ -35,6 +37,7 @@ runs: fi - name: Verify PINECONE_API_KEY + shell: bash run: | if [[ -z "${{ secrets.PINECONE_API_KEY }}" ]]; then echo "PINECONE_API_KEY is not set" >&2 @@ -42,6 +45,7 @@ runs: fi - name: Verify PINECONE_ENVIRONMENT + shell: bash run: | if [[ -z "${{ secrets.PINECONE_ENVIRONMENT }}" ]]; then echo "PINECONE_ENVIRONMENT is not set" >&2 @@ -49,13 +53,12 @@ runs: fi - name: Set up Python - if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' uses: actions/setup-python@v4 with: python-version: ${{ inputs.python-version }} }}} - name: Install dependencies - if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + shell: bash run: | python -m pip install --upgrade pip pip install -r ./requirements.txt @@ -63,7 +66,7 @@ runs: cp -R ./api/terraform/python/layer_genai/openai_utils venv/lib/python${{ inputs.python-version }}/site-packages/ - name: Create .env - if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + shell: bash run: | touch ./.env echo "OPENAI_API_ORGANIZATION=${{ secrets.OPENAI_API_ORGANIZATION }}" >> ./.env @@ -72,13 +75,13 @@ runs: echo "PINECONE_ENVIRONMENT=${{ secrets.PINECONE_ENVIRONMENT }}" >> ./.env - name: Test lambda_openai_v2 - if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + shell: bash run: | cd ./api/terraform/python/lambda_openai_v2 pytest -v -s tests/ - name: Test lambda_langchain - if: steps.file_changes.outputs.files_changed != '' || steps.file_changes.outputs.requirements_changed != '' + shell: bash run: | cd ./api/terraform/python/lambda_langchain pytest -v -s tests/ diff --git a/.github/actions/tests/reactjs/action.yml b/.github/actions/tests/reactjs/action.yml index a715b062..ce2bbffb 100644 --- a/.github/actions/tests/reactjs/action.yml +++ b/.github/actions/tests/reactjs/action.yml @@ -13,13 +13,8 @@ runs: id: checkout uses: actions/checkout@v4 - - name: Check for ReactJS files changed - id: file_changes - run: | - echo "::set-output name=reactjs_files_changed::$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep '^client/' || true)" - - name: Run ReactJS Tests - if: steps.file_changes.outputs.reactjs_files_changed != '' id: reactjs_tests + shell: bash run: | echo "Test scaffolding for ReactJS" diff --git a/.github/actions/tests/terraform/action.yml b/.github/actions/tests/terraform/action.yml index 69d7d2d9..7a03189b 100644 --- a/.github/actions/tests/terraform/action.yml +++ b/.github/actions/tests/terraform/action.yml @@ -14,7 +14,6 @@ runs: uses: actions/checkout@v4 - name: Configure AWS credentials - if: steps.file_changes.outputs.terraform_files_changed != '' uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -22,19 +21,19 @@ runs: aws-region: ${{ secrets.AWS_REGION }} - name: Terraform Init - if: steps.file_changes.outputs.terraform_files_changed != '' + shell: bash run: | cd api/terraform terraform init - name: Terraform Validate - if: steps.file_changes.outputs.terraform_files_changed != '' + shell: bash run: | cd api/terraform terraform validate - name: Terraform Format - if: steps.file_changes.outputs.terraform_files_changed != '' + shell: bash run: | cd api/terraform terraform fmt -check From aa65b3f54a9a70d21979f9892c8c58d71637749b Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Fri, 17 Nov 2023 09:16:48 -0600 Subject: [PATCH 3/4] chore: validate that AWS key-pair secrets are set --- .github/actions/tests/terraform/action.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/actions/tests/terraform/action.yml b/.github/actions/tests/terraform/action.yml index 7a03189b..1b02a99b 100644 --- a/.github/actions/tests/terraform/action.yml +++ b/.github/actions/tests/terraform/action.yml @@ -13,6 +13,30 @@ runs: id: checkout uses: actions/checkout@v4 + - name: Verify AWS_ACCESS_KEY_ID + shell: bash + run: | + if [[ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ]]; then + echo "AWS_ACCESS_KEY_ID is not set" >&2 + exit 1 + fi + + - name: Verify AWS_SECRET_ACCESS_KEY + shell: bash + run: | + if [[ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then + echo "AWS_SECRET_ACCESS_KEY is not set" >&2 + exit 1 + fi + + - name: Verify AWS_REGION + shell: bash + run: | + if [[ -z "${{ secrets.AWS_REGION }}" ]]; then + echo "AWS_REGION is not set" >&2 + exit 1 + fi + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: From 98e74fadcd23d875731bd0f844aa1095e57fe00d Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Fri, 17 Nov 2023 09:52:05 -0600 Subject: [PATCH 4/4] chore: add more validations --- .github/actions/tests/pre-commit/action.yml | 34 +++++++++++++++++++++ .github/actions/tests/python/action.yml | 28 +++++++++++++++++ .github/actions/tests/terraform/action.yml | 8 +++++ 3 files changed, 70 insertions(+) diff --git a/.github/actions/tests/pre-commit/action.yml b/.github/actions/tests/pre-commit/action.yml index c15328f9..beef5482 100644 --- a/.github/actions/tests/pre-commit/action.yml +++ b/.github/actions/tests/pre-commit/action.yml @@ -18,11 +18,45 @@ runs: id: checkout uses: actions/checkout@v4 + - name: Check for pre-commit in requirements + run: | + if ! grep -q "pre-commit" ./requirements.txt; then + echo "pre-commit not found in requirements.txt" >&2 + exit 1 + fi + + - name: Check for black in requirements + run: | + if ! grep -q "black" ./requirements.txt; then + echo "black not found in requirements.txt" >&2 + exit 1 + fi + + - name: Check for flake8 in requirements + run: | + if ! grep -q "flake8" ./requirements.txt; then + echo "flake8 not found in requirements.txt" >&2 + exit 1 + fi + + - name: Check for flake8-coding in requirements + run: | + if ! grep -q "flake8-coding" ./requirements.txt; then + echo "flake8-coding not found in requirements.txt" >&2 + exit 1 + fi + - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ inputs.python-version }} + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install -r ./requirements.txt + # see: https://pre-commit.ci/lite.html - name: pre-commit ci id: pre-commit-ci diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml index 85a073c6..b4500e03 100644 --- a/.github/actions/tests/python/action.yml +++ b/.github/actions/tests/python/action.yml @@ -52,6 +52,34 @@ runs: exit 1 fi + - name: Check for openai in requirements + run: | + if ! grep -q "openai" ./requirements.txt; then + echo "openai not found in requirements.txt" >&2 + exit 1 + fi + + - name: Check for langchain in requirements + run: | + if ! grep -q "langchain" ./requirements.txt; then + echo "langchain not found in requirements.txt" >&2 + exit 1 + fi + + - name: Check for langchain-experimental in requirements + run: | + if ! grep -q "langchain-experimental" ./requirements.txt; then + echo "langchain-experimental not found in requirements.txt" >&2 + exit 1 + fi + + - name: Check for pinecone-client in requirements + run: | + if ! grep -q "pinecone-client" ./requirements.txt; then + echo "pinecone-client not found in requirements.txt" >&2 + exit 1 + fi + - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/actions/tests/terraform/action.yml b/.github/actions/tests/terraform/action.yml index 1b02a99b..19ea6314 100644 --- a/.github/actions/tests/terraform/action.yml +++ b/.github/actions/tests/terraform/action.yml @@ -37,6 +37,14 @@ runs: exit 1 fi + - name: Verify Terraform is installed + run: | + if ! command -v terraform &> /dev/null + then + echo "Terraform could not be found" + exit 1 + fi + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: