Test Github Action unit tests

.github/actions/tests/pre-commit/action.yml

@@ -19,27 +19,31 @@ runs:
         uses: actions/checkout@v4
 
       - name: Check for pre-commit in requirements
+        shell: bash
         run: |
           if ! grep -q "pre-commit" ./requirements.txt; then
             echo "pre-commit not found in requirements.txt" >&2
             exit 1
           fi
 
       - name: Check for black in requirements
+        shell: bash
         run: |
           if ! grep -q "black" ./requirements.txt; then
             echo "black not found in requirements.txt" >&2
             exit 1
           fi
 
       - name: Check for flake8 in requirements
+        shell: bash
         run: |
           if ! grep -q "flake8" ./requirements.txt; then
             echo "flake8 not found in requirements.txt" >&2
             exit 1
           fi
 
       - name: Check for flake8-coding in requirements
+        shell: bash
         run: |
           if ! grep -q "flake8-coding" ./requirements.txt; then
             echo "flake8-coding not found in requirements.txt" >&2

.github/actions/tests/python/action.yml

@@ -10,6 +10,23 @@ inputs:
     description: 'The version of Python to use, such as 3.11.0'
     required: true
     type: string
+openai-api-organization:
+  description: 'The OpenAI API organization'
+  required: true
+  type: string
+openai-api-key:
+  description: 'The OpenAI API key'
+  required: true
+  type: string
+pinecone-api-key:
+  description: 'The Pinecone API key'
+  required: true
+  type: string
+pinecone-environment:
+  description: 'The Pinecone environment'
+  required: true
+  type: string
+
 env:
   REQUIREMENTS_PATH: 'api/terraform/python/layer_genai/requirements.txt'
 
@@ -20,60 +37,32 @@ runs:
         id: checkout
         uses: actions/checkout@v4
 
-      - name: Verify OPENAI_API_ORGANIZATION
-        shell: bash
-        run: |
-          if [[ -z "${{ secrets.OPENAI_API_ORGANIZATION }}" ]]; then
-            echo "OPENAI_API_ORGANIZATION is not set" >&2
-            exit 1
-          fi
-
-      - name: Verify OPENAI_API_KEY
-        shell: bash
-        run: |
-          if [[ -z "${{ secrets.OPENAI_API_KEY }}" ]]; then
-            echo "OPENAI_API_KEY is not set" >&2
-            exit 1
-          fi
-
-      - name: Verify PINECONE_API_KEY
-        shell: bash
-        run: |
-          if [[ -z "${{ secrets.PINECONE_API_KEY }}" ]]; then
-            echo "PINECONE_API_KEY is not set" >&2
-            exit 1
-          fi
-
-      - name: Verify PINECONE_ENVIRONMENT
-        shell: bash
-        run: |
-          if [[ -z "${{ secrets.PINECONE_ENVIRONMENT }}" ]]; then
-            echo "PINECONE_ENVIRONMENT is not set" >&2
-            exit 1
-          fi
-
       - name: Check for openai in requirements
+        shell: bash
         run: |
           if ! grep -q "openai" ./requirements.txt; then
             echo "openai not found in requirements.txt" >&2
             exit 1
           fi
 
       - name: Check for langchain in requirements
+        shell: bash
         run: |
           if ! grep -q "langchain" ./requirements.txt; then
             echo "langchain not found in requirements.txt" >&2
             exit 1
           fi
 
       - name: Check for langchain-experimental in requirements
+        shell: bash
         run: |
           if ! grep -q "langchain-experimental" ./requirements.txt; then
             echo "langchain-experimental not found in requirements.txt" >&2
             exit 1
           fi
 
       - name: Check for pinecone-client in requirements
+        shell: bash
         run: |
           if ! grep -q "pinecone-client" ./requirements.txt; then
             echo "pinecone-client not found in requirements.txt" >&2
@@ -83,24 +72,39 @@ runs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ inputs.python-version }} }}}
+          python-version: ${{ inputs.python-version }}
 
-      - name: Install dependencies
+      - name: locate site-packages path
+        shell: bash
+        run: |
+          echo "SITE_PACKAGES_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')" >> $GITHUB_ENV
+
+      - name: Install pip
         shell: bash
         run: |
           python -m pip install --upgrade pip
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          cp -R ./api/terraform/python/layer_genai/openai_utils ${{ env.SITE_PACKAGES_PATH }}
           pip install -r ./requirements.txt
-          pip install -r ${{ env.REQUIREMENTS_PATH }}
-          cp -R ./api/terraform/python/layer_genai/openai_utils venv/lib/python${{ inputs.python-version }}/site-packages/
+        env:
+          SITE_PACKAGES_PATH: ${{ env.SITE_PACKAGES_PATH }}
 
       - name: Create .env
         shell: bash
         run: |
           touch ./.env
-          echo "OPENAI_API_ORGANIZATION=${{ secrets.OPENAI_API_ORGANIZATION }}" >> ./.env
-          echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ./.env
-          echo "PINECONE_API_KEY=${{ secrets.PINECONE_API_KEY }}" >> ./.env
-          echo "PINECONE_ENVIRONMENT=${{ secrets.PINECONE_ENVIRONMENT }}" >> ./.env
+          echo "OPENAI_API_ORGANIZATION=${{ env.OPENAI_API_ORGANIZATION }}" >> ./.env
+          echo "OPENAI_API_KEY=${{ env.OPENAI_API_KEY }}" >> ./.env
+          echo "PINECONE_API_KEY=${{ env.PINECONE_API_KEY }}" >> ./.env
+          echo "PINECONE_ENVIRONMENT=${{ env.PINECONE_ENVIRONMENT }}" >> ./.env
+        env:
+          OPENAI_API_ORGANIZATION: ${{ inputs.openai-api-organization }}
+          OPENAI_API_KEY: ${{ inputs.openai-api-key }}
+          PINECONE_API_KEY: ${{ inputs.pinecone-api-key }}
+          PINECONE_ENVIRONMENT: ${{ inputs.pinecone-environment }}
 
       - name: Test lambda_openai_v2
         shell: bash

.github/workflows/pullRequestController.yml

@@ -118,6 +118,10 @@ jobs:
         uses: ./.github/actions/tests/python
         with:
           python-version: "${{ env.python-version}}"
+          openai-api-organization: "${{ secrets.OPENAI_API_ORGANIZATION }}"
+          openai-api-key: "${{ secrets.OPENAI_API_KEY }}"
+          pinecone-api-key: "${{ secrets.PINECONE_API_KEY }}"
+          pinecone-environment: "${{ secrets.PINECONE_ENVIRONMENT }}"
 
   terraform_tests:
     needs: check_for_pending_release

.github/workflows/runTests.yml

@@ -20,6 +20,10 @@ jobs:
         uses: ./.github/actions/tests/python
         with:
           python-version: "${{ env.python-version}}"
+          openai-api-organization: "${{ secrets.OPENAI_API_ORGANIZATION }}"
+          openai-api-key: "${{ secrets.OPENAI_API_KEY }}"
+          pinecone-api-key: "${{ secrets.PINECONE_API_KEY }}"
+          pinecone-environment: "${{ secrets.PINECONE_ENVIRONMENT }}"
 
       - name: Run Terraform tests
         uses: ./.github/actions/tests/reactjs

api/terraform/python/layer_genai/openai_utils/setup.py

@@ -2,7 +2,7 @@
 """Setup for openai_utils package."""
 from setuptools import setup, find_packages
 
-from openai_utils.setup_utils import get_semantic_version
+from setup_utils import get_semantic_version
 
 setup(
     name="openai_utils",
@@ -11,5 +11,8 @@
     author="Lawrence McDaniel",
     author_email="[email protected]",
     packages=find_packages(),
+    package_data={
+        "openai_utils": ["*.md", "data/*"],
+    },
     install_requires=["openai>=0.28"],
 )

requirements.txt

@@ -13,8 +13,10 @@
 boto3==1.29.2
 pytest==7.4.3
 pytest_mock==3.12.0
+black==23.11.0
 flake8==6.1.0
 flake8-coding==1.3.2
+pre-commit==3.5.0
 
 # production
 # ------------
@@ -25,4 +27,4 @@ langchain-experimental==0.0.37
 pinecone-client==2.2.4
 
 # local packages
-./api/terraform/python/layer_genai/openai_utils
+-e ./api/terraform/python/layer_genai/openai_utils