diff --git a/.github/actions/tests/pre-commit/action.yml b/.github/actions/tests/pre-commit/action.yml index beef5482..25e7801c 100644 --- a/.github/actions/tests/pre-commit/action.yml +++ b/.github/actions/tests/pre-commit/action.yml @@ -19,6 +19,7 @@ runs: uses: actions/checkout@v4 - name: Check for pre-commit in requirements + shell: bash run: | if ! grep -q "pre-commit" ./requirements.txt; then echo "pre-commit not found in requirements.txt" >&2 @@ -26,6 +27,7 @@ runs: fi - name: Check for black in requirements + shell: bash run: | if ! grep -q "black" ./requirements.txt; then echo "black not found in requirements.txt" >&2 @@ -33,6 +35,7 @@ runs: fi - name: Check for flake8 in requirements + shell: bash run: | if ! grep -q "flake8" ./requirements.txt; then echo "flake8 not found in requirements.txt" >&2 @@ -40,6 +43,7 @@ runs: fi - name: Check for flake8-coding in requirements + shell: bash run: | if ! grep -q "flake8-coding" ./requirements.txt; then echo "flake8-coding not found in requirements.txt" >&2 diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml index b4500e03..3776ddc3 100644 --- a/.github/actions/tests/python/action.yml +++ b/.github/actions/tests/python/action.yml @@ -10,6 +10,23 @@ inputs: description: 'The version of Python to use, such as 3.11.0' required: true type: string +openai-api-organization: + description: 'The OpenAI API organization' + required: true + type: string +openai-api-key: + description: 'The OpenAI API key' + required: true + type: string +pinecone-api-key: + description: 'The Pinecone API key' + required: true + type: string +pinecone-environment: + description: 'The Pinecone environment' + required: true + type: string + env: REQUIREMENTS_PATH: 'api/terraform/python/layer_genai/requirements.txt' @@ -20,39 +37,8 @@ runs: id: checkout uses: actions/checkout@v4 - - name: Verify OPENAI_API_ORGANIZATION - shell: bash - run: | - if [[ -z "${{ secrets.OPENAI_API_ORGANIZATION }}" ]]; then - echo "OPENAI_API_ORGANIZATION is not set" >&2 - exit 1 - fi - - - name: Verify OPENAI_API_KEY - shell: bash - run: | - if [[ -z "${{ secrets.OPENAI_API_KEY }}" ]]; then - echo "OPENAI_API_KEY is not set" >&2 - exit 1 - fi - - - name: Verify PINECONE_API_KEY - shell: bash - run: | - if [[ -z "${{ secrets.PINECONE_API_KEY }}" ]]; then - echo "PINECONE_API_KEY is not set" >&2 - exit 1 - fi - - - name: Verify PINECONE_ENVIRONMENT - shell: bash - run: | - if [[ -z "${{ secrets.PINECONE_ENVIRONMENT }}" ]]; then - echo "PINECONE_ENVIRONMENT is not set" >&2 - exit 1 - fi - - name: Check for openai in requirements + shell: bash run: | if ! grep -q "openai" ./requirements.txt; then echo "openai not found in requirements.txt" >&2 @@ -60,6 +46,7 @@ runs: fi - name: Check for langchain in requirements + shell: bash run: | if ! grep -q "langchain" ./requirements.txt; then echo "langchain not found in requirements.txt" >&2 @@ -67,6 +54,7 @@ runs: fi - name: Check for langchain-experimental in requirements + shell: bash run: | if ! grep -q "langchain-experimental" ./requirements.txt; then echo "langchain-experimental not found in requirements.txt" >&2 @@ -74,6 +62,7 @@ runs: fi - name: Check for pinecone-client in requirements + shell: bash run: | if ! grep -q "pinecone-client" ./requirements.txt; then echo "pinecone-client not found in requirements.txt" >&2 @@ -83,24 +72,39 @@ runs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ inputs.python-version }} }}} + python-version: ${{ inputs.python-version }} - - name: Install dependencies + - name: locate site-packages path + shell: bash + run: | + echo "SITE_PACKAGES_PATH=$(python -c 'import site; print(site.getsitepackages()[0])')" >> $GITHUB_ENV + + - name: Install pip shell: bash run: | python -m pip install --upgrade pip + + - name: Install dependencies + shell: bash + run: | + cp -R ./api/terraform/python/layer_genai/openai_utils ${{ env.SITE_PACKAGES_PATH }} pip install -r ./requirements.txt - pip install -r ${{ env.REQUIREMENTS_PATH }} - cp -R ./api/terraform/python/layer_genai/openai_utils venv/lib/python${{ inputs.python-version }}/site-packages/ + env: + SITE_PACKAGES_PATH: ${{ env.SITE_PACKAGES_PATH }} - name: Create .env shell: bash run: | touch ./.env - echo "OPENAI_API_ORGANIZATION=${{ secrets.OPENAI_API_ORGANIZATION }}" >> ./.env - echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ./.env - echo "PINECONE_API_KEY=${{ secrets.PINECONE_API_KEY }}" >> ./.env - echo "PINECONE_ENVIRONMENT=${{ secrets.PINECONE_ENVIRONMENT }}" >> ./.env + echo "OPENAI_API_ORGANIZATION=${{ env.OPENAI_API_ORGANIZATION }}" >> ./.env + echo "OPENAI_API_KEY=${{ env.OPENAI_API_KEY }}" >> ./.env + echo "PINECONE_API_KEY=${{ env.PINECONE_API_KEY }}" >> ./.env + echo "PINECONE_ENVIRONMENT=${{ env.PINECONE_ENVIRONMENT }}" >> ./.env + env: + OPENAI_API_ORGANIZATION: ${{ inputs.openai-api-organization }} + OPENAI_API_KEY: ${{ inputs.openai-api-key }} + PINECONE_API_KEY: ${{ inputs.pinecone-api-key }} + PINECONE_ENVIRONMENT: ${{ inputs.pinecone-environment }} - name: Test lambda_openai_v2 shell: bash diff --git a/.github/workflows/pullRequestController.yml b/.github/workflows/pullRequestController.yml index e5f87138..6361cdfe 100644 --- a/.github/workflows/pullRequestController.yml +++ b/.github/workflows/pullRequestController.yml @@ -118,6 +118,10 @@ jobs: uses: ./.github/actions/tests/python with: python-version: "${{ env.python-version}}" + openai-api-organization: "${{ secrets.OPENAI_API_ORGANIZATION }}" + openai-api-key: "${{ secrets.OPENAI_API_KEY }}" + pinecone-api-key: "${{ secrets.PINECONE_API_KEY }}" + pinecone-environment: "${{ secrets.PINECONE_ENVIRONMENT }}" terraform_tests: needs: check_for_pending_release diff --git a/.github/workflows/runTests.yml b/.github/workflows/runTests.yml index 3f121e9d..52324192 100644 --- a/.github/workflows/runTests.yml +++ b/.github/workflows/runTests.yml @@ -20,6 +20,10 @@ jobs: uses: ./.github/actions/tests/python with: python-version: "${{ env.python-version}}" + openai-api-organization: "${{ secrets.OPENAI_API_ORGANIZATION }}" + openai-api-key: "${{ secrets.OPENAI_API_KEY }}" + pinecone-api-key: "${{ secrets.PINECONE_API_KEY }}" + pinecone-environment: "${{ secrets.PINECONE_ENVIRONMENT }}" - name: Run Terraform tests uses: ./.github/actions/tests/reactjs diff --git a/api/terraform/python/layer_genai/openai_utils/setup.py b/api/terraform/python/layer_genai/openai_utils/setup.py index 58d298b4..6612cffd 100644 --- a/api/terraform/python/layer_genai/openai_utils/setup.py +++ b/api/terraform/python/layer_genai/openai_utils/setup.py @@ -2,7 +2,7 @@ """Setup for openai_utils package.""" from setuptools import setup, find_packages -from openai_utils.setup_utils import get_semantic_version +from setup_utils import get_semantic_version setup( name="openai_utils", @@ -11,5 +11,8 @@ author="Lawrence McDaniel", author_email="lpm0073@gmail.com", packages=find_packages(), + package_data={ + "openai_utils": ["*.md", "data/*"], + }, install_requires=["openai>=0.28"], ) diff --git a/requirements.txt b/requirements.txt index 2214f17d..907b9f01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,10 @@ boto3==1.29.2 pytest==7.4.3 pytest_mock==3.12.0 +black==23.11.0 flake8==6.1.0 flake8-coding==1.3.2 +pre-commit==3.5.0 # production # ------------ @@ -25,4 +27,4 @@ langchain-experimental==0.0.37 pinecone-client==2.2.4 # local packages -./api/terraform/python/layer_genai/openai_utils +-e ./api/terraform/python/layer_genai/openai_utils