Feature/feature store 1.0.4 (#370)

oracle · Oct 27, 2023 · 81d11d3 · 81d11d3
2 parents 34c7de9 + 6393b3a
commit 81d11d3
Show file tree

Hide file tree

Showing 104 changed files with 4,598 additions and 2,763 deletions.
diff --git a/.github/workflows/create-more-space/action.yml b/.github/workflows/create-more-space/action.yml
@@ -0,0 +1,14 @@
+# We are running out space on the disk, solution described here:
+# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+
+name: "Create more disk space"
+description: "Removing some folders to create more disk space"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
diff --git a/.github/workflows/run-unittests-default_setup.yml b/.github/workflows/run-unittests-default_setup.yml
@@ -1,18 +1,13 @@
-name: tests/unitary/default_setup/**
+name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
 
 on:
   workflow_dispatch:
   pull_request:
-    branches:
-      - main
-      - "release/**"
-      - develop
     paths:
       - "ads/**"
       - pyproject.toml
       - "**requirements.txt"
-      - .github/workflows/run-unittests.yml
-      - .github/workflows/run-unittests-default_setup.yml
+      - ".github/workflows/run-unittests*.yml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -41,37 +36,16 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      # Caching python libraries installed with pip
-      # https://github.com/actions/cache/blob/main/examples.md#python---pip
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
 
-      - name: "Test config setup"
-        shell: bash
-        env:
-          HOME_RUNNER_DIR: /home/runner
-        run: |
-          set -x # print commands that are executed
-          mkdir -p "$HOME_RUNNER_DIR"/.oci
-          openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
-          cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
-          [DEFAULT]
-          user=ocid1.user.oc1..xxx
-          fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
-          tenancy=ocid1.tenancy.oc1..xxx
-          region=test_region
-          key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
-          EOT
-          ls -lha "$HOME_RUNNER_DIR"/.oci
-          echo "Test config file:"
-          cat $HOME_RUNNER_DIR/.oci/config
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
 
       - name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
         timeout-minutes: 15

diff --git a/.github/workflows/run-unittests.yml → ...rkflows/run-unittests-py38-cov-report.yml b/.github/workflows/run-unittests.yml → ...rkflows/run-unittests-py38-cov-report.yml
@@ -1,18 +1,13 @@
-name: tests/unitary/**
+name: "[Py3.8][COV REPORT] tests/unitary/**"
 
 on:
   workflow_dispatch:
   pull_request:
-    branches:
-      - main
-      - "release/**"
-      - develop
     paths:
       - "ads/**"
       - pyproject.toml
       - "**requirements.txt"
-      - .github/workflows/run-unittests.yml
-      - .github/workflows/run-unittests-default_setup.yml
+      - ".github/workflows/run-unittests*.yml"
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -30,58 +25,37 @@ env:
 
 jobs:
   test:
-    name: python ${{ matrix.python-version }}, ${{ matrix.name }}
+    name: python 3.8, ${{ matrix.name }}
     runs-on: ubuntu-latest
     timeout-minutes: 90
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
         test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
         include:
           - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
             ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
             name: "unitary"
           - test-path: "tests/unitary/with_extras/model"
             name: "model"
-          - python-version: "3.8"
-            cov-reports: --cov=ads --cov-report=xml --cov-report=html
 
     steps:
       - uses: actions/checkout@v3
 
-      # Caching python libraries installed with pip
-      # https://github.com/actions/cache/blob/main/examples.md#python---pip
-      - uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+      - uses: ./.github/workflows/create-more-space
+        name: "Create more disk space"
+
       - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.8"
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
 
-      - name: "Test config setup"
-        shell: bash
-        env:
-          HOME_RUNNER_DIR: /home/runner
-        run: |
-          set -x # print commands that are executed
-          mkdir -p "$HOME_RUNNER_DIR"/.oci
-          openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
-          cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
-          [DEFAULT]
-          user=ocid1.user.oc1..xxx
-          fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
-          tenancy=ocid1.tenancy.oc1..xxx
-          region=test_region
-          key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
-          EOT
-          ls -lha "$HOME_RUNNER_DIR"/.oci
-          echo "Test config file:"
-          cat $HOME_RUNNER_DIR/.oci/config
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
 
       - name: "Test env setup"
         timeout-minutes: 20
@@ -104,22 +78,19 @@ jobs:
           set -x # print commands that are executed
 
           # Setup project and tests folder for cov reports to not be overwritten by another parallel step
-          if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
-            mkdir -p cov-${{ matrix.name }}
-            cd cov-${{ matrix.name }}
-            ln -s ../tests tests
-            ln -s ../ads ads
-            ln -s ../.coveragerc .coveragerc
-          fi
+          mkdir -p cov-${{ matrix.name }}
+          cd cov-${{ matrix.name }}
+          ln -s ../tests tests
+          ln -s ../ads ads
+          ln -s ../.coveragerc .coveragerc
 
           # Run tests
           python -m pytest -v -p no:warnings --durations=5 \
-          -n auto --dist loadfile ${{ matrix.cov-reports }} \
+          -n auto --dist loadfile --cov=ads --cov-report=xml --cov-report=html \
           ${{ matrix.test-path }} ${{ matrix.ignore-path }}
 
       - name: "Save coverage files"
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.cov-reports }}
         with:
           name: cov-reports-${{ matrix.name }}
           path: |
@@ -132,7 +103,7 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     needs: test
-    if: ${{ success() }} && ${{ github.event.issue.pull_request }}
+    if: ${{ success() }} && && github.event_name == 'pull_request'
     env:
       COMPARE_BRANCH: main
 

diff --git a/.github/workflows/run-unittests-py39-py310.yml b/.github/workflows/run-unittests-py39-py310.yml
@@ -0,0 +1,84 @@
+name: "[Py3.9][Py3.10] - tests/unitary/**"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "ads/**"
+      - pyproject.toml
+      - "**requirements.txt"
+      - ".github/workflows/run-unittests*.yml"
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}, ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10"]
+        test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
+        include:
+          - test-path: "tests/unitary/with_extras tests/unitary/default_setup"
+            ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
+            name: "unitary"
+          - test-path: "tests/unitary/with_extras/model"
+            name: "model"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: ./.github/workflows/create-more-space
+        name: "Create more disk space"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
+
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
+
+      - name: "Test env setup"
+        timeout-minutes: 20
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+
+          sudo apt-get install libkrb5-dev graphviz
+          $CONDA/bin/conda init
+          source /home/runner/.bashrc
+
+          pip install -r dev-requirements.txt
+
+      - name: "Run unitary tests folder with maximum ADS dependencies"
+        timeout-minutes: 60
+        shell: bash
+        env:
+          CONDA_PREFIX: /usr/share/miniconda
+        run: |
+          set -x # print commands that are executed
+
+          # Run tests
+          python -m pytest -v -p no:warnings --durations=5 \
+          -n auto --dist loadfile \
+          ${{ matrix.test-path }} ${{ matrix.ignore-path }}
diff --git a/.github/workflows/set-dummy-conf/action.yml b/.github/workflows/set-dummy-conf/action.yml
@@ -0,0 +1,26 @@
+# This composite action is to avoid duplicating code across test actions.
+# Ref: https://docs.github.com/en/actions/creating-actions/creating-a-composite-action
+
+name: "Create dummy oci config"
+description: "Creates  dummy oci config folder in ~/.oci"
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        HOME_RUNNER_DIR: /home/runner
+      run: |
+        set -x # print commands that are executed
+        mkdir -p "$HOME_RUNNER_DIR"/.oci
+        openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
+        cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
+        [DEFAULT]
+        user=ocid1.user.oc1..xxx
+        fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
+        tenancy=ocid1.tenancy.oc1..xxx
+        region=test_region
+        key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
+        EOT
+        ls -lha "$HOME_RUNNER_DIR"/.oci
+        echo "Test config file:"
+        cat $HOME_RUNNER_DIR/.oci/config
diff --git a/.gitleaks.toml b/.gitleaks.toml
@@ -13,6 +13,7 @@ useDefault = true
         '''example-password''',
         '''this-is-not-the-secret''',
         '''<redacted>''',
+        '''security_token''',
         # NVIDIA_GPGKEY_SUM from public documentation:
         # https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/10.1/centos7/base/Dockerfile
         '''d0664fbbdb8c32356d45de36c5984617217b2d0bef41b93ccecd326ba3b80c87'''

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -39,6 +39,7 @@ repos:
     rev: v8.17.0
     hooks:
     -   id: gitleaks
+        exclude: .github/workflows/reusable-actions/set-dummy-conf.yml
 # Oracle copyright checker
 -   repo: https://github.com/oracle-samples/oci-data-science-ai-samples/
     rev: cbe0136f7aaffe463b31ddf3f34b0e16b4b124ff

diff --git a/ads/common/auth.py b/ads/common/auth.py
@@ -416,7 +416,7 @@ def create_signer(
     >>> auth = ads.auth.create_signer(auth_type="security_token", config=config) # security token authentication created based on provided config
     """
     if signer or signer_callable:
-        configuration = ads.telemetry.update_oci_client_config()
+        configuration = ads.telemetry.update_oci_client_config(config)
         if signer_callable:
             signer = signer_callable(**signer_kwargs)
         signer_dict = {
@@ -479,7 +479,7 @@ def default_signer(client_kwargs: Optional[Dict] = None) -> Dict:
     """
     auth_state = AuthState()
     if auth_state.oci_signer or auth_state.oci_signer_callable:
-        configuration = ads.telemetry.update_oci_client_config()
+        configuration = ads.telemetry.update_oci_client_config(auth_state.oci_config)
         signer = auth_state.oci_signer
         if auth_state.oci_signer_callable:
             signer_kwargs = auth_state.oci_signer_kwargs or {}