Skip to content

Commit

Permalink
Merge branch 'feature/feature-store-1.0.4' of github.com:oracle/accel…
Browse files Browse the repository at this point in the history
…erated-data-science into ODSC-47451-streaming-dataframe-support

backmerged the code
  • Loading branch information
yogesh266 committed Oct 15, 2023
2 parents 6bbb58a + 1ec5922 commit 3ec41ae
Show file tree
Hide file tree
Showing 176 changed files with 3,178 additions and 1,439 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/add-3plicense-warning.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: "Add 3P License Warning to PR"
on:
pull_request:
paths:
- setup.py
- pyproject.toml

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
Expand All @@ -23,9 +23,9 @@ jobs:
steps:
- run: |
BODY_MSG=$(cat << EOF
⚠️ This PR changed **setup.py** file. ⚠️
- PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **setup.py**.
- PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **setup.py**.
⚠️ This PR changed **pyproject.toml** file. ⚠️
- PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **pyproject.toml**.
- PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **pyproject.toml**.
EOF
)
echo "BODY_MSG<<EOF" >> $GITHUB_ENV
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/create-more-space/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# We are running out space on the disk, solution described here:
# https://github.com/actions/runner-images/issues/2840#issuecomment-790492173

name: "Create more disk space"
description: "Removing some folders to create more disk space"
runs:
using: composite
steps:
- shell: bash
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
6 changes: 3 additions & 3 deletions .github/workflows/publish-to-pypi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ jobs:
python-version: "3.x"
- name: Build distribution 📦
run: |
pip install wheel
pip install build
make dist
- name: Validate
run: |
pip install dist/*.whl
python -c "import ads;"
## To run publish to test PyPI secret with token needs to be added,
## To run publish to test PyPI a secret with token needs to be added,
## this one GH_ADS_TESTPYPI_TOKEN - removed after initial test.
## Project name also needed to be updated in setup.py - setup(name="test_oracle_ads", ...),
## Project name also needed to be updated in pyproject.toml - name = "test_oracle_ads" in [project] section
## regular name is occupied by former developer and can't be used for testing
# - name: Publish distribution 📦 to Test PyPI
# env:
Expand Down
45 changes: 9 additions & 36 deletions .github/workflows/run-unittests-default_setup.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
name: tests/unitary/default_setup/**
name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"

on:
workflow_dispatch:
pull_request:
branches:
- main
- "release/**"
- develop
paths:
- "ads/**"
- "!ads/ads_version.json"
- setup.py
- pyproject.toml
- "**requirements.txt"
- .github/workflows/run-unittests.yml
- .github/workflows/run-unittests-default_setup.yml
- ".github/workflows/run-unittests*.yml"

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
Expand Down Expand Up @@ -42,37 +36,16 @@ jobs:
steps:
- uses: actions/checkout@v3

# Caching python libraries installed with pip
# https://github.com/actions/cache/blob/main/examples.md#python---pip
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: |
pyproject.toml
"**requirements.txt"
- name: "Test config setup"
shell: bash
env:
HOME_RUNNER_DIR: /home/runner
run: |
set -x # print commands that are executed
mkdir -p "$HOME_RUNNER_DIR"/.oci
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
[DEFAULT]
user=ocid1.user.oc1..xxx
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
tenancy=ocid1.tenancy.oc1..xxx
region=test_region
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
EOT
ls -lha "$HOME_RUNNER_DIR"/.oci
echo "Test config file:"
cat $HOME_RUNNER_DIR/.oci/config
- uses: ./.github/workflows/set-dummy-conf
name: "Test config setup"

- name: "Run default_setup tests folder ONLY with minimum ADS dependencies"
timeout-minutes: 15
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
name: tests/unitary/**
name: "[Py3.8][COV REPORT] tests/unitary/**"

on:
workflow_dispatch:
pull_request:
branches:
- main
- "release/**"
- develop
paths:
- "ads/**"
- "!ads/ads_version.json"
- setup.py
- pyproject.toml
- "**requirements.txt"
- .github/workflows/run-unittests.yml
- .github/workflows/run-unittests-default_setup.yml
- ".github/workflows/run-unittests*.yml"

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
Expand All @@ -31,58 +25,37 @@ env:

jobs:
test:
name: python ${{ matrix.python-version }}, ${{ matrix.name }}
name: python 3.8, ${{ matrix.name }}
runs-on: ubuntu-latest
timeout-minutes: 90

strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10"]
test-path: ["tests/unitary", "tests/unitary/with_extras/model"]
test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
include:
- test-path: "tests/unitary"
- test-path: "tests/unitary/with_extras tests/unitary/default_setup"
ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
name: "unitary"
- test-path: "tests/unitary/with_extras/model"
name: "model"
- python-version: "3.8"
cov-reports: --cov=ads --cov-report=xml --cov-report=html

steps:
- uses: actions/checkout@v3

# Caching python libraries installed with pip
# https://github.com/actions/cache/blob/main/examples.md#python---pip
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- uses: ./.github/workflows/create-more-space
name: "Create more disk space"

- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
python-version: "3.8"
cache: "pip"
cache-dependency-path: |
pyproject.toml
"**requirements.txt"
- name: "Test config setup"
shell: bash
env:
HOME_RUNNER_DIR: /home/runner
run: |
set -x # print commands that are executed
mkdir -p "$HOME_RUNNER_DIR"/.oci
openssl genrsa -out $HOME_RUNNER_DIR/.oci/oci_ads_user.pem 2048
cat <<EOT >> "$HOME_RUNNER_DIR/.oci/config"
[DEFAULT]
user=ocid1.user.oc1..xxx
fingerprint=00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
tenancy=ocid1.tenancy.oc1..xxx
region=test_region
key_file=$HOME_RUNNER_DIR/.oci/oci_ads_user.pem
EOT
ls -lha "$HOME_RUNNER_DIR"/.oci
echo "Test config file:"
cat $HOME_RUNNER_DIR/.oci/config
- uses: ./.github/workflows/set-dummy-conf
name: "Test config setup"

- name: "Test env setup"
timeout-minutes: 20
Expand All @@ -105,22 +78,19 @@ jobs:
set -x # print commands that are executed
# Setup project and tests folder for cov reports to not be overwritten by another parallel step
if [[ ! -z "${{ matrix.cov-reports }}" ]]; then
mkdir -p cov-${{ matrix.name }}
cd cov-${{ matrix.name }}
ln -s ../tests tests
ln -s ../ads ads
ln -s ../.coveragerc .coveragerc
fi
mkdir -p cov-${{ matrix.name }}
cd cov-${{ matrix.name }}
ln -s ../tests tests
ln -s ../ads ads
ln -s ../.coveragerc .coveragerc
# Run tests
python -m pytest -v -p no:warnings --durations=5 \
-n auto --dist loadfile ${{ matrix.cov-reports }} \
-n auto --dist loadfile --cov=ads --cov-report=xml --cov-report=html \
${{ matrix.test-path }} ${{ matrix.ignore-path }}
- name: "Save coverage files"
uses: actions/upload-artifact@v3
if: ${{ matrix.cov-reports }}
with:
name: cov-reports-${{ matrix.name }}
path: |
Expand All @@ -133,7 +103,7 @@ jobs:
runs-on: ubuntu-latest
continue-on-error: true
needs: test
if: ${{ success() }} && ${{ github.event.issue.pull_request }}
if: ${{ success() }} && && github.event_name == 'pull_request'
env:
COMPARE_BRANCH: main

Expand All @@ -148,15 +118,15 @@ jobs:
run: |
set -x # print commands that are executed
# Prepare default cov body text
# Prepare default cov body text
COV_BODY_INTRO="📌 Overall coverage:\n\n"
echo COV_BODY="$COV_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
# Prepare file paths to .coverage files
# Filenames taken from job.test last step with name - "Save coverage files"
FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
FILE_UNITARY="cov-reports-unitary/.coverage"; [[ ! -f $FILE_UNITARY ]] && FILE_UNITARY=""
FILE_MODEL="cov-reports-model/.coverage"; [[ ! -f $FILE_MODEL ]] && FILE_MODEL=""
# Combine coverage files
pip install coverage
coverage combine $FILE_UNITARY $FILE_MODEL
Expand All @@ -166,7 +136,7 @@ jobs:
# Calculate overall coverage and update body message
COV=$(grep -E 'pc_cov' htmlcov/index.html | cut -d'>' -f 2 | cut -d'%' -f 1)
if [[ ! -z $COV ]]; then
if [[ ! -z $COV ]]; then
if [[ $COV -lt 50 ]]; then COLOR=red; elif [[ $COV -lt 80 ]]; then COLOR=yellow; else COLOR=green; fi
echo COV_BODY="$COV_BODY_INTRO ![Coverage-$COV%](https://img.shields.io/badge/coverage-$COV%25-$COLOR)" >> $GITHUB_ENV
fi
Expand All @@ -176,23 +146,23 @@ jobs:
run: |
set -x # print commands that are executed
# Prepare default diff body text
# Prepare default diff body text
DIFF_BODY_INTRO="📌 Cov diff with **${{ env.COMPARE_BRANCH }}**:\n\n"
echo DIFF_BODY="$DIFF_BODY_INTRO No success to gather report. 😿" >> $GITHUB_ENV
# Prepare file paths to coverage xml files
# Filenames taken from job.test last step with name - "Save coverage files"
FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
FILE1="cov-reports-unitary/coverage.xml"; [[ ! -f $FILE1 ]] && FILE1=""
FILE2="cov-reports-model/coverage.xml"; [[ ! -f $FILE2 ]] && FILE2=""
echo "FILE1=$FILE1" >> $GITHUB_ENV
echo "FILE2=$FILE2" >> $GITHUB_ENV
# Calculate coverage diff and update body message
pip install diff_cover
diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
diff-cover $FILE1 $FILE2 --compare-branch=origin/${{ env.COMPARE_BRANCH }}
DIFF=$(diff-cover $FILE1 $FILE2 \
--compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep Coverage: | cut -d' ' -f 2 | cut -d'%' -f 1)
if [[ -z $DIFF ]]; then
if [[ -z $DIFF ]]; then
DIFF_INFO=$(diff-cover $FILE1 $FILE2 \
--compare-branch=origin/${{ env.COMPARE_BRANCH }} | grep "No lines");
echo DIFF_BODY="$DIFF_BODY_INTRO $DIFF_INFO">> $GITHUB_ENV
Expand Down
84 changes: 84 additions & 0 deletions .github/workflows/run-unittests-py39-py310.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
name: "[Py3.9][Py3.10] - tests/unitary/**"

on:
workflow_dispatch:
pull_request:
paths:
- "ads/**"
- pyproject.toml
- "**requirements.txt"
- ".github/workflows/run-unittests*.yml"

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

permissions:
contents: read
pull-requests: write

# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5

jobs:
test:
name: python ${{ matrix.python-version }}, ${{ matrix.name }}
runs-on: ubuntu-latest
timeout-minutes: 90

strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10"]
test-path: ["tests/unitary/with_extras tests/unitary/default_setup", "tests/unitary/with_extras/model"]
include:
- test-path: "tests/unitary/with_extras tests/unitary/default_setup"
ignore-path: "--ignore tests/unitary/with_extras/model --ignore tests/unitary/with_extras/feature_store"
name: "unitary"
- test-path: "tests/unitary/with_extras/model"
name: "model"

steps:
- uses: actions/checkout@v3

- uses: ./.github/workflows/create-more-space
name: "Create more disk space"

- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: |
pyproject.toml
"**requirements.txt"
- uses: ./.github/workflows/set-dummy-conf
name: "Test config setup"

- name: "Test env setup"
timeout-minutes: 20
shell: bash
run: |
set -x # print commands that are executed
sudo apt-get install libkrb5-dev graphviz
$CONDA/bin/conda init
source /home/runner/.bashrc
pip install -r dev-requirements.txt
- name: "Run unitary tests folder with maximum ADS dependencies"
timeout-minutes: 60
shell: bash
env:
CONDA_PREFIX: /usr/share/miniconda
run: |
set -x # print commands that are executed
# Run tests
python -m pytest -v -p no:warnings --durations=5 \
-n auto --dist loadfile \
${{ matrix.test-path }} ${{ matrix.ignore-path }}
Loading

0 comments on commit 3ec41ae

Please sign in to comment.