Skip to content

treon_nightly

treon_nightly #19

Workflow file for this run

name: treon_nightly
on:
workflow_dispatch:
paths:
- 'notebooks/**.ipynb'
- 'notebooks/**.py'
- 'requirements.txt'
- '.ci/*'
- '.github/workflows/*.yml'
- '.github/workflows/.env'
schedule:
- cron: '30 8 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
build_treon:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, ubuntu-22.04, windows-2019, macos-12]
python: ["3.8", "3.9", "3.10"]
steps:
#### Installation/preparation ####
#
# These steps are also copied to convert_notebooks.yml
- name: Checkout repository
uses: actions/checkout@v4
- name: Dotenv Action
id: dotenv
uses: xom9ikk/[email protected]
with:
path: ./.github/workflows
- name: Install required packages
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
sudo apt-get install libsndfile1 -y
fi
shell: bash
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Cache OpenVINO Pip Packages
id: cachepip
uses: actions/cache@v3
with:
path: |
pipcache
key: ${{ env.PIP_CACHE_KEY }}-${{ matrix.os }}-${{ matrix.python }}
# Cache specific files to reduce downloads or prevent network issues
- name: Cache Files
id: cachefiles
uses: actions/cache@v3
with:
path: |
# NOTE: when modifying cache paths, update FILES_CACHE_KEY in .env
# and change cache paths in both treon.yml and convert_notebooks.yml
case_00030.zip
notebooks/110-ct-segmentation-quantize/kits19_frames_1
notebooks/112-pytorch-post-training-quantization-nncf/output/tiny-imagenet-200.zip
# 208 omz cache location is set to this with test_replace
notebooks/208-optical-character-recognition/open_model_zoo_cache
notebooks/110-ct-scan-live-inference/kits19_frames_1
notebooks/302-pytorch-quantization-aware-training/data/tiny-imagenet-200.zip
key: ${{ env.FILES_CACHE_KEY }}
# PaddleGAN stores cache in ppgan directory in CACHE_DIR
- name: Set CACHE_DIR
shell: bash
run: |
python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))'
# replace backslashes with forward slashes for Windows paths
python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))' | sed -e 's/\\/\//g' >> $GITHUB_ENV
# PaddleHub stores cache in directory pointed to by HUB_HOME environment variable
- name: Set HUB_HOME
shell: bash
run: |
echo HUB_HOME=${{ env.CACHE_DIR }}/.paddlehub >> $GITHUB_ENV
# Cache PaddlePaddle cache directories to prevent CI failing due to network/download issues
- name: Cache PaddlePaddle cache directories
id: cacheusercache
uses: actions/cache@v3
with:
path: |
${{ env.HUB_HOME }}
${{ env.CACHE_DIR }}/paddle
${{ env.CACHE_DIR }}/ppgan
key: ${{ env.USER_CACHE_KEY }}-${{ runner.os }}
- name: Cache openvino packages
if: steps.cachepip.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade pip
mkdir pipcache
python -m pip install --cache-dir pipcache --no-deps openvino openvino-dev nncf
cp -r pipcache pipcache_openvino
python -m pip uninstall -y openvino openvino-dev nncf
# Download a small dataset to use for testing purposes in monai-kidney training notebook
- name: Download CT files
if: steps.cachefiles.outputs.cache-hit != 'true'
run: |
curl -O https://storage.openvinotoolkit.org/data/test_data/openvino_notebooks/kits19/case_00030.zip
- name: Copy CT files
run: |
mkdir notebooks/110-ct-segmentation-quantize/kits19
mkdir notebooks/110-ct-segmentation-quantize/kits19/kits19_frames
unzip case_00030.zip
cp -r case_00030 case_00001
mv case_00030 notebooks/110-ct-segmentation-quantize/kits19/kits19_frames
mv case_00001 notebooks/110-ct-segmentation-quantize/kits19/kits19_frames
# Prevent test aborting by timeout
- name: Download long loading models
run: |
mkdir notebooks/203-meter-reader/model
curl -o notebooks/203-meter-reader/model/meter_det_model.tar.gz 'https://bj.bcebos.com/paddlex/examples2/meter_reader/meter_det_model.tar.gz'
curl -o notebooks/203-meter-reader/model/meter_seg_model.tar.gz 'https://bj.bcebos.com/paddlex/examples2/meter_reader/meter_seg_model.tar.gz'
mkdir notebooks/405-paddle-ocr-webcam/model
curl -o notebooks/405-paddle-ocr-webcam/model/ch_PP-OCRv3_det_infer.tar 'https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_det_infer.tar'
curl -o notebooks/405-paddle-ocr-webcam/model/ch_PP-OCRv3_rec_infer.tar 'https://paddleocr.bj.bcebos.com/PP-OCRv3/chinese/ch_PP-OCRv3_rec_infer.tar'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r .ci/dev-requirements.txt --cache-dir pipcache
python -m ipykernel install --user --name openvino_env
# Cache OpenVINO packages. mv works cross-platform
- name: Make pipcache directory with OpenVINO packages
if: steps.cachepip.outputs.cache-hit != 'true'
run: |
mv pipcache pipcache_full
mv pipcache_openvino pipcache
# Create list of installed pip packages that can be downloaded as artifacts
# to verify the exact environment of a specific test run
- name: Pip freeze
run: |
python -m pip freeze
python -m pip freeze > pip-freeze-${{ github.sha }}-${{matrix.os}}-${{ matrix.python }}.txt
- name: Archive pip freeze
uses: actions/upload-artifact@v4
with:
name: pip-freeze-${{matrix.os}}-${{ matrix.python }}
path: pip-freeze-${{ github.sha }}-${{matrix.os}}-${{ matrix.python }}.txt
#### End installation/preparation
- name: Check install
run: |
python check_install.py
# Patch long running cells to run faster
- name: Patch notebooks
run: |
python .ci/patch_notebooks.py .
# Test that JupyterLab runs without errors
- name: Test Jupyterlab
run: |
jupyter lab notebooks --help
# Main notebooks test. Verifies that all notebooks run without errors
- name: Analysing with treon (Windows)
if: runner.os == 'Windows'
run: |
python .ci/validate_notebooks.py --ignore_list .ci/ignore_treon_win.txt --report_dir test_report/${{matrix.os}}-${{ matrix.python }} --move_notebooks_dir c:/notebooks --timeout 1200
shell: bash
- name: Analysing with treon (Linux)
if: runner.os == 'Linux'
run: |
python .ci/validate_notebooks.py --ignore_list .ci/ignore_treon_linux.txt --report_dir test_report/${{matrix.os}}-${{ matrix.python }} --timeout 1200
shell: bash
- name: Analysing with treon (MacOS)
if: runner.os == 'MacOS'
run: |
python .ci/validate_notebooks.py --ignore_list .ci/ignore_treon_mac.txt --report_dir test_report/${{matrix.os}}-${{ matrix.python }} --timeout 1200
shell: bash
- name: Archive test report
uses: actions/upload-artifact@v4
with:
name: test_report.csv-${{matrix.os}}-${{ matrix.python }}
path: test_report.csv
- name: Archive notebook test report
uses: actions/upload-artifact@v4
with:
name: test_report-${{matrix.os}}-${{ matrix.python }}
path: test_report/
# Show the cache after running the notebooks
- name: Show cache
run: |
ls -laR ${{ env.CACHE_DIR }}
du -sh ${{ env.CACHE_DIR }}
if: runner.os != 'Windows'