Bump required Python 3.7 -> 3.8, drop TF1 support #5907
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI | |
on: | |
push: | |
branches: | |
- master | |
- main | |
pull_request: | |
workflow_dispatch: | |
jobs: | |
pycharm-inspect: | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: | |
- "3.10" | |
torch-version: | |
# Some code uses newer features. | |
- 2.0.0 | |
tf-version: | |
- 2.10.0 | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Cache dependencies | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache | |
~/.local | |
key: cache-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-tf${{ matrix.tf-version }} | |
restore-keys: | | |
cache-${{ runner.os }}-py${{ matrix.python-version }}- | |
- name: Setup dependencies | |
run: | | |
echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV | |
echo "Python ${{matrix.python-version}}" | |
echo "pip: $(type pip)" | |
pip install --user --upgrade pip setuptools wheel | |
pip install --user typing # needed for Python 2 in some cases | |
pip install --user pytest | |
pip install --user dm-tree h5py typing | |
pip install --user --progress-bar=off "numpy<2" | |
pip install --user --progress-bar=off scipy # for some tests | |
pip install --user --upgrade --progress-bar=off torch==${{ matrix.torch-version }} | |
# Retry several times in case download breaks. https://github.com/pypa/pip/issues/4796 | |
for ((i=0; i<3; i++)); do | |
if pip install --user --upgrade --progress-bar=off tensorflow==${{ matrix.tf-version }}; then break; fi | |
done | |
# Used in some tests. | |
pip install --user --progress-bar=off transformers | |
pip install --user --progress-bar=off espnet | |
- name: Test Python/Numpy/TF versions. | |
run: | | |
echo "Python env: $(type python) $(python --version)" | |
python -c "import numpy; print('NumPy:', numpy.version.full_version)" | |
python -c "import torch; print('PyTorch:', torch.__version__, torch.__file__)" | |
# torch.__version__ is for example "1.13.1+cu117" | |
python -c "import torch; assert (torch.__version__ + '+').startswith('${{ matrix.torch-version }}+')" | |
python -c "import tensorflow as tf; print('TensorFlow:', tf.__git_version__, tf.__version__, tf.__file__)" | |
python -c "import tensorflow as tf; assert tf.__version__ == '${{ matrix.tf-version }}'" | |
- name: Cache PyCharm | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache/JetBrains | |
key: cache-pycharm-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-tf${{ matrix.tf-version }} | |
- name: Run PyCharm inspect | |
run: | | |
tests/pycharm-inspect.py | |
base-tests: | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: | |
- 3.8 | |
action: | |
- TEST=Dataset | |
- TEST=fork_exec | |
- TEST=GeneratingDataset | |
- TEST=hdf_dump | |
- TEST=HDFDataset | |
- TEST=LearningRateControl | |
- TEST=MultiProcDataset | |
- TEST=Pretrain | |
- TEST=SprintDataset | |
- TEST=TaskSystem | |
- TEST=TaskSystem_SharedMem | |
- TEST=tensor | |
- TEST=TranslationDataset | |
- TEST=Util | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Cache dependencies | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache | |
~/.local | |
key: cache-${{ runner.os }}-py${{ matrix.python-version }}-no-tf | |
restore-keys: | | |
cache-${{ runner.os }}-py${{ matrix.python-version }}- | |
- name: Setup dependencies | |
run: | | |
echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV | |
echo "Python ${{matrix.python-version}}" | |
echo "pip: $(type pip)" | |
pip install --user --upgrade pip setuptools wheel | |
pip install --user typing # needed for Python 2 in some cases | |
pip install --user pytest | |
pip install --user --progress-bar=off -r requirements.txt | |
- name: Test Python/Numpy/TF versions. | |
run: | | |
echo "Python env: $(type python) $(python --version)" | |
python -c "import numpy; print('NumPy:', numpy.version.full_version)" | |
echo "Pytest env:" | |
python -m pytest -V | |
- name: Run test | |
run: | | |
export ${{ matrix.action }} | |
python -m pytest tests/test_$TEST.py | |
tf-tests: | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: | |
- 3.8 | |
tf-version: | |
- 2.10.0 | |
action: | |
- TEST=Config | |
- TEST=Fsa | |
- TEST=Log | |
- TEST=SprintInterface | |
- TEST=TFEngine | |
- TEST=TFNativeOp | |
- TEST=TFNetworkLayer | |
- TEST=TFNetworkRecLayer | |
- TEST=TFUpdater | |
- TEST=TFUtil | |
- TEST=tools | |
include: | |
- action: TEST=TFUtil | |
python-version: 3.8 | |
tf-version: 1.15.3 | |
- action: TEST=TFEngine | |
python-version: 3.8 | |
tf-version: 1.15.3 | |
- action: TEST=TFEngine | |
python-version: 3.8 | |
tf-version: 2.3.0 | |
- action: TEST=TFNativeOp | |
python-version: 3.8 | |
tf-version: 1.14.0 | |
- action: TEST=TFNetworkSigProcLayer | |
python-version: 3.8 | |
tf-version: 1.15.3 | |
- action: TEST=demos RETURNN_DISABLE_TORCH=1 | |
python-version: 3.8 | |
tf-version: 2.3.0 | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Cache dependencies | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache | |
~/.local | |
key: cache-${{ runner.os }}-py${{ matrix.python-version }}-tf${{ matrix.tf-version }} | |
restore-keys: | | |
cache-${{ runner.os }}-py${{ matrix.python-version }}- | |
- name: Setup dependencies | |
run: | | |
echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV | |
echo "Python ${{matrix.python-version}}" | |
echo "pip: $(type pip)" | |
pip install --user --upgrade pip setuptools wheel | |
pip install --user typing # needed for Python 2 in some cases | |
pip install --user pytest | |
if [[ "${{matrix.tf-version}}" == 2.[0123].* || "${{matrix.tf-version}}" == 1.* ]]; then | |
# Older TF needs older NumPy version. | |
# https://github.com/rwth-i6/returnn/pull/1160#issuecomment-1284537803 | |
pip install --user numpy==1.19.5 | |
# Older TF needs also older protobuf version. | |
# https://github.com/rwth-i6/returnn/issues/1209 | |
pip install --user "protobuf<=3.20.1" | |
fi | |
if [[ "${{matrix.tf-version}}" == 2.10.* ]]; then | |
# TF 2.10 requires gast<=0.4.0,>=0.2.1. But for example, with gast 0.2.2, we get some AutoGraph error: | |
# Cause: module 'gast' has no attribute 'Constant' | |
# Similar like: https://github.com/tensorflow/tensorflow/issues/47802 | |
pip install --user --upgrade "gast<=0.4.0" | |
fi | |
pip install --user --progress-bar=off -r requirements.txt | |
pip install --user --progress-bar=off scipy # for some tests | |
pip install --user --upgrade --progress-bar=off tensorflow==${{ matrix.tf-version }} | |
- name: Test Python/Numpy/TF versions. | |
run: | | |
echo "Python env: $(type python) $(python --version)" | |
python -c "import numpy; print('NumPy:', numpy.version.full_version)" | |
python -c "import tensorflow as tf; print('TensorFlow:', tf.__git_version__, tf.__version__, tf.__file__)" | |
python -c "import tensorflow as tf; assert tf.__version__ == '${{ matrix.tf-version }}'" | |
echo "Pytest env:" | |
python -m pytest -V | |
- name: Run test | |
run: | | |
export ${{ matrix.action }} | |
python -m pytest tests/test_$TEST.py | |
torch-tests: | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: | |
- "3.10" | |
torch-version: | |
- 1.13.1 | |
- 2.0.0 | |
action: | |
- TEST=demos RETURNN_DISABLE_TF=1 | |
- TEST=torch_dataset | |
- TEST=torch_engine | |
- TEST=torch_frontend | |
- TEST=torch_internal_frontend | |
- TEST=torch_util | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Cache dependencies | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache | |
~/.local | |
key: cache-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }} | |
restore-keys: | | |
cache-${{ runner.os }}-py${{ matrix.python-version }}- | |
- name: Setup dependencies | |
run: | | |
echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV | |
echo "Python ${{matrix.python-version}}" | |
echo "pip: $(type pip)" | |
pip install --user --upgrade pip setuptools wheel | |
pip install --user typing # needed for Python 2 in some cases | |
pip install --user pytest | |
pip install --user dm-tree h5py typing | |
pip install --user --progress-bar=off "numpy<2" | |
pip install --user --progress-bar=off scipy # for some tests | |
pip install --user --upgrade --progress-bar=off torch==${{ matrix.torch-version }} | |
pip install --user --progress-bar=off onnx onnxruntime | |
pip install --user --progress-bar=off lovely_tensors | |
- name: Test Python/Numpy/PyTorch versions. | |
run: | | |
echo "Python env: $(type python) $(python --version)" | |
python -c "import numpy; print('NumPy:', numpy.version.full_version)" | |
python -c "import torch; print('PyTorch:', torch.__version__, torch.__file__)" | |
# torch.__version__ is for example "1.13.1+cu117" | |
python -c "import torch; assert (torch.__version__ + '+').startswith('${{ matrix.torch-version }}+')" | |
echo "Pytest env:" | |
python -m pytest -V | |
- name: Run test | |
run: | | |
export ${{ matrix.action }} | |
python -m pytest tests/test_$TEST.py | |
rf-tests: | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
python-version: | |
- "3.10" | |
torch-version: | |
- 2.0.0 | |
tf-version: | |
- 2.10.0 | |
action: | |
- TEST=rf_array | |
- TEST=rf_attention | |
- TEST=rf_base | |
- TEST=rf_cond | |
- TEST=rf_const | |
- TEST=rf_container | |
- TEST=rf_conv | |
- TEST=rf_decoder_transformer | |
- TEST=rf_encoder_conformer | |
- TEST=rf_gradient | |
- TEST=rf_label_smoothing | |
- TEST=rf_loop | |
- TEST=rf_math | |
- TEST=rf_normalization | |
- TEST=rf_piecewise_linear | |
- TEST=rf_rec | |
- TEST=rf_reduce | |
- TEST=rf_signal | |
include: | |
# Some selected tests (but not all) with some other versions. | |
- action: TEST=rf_base | |
python-version: 3.8 | |
torch-version: 1.13.1 | |
tf-version: 2.10.0 | |
steps: | |
- uses: actions/checkout@v4 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Cache dependencies | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache | |
~/.local | |
key: cache-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-tf${{ matrix.tf-version }} | |
restore-keys: | | |
cache-${{ runner.os }}-py${{ matrix.python-version }}- | |
- name: Setup dependencies | |
run: | | |
echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV | |
echo "Python ${{matrix.python-version}}" | |
echo "pip: $(type pip)" | |
pip install --user --upgrade pip setuptools wheel | |
pip install --user typing # needed for Python 2 in some cases | |
pip install --user pytest | |
pip install --user dm-tree h5py typing | |
pip install --user --progress-bar=off "numpy<2" | |
pip install --user --progress-bar=off scipy # for some tests | |
pip install --user --upgrade torch==${{ matrix.torch-version }} | |
pip install --user --progress-bar=off lovely_tensors | |
if [[ "${{matrix.tf-version}}" == 2.10.* ]]; then | |
# TF 2.10 requires gast<=0.4.0,>=0.2.1. But for example, with gast 0.2.2, we get some AutoGraph error: | |
# Cause: module 'gast' has no attribute 'Constant' | |
# Similar like: https://github.com/tensorflow/tensorflow/issues/47802 | |
pip install --user --upgrade "gast<=0.4.0" | |
fi | |
# Retry several times in case download breaks. https://github.com/pypa/pip/issues/4796 | |
for ((i=0; i<3; i++)); do | |
if pip install --user --upgrade --progress-bar=off tensorflow==${{ matrix.tf-version }}; then break; fi | |
done | |
# Needed for some tests. | |
pip install --user --progress-bar=off transformers | |
pip install --user --progress-bar=off espnet | |
# TorchAudio needed by ESPnet. | |
# https://pytorch.org/audio/stable/installation.html#compatibility-matrix | |
if [[ "${{matrix.torch-version}}" == 2.0.0 ]]; then | |
pip install --user --progress-bar=off torchaudio==2.0.1 | |
elif [[ "${{matrix.torch-version}}" == 1.13.1 ]]; then | |
pip install --user --progress-bar=off torchaudio==0.13.1 | |
else | |
pip install --user --progress-bar=off torchaudio==${{matrix.torch-version}} | |
fi | |
- name: Test Python/Numpy/PyTorch versions. | |
run: | | |
echo "Python env: $(type python) $(python --version)" | |
python -c "import numpy; print('NumPy:', numpy.version.full_version)" | |
python -c "import torch; print('PyTorch:', torch.__version__, torch.__file__)" | |
# torch.__version__ is for example "1.13.1+cu117" | |
python -c "import torch; assert (torch.__version__ + '+').startswith('${{ matrix.torch-version }}+')" | |
python -c "import tensorflow as tf; print('TensorFlow:', tf.__git_version__, tf.__version__, tf.__file__)" | |
python -c "import tensorflow as tf; assert tf.__version__ == '${{ matrix.tf-version }}'" | |
echo "Pytest env:" | |
python -m pytest -V | |
- name: Run test | |
run: | | |
export ${{ matrix.action }} | |
python -m pytest tests/test_$TEST.py |