Skip to content

ci: updates for Rust 1.81 (#4533) #15

ci: updates for Rust 1.81 (#4533)

ci: updates for Rust 1.81 (#4533) #15

Workflow file for this run

name: CI
on:
push:
branches:
- main
pull_request:
merge_group:
types: [checks_requested]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
fmt:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: python -m pip install --upgrade pip && pip install nox
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
- name: Check python formatting and lints (ruff)
run: nox -s ruff
- name: Check rust formatting (rustfmt)
run: nox -s rustfmt
resolve:
runs-on: ubuntu-latest
outputs:
MSRV: ${{ steps.resolve-msrv.outputs.MSRV }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: resolve MSRV
id: resolve-msrv
run:
echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT
semver-checks:
if: github.ref != 'refs/heads/main'
needs: [fmt]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: obi1kenobi/cargo-semver-checks-action@v2
check-msrv:
needs: [fmt, resolve]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ needs.resolve.outputs.MSRV }}
components: rust-src
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- run: python -m pip install --upgrade pip && pip install nox
# This is a smoke test to confirm that CI will run on MSRV (including dev dependencies)
- name: Check with MSRV package versions
run: |
nox -s set-msrv-package-versions
nox -s check-all
env:
CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu
clippy:
needs: [fmt]
runs-on: ${{ matrix.platform.os }}
strategy:
# If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
rust: [stable]
platform: [
{
os: "macos-14", # first available arm macos runner
python-architecture: "arm64",
rust-target: "aarch64-apple-darwin",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "powerpc64le-unknown-linux-gnu",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "s390x-unknown-linux-gnu",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "wasm32-wasi",
},
{
os: "windows-latest",
python-architecture: "x64",
rust-target: "x86_64-pc-windows-msvc",
},
{
os: "windows-latest",
python-architecture: "x86",
rust-target: "i686-pc-windows-msvc",
},
]
include:
# Run beta clippy as a way to detect any incoming lints which may affect downstream users
- rust: beta
platform:
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
}
name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }}
continue-on-error: ${{ matrix.rust != 'stable' }}
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ matrix.rust }}
targets: ${{ matrix.platform.rust-target }}
components: clippy,rust-src
- uses: actions/setup-python@v5
with:
python-version: '3.12'
architecture: ${{ matrix.platform.python-architecture }}
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- run: python -m pip install --upgrade pip && pip install nox
- run: nox -s clippy-all
env:
CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }}
build-pr:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }}
name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
needs: [fmt, resolve]
uses: ./.github/workflows/build.yml
with:
os: ${{ matrix.platform.os }}
python-version: ${{ matrix.python-version }}
python-architecture: ${{ matrix.platform.python-architecture }}
rust: ${{ matrix.rust }}
rust-target: ${{ matrix.platform.rust-target }}
MSRV: ${{ needs.resolve.outputs.MSRV }}
secrets: inherit
strategy:
# If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
rust: [stable]
python-version: ["3.12"]
platform:
[
{
os: "macos-14", # first available arm macos runner
python-architecture: "arm64",
rust-target: "aarch64-apple-darwin",
},
{
os: "macos-13", # last available x86_64 macos runner
python-architecture: "x64",
rust-target: "x86_64-apple-darwin",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
},
{
os: "windows-latest",
python-architecture: "x64",
rust-target: "x86_64-pc-windows-msvc",
},
{
os: "windows-latest",
python-architecture: "x86",
rust-target: "i686-pc-windows-msvc",
},
]
include:
# Test nightly Rust on PRs so that PR authors have a chance to fix nightly
# failures, as nightly does not block merge.
- rust: nightly
python-version: "3.12"
platform:
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
}
build-full:
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
needs: [fmt, resolve]
uses: ./.github/workflows/build.yml
with:
os: ${{ matrix.platform.os }}
python-version: ${{ matrix.python-version }}
python-architecture: ${{ matrix.platform.python-architecture }}
rust: ${{ matrix.rust }}
rust-target: ${{ matrix.platform.rust-target }}
MSRV: ${{ needs.resolve.outputs.MSRV }}
secrets: inherit
strategy:
# If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
rust: [stable]
python-version: [
"3.7",
"3.8",
"3.9",
"3.10",
"3.11",
"3.12",
"3.13-dev",
"pypy3.7",
"pypy3.8",
"pypy3.9",
"pypy3.10",
"graalpy24.0",
]
platform:
[
# for the full matrix, use x86_64 macos runners because not all Python versions
# PyO3 supports are available for arm on GitHub Actions. (Availability starts
# around Python 3.10, can switch the full matrix to arm once earlier versions
# are dropped.)
# NB: if this switches to arm, switch the arm job below in the `include` to x86_64
{
os: "macos-13",
python-architecture: "x64",
rust-target: "x86_64-apple-darwin",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
},
{
os: "windows-latest",
python-architecture: "x64",
rust-target: "x86_64-pc-windows-msvc",
},
]
include:
# Test minimal supported Rust version
- rust: ${{ needs.resolve.outputs.MSRV }}
python-version: "3.12"
platform:
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
}
# Test the `nightly` feature
- rust: nightly
python-version: "3.12"
platform:
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
}
# Run rust beta to help catch toolchain regressions
- rust: beta
python-version: "3.12"
platform:
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
}
# Test 32-bit Windows only with the latest Python version
- rust: stable
python-version: "3.12"
platform:
{
os: "windows-latest",
python-architecture: "x86",
rust-target: "i686-pc-windows-msvc",
}
# test arm macos runner with the latest Python version
# NB: if the full matrix switchess to arm, switch to x86_64 here
- rust: stable
python-version: "3.12"
platform:
{
os: "macos-14",
python-architecture: "arm64",
rust-target: "aarch64-apple-darwin",
}
valgrind:
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
needs: [fmt]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
# FIXME valgrind detects an issue with Python 3.12.5, needs investigation
# whether it's a PyO3 issue or upstream CPython.
python-version: '3.12.4'
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@stable
- uses: taiki-e/install-action@valgrind
- run: python -m pip install --upgrade pip && pip install nox
- run: nox -s test-rust -- release skip-full
env:
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: valgrind --leak-check=no --error-exitcode=1
RUST_BACKTRACE: 1
TRYBUILD: overwrite
careful:
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
needs: [fmt]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
- uses: taiki-e/install-action@cargo-careful
- run: python -m pip install --upgrade pip && pip install nox
- run: nox -s test-rust -- careful skip-full
env:
RUST_BACKTRACE: 1
TRYBUILD: overwrite
docsrs:
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
needs: [fmt]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
- run: cargo rustdoc --lib --no-default-features --features full -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]"
coverage:
if : ${{ github.event_name != 'merge_group' }}
needs: [fmt]
name: coverage ${{ matrix.os }}
strategy:
matrix:
os: ["windows-latest", "macos-14", "ubuntu-latest"] # first available arm macos runner
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
# Use the PR head, not the merge commit, because the head commit (and the base)
# is what codecov uses to calculate diffs.
ref: ${{ github.event.pull_request.head.sha }}
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview,rust-src
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- run: python -m pip install --upgrade pip && pip install nox
- run: nox -s coverage
- uses: codecov/codecov-action@v4
with:
file: coverage.json
name: ${{ matrix.os }}
token: ${{ secrets.CODECOV_TOKEN }}
emscripten:
name: emscripten
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
needs: [fmt]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
# TODO bump emscripten builds to test on 3.12
python-version: 3.11
id: setup-python
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-unknown-emscripten
components: rust-src
- uses: actions/setup-node@v4
with:
node-version: 14
- run: python -m pip install --upgrade pip && pip install nox
- uses: actions/cache@v4
id: cache
with:
path: |
.nox/emscripten
key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- name: Build
if: steps.cache.outputs.cache-hit != 'true'
run: nox -s build-emscripten
- name: Test
run: nox -s test-emscripten
test-debug:
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
needs: [fmt]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@stable
with:
components: rust-src
- name: Install python3 standalone debug build with nox
run: |
PBS_RELEASE="20231002"
PBS_PYTHON_VERSION="3.12.0"
PBS_ARCHIVE="cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-x86_64-unknown-linux-gnu-debug-full.tar.zst"
wget "https://github.com/indygreg/python-build-standalone/releases/download/${PBS_RELEASE}/${PBS_ARCHIVE}"
tar -I zstd -xf "${PBS_ARCHIVE}"
ls -l $(pwd)/python/install/bin
ls -l $(pwd)/python/install/lib
echo PATH=$(pwd)/python/install/bin:$PATH >> $GITHUB_ENV
echo LD_LIBRARY_PATH=$(pwd)/python/install/lib:$LD_LIBRARY_PATH >> $GITHUB_ENV
echo PYTHONHOME=$(pwd)/python/install >> $GITHUB_ENV
echo PYO3_PYTHON=$(pwd)/python/install/bin/python3 >> $GITHUB_ENV
- run: python3 -m sysconfig
- run: python3 -m pip install --upgrade pip && pip install nox
- run: |
PYO3_CONFIG_FILE=$(mktemp)
cat > $PYO3_CONFIG_FILE << EOF
implementation=CPython
version=3.12
shared=true
abi3=false
lib_name=python3.12d
lib_dir=${{ github.workspace }}/python/install/lib
executable=${{ github.workspace }}/python/install/bin/python3
pointer_width=64
build_flags=Py_DEBUG,Py_REF_DEBUG
suppress_build_script_link_lines=false
EOF
echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV
- run: python3 -m nox -s test
test-free-threaded:
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
needs: [fmt]
runs-on: ubuntu-latest
env:
UNSAFE_PYO3_BUILD_FREE_THREADED: 1
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@stable
with:
components: rust-src
# TODO: replace with setup-python when there is support
- uses: deadsnakes/[email protected]
with:
python-version: '3.13-dev'
nogil: true
- run: python3 -m sysconfig
- run: python3 -m pip install --upgrade pip && pip install nox
- run: nox -s ffi-check
- name: Run default nox sessions that should pass
run: nox -s clippy docs rustfmt ruff
- name: Run PyO3 tests with free-threaded Python (can fail)
# TODO fix the test crashes so we can unset this
continue-on-error: true
run: nox -s test
test-version-limits:
needs: [fmt]
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@stable
- run: python3 -m pip install --upgrade pip && pip install nox
- run: python3 -m nox -s test-version-limits
check-feature-powerset:
needs: [fmt, resolve]
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
runs-on: ubuntu-latest
name: check-feature-powerset ${{ matrix.rust }}
strategy:
# run on stable and MSRV to check that all combinations of features are expected to build fine on our supported
# range of compilers
matrix:
rust: ["stable"]
include:
- rust: ${{ needs.resolve.outputs.MSRV }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
- uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
- uses: taiki-e/install-action@v2
with:
tool: cargo-hack,cargo-minimal-versions
- run: python3 -m pip install --upgrade pip && pip install nox
- run: python3 -m nox -s check-feature-powerset -- ${{ matrix.rust != 'stable' && 'minimal-versions' || '' }}
test-cross-compilation:
needs: [fmt]
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
runs-on: ${{ matrix.os }}
name: test-cross-compilation ${{ matrix.os }} -> ${{ matrix.target }}
strategy:
# If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present
fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }}
matrix:
include:
# ubuntu "cross compile" to itself
- os: "ubuntu-latest"
target: "x86_64-unknown-linux-gnu"
flags: "-i python3.12"
manylinux: auto
# ubuntu x86_64 -> aarch64
- os: "ubuntu-latest"
target: "aarch64-unknown-linux-gnu"
flags: "-i python3.12"
manylinux: auto
# ubuntu x86_64 -> windows x86_64
- os: "ubuntu-latest"
target: "x86_64-pc-windows-gnu"
flags: "-i python3.12 --features abi3 --features generate-import-lib"
manylinux: off
# macos x86_64 -> aarch64
- os: "macos-13" # last x86_64 macos runners
target: "aarch64-apple-darwin"
# macos aarch64 -> x86_64
- os: "macos-14" # aarch64 macos runners
target: "x86_64-apple-darwin"
steps:
- uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
with:
workspaces:
examples/maturin-starter
save-if: ${{ github.event_name != 'merge_group' }}
key: ${{ matrix.target }}
- name: Setup cross-compiler
if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }}
run: sudo apt-get install -y mingw-w64 llvm
- uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux }}
args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }}
test-cross-compilation-windows:
needs: [fmt]
if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: Swatinem/rust-cache@v2
with:
workspaces:
examples/maturin-starter
save-if: ${{ github.event_name != 'merge_group' }}
- uses: actions/cache/restore@v4
with:
# https://github.com/PyO3/maturin/discussions/1953
path: ~/.cache/cargo-xwin
key: cargo-xwin-cache
- name: Test cross compile to Windows
env:
XWIN_ARCH: x86_64
run: |
set -ex
sudo apt-get install -y mingw-w64 llvm
rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc
pip install cargo-xwin
# abi3
cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu
cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc
# non-abi3
export PYO3_CROSS_PYTHON_VERSION=3.12
cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu
cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc
- if: ${{ github.ref == 'refs/heads/main' }}
uses: actions/cache/save@v4
with:
path: ~/.cache/cargo-xwin
key: cargo-xwin-cache
conclusion:
needs:
- fmt
- check-msrv
- clippy
- build-pr
- build-full
- valgrind
- careful
- docsrs
- coverage
- emscripten
- test-debug
- test-free-threaded
- test-version-limits
- check-feature-powerset
- test-cross-compilation
- test-cross-compilation-windows
if: always()
runs-on: ubuntu-latest
steps:
- name: Result
run: |
jq -C <<< "${needs}"
# Check if all needs were successful or skipped.
"$(jq -r 'all(.result as $result | (["success", "skipped"] | contains([$result])))' <<< "${needs}")"
env:
needs: ${{ toJson(needs) }}