From 28c8d2bf79f2a6917ed4812a88a0d2d8eafd5ca4 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 27 Feb 2024 14:24:09 +0000 Subject: [PATCH] Add CI --- .github/actions/setup-python/action.yml | 47 +++++++++++++++++++++++++ .github/actions/setup-rust/action.yml | 44 +++++++++++++++++++++++ .github/actions/setup-zig/action.yml | 23 ++++++++++++ .github/workflows/bench-pr.yml | 42 ++++++++++++++++++++++ .github/workflows/bench.yml | 35 ++++++++++++++++++ .github/workflows/ci.yml | 46 ++++++++++++++++++++++++ pyenc/test/test_array.py | 3 +- pyenc/test/test_compress.py | 3 +- pyenc/test/test_serde.py | 3 +- pyproject.toml | 4 ++- 10 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 .github/actions/setup-python/action.yml create mode 100644 .github/actions/setup-rust/action.yml create mode 100644 .github/actions/setup-zig/action.yml create mode 100644 .github/workflows/bench-pr.yml create mode 100644 .github/workflows/bench.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml new file mode 100644 index 0000000000..2d1998905c --- /dev/null +++ b/.github/actions/setup-python/action.yml @@ -0,0 +1,47 @@ +name: 'Setup Rust' +description: 'Toolchain setup and Initial compilation' +inputs: + rye-version: + description: 'Rye version to use' + required: true + default: '0.16.0' +runs: + using: "composite" + steps: + - name: Rye Cache + id: rye-cache + uses: actions/cache@v4 + with: + path: ~/.rye + key: "rye-${{ runner.os }}-${{ inputs.rye-version }}" + + - name: Rye Install + shell: bash + run: curl -sSf https://rye-up.com/get | bash + if: steps.rye-cache.outputs.cache-hit != 'true' + env: + RYE_VERSION: "${{ inputs.rye-version }}" + RYE_INSTALL_OPTION: "--yes" + + - name: Rye Shims + shell: bash + run: echo "~/.rye/shims" >> $GITHUB_PATH + + - name: Venv Cache + id: venv-cache + uses: actions/cache@v4 + with: + path: .venv + key: "venv-${{ runner.os }}-${{ hashFiles('requirements**.lock') }}" + + - name: Rye Sync + shell: bash + # --no-lock prevents resolution of the lock file. The locks are still respected. + # We always run `rye sync` even if the cache fetch was successful since it builds our Rust extensions for us. + run: rye sync --no-lock + env: + MATURIN_PEP517_ARGS: "--profile dev" + + - name: Export Path + shell: bash + run: echo "PATH=$PATH" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 0000000000..100b184cba --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,44 @@ +name: 'Setup Rust' +description: 'Toolchain setup and Initial compilation' +inputs: + rust-toolchain: # id of input + description: 'Rust toolchain version to use' + required: true + default: stable +runs: + using: "composite" + steps: + - name: Rust Toolchain Cache + id: rustup-cache + uses: actions/cache@v4 + with: + path: ~/.rustup + key: "rustup-${{ runner.os }}-${{ inputs.rust-toolchain }}" + + - name: Rust Toolchain + uses: dtolnay/rust-toolchain@stable + if: steps.rustup-cache.outputs.cache-hit != 'true' + with: + toolchain: "${{ inputs.rust-toolchain }}" + components: clippy, rustfmt + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "shared" # To allow reuse across jobs + + - name: Rust Compile Cache + uses: mozilla-actions/sccache-action@v0.0.4 + - name: Rust Compile Cache Config + shell: bash + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV + + - name: Rust Build + shell: bash + run: cargo build --all-targets + + - name: Export Path + shell: bash + run: echo "PATH=$PATH" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/setup-zig/action.yml b/.github/actions/setup-zig/action.yml new file mode 100644 index 0000000000..33486f0f81 --- /dev/null +++ b/.github/actions/setup-zig/action.yml @@ -0,0 +1,23 @@ +name: 'Setup Zig' +description: 'Toolchain setup and Initial compilation' +runs: + using: "composite" + steps: + - name: Zig Version + id: zig-version + shell: bash + run: echo "version=$(cat .zig-version | tr -d '\n ')" >> $GITHUB_OUTPUT + + - name: Zig Setup + uses: goto-bus-stop/setup-zig@v2 + with: + version: "${{ steps.zig-version.outputs.version }}" + + # TODO(ngates): should we cache zig-cache? Or zig-out? + - name: Zig Build + shell: bash + run: zig build + + - name: Export Path + shell: bash + run: echo "PATH=$PATH" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml new file mode 100644 index 0000000000..d52ccfffe8 --- /dev/null +++ b/.github/workflows/bench-pr.yml @@ -0,0 +1,42 @@ +name: PR Benchmarks + +on: + pull_request: + types: [ labeled, synchronize ] + branches: [ "develop" ] + workflow_dispatch: { } + +permissions: + actions: write + contents: read + pull-requests: write + +jobs: + bench: + runs-on: ubuntu-latest-large + if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }} + steps: + # We remove the benchmark label first so that the workflow can be re-triggered. + - uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: benchmark + + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-zig + - uses: ./.github/actions/setup-rust + + - name: Bench - Vortex + run: cargo bench | tee bench.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.19.3 + with: + name: Vortex Benchmarks + tool: cargo + github-token: ${{ secrets.GITHUB_TOKEN }} + output-file-path: bench.txt + summary-always: true + auto-push: true + fail-on-alert: false + diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000000..5180858bef --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,35 @@ +name: Benchmarks + +on: + push: + branches: [ "develop" ] + workflow_dispatch: { } + +permissions: + actions: read + contents: write + deployments: write + +jobs: + bench: + runs-on: ubuntu-latest-large + if: ${{ github.event_name == 'workflow_dispatch' || (contains(github.event.head_commit.message, '[benchmark]') && github.ref_name == 'develop') }} + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-zig + - uses: ./.github/actions/setup-rust + + - name: Bench - Vortex + run: cargo bench | tee bench.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.19.3 + with: + name: Vortex Benchmarks + tool: cargo + github-token: ${{ secrets.GITHUB_TOKEN }} + output-file-path: bench.txt + summary-always: true + auto-push: true + fail-on-alert: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..b3eab6a16a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +name: CI + +on: + push: + branches: [ "develop" ] + pull_request: + branches: [ "develop" ] + workflow_dispatch: { } + +permissions: + actions: read + contents: read + +jobs: + build: + name: 'build' + runs-on: ubuntu-latest-medium + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-zig + - uses: ./.github/actions/setup-rust + - uses: ./.github/actions/setup-python + + - name: Python Lint - Format + run: rye run ruff format --check . + - name: Python Lint - Ruff + run: rye run ruff . + + - name: Rust Lint - Format + run: cargo fmt --all --check + - name: Rust Lint - Clippy + run: cargo clippy --all-targets + + - name: Zig Lint - Fmt + run: zig fmt --check zig/ + + - name: Rust Test + run: cargo test --all + + - name: Zig Test + run: zig build test + + - name: Pytest - PyEnc + run: rye run pytest --benchmark-disable test/ + working-directory: pyenc/ + diff --git a/pyenc/test/test_array.py b/pyenc/test/test_array.py index 889e2452e7..5686b4ac0c 100644 --- a/pyenc/test/test_array.py +++ b/pyenc/test/test_array.py @@ -1,7 +1,8 @@ -import enc import pyarrow as pa import pytest +import enc + def test_primitive_array_round_trip(): a = pa.array([0, 1, 2, 3]) diff --git a/pyenc/test/test_compress.py b/pyenc/test/test_compress.py index 2ad4b96c73..fb94d95a6b 100644 --- a/pyenc/test/test_compress.py +++ b/pyenc/test/test_compress.py @@ -1,7 +1,8 @@ -import enc import numpy as np import pyarrow as pa +import enc + def test_primitive_compress(): a = pa.array([0, 0, 0, 0, 9, 9, 9, 9, 1, 5]) diff --git a/pyenc/test/test_serde.py b/pyenc/test/test_serde.py index b6e0d78399..c30a5dd208 100644 --- a/pyenc/test/test_serde.py +++ b/pyenc/test/test_serde.py @@ -1,7 +1,8 @@ -import enc import pyarrow as pa from pyarrow import fs +import enc + local = fs.LocalFileSystem() diff --git a/pyproject.toml b/pyproject.toml index a6515e6a65..cffda950cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,10 +40,12 @@ members = [ [tool.ruff] line-length = 120 +extend-exclude = [".venv"] + +[tool.ruff.lint] select = ["F", "E", "W", "UP", "I"] # Do not auto-fix unused variables. This is really annoying when IntelliJ runs autofix while editing. unfixable = ["F841"] -extend-exclude = [".venv"] [tool.pytest.ini_options] log_cli = true