From 95233e2a53fe422ba969d519974142d5bb18c0ee Mon Sep 17 00:00:00 2001 From: Matt Green Date: Tue, 29 Oct 2024 16:24:15 -0700 Subject: [PATCH] Add maturin build step for ci (#52) --- .github/workflows/python.yml | 181 ++++++++++++++++++ Cargo.lock | 141 +++----------- Cargo.toml | 6 +- crates/core/Cargo.toml | 7 +- crates/core/src/state_backend/mod.rs | 2 +- .../python/examples/stream_aggregate.py | 5 +- py-denormalized/src/lib.rs | 2 +- 7 files changed, 224 insertions(+), 120 deletions(-) create mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..898cba6 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,181 @@ +# This file is autogenerated by maturin v1.7.4 +# To update, run +# +# maturin generate-ci -m py-denormalized/Cargo.toml github +# +name: Python + +on: + push: + branches: + - main + - master + tags: + - "*" + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: aarch64 + - runner: ubuntu-latest + target: s390x + - runner: ubuntu-latest + target: ppc64le + steps: + # - name: Run sccache-cache + # uses: mozilla-actions/sccache-action@v0.0.6 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --zig --out dist --manifest-path py-denormalized/Cargo.toml + # sccache: 'true' + manylinux: 2_28 + # - name: Upload wheels + # uses: actions/upload-artifact@v4 + # with: + # name: wheels-linux-${{ matrix.platform.target }} + # path: dist + + musllinux: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64 + - runner: ubuntu-latest + target: aarch64 + # - runner: ubuntu-latest + # target: armv7 + steps: + # - name: Run sccache-cache + # uses: mozilla-actions/sccache-action@v0.0.6 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --zig --out dist --manifest-path py-denormalized/Cargo.toml + # sccache: 'true' + manylinux: musllinux_1_2 + rust-toolchain: 1.81.0 + + # - name: Upload wheels + # uses: actions/upload-artifact@v4 + # with: + # name: wheels-musllinux-${{ matrix.platform.target }} + # path: dist + + # windows: + # runs-on: ${{ matrix.platform.runner }} + # strategy: + # matrix: + # platform: + # - runner: windows-latest + # target: x64 + # - runner: windows-latest + # target: x86 + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-python@v5 + # with: + # python-version: 3.x + # architecture: ${{ matrix.platform.target }} + # - name: Build wheels + # uses: PyO3/maturin-action@v1 + # with: + # target: ${{ matrix.platform.target }} + # args: --release --out dist --manifest-path py-denormalized/Cargo.toml + # # sccache: 'true' + # # - name: Upload wheels + # # uses: actions/upload-artifact@v4 + # # with: + # # name: wheels-windows-${{ matrix.platform.target }} + # # path: dist + + macos: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - runner: macos-12 + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --manifest-path py-denormalized/Cargo.toml + # sccache: 'true' + # - name: Upload wheels + # uses: actions/upload-artifact@v4 + # with: + # name: wheels-macos-${{ matrix.platform.target }} + # path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist --manifest-path py-denormalized/Cargo.toml + # - name: Upload sdist + # uses: actions/upload-artifact@v4 + # with: + # name: wheels-sdist + # path: dist + + # release: + # name: Release + # runs-on: ubuntu-latest + # if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + # needs: [linux, musllinux, windows, macos, sdist] + # permissions: + # # Use to sign the release artifacts + # id-token: write + # # Used to upload release artifacts + # contents: write + # # Used to generate artifact attestation + # attestations: write + # steps: + # - uses: actions/download-artifact@v4 + # - name: Generate artifact attestation + # uses: actions/attest-build-provenance@v1 + # with: + # subject-path: 'wheels-*/*' + # - name: Publish to PyPI + # if: "startsWith(github.ref, 'refs/tags/')" + # uses: PyO3/maturin-action@v1 + # env: + # MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + # with: + # command: upload + # args: --non-interactive --skip-existing wheels-*/* diff --git a/Cargo.lock b/Cargo.lock index a62eb7c..ab6bf26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,26 +612,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.6.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.79", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -752,15 +732,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -803,17 +774,6 @@ dependencies = [ "phf_codegen", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.5.20" @@ -854,6 +814,15 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "cmsketch" version = "0.2.1" @@ -1203,7 +1172,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.6.0", - "itertools 0.13.0", + "itertools", "log", "num-traits", "num_cpus", @@ -1338,7 +1307,7 @@ dependencies = [ "datafusion-expr", "hashbrown 0.14.5", "hex", - "itertools 0.13.0", + "itertools", "log", "md-5", "rand", @@ -1397,7 +1366,7 @@ dependencies = [ "datafusion-functions", "datafusion-functions-aggregate", "datafusion-physical-expr-common", - "itertools 0.13.0", + "itertools", "log", "paste", "rand", @@ -1427,7 +1396,7 @@ dependencies = [ "datafusion-physical-expr", "hashbrown 0.14.5", "indexmap 2.6.0", - "itertools 0.13.0", + "itertools", "log", "paste", "regex-syntax 0.8.5", @@ -1457,7 +1426,7 @@ dependencies = [ "hashbrown 0.14.5", "hex", "indexmap 2.6.0", - "itertools 0.13.0", + "itertools", "log", "paste", "petgraph", @@ -1487,7 +1456,7 @@ dependencies = [ "datafusion-execution", "datafusion-physical-expr", "datafusion-physical-plan", - "itertools 0.13.0", + "itertools", ] [[package]] @@ -1515,7 +1484,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap 2.6.0", - "itertools 0.13.0", + "itertools", "log", "once_cell", "parking_lot", @@ -1591,6 +1560,7 @@ dependencies = [ "bincode", "bytes", "chrono", + "cmake", "crossbeam", "datafusion", "delegate", @@ -1600,11 +1570,10 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "itertools 0.13.0", + "itertools", "log", "object_store", "rdkafka", - "rocksdb", "serde", "serde_json", "slatedb", @@ -1896,7 +1865,7 @@ dependencies = [ "fastrace", "futures", "hashbrown 0.14.5", - "itertools 0.13.0", + "itertools", "madsim-tokio", "metrics", "parking_lot", @@ -1911,7 +1880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01dbfd6763227809019a1dc01c98b6a78949c63d3a204d0a6f8e0325f077ab5c" dependencies = [ "foyer-common", - "itertools 0.13.0", + "itertools", ] [[package]] @@ -1928,7 +1897,7 @@ dependencies = [ "foyer-intrusive", "futures", "hashbrown 0.14.5", - "itertools 0.13.0", + "itertools", "madsim-tokio", "parking_lot", "pin-project", @@ -1959,7 +1928,7 @@ dependencies = [ "foyer-memory", "fs4", "futures", - "itertools 0.13.0", + "itertools", "libc", "lz4", "madsim-tokio", @@ -2414,15 +2383,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2462,12 +2422,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "lexical-core" version = "1.0.2" @@ -2562,16 +2516,6 @@ dependencies = [ "rle-decode-fast", ] -[[package]] -name = "libloading" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" -dependencies = [ - "cfg-if", - "windows-targets", -] - [[package]] name = "libm" version = "0.2.8" @@ -2588,22 +2532,6 @@ dependencies = [ "libc", ] -[[package]] -name = "librocksdb-sys" -version = "0.16.0+8.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" -dependencies = [ - "bindgen", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "lz4-sys", - "zstd-sys", -] - [[package]] name = "libz-sys" version = "1.1.20" @@ -3019,7 +2947,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.13.0", + "itertools", "md-5", "parking_lot", "percent-encoding", @@ -3315,7 +3243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools", "proc-macro2", "quote", "syn 2.0.79", @@ -3434,7 +3362,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "socket2", "thiserror", @@ -3451,7 +3379,7 @@ dependencies = [ "bytes", "rand", "ring", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "slab", "thiserror", @@ -3562,6 +3490,7 @@ version = "4.7.0+2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ + "cmake", "libc", "libz-sys", "num_enum", @@ -3693,16 +3622,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" -[[package]] -name = "rocksdb" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" -dependencies = [ - "libc", - "librocksdb-sys", -] - [[package]] name = "rtrb" version = "0.3.1" @@ -3715,12 +3634,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 4b6da32..f3900f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,10 @@ [workspace] resolver = "2" -members = ["crates/*", "py-denormalized", "examples"] +members = [ + "crates/*", + "examples", + "py-denormalized", +] [workspace.package] authors = [ diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 9bdbd30..ce4d5c6 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -21,7 +21,7 @@ arrow-ord = { workspace = true } apache-avro = { workspace = true } base64 = { workspace = true } -rdkafka = { workspace = true } +rdkafka = { workspace = true, features = ["cmake-build"] } futures = { workspace = true } async-trait = { workspace = true } tracing = { workspace = true } @@ -31,7 +31,7 @@ log = { workspace = true } chrono = { workspace = true } itertools = { workspace = true } serde.workspace = true -rocksdb = "0.22.0" +# rocksdb = "0.22.0" bincode = "1.3.3" half = "2.4.1" delegate = "0.12.0" @@ -42,3 +42,6 @@ crossbeam = "0.8.4" slatedb = { workspace = true } # "0.2.0" object_store = "0.11.0" bytes = "1.7.2" + +[build-dependencies] +cmake = "0.1" diff --git a/crates/core/src/state_backend/mod.rs b/crates/core/src/state_backend/mod.rs index e64865d..90aba83 100644 --- a/crates/core/src/state_backend/mod.rs +++ b/crates/core/src/state_backend/mod.rs @@ -1,2 +1,2 @@ -pub mod rocksdb_backend; +// pub mod rocksdb_backend; pub mod slatedb; diff --git a/py-denormalized/python/examples/stream_aggregate.py b/py-denormalized/python/examples/stream_aggregate.py index 7c1c548..99c769d 100644 --- a/py-denormalized/python/examples/stream_aggregate.py +++ b/py-denormalized/python/examples/stream_aggregate.py @@ -3,6 +3,7 @@ import json import signal import sys +import pprint as pp from denormalized import Context from denormalized.datafusion import col @@ -27,7 +28,7 @@ def signal_handler(sig, frame): def print_batch(rb): - print(rb) + pp.pprint(rb.to_pydict()) ctx = Context() @@ -41,6 +42,8 @@ def print_batch(rb): f.min(col("reading")).alias("min"), f.max(col("reading")).alias("max"), f.avg(col("reading")).alias("average"), + f.median(col("reading")).alias("median"), + f.stddev(col("reading")).alias("stddev"), ], 1000, None, diff --git a/py-denormalized/src/lib.rs b/py-denormalized/src/lib.rs index fd92024..a040639 100644 --- a/py-denormalized/src/lib.rs +++ b/py-denormalized/src/lib.rs @@ -12,7 +12,7 @@ pub(crate) struct TokioRuntime(tokio::runtime::Runtime); /// A Python module implemented in Rust. #[pymodule] -fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { +fn _py_denormalized_internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?;