From 578f67084070eab93cb3d5f311a3735ed1dd7580 Mon Sep 17 00:00:00 2001 From: Congyu Date: Sat, 2 Mar 2024 11:18:56 +0800 Subject: [PATCH 1/4] enable cross-language lto for linux build --- .github/workflows/CI.yml | 34 ++++++++++++++++++++++++++++------ .github/workflows/PR.yml | 34 ++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index cf04287..ad9cc23 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -120,9 +120,17 @@ jobs: target: ${{ matrix.target }} manylinux: 2_28 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' - container: messense/manylinux_2_28-cross:${{ matrix.target }} + container: congyuwang/manylinux_2_28_x86_64:llvm-17.0.6 + env: + LIBCLANG_PATH: /usr/local/lib + CC: /usr/local/bin/clang + CXX: /usr/local/bin/clang++ + AR: /usr/local/bin/llvm-ar + CFLAGS: "-flto=thin -O3" + CXXFLAGS: "-flto=thin -O3" + RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" + - name: Install built wheel and test - if: matrix.target == 'x86_64' run: | pip install rocksdict --no-index --find-links dist --force-reinstall || true pip install speedict --no-index --find-links dist --force-reinstall || true @@ -153,9 +161,16 @@ jobs: target: ${{ matrix.target }} manylinux: 2014 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' - container: congyuwang/manylinux2014:${{ matrix.target }} + container: congyuwang/manylinux2014_x86_64:llvm-17.0.6 + env: + LIBCLANG_PATH: /usr/local/lib + CC: /usr/local/bin/clang + CXX: /usr/local/bin/clang++ + AR: /usr/local/bin/llvm-ar + CFLAGS: "-flto=thin -O3" + CXXFLAGS: "-flto=thin -O3" + RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test - if: matrix.target == 'x86_64' run: | pip install rocksdict --no-index --find-links dist --force-reinstall || true pip install speedict --no-index --find-links dist --force-reinstall || true @@ -185,9 +200,16 @@ jobs: target: ${{ matrix.target }} manylinux: 2_28 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' - container: congyuwang/manylinux_2_28-cross:${{ matrix.target }} + container: congyuwang/manylinux_2_28_aarch64:llvm-17.0.6 + env: + LIBCLANG_PATH: /usr/local/lib + CC: /usr/local/bin/clang + CXX: /usr/local/bin/clang++ + AR: /usr/local/bin/llvm-ar + CFLAGS: "-flto=thin -O3" + CXXFLAGS: "-flto=thin -O3" + RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - uses: uraimo/run-on-arch-action@v2 - if: matrix.target != 'ppc64' name: Install built wheel and test with: arch: ${{ matrix.target }} diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index a67bffc..2b51105 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -120,9 +120,17 @@ jobs: target: ${{ matrix.target }} manylinux: 2_28 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' - container: messense/manylinux_2_28-cross:${{ matrix.target }} + container: congyuwang/manylinux_2_28_x86_64:llvm-17.0.6 + env: + LIBCLANG_PATH: /usr/local/lib + CC: /usr/local/bin/clang + CXX: /usr/local/bin/clang++ + AR: /usr/local/bin/llvm-ar + CFLAGS: "-flto=thin -O3" + CXXFLAGS: "-flto=thin -O3" + RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" + - name: Install built wheel and test - if: matrix.target == 'x86_64' run: | pip install rocksdict --no-index --find-links dist --force-reinstall || true pip install speedict --no-index --find-links dist --force-reinstall || true @@ -153,9 +161,16 @@ jobs: target: ${{ matrix.target }} manylinux: 2014 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' - container: congyuwang/manylinux2014:${{ matrix.target }} + container: congyuwang/manylinux2014_x86_64:llvm-17.0.6 + env: + LIBCLANG_PATH: /usr/local/lib + CC: /usr/local/bin/clang + CXX: /usr/local/bin/clang++ + AR: /usr/local/bin/llvm-ar + CFLAGS: "-flto=thin -O3" + CXXFLAGS: "-flto=thin -O3" + RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test - if: matrix.target == 'x86_64' run: | pip install rocksdict --no-index --find-links dist --force-reinstall || true pip install speedict --no-index --find-links dist --force-reinstall || true @@ -185,9 +200,16 @@ jobs: target: ${{ matrix.target }} manylinux: 2_28 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' - container: congyuwang/manylinux_2_28-cross:${{ matrix.target }} + container: congyuwang/manylinux_2_28_aarch64:llvm-17.0.6 + env: + LIBCLANG_PATH: /usr/local/lib + CC: /usr/local/bin/clang + CXX: /usr/local/bin/clang++ + AR: /usr/local/bin/llvm-ar + CFLAGS: "-flto=thin -O3" + CXXFLAGS: "-flto=thin -O3" + RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - uses: uraimo/run-on-arch-action@v2 - if: matrix.target != 'ppc64' name: Install built wheel and test with: arch: ${{ matrix.target }} From 319bb88a98c0d7bc83fefe26214c1137a3a359f8 Mon Sep 17 00:00:00 2001 From: Congyu Date: Sat, 2 Mar 2024 11:37:00 +0800 Subject: [PATCH 2/4] use libc++ for clang --- .github/workflows/CI.yml | 4 ++++ .github/workflows/PR.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ad9cc23..9f93231 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -36,6 +36,7 @@ jobs: export AR=${HOMEBREW_PREFIX}/opt/llvm/bin/llvm-ar export CFLAGS="-flto=thin -O3" export CXXFLAGS="-flto=thin -O3" + export CXXSTDLIB="c++" export LLD=${HOMEBREW_PREFIX}/opt/llvm/bin/ld64.lld export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}" maturin build --release --target aarch64-apple-darwin --out dist -i python${{ matrix.py_version }} @@ -128,6 +129,7 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" + CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test @@ -169,6 +171,7 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" + CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test run: | @@ -208,6 +211,7 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" + CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - uses: uraimo/run-on-arch-action@v2 name: Install built wheel and test diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 2b51105..6870545 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -36,6 +36,7 @@ jobs: export AR=${HOMEBREW_PREFIX}/opt/llvm/bin/llvm-ar export CFLAGS="-flto=thin -O3" export CXXFLAGS="-flto=thin -O3" + export CXXSTDLIB="c++" export LLD=${HOMEBREW_PREFIX}/opt/llvm/bin/ld64.lld export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}" maturin build --release --target aarch64-apple-darwin --out dist -i python${{ matrix.py_version }} @@ -128,6 +129,7 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" + CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test @@ -169,6 +171,7 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" + CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test run: | @@ -208,6 +211,7 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" + CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - uses: uraimo/run-on-arch-action@v2 name: Install built wheel and test From 45a40784eacb0977d274aec59e44e4e319e7f3c3 Mon Sep 17 00:00:00 2001 From: Congyu Date: Sat, 2 Mar 2024 11:45:42 +0800 Subject: [PATCH 3/4] Revert "use libc++ for clang" This reverts commit 319bb88a98c0d7bc83fefe26214c1137a3a359f8. --- .github/workflows/CI.yml | 4 ---- .github/workflows/PR.yml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9f93231..ad9cc23 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -36,7 +36,6 @@ jobs: export AR=${HOMEBREW_PREFIX}/opt/llvm/bin/llvm-ar export CFLAGS="-flto=thin -O3" export CXXFLAGS="-flto=thin -O3" - export CXXSTDLIB="c++" export LLD=${HOMEBREW_PREFIX}/opt/llvm/bin/ld64.lld export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}" maturin build --release --target aarch64-apple-darwin --out dist -i python${{ matrix.py_version }} @@ -129,7 +128,6 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" - CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test @@ -171,7 +169,6 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" - CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test run: | @@ -211,7 +208,6 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" - CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - uses: uraimo/run-on-arch-action@v2 name: Install built wheel and test diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 6870545..2b51105 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -36,7 +36,6 @@ jobs: export AR=${HOMEBREW_PREFIX}/opt/llvm/bin/llvm-ar export CFLAGS="-flto=thin -O3" export CXXFLAGS="-flto=thin -O3" - export CXXSTDLIB="c++" export LLD=${HOMEBREW_PREFIX}/opt/llvm/bin/ld64.lld export RUSTFLAGS="-Clinker-plugin-lto -Clinker=$PWD/macos-linker.sh -Clink-arg=-fuse-ld=${LLD}" maturin build --release --target aarch64-apple-darwin --out dist -i python${{ matrix.py_version }} @@ -129,7 +128,6 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" - CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test @@ -171,7 +169,6 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" - CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - name: Install built wheel and test run: | @@ -211,7 +208,6 @@ jobs: AR: /usr/local/bin/llvm-ar CFLAGS: "-flto=thin -O3" CXXFLAGS: "-flto=thin -O3" - CXXSTDLIB: "c++" RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" - uses: uraimo/run-on-arch-action@v2 name: Install built wheel and test From 6caa3866924d08889515994508cf5455ed6bc4c8 Mon Sep 17 00:00:00 2001 From: Congyu Date: Sat, 2 Mar 2024 13:15:21 +0800 Subject: [PATCH 4/4] fix linux-cross CI --- .github/workflows/CI.yml | 28 ++++++++++++++++++++++------ .github/workflows/PR.yml | 28 ++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ad9cc23..d749157 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -201,14 +201,30 @@ jobs: manylinux: 2_28 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' container: congyuwang/manylinux_2_28_aarch64:llvm-17.0.6 + docker-options: "-e BINDGEN_EXTRA_CLANG_ARGS" + before-script-linux: | + ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/7.5.0/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o + ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/7.5.0/crtendS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtendS.o env: LIBCLANG_PATH: /usr/local/lib - CC: /usr/local/bin/clang - CXX: /usr/local/bin/clang++ - AR: /usr/local/bin/llvm-ar - CFLAGS: "-flto=thin -O3" - CXXFLAGS: "-flto=thin -O3" - RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" + CC_aarch64_unknown_linux_gnu: /usr/local/bin/clang + CXX_aarch64_unknown_linux_gnu: /usr/local/bin/clang++ + AR_aarch64_unknown_linux_gnu: /usr/local/bin/llvm-ar + CFLAGS_aarch64_unknown_linux_gnu: > + -flto=thin -O3 --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ + CXXFLAGS_aarch64_unknown_linux_gnu: > + -flto=thin -O3 --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ + -I /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/include/c++/7.5.0/ + -I /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/include/c++/7.5.0/aarch64-unknown-linux-gnu/ + BINDGEN_EXTRA_CLANG_ARGS: > + --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: > + /usr/local/bin/clang + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS: > + -Clinker-plugin-lto + -Clinker=/usr/local/bin/clang + -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld + -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ - uses: uraimo/run-on-arch-action@v2 name: Install built wheel and test with: diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 2b51105..56312db 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -201,14 +201,30 @@ jobs: manylinux: 2_28 args: --release --out dist --interpreter '3.7 3.8 3.9 3.10 3.11 3.12' container: congyuwang/manylinux_2_28_aarch64:llvm-17.0.6 + docker-options: "-e BINDGEN_EXTRA_CLANG_ARGS" + before-script-linux: | + ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/7.5.0/crtbeginS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtbeginS.o + ln -s /usr/aarch64-unknown-linux-gnu/lib/gcc/aarch64-unknown-linux-gnu/7.5.0/crtendS.o /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/usr/lib/crtendS.o env: LIBCLANG_PATH: /usr/local/lib - CC: /usr/local/bin/clang - CXX: /usr/local/bin/clang++ - AR: /usr/local/bin/llvm-ar - CFLAGS: "-flto=thin -O3" - CXXFLAGS: "-flto=thin -O3" - RUSTFLAGS: "-Clinker-plugin-lto -Clinker=/usr/local/bin/clang -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld" + CC_aarch64_unknown_linux_gnu: /usr/local/bin/clang + CXX_aarch64_unknown_linux_gnu: /usr/local/bin/clang++ + AR_aarch64_unknown_linux_gnu: /usr/local/bin/llvm-ar + CFLAGS_aarch64_unknown_linux_gnu: > + -flto=thin -O3 --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ + CXXFLAGS_aarch64_unknown_linux_gnu: > + -flto=thin -O3 --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ + -I /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/include/c++/7.5.0/ + -I /usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/include/c++/7.5.0/aarch64-unknown-linux-gnu/ + BINDGEN_EXTRA_CLANG_ARGS: > + --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: > + /usr/local/bin/clang + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS: > + -Clinker-plugin-lto + -Clinker=/usr/local/bin/clang + -Clink-arg=-fuse-ld=/usr/local/bin/ld.lld + -Clink-arg=--sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot/ - uses: uraimo/run-on-arch-action@v2 name: Install built wheel and test with: