From 392a5049f0ba33c3009b266cfff67cdb4197efd9 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 14 Oct 2024 14:48:00 +0200 Subject: [PATCH 01/31] Cross compile aarch64 inside manylinux container --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 22c722085ce..3df145fdaae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,6 +60,7 @@ jobs: os: ubuntu-24.04 cmake_args: -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$PWD/cmake/arm64_linux_toolchain.cmake triplet: arm64-linux-release + manylinux: quay.io/pypa/manylinux2014_x86_64 - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 From 52f8fd485eaf2e9a2ed2da5c7b7d7ded33d5689f Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 14 Oct 2024 15:03:14 +0200 Subject: [PATCH 02/31] Do not use sudo --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3df145fdaae..d3384fc7f28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,7 +102,7 @@ jobs: - name: Install aarch64 cross compilation prerequisites if: ${{ matrix.platform == 'linux-aarch64' }} run: | - sudo apt-get install -y cmake gcc g++ ninja-build libtool autoconf pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu curl zip unzip tar + apt-get install -y cmake gcc g++ ninja-build libtool autoconf pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic - name: Install manylinux prerequisites From 208cc05895b82e1b5f6470487b9d17b7eba0f4eb Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 14 Oct 2024 15:13:26 +0200 Subject: [PATCH 03/31] Install aarch64 deps using yum --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d3384fc7f28..030c90e3e30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,11 +102,11 @@ jobs: - name: Install aarch64 cross compilation prerequisites if: ${{ matrix.platform == 'linux-aarch64' }} run: | - apt-get install -y cmake gcc g++ ninja-build libtool autoconf pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu curl zip unzip tar + yum install -y cmake devtoolset-7 ninja-build libtool autoconf gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic - name: Install manylinux prerequisites - if: matrix.manylinux + if: ${{ startsWith(matrix.platform, 'linux-x86_64') == true }} run: | set -e pipefail yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd From 1cec4bc3115ec72303b92e92c05ea262a1d708ee Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 14 Oct 2024 16:27:34 +0200 Subject: [PATCH 04/31] Install glibc --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 030c90e3e30..1c6fb06d44b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,7 +102,7 @@ jobs: - name: Install aarch64 cross compilation prerequisites if: ${{ matrix.platform == 'linux-aarch64' }} run: | - yum install -y cmake devtoolset-7 ninja-build libtool autoconf gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu curl zip unzip tar + yum install -y cmake devtoolset-7 ninja-build libtool autoconf gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu glibc-devel.i686 curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic - name: Install manylinux prerequisites From b551e056679936d851b35e35010de79377e0a59c Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 12:53:33 +0200 Subject: [PATCH 05/31] Use newer version of manylinux container --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1c6fb06d44b..1a21a5b3f2a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,7 +60,7 @@ jobs: os: ubuntu-24.04 cmake_args: -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$PWD/cmake/arm64_linux_toolchain.cmake triplet: arm64-linux-release - manylinux: quay.io/pypa/manylinux2014_x86_64 + manylinux: quay.io/pypa/manylinux_2_28_x86_64 - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 From 0bc21be87f2ac758ae79026fa9de348d7f1d671a Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 13:37:24 +0200 Subject: [PATCH 06/31] Use manylinux-cross --- .github/workflows/release.yml | 2 +- cmake/arm64_linux_toolchain.cmake | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a21a5b3f2a..7a6210c53e0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,7 +60,7 @@ jobs: os: ubuntu-24.04 cmake_args: -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$PWD/cmake/arm64_linux_toolchain.cmake triplet: arm64-linux-release - manylinux: quay.io/pypa/manylinux_2_28_x86_64 + manylinux: "messense/manylinux_2_28-cross:aarch64-amd64" - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake index d09bc783c8b..735ee57673a 100644 --- a/cmake/arm64_linux_toolchain.cmake +++ b/cmake/arm64_linux_toolchain.cmake @@ -5,6 +5,17 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(TARGET_CC $ENV{TARGET_CC}) +set(TARGET_CXX $ENV{TARGET_CXX}) + +if (NOT TARGET_CC) + set(TARGET_CC "aarch64-linux-gnu-gcc") +endif() + +if (NOT TARGET_CXX) + set(TARGET_CXX "aarch64-linux-gnu-g++") +endif() + # Name of C compiler. -set(CMAKE_C_COMPILER "aarch64-linux-gnu-gcc") -set(CMAKE_CXX_COMPILER "aarch64-linux-gnu-g++") +set(CMAKE_C_COMPILER "${TARGET_CC}") +set(CMAKE_CXX_COMPILER "${TARGET_CXX}") From d200f926906a622dfb798d828d2292e667ab748f Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 13:40:50 +0200 Subject: [PATCH 07/31] Install dependencies from apt --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a6210c53e0..6e3881bc463 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,7 +102,7 @@ jobs: - name: Install aarch64 cross compilation prerequisites if: ${{ matrix.platform == 'linux-aarch64' }} run: | - yum install -y cmake devtoolset-7 ninja-build libtool autoconf gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu glibc-devel.i686 curl zip unzip tar + apt-get install -y cmake ninja-build libtool autoconf pkg-config curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic - name: Install manylinux prerequisites From 95f3deede53b4807588c05afaf6b7f4b51797ad1 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 13:43:43 +0200 Subject: [PATCH 08/31] Apt update --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e3881bc463..91189c48e13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,6 +102,7 @@ jobs: - name: Install aarch64 cross compilation prerequisites if: ${{ matrix.platform == 'linux-aarch64' }} run: | + apt-get update apt-get install -y cmake ninja-build libtool autoconf pkg-config curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic From ce5e6174d07d846f6d169561445cec8ae1ace993 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 14:05:00 +0200 Subject: [PATCH 09/31] Use x86_64 toolchain image --- .github/workflows/release.yml | 4 ++-- cmake/arm64_linux_toolchain.cmake | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91189c48e13..1f901aedc29 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,7 +60,7 @@ jobs: os: ubuntu-24.04 cmake_args: -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$PWD/cmake/arm64_linux_toolchain.cmake triplet: arm64-linux-release - manylinux: "messense/manylinux_2_28-cross:aarch64-amd64" + manylinux: "messense/manylinux_2_28-cross:x86_64" - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 @@ -103,7 +103,7 @@ jobs: if: ${{ matrix.platform == 'linux-aarch64' }} run: | apt-get update - apt-get install -y cmake ninja-build libtool autoconf pkg-config curl zip unzip tar + apt-get install -y cmake gcc g++ ninja-build libtool autoconf pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic - name: Install manylinux prerequisites diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake index 735ee57673a..f965419a3a0 100644 --- a/cmake/arm64_linux_toolchain.cmake +++ b/cmake/arm64_linux_toolchain.cmake @@ -5,8 +5,8 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) -set(TARGET_CC $ENV{TARGET_CC}) -set(TARGET_CXX $ENV{TARGET_CXX}) +# set(TARGET_CC $ENV{TARGET_CC}) +# set(TARGET_CXX $ENV{TARGET_CXX}) if (NOT TARGET_CC) set(TARGET_CC "aarch64-linux-gnu-gcc") From 01eee2bfc6990cb68b2edbaa7d2dfedefaaea601 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 15:18:02 +0200 Subject: [PATCH 10/31] Try out aarch64-amd64 manylinux --- .github/workflows/release.yml | 4 ++-- cmake/arm64_linux_toolchain.cmake | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f901aedc29..91189c48e13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,7 +60,7 @@ jobs: os: ubuntu-24.04 cmake_args: -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$PWD/cmake/arm64_linux_toolchain.cmake triplet: arm64-linux-release - manylinux: "messense/manylinux_2_28-cross:x86_64" + manylinux: "messense/manylinux_2_28-cross:aarch64-amd64" - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 @@ -103,7 +103,7 @@ jobs: if: ${{ matrix.platform == 'linux-aarch64' }} run: | apt-get update - apt-get install -y cmake gcc g++ ninja-build libtool autoconf pkg-config gcc-aarch64-linux-gnu g++-aarch64-linux-gnu curl zip unzip tar + apt-get install -y cmake ninja-build libtool autoconf pkg-config curl zip unzip tar # Use upstream libmagic port which supports cross-compilation. rm -rf ports/libmagic - name: Install manylinux prerequisites diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake index f965419a3a0..735ee57673a 100644 --- a/cmake/arm64_linux_toolchain.cmake +++ b/cmake/arm64_linux_toolchain.cmake @@ -5,8 +5,8 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) -# set(TARGET_CC $ENV{TARGET_CC}) -# set(TARGET_CXX $ENV{TARGET_CXX}) +set(TARGET_CC $ENV{TARGET_CC}) +set(TARGET_CXX $ENV{TARGET_CXX}) if (NOT TARGET_CC) set(TARGET_CC "aarch64-linux-gnu-gcc") From 3619bc18dce8d9c32537eb6c0599abd420e0f830 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 15:43:27 +0200 Subject: [PATCH 11/31] Do not source error log print --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91189c48e13..56f74d55096 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -142,7 +142,7 @@ jobs: build/tiledb-*.zip* - name: "Print log files (failed build only)" run: | - source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh + $GITHUB_WORKSPACE/scripts/ci/print_logs.sh if: failure() # only run this job if the build step failed Test-Release-Artifacts: From 6df111e8e5de559c09ef173e0af91f0e3bf3e803 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 15:46:00 +0200 Subject: [PATCH 12/31] Use sh instead of source --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56f74d55096..93a035b4e53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -142,7 +142,7 @@ jobs: build/tiledb-*.zip* - name: "Print log files (failed build only)" run: | - $GITHUB_WORKSPACE/scripts/ci/print_logs.sh + sh $GITHUB_WORKSPACE/scripts/ci/print_logs.sh if: failure() # only run this job if the build step failed Test-Release-Artifacts: From bca7be5a91d1aaf0020eecccba7db4e08fd189ec Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 15:51:19 +0200 Subject: [PATCH 13/31] TEMP: Change default compiler values --- cmake/arm64_linux_toolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake index 735ee57673a..b19ec080046 100644 --- a/cmake/arm64_linux_toolchain.cmake +++ b/cmake/arm64_linux_toolchain.cmake @@ -9,11 +9,11 @@ set(TARGET_CC $ENV{TARGET_CC}) set(TARGET_CXX $ENV{TARGET_CXX}) if (NOT TARGET_CC) - set(TARGET_CC "aarch64-linux-gnu-gcc") + set(TARGET_CC "aarch64-unknown-linux-gnu-gcc") endif() if (NOT TARGET_CXX) - set(TARGET_CXX "aarch64-linux-gnu-g++") + set(TARGET_CXX "aarch64-unknown-linux-gnu-g++") endif() # Name of C compiler. From 6c750b7143ab91ced576bb9b89634a7d9f43c11b Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Wed, 16 Oct 2024 15:55:22 +0200 Subject: [PATCH 14/31] TEMP: Set hardcoded compiler --- cmake/arm64_linux_toolchain.cmake | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake index b19ec080046..02ab153134d 100644 --- a/cmake/arm64_linux_toolchain.cmake +++ b/cmake/arm64_linux_toolchain.cmake @@ -5,17 +5,6 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) -set(TARGET_CC $ENV{TARGET_CC}) -set(TARGET_CXX $ENV{TARGET_CXX}) - -if (NOT TARGET_CC) - set(TARGET_CC "aarch64-unknown-linux-gnu-gcc") -endif() - -if (NOT TARGET_CXX) - set(TARGET_CXX "aarch64-unknown-linux-gnu-g++") -endif() - # Name of C compiler. -set(CMAKE_C_COMPILER "${TARGET_CC}") -set(CMAKE_CXX_COMPILER "${TARGET_CXX}") +set(CMAKE_C_COMPILER "aarch64-unknown-linux-gnu-gcc") +set(CMAKE_CXX_COMPILER "aarch64-unknown-linux-gnu-g++") From 2af917fe93ddbc619639f85f1da9b7d81b513691 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Thu, 17 Oct 2024 14:34:37 +0200 Subject: [PATCH 15/31] Use native arm64 runner --- .github/workflows/release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93a035b4e53..d7a72c38e3b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,9 +58,9 @@ jobs: manylinux: quay.io/pypa/manylinux2014_x86_64 - platform: linux-aarch64 os: ubuntu-24.04 - cmake_args: -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$PWD/cmake/arm64_linux_toolchain.cmake triplet: arm64-linux-release - manylinux: "messense/manylinux_2_28-cross:aarch64-amd64" + manylinux: quay.io/pypa/manylinux2014_aarch64 + runner_name: linux-arm64-ubuntu24 - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 @@ -72,6 +72,7 @@ jobs: triplet: arm64-osx-release runs-on: ${{ matrix.os }} container: ${{ matrix.manylinux || '' }} + runner-name: ${{ matrix.runner_name || '' }} env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.MACOSX_DEPLOYMENT_TARGET }} VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' @@ -102,10 +103,9 @@ jobs: - name: Install aarch64 cross compilation prerequisites if: ${{ matrix.platform == 'linux-aarch64' }} run: | - apt-get update - apt-get install -y cmake ninja-build libtool autoconf pkg-config curl zip unzip tar - # Use upstream libmagic port which supports cross-compilation. - rm -rf ports/libmagic + set -e pipefail + yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd + echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc - name: Install manylinux prerequisites if: ${{ startsWith(matrix.platform, 'linux-x86_64') == true }} run: | From 632a1dae69f7b0587ad255b5988f601b29752e2b Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Thu, 17 Oct 2024 14:43:43 +0200 Subject: [PATCH 16/31] Fix runs-on --- .github/workflows/release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7a72c38e3b..127fde875b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,10 +57,9 @@ jobs: triplet: x64-linux-release manylinux: quay.io/pypa/manylinux2014_x86_64 - platform: linux-aarch64 - os: ubuntu-24.04 + os: linux-arm64-ubuntu24 triplet: arm64-linux-release manylinux: quay.io/pypa/manylinux2014_aarch64 - runner_name: linux-arm64-ubuntu24 - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 @@ -72,7 +71,6 @@ jobs: triplet: arm64-osx-release runs-on: ${{ matrix.os }} container: ${{ matrix.manylinux || '' }} - runner-name: ${{ matrix.runner_name || '' }} env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.MACOSX_DEPLOYMENT_TARGET }} VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' From 022783ec8bb54445fb130b6208501cd9e95f4959 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Fri, 18 Oct 2024 11:20:33 +0200 Subject: [PATCH 17/31] Modify release CI to run in PR --- .github/workflows/release.yml | 125 +++++++++++++++++----------------- 1 file changed, 63 insertions(+), 62 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 127fde875b0..bf555ff9038 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,6 +2,7 @@ name: Release on: workflow_dispatch: + pull_request: push: branches: - dev @@ -160,65 +161,65 @@ jobs: exit 1 fi - Publish-Release: - needs: - - Test-Release-Artifacts - - Package-Source-Release - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - steps: - - name: Download release artifacts - uses: actions/download-artifact@v3 - with: - name: release - path: dist - - name: Publish release artifacts - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const repo = context.repo; - const release = await github.rest.repos.getReleaseByTag({ - owner: repo.owner, - repo: repo.repo, - tag: "${{ github.ref_name }}" - }); - const globber = await glob.create('dist/*'); - for await (const file of globber.globGenerator()) { - await github.rest.repos.uploadReleaseAsset({ - owner: repo.owner, - repo: repo.repo, - release_id: release.data.id, - headers: { - 'content-type': 'application/octet-stream', - 'content-length': fs.statSync(file).size - }, - name: path.basename(file), - data: fs.readFileSync(file) - }); - } - - Generate-Release-List: - needs: - - Publish-Release - uses: ./.github/workflows/append-release-cmake.yml - with: - ref: ${{ github.ref_name }} - - Create-Issue-On-Fail: - permissions: - issues: write - runs-on: ubuntu-latest - needs: Publish-Release - if: (failure() || cancelled()) && github.event_name == 'push' - steps: - # The open-issue action requires to clone the repository. - - name: Checkout TileDB - uses: actions/checkout@v4 - - name: Create Issue if Build Fails - uses: TileDB-Inc/github-actions/open-issue@main - with: - name: Release failed - label: release - assignee: KiterLuc,teo-tsirpanis,davisp + # Publish-Release: + # needs: + # - Test-Release-Artifacts + # - Package-Source-Release + # if: startsWith(github.ref, 'refs/tags/') + # runs-on: ubuntu-latest + # steps: + # - name: Download release artifacts + # uses: actions/download-artifact@v3 + # with: + # name: release + # path: dist + # - name: Publish release artifacts + # uses: actions/github-script@v6 + # with: + # script: | + # const fs = require('fs'); + # const path = require('path'); + # const repo = context.repo; + # const release = await github.rest.repos.getReleaseByTag({ + # owner: repo.owner, + # repo: repo.repo, + # tag: "${{ github.ref_name }}" + # }); + # const globber = await glob.create('dist/*'); + # for await (const file of globber.globGenerator()) { + # await github.rest.repos.uploadReleaseAsset({ + # owner: repo.owner, + # repo: repo.repo, + # release_id: release.data.id, + # headers: { + # 'content-type': 'application/octet-stream', + # 'content-length': fs.statSync(file).size + # }, + # name: path.basename(file), + # data: fs.readFileSync(file) + # }); + # } + # + # Generate-Release-List: + # needs: + # - Publish-Release + # uses: ./.github/workflows/append-release-cmake.yml + # with: + # ref: ${{ github.ref_name }} + # + # Create-Issue-On-Fail: + # permissions: + # issues: write + # runs-on: ubuntu-latest + # needs: Publish-Release + # if: (failure() || cancelled()) && github.event_name == 'push' + # steps: + # # The open-issue action requires to clone the repository. + # - name: Checkout TileDB + # uses: actions/checkout@v4 + # - name: Create Issue if Build Fails + # uses: TileDB-Inc/github-actions/open-issue@main + # with: + # name: Release failed + # label: release + # assignee: KiterLuc,teo-tsirpanis,davisp From d105286998993498636cb49255d5fbbb2834d0dd Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Fri, 25 Oct 2024 10:23:57 +0200 Subject: [PATCH 18/31] Install ninja-build on linux --- .github/workflows/release.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bf555ff9038..b600ed950c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,17 +99,11 @@ jobs: ref=${{ github.head_ref || github.ref_name }} echo "release_version=${ref##*/}-$release_hash" >> $GITHUB_OUTPUT shell: bash - - name: Install aarch64 cross compilation prerequisites - if: ${{ matrix.platform == 'linux-aarch64' }} - run: | - set -e pipefail - yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd - echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc - name: Install manylinux prerequisites - if: ${{ startsWith(matrix.platform, 'linux-x86_64') == true }} + if: ${{ startsWith(matrix.platform, 'linux') == true }} run: | set -e pipefail - yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd + yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd ninja-build echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc - name: Configure TileDB run: | From 61634e35b592594c8a653490e9474ba7eb265555 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Fri, 25 Oct 2024 10:36:05 +0200 Subject: [PATCH 19/31] Install ninja using pip --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b600ed950c2..530e24777f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,10 +103,12 @@ jobs: if: ${{ startsWith(matrix.platform, 'linux') == true }} run: | set -e pipefail - yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd ninja-build + yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc + python3.9 -m pip install ninja - name: Configure TileDB run: | + export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ From 124971a11a92741240c89e204d3568a142e1ea58 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Fri, 25 Oct 2024 10:46:07 +0200 Subject: [PATCH 20/31] Add env variable for arm64 specific build --- .github/workflows/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 530e24777f2..5a4c3e22f1c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,8 @@ jobs: os: linux-arm64-ubuntu24 triplet: arm64-linux-release manylinux: quay.io/pypa/manylinux2014_aarch64 + env: + VCPKG_FORCE_SYSTEM_BINARIES: ON - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 @@ -107,6 +109,7 @@ jobs: echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc python3.9 -m pip install ninja - name: Configure TileDB + env: {{ matrix.env }} run: | export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ From ae0c420e3fe7abc575e8634684b0734f236d1666 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 28 Oct 2024 15:18:07 +0100 Subject: [PATCH 21/31] Remove env --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5a4c3e22f1c..2b53b236145 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -109,7 +109,6 @@ jobs: echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc python3.9 -m pip install ninja - name: Configure TileDB - env: {{ matrix.env }} run: | export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ From 02808e6c340486302d91d3137f39e32fba584547 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 28 Oct 2024 15:30:57 +0100 Subject: [PATCH 22/31] properly set VCPKG_FORCE_SYSTEM_BINARIES --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b53b236145..28a08065cad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,8 +61,6 @@ jobs: os: linux-arm64-ubuntu24 triplet: arm64-linux-release manylinux: quay.io/pypa/manylinux2014_aarch64 - env: - VCPKG_FORCE_SYSTEM_BINARIES: ON - platform: macos-x86_64 os: macos-12 MACOSX_DEPLOYMENT_TARGET: 11 @@ -109,6 +107,8 @@ jobs: echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc python3.9 -m pip install ninja - name: Configure TileDB + env: + VCPKG_FORCE_SYSTEM_BINARIES: {{ startsWith(matrix.platform, 'linux-aarch64') ? 'YES' : 'NO' }} run: | export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ From 378a7fb306c84c203fbabb51820f4a0240320970 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 28 Oct 2024 15:32:53 +0100 Subject: [PATCH 23/31] Fix ternary operator --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28a08065cad..a6020053e2f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,7 +108,7 @@ jobs: python3.9 -m pip install ninja - name: Configure TileDB env: - VCPKG_FORCE_SYSTEM_BINARIES: {{ startsWith(matrix.platform, 'linux-aarch64') ? 'YES' : 'NO' }} + VCPKG_FORCE_SYSTEM_BINARIES: {{ startsWith(matrix.platform, 'linux-aarch64') && 'YES' || 'NO' }} run: | export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ From 1fca2bc3833c6911a210be66b746d4d1d6ba3f72 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Mon, 28 Oct 2024 15:34:18 +0100 Subject: [PATCH 24/31] Fix --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6020053e2f..20cbf96088d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,7 +108,7 @@ jobs: python3.9 -m pip install ninja - name: Configure TileDB env: - VCPKG_FORCE_SYSTEM_BINARIES: {{ startsWith(matrix.platform, 'linux-aarch64') && 'YES' || 'NO' }} + VCPKG_FORCE_SYSTEM_BINARIES: ${{ startsWith(matrix.platform, 'linux-aarch64') && 'YES' || 'NO' }} run: | export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ From ed56602cfbd9ec240b959ca9c89428d761116b9c Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Tue, 29 Oct 2024 08:52:28 +0100 Subject: [PATCH 25/31] Use bash in print on error for Windows --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20cbf96088d..20c4a950469 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,6 +138,7 @@ jobs: build/tiledb-*.tar.gz* build/tiledb-*.zip* - name: "Print log files (failed build only)" + shell: bash run: | sh $GITHUB_WORKSPACE/scripts/ci/print_logs.sh if: failure() # only run this job if the build step failed From 972010ba22a05637dff9072fab84e46d5ee1a5c2 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Tue, 29 Oct 2024 09:29:32 +0100 Subject: [PATCH 26/31] Brew install ninja --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d02ac2e2c8a..c5e9749a54c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -85,7 +85,7 @@ jobs: # v4 uses node 20 which is incompatible with the libc version of the manylinux image uses: actions/checkout@v3 - name: 'Homebrew setup' - run: brew install automake pkg-config + run: brew install automake pkg-config ninja if: ${{ startsWith(matrix.os, 'macos-') == true }} - name: Export GitHub Actions cache variables uses: actions/github-script@v6 From de7935d14f4b45da1ec01fd65b977d4d6ff53790 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Tue, 29 Oct 2024 10:29:59 +0100 Subject: [PATCH 27/31] Set path only for linux builds --- .github/workflows/release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5e9749a54c..ed942135fd6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -107,11 +107,10 @@ jobs: yum install -y redhat-lsb-core centos-release-scl devtoolset-7 perl-IPC-Cmd echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc python3.9 -m pip install ninja + echo "/opt/_internal/cpython-3.9.20/bin" >> $GITHUB_PATH + echo "VCPKG_FORCE_SYSTEM_BINARIES=YES" >> $GITHUB_ENV - name: Configure TileDB - env: - VCPKG_FORCE_SYSTEM_BINARIES: ${{ startsWith(matrix.platform, 'linux-aarch64') && 'YES' || 'NO' }} run: | - export PATH=$PATH:/opt/_internal/cpython-3.9.20/bin cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ From 342d87e491ea7d8929c2f4518bb1b4851f111743 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Tue, 29 Oct 2024 12:12:56 +0100 Subject: [PATCH 28/31] Rename C and CXX compiler for aarch64 --- cmake/arm64_linux_toolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake index 02ab153134d..d09bc783c8b 100644 --- a/cmake/arm64_linux_toolchain.cmake +++ b/cmake/arm64_linux_toolchain.cmake @@ -6,5 +6,5 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) # Name of C compiler. -set(CMAKE_C_COMPILER "aarch64-unknown-linux-gnu-gcc") -set(CMAKE_CXX_COMPILER "aarch64-unknown-linux-gnu-g++") +set(CMAKE_C_COMPILER "aarch64-linux-gnu-gcc") +set(CMAKE_CXX_COMPILER "aarch64-linux-gnu-g++") From 2888cae47c2811e4ff85e4bd78996b331bd5d809 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Tue, 29 Oct 2024 12:15:15 +0100 Subject: [PATCH 29/31] Remove unused arm64 toolchain --- cmake/arm64_linux_toolchain.cmake | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 cmake/arm64_linux_toolchain.cmake diff --git a/cmake/arm64_linux_toolchain.cmake b/cmake/arm64_linux_toolchain.cmake deleted file mode 100644 index d09bc783c8b..00000000000 --- a/cmake/arm64_linux_toolchain.cmake +++ /dev/null @@ -1,10 +0,0 @@ -# -# CMake Toolchain file for crosscompiling on ARM. -# -# Target operating system name. -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_PROCESSOR aarch64) - -# Name of C compiler. -set(CMAKE_C_COMPILER "aarch64-linux-gnu-gcc") -set(CMAKE_CXX_COMPILER "aarch64-linux-gnu-g++") From 6299b48dc3903f7a191c8a78eb936d3851816b27 Mon Sep 17 00:00:00 2001 From: Dusan Baran Date: Tue, 29 Oct 2024 12:24:45 +0100 Subject: [PATCH 30/31] Uncomment release logic --- .github/workflows/release.yml | 125 +++++++++++++++++----------------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed942135fd6..a04b37a1c85 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,6 @@ name: Release on: workflow_dispatch: - pull_request: push: branches: - dev @@ -160,65 +159,65 @@ jobs: exit 1 fi - # Publish-Release: - # needs: - # - Test-Release-Artifacts - # - Package-Source-Release - # if: startsWith(github.ref, 'refs/tags/') - # runs-on: ubuntu-latest - # steps: - # - name: Download release artifacts - # uses: actions/download-artifact@v3 - # with: - # name: release - # path: dist - # - name: Publish release artifacts - # uses: actions/github-script@v6 - # with: - # script: | - # const fs = require('fs'); - # const path = require('path'); - # const repo = context.repo; - # const release = await github.rest.repos.getReleaseByTag({ - # owner: repo.owner, - # repo: repo.repo, - # tag: "${{ github.ref_name }}" - # }); - # const globber = await glob.create('dist/*'); - # for await (const file of globber.globGenerator()) { - # await github.rest.repos.uploadReleaseAsset({ - # owner: repo.owner, - # repo: repo.repo, - # release_id: release.data.id, - # headers: { - # 'content-type': 'application/octet-stream', - # 'content-length': fs.statSync(file).size - # }, - # name: path.basename(file), - # data: fs.readFileSync(file) - # }); - # } - # - # Generate-Release-List: - # needs: - # - Publish-Release - # uses: ./.github/workflows/append-release-cmake.yml - # with: - # ref: ${{ github.ref_name }} - # - # Create-Issue-On-Fail: - # permissions: - # issues: write - # runs-on: ubuntu-latest - # needs: Publish-Release - # if: (failure() || cancelled()) && github.event_name == 'push' - # steps: - # # The open-issue action requires to clone the repository. - # - name: Checkout TileDB - # uses: actions/checkout@v4 - # - name: Create Issue if Build Fails - # uses: TileDB-Inc/github-actions/open-issue@main - # with: - # name: Release failed - # label: release - # assignee: KiterLuc,teo-tsirpanis,davisp + Publish-Release: + needs: + - Test-Release-Artifacts + - Package-Source-Release + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - name: Download release artifacts + uses: actions/download-artifact@v3 + with: + name: release + path: dist + - name: Publish release artifacts + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const repo = context.repo; + const release = await github.rest.repos.getReleaseByTag({ + owner: repo.owner, + repo: repo.repo, + tag: "${{ github.ref_name }}" + }); + const globber = await glob.create('dist/*'); + for await (const file of globber.globGenerator()) { + await github.rest.repos.uploadReleaseAsset({ + owner: repo.owner, + repo: repo.repo, + release_id: release.data.id, + headers: { + 'content-type': 'application/octet-stream', + 'content-length': fs.statSync(file).size + }, + name: path.basename(file), + data: fs.readFileSync(file) + }); + } + + Generate-Release-List: + needs: + - Publish-Release + uses: ./.github/workflows/append-release-cmake.yml + with: + ref: ${{ github.ref_name }} + + Create-Issue-On-Fail: + permissions: + issues: write + runs-on: ubuntu-latest + needs: Publish-Release + if: (failure() || cancelled()) && github.event_name == 'push' + steps: + # The open-issue action requires to clone the repository. + - name: Checkout TileDB + uses: actions/checkout@v4 + - name: Create Issue if Build Fails + uses: TileDB-Inc/github-actions/open-issue@main + with: + name: Release failed + label: release + assignee: KiterLuc,teo-tsirpanis,davisp From dd8d05a897cfefde6ee49e89445956750f68de4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Wed, 13 Nov 2024 11:16:38 +0100 Subject: [PATCH 31/31] Update .github/workflows/release.yml Co-authored-by: Theodore Tsirpanis --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a04b37a1c85..10e121d001e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,7 +139,7 @@ jobs: - name: "Print log files (failed build only)" shell: bash run: | - sh $GITHUB_WORKSPACE/scripts/ci/print_logs.sh + $GITHUB_WORKSPACE/scripts/ci/print_logs.sh if: failure() # only run this job if the build step failed Test-Release-Artifacts: