Multiplatform Build and Test #477

Workflow file for this run

.github/workflows/multiplatform-build-test.yml at c93bbd9

	name: Multiplatform Build and Test

	env:
	USE_EXISTING_BINARY_DATASET: 1
	PIP_BREAK_SYSTEM_PACKAGES: 1

	on:
	workflow_dispatch:
	schedule:
	- cron: "0 8 * * *"

	jobs:
	generate-binary-demo:
	runs-on: kuzu-self-hosted-testing
	env:
	NUM_THREADS: 32
	GEN: Ninja
	CC: gcc
	CXX: g++
	steps:
	- uses: actions/checkout@v4

	- name: Build
	run: make release

	- name: Generate datasets
	run: bash scripts/generate_binary_demo.sh

	- name: Upload binary-demo
	uses: actions/upload-artifact@v4
	with:
	name: binary-demo
	path: dataset/binary-demo

	macos-build-test:
	strategy:
	matrix:
	runner: [macos-13, macos-14]
	fail-fast: false
	name: ${{ matrix.runner }}
	runs-on: ${{ matrix.runner }}
	needs: [generate-binary-demo]
	defaults:
	run:
	shell: bash
	steps:
	- uses: actions/checkout@v4
	continue-on-error: true

	- name: Download binary-demo
	uses: actions/download-artifact@v4
	with:
	name: binary-demo
	path: ${{ github.workspace }}/dataset/binary-demo

	- uses: actions/setup-python@v4
	continue-on-error: true
	with:
	python-version: "3.10"

	- name: Ensure Python dependencies
	continue-on-error: true
	run: \|
	pip install torch~=2.2.0 --extra-index-url https://download.pytorch.org/whl/cpu
	pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.0+cpu.html

	- name: Ensure Node.js dependencies
	continue-on-error: true
	working-directory: tools/nodejs_api
	run: npm install --include=dev

	- name: Build
	continue-on-error: true
	run: \|
	set +e
	make release NUM_THREADS=$(sysctl -n hw.physicalcpu)
	echo "Build,$?" > status.txt

	- name: Test
	continue-on-error: true
	run: \|
	set +e
	make test NUM_THREADS=$(sysctl -n hw.physicalcpu)
	echo "Test,$?" >> status.txt

	- name: Remove ldbc-1
	run: rm -rf dataset/ldbc-1

	- name: C and C++ examples
	continue-on-error: true
	run: \|
	set +e
	make example NUM_THREADS=$(sysctl -n hw.physicalcpu)
	echo "C and C++ examples,$?" >> status.txt

	- name: Python test
	continue-on-error: true
	run: \|
	set +e
	make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu)
	echo "Python test,$?" >> status.txt

	- name: Node.js test
	continue-on-error: true
	run: \|
	set +e
	make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu)
	echo "Node.js test,$?" >> status.txt

	- name: Java test
	continue-on-error: true
	run: \|
	set +e
	make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu)
	echo "Java test,$?" >> status.txt

	- name: Cleanup
	run: make clean

	- name: Rust share build
	continue-on-error: true
	run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml

	- name: Rust set env
	continue-on-error: true
	run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV

	- name: Rust test
	continue-on-error: true
	working-directory: tools/rust_api
	run: \|
	set +e
	cargo test --release --features arrow -- --test-threads=12
	echo "Rust test,$?" >> ../../status.txt

	- name: Rust example
	continue-on-error: true
	working-directory: examples/rust
	run: \|
	set +e
	cargo build --release --features arrow
	echo "Rust example,$?" >> ../../status.txt

	- name: Rename status.txt
	continue-on-error: true
	run: mv status.txt ${{ matrix.runner }}.csv

	- uses: actions/upload-artifact@v4
	continue-on-error: true
	with:
	name: ${{ matrix.runner }}
	path: ${{ matrix.runner }}.csv

	windows-build-test:
	strategy:
	matrix:
	runner: [windows-2019, windows-2022]
	fail-fast: false
	name: ${{ matrix.runner }}
	needs: [generate-binary-demo]
	runs-on: ${{ matrix.runner }}
	steps:
	- name: Disable Windows Defender
	continue-on-error: true
	shell: powershell
	run: \|
	Set-MpPreference -DisableRealtimeMonitoring $true
	Set-MpPreference -DisableBehaviorMonitoring $true
	Set-MpPreference -DisableBlockAtFirstSeen $true
	Set-MpPreference -DisableIOAVProtection $true
	Set-MpPreference -DisablePrivacyMode $true
	Set-MpPreference -SignatureDisableUpdateOnStartupWithoutEngine $true
	Set-MpPreference -DisableArchiveScanning $true
	Set-MpPreference -DisableIntrusionPreventionSystem $true
	Set-MpPreference -DisableScriptScanning $true
	Set-MpPreference -SubmitSamplesConsent NeverSend

	- uses: actions/checkout@v4
	continue-on-error: true

	- name: Download binary-demo
	uses: actions/download-artifact@v4
	with:
	name: binary-demo
	path: ${{ github.workspace }}/dataset/binary-demo

	- uses: ilammy/msvc-dev-cmd@v1
	continue-on-error: true

	- uses: actions/setup-python@v4
	continue-on-error: true
	with:
	python-version: "3.10"

	- name: Ensure Python dependencies
	continue-on-error: true
	shell: cmd
	run: \|
	pip install torch~=2.2.0 --extra-index-url https://download.pytorch.org/whl/cpu
	pip install --user -r tools\python_api\requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.0+cpu.html

	- name: Ensure Node.js dependencies
	continue-on-error: true
	shell: cmd
	working-directory: .\tools\nodejs_api
	run: npm install --include=dev

	- name: Build
	continue-on-error: true
	shell: cmd
	run: \|
	make release NUM_THREADS=%NUMBER_OF_PROCESSORS%
	echo Build,%ERRORLEVEL% > status.txt

	- name: Test
	continue-on-error: true
	shell: cmd
	run: \|
	make test NUM_THREADS=%NUMBER_OF_PROCESSORS%
	echo Test,%ERRORLEVEL% >> status.txt

	- name: Remove ldbc-1
	run: Remove-Item -Recurse -Force dataset\ldbc-1

	- name: C and C++ examples
	continue-on-error: true
	shell: cmd
	run: \|
	make example NUM_THREADS=%NUMBER_OF_PROCESSORS%
	echo C and C++ examples,%ERRORLEVEL% >> status.txt

	- name: Python test
	continue-on-error: true
	shell: cmd
	run: \|
	make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS%
	echo Python test,%ERRORLEVEL% >> status.txt

	- name: Node.js test
	continue-on-error: true
	shell: cmd
	run: \|
	make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS%
	echo Node.js test,%ERRORLEVEL% >> status.txt

	- name: Java test
	continue-on-error: true
	shell: cmd
	run: \|
	make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS%
	echo Java test,%ERRORLEVEL% >> status.txt

	- name: Cleanup
	shell: cmd
	run: make clean

	- name: Rust share build
	continue-on-error: true
	shell: cmd
	run: (echo [workspace]& echo members = ["tools/rust_api","examples/rust"]) > Cargo.toml

	- name: Rust test
	continue-on-error: true
	shell: cmd
	run: \|
	make clean
	set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64
	set CXXFLAGS=/std:c++20
	set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS%
	cargo test --release --features arrow -- --test-threads=12
	echo Rust test,%ERRORLEVEL% >> status.txt

	- name: Rust example
	continue-on-error: true
	shell: cmd
	run: \|
	set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS%
	cargo build --release --features arrow
	echo Rust example,%ERRORLEVEL% >> status.txt

	- name: Rename status.txt
	continue-on-error: true
	shell: cmd
	run: \|
	rename status.txt ${{ matrix.runner }}.csv

	- uses: actions/upload-artifact@v4
	continue-on-error: true
	with:
	name: ${{ matrix.runner }}
	path: ${{ matrix.runner }}.csv

	debian-ubuntu-build-test:
	strategy:
	matrix:
	image: ["ubuntu:22.04", "ubuntu:24.04", "ubuntu:24.10", "debian:12", "debian:sid"]
	fail-fast: false
	name: ${{ matrix.image }}
	needs: [generate-binary-demo]
	runs-on: ubuntu-24.04
	container:
	image: ${{ matrix.image }}
	env:
	CC: gcc
	CXX: g++
	JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
	steps:
	- name: Setup Node.js
	if: ${{ matrix.image != 'debian:sid' }}
	continue-on-error: true
	run: \|
	apt-get update
	apt-get install -y ca-certificates curl gnupg
	mkdir -p /etc/apt/keyrings
	curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
	echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" \| tee /etc/apt/sources.list.d/nodesource.list

	- name: Install packages
	continue-on-error: true
	run: \|
	apt-get update
	apt-get install -y git build-essential cmake gcc g++ python3 python3-dev python3-pip openjdk-17-jdk nodejs ca-certificates curl gnupg

	- name: Install npm
	if: ${{ matrix.image == 'debian:sid' }}
	continue-on-error: true
	run: apt-get install -y npm

	- uses: actions/checkout@v4
	continue-on-error: true

	- name: Download binary-demo
	uses: actions/download-artifact@v4
	with:
	name: binary-demo
	path: ${{ github.workspace }}/dataset/binary-demo

	- name: Ensure Python dependencies
	continue-on-error: true
	run: \|
	pip install torch~=2.2.0 --extra-index-url https://download.pytorch.org/whl/cpu
	pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.0+cpu.html

	- name: Ensure Node.js dependencies
	working-directory: tools/nodejs_api
	continue-on-error: true
	run: npm install --include=dev

	- name: Install Rust
	continue-on-error: true
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --no-update-default-toolchain
	echo "$HOME/.cargo/bin" >> $GITHUB_PATH
	$HOME/.cargo/bin/rustup toolchain install 1.76

	- name: Build
	continue-on-error: true
	run: \|
	set +e
	make release NUM_THREADS=$(nproc)
	echo "Build,$?" > status.txt

	- name: Test
	continue-on-error: true
	run: \|
	set +e
	make test NUM_THREADS=$(nproc)
	echo "Test,$?" >> status.txt

	- name: Remove ldbc-1
	run: rm -rf dataset/ldbc-1

	- name: C and C++ examples
	continue-on-error: true
	run: \|
	set +e
	make example NUM_THREADS=$(nproc)
	echo "C and C++ examples,$?" >> status.txt

	- name: Python test
	continue-on-error: true
	run: \|
	set +e
	make pytest NUM_THREADS=$(nproc)
	echo "Python test,$?" >> status.txt

	- name: Node.js test
	continue-on-error: true
	run: \|
	set +e
	make nodejstest NUM_THREADS=$(nproc)
	echo "Node.js test,$?" >> status.txt

	- name: Java test
	continue-on-error: true
	run: \|
	set +e
	make javatest NUM_THREADS=$(nproc)
	echo "Java test,$?" >> status.txt

	- name: Cleanup
	run: make clean

	- name: Rust share build
	continue-on-error: true
	run: echo '[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml

	- name: Rust set env
	continue-on-error: true
	run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV

	- name: Rust test
	continue-on-error: true
	working-directory: tools/rust_api
	run: \|
	set +e
	cargo test --release --features arrow -- --test-threads=12
	echo "Rust test,$?" >> ../../status.txt

	- name: Rust example
	continue-on-error: true
	working-directory: examples/rust
	run: \|
	set +e
	cargo build --release --features arrow
	echo "Rust example,$?" >> ../../status.txt

	- name: Rename status.txt
	continue-on-error: true
	run: \|
	PLATFORM=$(echo ${{ matrix.image }} \| tr ':' '-')
	echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
	mv status.txt $PLATFORM.csv

	- uses: actions/upload-artifact@v4
	continue-on-error: true
	with:
	name: ${{env.PLATFORM}}
	path: ${{env.PLATFORM}}.csv

	rhel-fedora-build-test:
	strategy:
	matrix:
	image: ["rockylinux:8", "rockylinux:9", "fedora:38", "fedora:39"]
	fail-fast: false
	name: ${{ matrix.image }}
	needs: [generate-binary-demo]
	runs-on: ubuntu-24.04
	container:
	image: ${{ matrix.image }}
	env:
	CC: gcc
	CXX: g++
	JAVA_HOME: /usr/lib/jvm/java-17-openjdk
	HOME: /root
	steps:
	- name: Enable EPEL
	continue-on-error: true
	if: ${{ matrix.image == 'rockylinux:8' \|\| matrix.image == 'rockylinux:9' }}
	run: \|
	dnf update -y
	dnf install -y epel-release
	dnf update -y

	- name: Enable SHA-1 on Rocky Linux 9
	continue-on-error: true
	if: matrix.image == 'rockylinux:9'
	run: update-crypto-policies --set LEGACY

	- name: Install packages
	continue-on-error: true
	run: \|
	curl -fsSL https://rpm.nodesource.com/setup_20.x \| bash -
	dnf install -y git cmake ${{ matrix.image == 'rockylinux:8' && 'gcc-toolset-12 python3.11 python3.11-devel' \|\| 'gcc gcc-c++ python3-devel' }} java-17-openjdk-devel nodejs

	- name: Enable gcc-toolset-12 and python3.11 on Rocky Linux 8
	continue-on-error: true
	if: matrix.image == 'rockylinux:8'
	run: \|
	alternatives --set python /usr/bin/python3.11
	alternatives --set python3 /usr/bin/python3.11
	echo "PYTHON_EXECUTABLE=/usr/bin/python3.11" >> $GITHUB_ENV
	echo "PYBIND11_PYTHON_VERSION=3.11" >> $GITHUB_ENV
	source /opt/rh/gcc-toolset-12/enable
	echo $PATH >> $GITHUB_PATH

	- uses: actions/checkout@v4
	continue-on-error: true

	- name: Download binary-demo
	uses: actions/download-artifact@v4
	with:
	name: binary-demo
	path: ${{ github.workspace }}/dataset/binary-demo

	- name: Ensure Python dependencies
	continue-on-error: true
	run: \|
	python3 -m venv /opt/venv
	source /opt/venv/bin/activate
	pip3 install torch~=2.2.1 --extra-index-url https://download.pytorch.org/whl/cpu
	pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.1+cpu.html

	- name: Ensure Node.js dependencies
	continue-on-error: true
	working-directory: tools/nodejs_api
	run: npm install --include=dev

	- name: Install Rust
	continue-on-error: true
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --no-update-default-toolchain
	echo "$HOME/.cargo/bin" >> $GITHUB_PATH
	$HOME/.cargo/bin/rustup toolchain install 1.76

	- name: Build
	continue-on-error: true
	run: \|
	set +e
	make release NUM_THREADS=$(nproc)
	echo "Build,$?" > status.txt

	- name: Test
	continue-on-error: true
	run: \|
	set +e
	make test NUM_THREADS=$(nproc)
	echo "Test,$?" >> status.txt

	- name: Remove ldbc-1
	run: rm -rf dataset/ldbc-1

	- name: C and C++ examples
	continue-on-error: true
	run: \|
	set +e
	make example NUM_THREADS=$(nproc)
	echo "C and C++ examples,$?" >> status.txt

	- name: Python test
	continue-on-error: true
	run: \|
	set +e
	source /opt/venv/bin/activate
	make pytest NUM_THREADS=$(nproc)
	echo "Python test,$?" >> status.txt

	- name: Node.js test
	continue-on-error: true
	run: \|
	set +e
	make nodejstest NUM_THREADS=$(nproc)
	echo "Node.js test,$?" >> status.txt

	- name: Java test
	continue-on-error: true
	run: \|
	set +e
	make javatest NUM_THREADS=$(nproc)
	echo "Java test,$?" >> status.txt

	- name: Cleanup
	run: make clean

	- name: Rust share build
	continue-on-error: true
	run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml

	- name: Rust set env
	continue-on-error: true
	run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV

	- name: Rust test
	continue-on-error: true
	working-directory: tools/rust_api
	run: \|
	set +e
	cargo test --release --features arrow -- --test-threads=12
	echo "Rust test,$?" >> ../../status.txt

	- name: Rust example
	continue-on-error: true
	working-directory: examples/rust
	run: \|
	set +e
	cargo build --release --features arrow
	echo "Rust example,$?" >> ../../status.txt

	- name: Rename status.txt
	continue-on-error: true
	run: \|
	PLATFORM=$(echo ${{ matrix.image }} \| tr ':' '-')
	echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
	mv status.txt $PLATFORM.csv

	- uses: actions/upload-artifact@v4
	continue-on-error: true
	with:
	name: ${{env.PLATFORM}}
	path: ${{env.PLATFORM}}.csv

	archlinux-build-test:
	name: archlinux
	runs-on: ubuntu-24.04
	needs: [generate-binary-demo]
	container:
	image: archlinux:latest
	env:
	CC: gcc
	CXX: g++
	JAVA_HOME: /usr/lib/jvm/java-17-openjdk
	steps:
	- name: Install packages
	continue-on-error: true
	run: \|
	pacman -Syu --noconfirm
	pacman -S --needed --noconfirm git base-devel cmake gcc python python-pip npm jdk17-openjdk

	- uses: actions/checkout@v4
	continue-on-error: true

	- name: Download binary-demo
	uses: actions/download-artifact@v4
	with:
	name: binary-demo
	path: ${{ github.workspace }}/dataset/binary-demo

	- name: Ensure Python dependencies
	continue-on-error: true
	run: \|
	pip install torch~=2.2.0 --extra-index-url https://download.pytorch.org/whl/cpu
	pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.0+cpu.html

	- name: Ensure Node.js dependencies
	continue-on-error: true
	working-directory: tools/nodejs_api
	run: npm install --include=dev

	- name: Install Rust
	continue-on-error: true
	run: \|
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \| sh -s -- -y --no-update-default-toolchain
	echo "$HOME/.cargo/bin" >> $GITHUB_PATH
	$HOME/.cargo/bin/rustup toolchain install 1.76

	- name: Build
	continue-on-error: true
	run: \|
	set +e
	make release NUM_THREADS=$(nproc)
	echo "Build,$?" > status.txt

	- name: Test
	continue-on-error: true
	run: \|
	set +e
	make test NUM_THREADS=$(nproc)
	echo "Test,$?" >> status.txt

	- name: Remove ldbc-1
	run: rm -rf dataset/ldbc-1

	- name: C and C++ examples
	continue-on-error: true
	run: \|
	set +e
	make example NUM_THREADS=$(nproc)
	echo "C and C++ examples,$?" >> status.txt

	- name: Python test
	continue-on-error: true
	run: \|
	set +e
	make pytest NUM_THREADS=$(nproc)
	echo "Python test,$?" >> status.txt

	- name: Node.js test
	continue-on-error: true
	run: \|
	set +e
	make nodejstest NUM_THREADS=$(nproc)
	echo "Node.js test,$?" >> status.txt

	- name: Java test
	continue-on-error: true
	run: \|
	set +e
	make javatest NUM_THREADS=$(nproc)
	echo "Java test,$?" >> status.txt

	- name: Cleanup
	run: make clean

	- name: Rust share build
	continue-on-error: true
	run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml

	- name: Rust set env
	continue-on-error: true
	run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV

	- name: Rust test
	working-directory: tools/rust_api
	continue-on-error: true
	run: \|
	set +e
	cargo test --release --features arrow -- --test-threads=12
	echo "Rust test,$?" >> ../../status.txt

	- name: Rust example
	working-directory: examples/rust
	continue-on-error: true
	run: \|
	set +e
	cargo build --release --features arrow
	echo "Rust example,$?" >> ../../status.txt

	- name: Rename status.txt
	continue-on-error: true
	run: \|
	mv status.txt archlinux.csv

	- uses: actions/upload-artifact@v4
	continue-on-error: true
	with:
	name: archlinux
	path: archlinux.csv

	collect-results:
	runs-on: ubuntu-22.04
	env:
	DISCORD_CHANNEL_ID: ${{ secrets.DISCORD_CHANNEL_ID }}
	DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
	GITHUB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
	needs:
	[
	macos-build-test,
	windows-build-test,
	debian-ubuntu-build-test,
	rhel-fedora-build-test,
	archlinux-build-test,
	]
	steps:
	- uses: actions/checkout@v4

	- name: Download artifacts
	uses: actions/download-artifact@v4
	with:
	path: results

	- name: List artifacts
	run: ls -R results

	- name: Create summary
	run: \|
	pip3 install pandas tabulate discord.py
	python3 scripts/multiplatform-test-helper/collect-results.py results
	python3 scripts/multiplatform-test-helper/notify-discord.py results.json

	- name: Upload summary
	uses: actions/upload-artifact@v4
	with:
	name: results
	path: results.md

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Multiplatform Build and Test #477

Workflow file

Multiplatform Build and Test #477

Jobs

Run details

Workflow file for this run