split builds into core vs substrait

sundeck-io · Oct 14, 2024 · f8db63a · f8db63a
1 parent 54f81df
commit f8db63a
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 17 deletions.
diff --git a/.github/workflows/deps.yaml b/.github/workflows/deps.yaml
@@ -21,47 +21,80 @@ jobs:
           name: includes
           path: include/*
           retention-days: 1
+
   darwin_amd64:
     runs-on: macos-13
+    strategy:
+      matrix:
+        build_type:
+          - { name: 'core', BUILD_CORE: 'TRUE', BUILD_SUBSTRAIT: 'FALSE' }
+          - { name: 'substrait', BUILD_CORE: 'FALSE', BUILD_SUBSTRAIT: 'TRUE' }
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
       - shell: bash
+        env:
+          BUILD_CORE: ${{ matrix.build_type.BUILD_CORE }}
+          BUILD_SUBSTRAIT: ${{ matrix.build_type.BUILD_SUBSTRAIT }}
         run: make deps.darwin.amd64
       - uses: actions/upload-artifact@v4
         with:
-          name: darwin_amd64
+          name: darwin_amd64_${{ matrix.build_type.name }}
           path: deps/darwin_amd64/*
           retention-days: 1
+
   darwin_arm64:
     runs-on: macos-latest
+    strategy:
+      matrix:
+        build_type:
+          - { name: 'core', BUILD_CORE: 'TRUE', BUILD_SUBSTRAIT: 'FALSE' }
+          - { name: 'substrait', BUILD_CORE: 'FALSE', BUILD_SUBSTRAIT: 'TRUE' }
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
       - shell: bash
+        env:
+          BUILD_CORE: ${{ matrix.build_type.BUILD_CORE }}
+          BUILD_SUBSTRAIT: ${{ matrix.build_type.BUILD_SUBSTRAIT }}
         run: make deps.darwin.arm64
       - uses: actions/upload-artifact@v4
         with:
-          name: darwin_arm64
+          name: darwin_arm64_${{ matrix.build_type.name }}
           path: deps/darwin_arm64/*
           retention-days: 1
+
   linux_amd64:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        build_type:
+          - { name: 'core', BUILD_CORE: 'TRUE', BUILD_SUBSTRAIT: 'FALSE' }
+          - { name: 'substrait', BUILD_CORE: 'FALSE', BUILD_SUBSTRAIT: 'TRUE' }
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
       - shell: bash
+        env:
+          BUILD_CORE: ${{ matrix.build_type.BUILD_CORE }}
+          BUILD_SUBSTRAIT: ${{ matrix.build_type.BUILD_SUBSTRAIT }}
         run: make deps.linux.amd64
       - uses: actions/upload-artifact@v4
         with:
-          name: linux_amd64
+          name: linux_amd64_${{ matrix.build_type.name }}
           path: deps/linux_amd64/*
           retention-days: 1
+
   linux_arm64:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        build_type:
+          - { name: 'core', BUILD_CORE: 'TRUE', BUILD_SUBSTRAIT: 'FALSE' }
+          - { name: 'substrait', BUILD_CORE: 'FALSE', BUILD_SUBSTRAIT: 'TRUE' }
     steps:
       - uses: actions/checkout@v4
         with:
@@ -72,10 +105,13 @@ jobs:
           sudo apt update
           sudo apt install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
       - shell: bash
+        env:
+          BUILD_CORE: ${{ matrix.build_type.BUILD_CORE }}
+          BUILD_SUBSTRAIT: ${{ matrix.build_type.BUILD_SUBSTRAIT }}
         run: make deps.linux.arm64
       - uses: actions/upload-artifact@v4
         with:
-          name: linux_arm64
+          name: linux_arm64_${{ matrix.build_type.name }}
           path: deps/linux_arm64/*
           retention-days: 1
 
@@ -101,21 +137,37 @@ jobs:
           path: include
       - uses: actions/download-artifact@v4
         with:
-          name: darwin_amd64
+          name: darwin_amd64_core
           path: deps/darwin_amd64
       - uses: actions/download-artifact@v4
         with:
-          name: darwin_arm64
+          name: darwin_amd64_substrait
+          path: deps/darwin_amd64
+      - uses: actions/download-artifact@v4
+        with:
+          name: darwin_arm64_core
+          path: deps/darwin_arm64
+      - uses: actions/download-artifact@v4
+        with:
+          name: darwin_arm64_substrait
           path: deps/darwin_arm64
       - uses: actions/download-artifact@v4
         with:
-          name: linux_amd64
+          name: linux_amd64_core
+          path: deps/linux_amd64
+      - uses: actions/download-artifact@v4
+        with:
+          name: linux_amd64_substrait
           path: deps/linux_amd64
       - uses: actions/download-artifact@v4
         with:
-          name: linux_arm64
+          name: linux_arm64_core
+          path: deps/linux_arm64
+      - uses: actions/download-artifact@v4
+        with:
+          name: linux_arm64_substrait
           path: deps/linux_arm64
       - name: Push static libraries
         uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit_message: Re-build static libraries
+          commit_message: Re-build static libraries
diff --git a/Makefile b/Makefile
@@ -13,19 +13,30 @@ DEPS_PATH = ""
 DUCKDB_COMMON_BUILD_FLAGS := BUILD_SHELL=0 BUILD_UNITTESTS=0 DUCKDB_PLATFORM=any
 CHECK_DARWIN = if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "darwin" ]; then echo "Error: must run build on darwin"; false; fi
 CHECK_LINUX = if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "linux" ]; then echo "Error: must run build on linux"; false; fi
-WORK_COMMAND =  \
-	mkdir -p deps/$(DEPS_PATH) && \
+MKDIR_COMMAND = mkdir -p deps/$(DEPS_PATH)
+CORE_COMMAND =  \
 	cd duckdb && \
 	mkdir build && \
 	cd build && \
 	cmake -DBUILD_EXTENSIONS="icu;parquet;tpch;tpcds;json" -DBUILD_ONLY_EXTENSIONS=TRUE .. && \
-	CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" CC="${CC}" CXX="${CXX}" make icu_extension tpch_extension tpcds_extension json_extension parquet_extension -j 2 && \
+	CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" CC="${CC}" CXX="${CXX}" ${DUCKDB_COMMON_BUILD_FLAGS} make icu_extension tpch_extension tpcds_extension json_extension parquet_extension -j 2 && \
 	cd ../.. && \
 	find duckdb/build/ -type f -name '*extension*.a' -exec cp {} deps/$(DEPS_PATH) \; && \
 	cd substrait && \
-	CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" CC="${CC}" CXX="${CXX}" make -j 2 && \
+	CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" CC="${CC}" CXX="${CXX}" ${DUCKDB_COMMON_BUILD_FLAGS} make -j 2 && \
 	cd .. && \
 	cp substrait/build/release/extension/substrait/libsubstrait_extension.a deps/$(DEPS_PATH)
+SUBSTRAIT_COMMAND = \
+	cd substrait && \
+	CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" CC="${CC}" CXX="${CXX}" ${DUCKDB_COMMON_BUILD_FLAGS} make -j 2 && \
+	cd .. && \
+	cp substrait/build/release/extension/substrait/libsubstrait_extension.a deps/$(DEPS_PATH)
+
+
+define get_build_commands
+$(if $(filter TRUE,$(or $(BUILD_CORE),TRUE)), $(CORE_COMMAND))
+$(if $(filter TRUE,$(BUILD_SUBSTRAIT)), $(SUBSTRAIT_COMMAND))
+endef
 
 .PHONY: duckdb
 duckdb:
@@ -53,26 +64,29 @@ deps.darwin.amd64: CXXFLAGS += -target x86_64-apple-macos11
 deps.darwin.amd64: DEPS_PATH = darwin_amd64
 deps.darwin.amd64: duckdb substrait
 	$(CHECK_DARWIN)
-	$(WORK_COMMAND)
+	$(get_build_commands)
 
 .PHONY: deps.darwin.arm64
 deps.darwin.arm64: CFLAGS += -target arm64-apple-macos11
 deps.darwin.arm64: CXXFLAGS += -target arm64-apple-macos11
 deps.darwin.arm64: DEPS_PATH = darwin_arm64
 deps.darwin.arm64: duckdb substrait
 	$(CHECK_DARWIN)
-	$(WORK_COMMAND)
+	$(get_build_commands)
+
 
 .PHONY: deps.linux.amd64
 deps.linux.amd64: DEPS_PATH = linux_amd64
 deps.linux.amd64: duckdb substrait
 	$(CHECK_LINUX)
-	$(WORK_COMMAND)
+	$(get_build_commands)
+
 
 .PHONY: deps.linux.arm64
 deps.linux.arm64: CC = aarch64-linux-gnu-gcc
 deps.linux.arm64: CXX = aarch64-linux-gnu-g++
 deps.linux.arm64: DEPS_PATH = linux_arm64
 deps.linux.arm64: duckdb substrait
 	$(CHECK_LINUX)
-	$(WORK_COMMAND)
+	$(get_build_commands)
+