[Spark] Optimize fileInfo Scan (#549) #110
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2023 LakeSoul Contributors | |
| # | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: Cross build | |
| on: | |
| push: | |
| paths-ignore: | |
| - "javadoc/**" | |
| - "website/**" | |
| - "**.md" | |
| branches: | |
| - 'main' | |
| pull_request: | |
| paths-ignore: | |
| - "javadoc/**" | |
| - "website/**" | |
| - "**.md" | |
| branches: | |
| - 'main' | |
| - 'release/**' | |
| workflow_dispatch: | |
| jobs: | |
| build-linux-x86_64: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| cache: maven | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: stable | |
| default: true | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: "./rust -> target" | |
| env-vars: "JAVA_HOME" | |
| - name: Cache Docker images | |
| uses: ScribeMD/[email protected] | |
| with: | |
| key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml') }} | |
| - uses: actions-rs/cargo@v1 | |
| with: | |
| use-cross: true | |
| command: build | |
| args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features' | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lakesoul-nativeio-x86_64-unknown-linux-gnu | |
| path: ./rust/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu | |
| path: ./rust/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so | |
| build-windows-x86_64: | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| cache: maven | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v2 | |
| with: | |
| version: "23.x" | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: stable | |
| default: true | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: "./rust -> target" | |
| env-vars: "JAVA_HOME" | |
| - uses: actions-rs/cargo@v1 | |
| with: | |
| command: build | |
| args: '--manifest-path rust/Cargo.toml --release --all-features' | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lakesoul-nativeio-x86_64-pc-windows-msvc | |
| path: ./rust/target/release/lakesoul_io_c.dll | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lakesoul-nativemetadata-x86_64-pc-windows-msvc | |
| path: ./rust/target/release/lakesoul_metadata_c.dll | |
| build-macos-x86_64: | |
| runs-on: macos-latest | |
| steps: | |
| - name: Install automake | |
| run: brew install automake | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| cache: maven | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v2 | |
| with: | |
| version: "23.x" | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions-rs/toolchain@v1 | |
| with: | |
| profile: minimal | |
| toolchain: stable | |
| default: true | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: "./rust -> target" | |
| env-vars: "JAVA_HOME" | |
| - uses: actions-rs/cargo@v1 | |
| with: | |
| command: build | |
| args: '--manifest-path rust/Cargo.toml --release --all-features' | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lakesoul-nativeio-x86_64-apple-darwin | |
| path: ./rust/target/release/liblakesoul_io_c.dylib | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: lakesoul-nativemetadata-x86_64-apple-darwin | |
| path: ./rust/target/release/liblakesoul_metadata_c.dylib | |
| build-maven-package: | |
| runs-on: ubuntu-latest | |
| needs: [ build-linux-x86_64, build-windows-x86_64, build-macos-x86_64 ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lakesoul-nativeio-x86_64-unknown-linux-gnu | |
| path: ./rust/target/release/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lakesoul-nativeio-x86_64-apple-darwin | |
| path: ./rust/target/release/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lakesoul-nativeio-x86_64-pc-windows-msvc | |
| path: ./rust/target/release/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu | |
| path: ./rust/target/release/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lakesoul-nativemetadata-x86_64-apple-darwin | |
| path: ./rust/target/release/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: lakesoul-nativemetadata-x86_64-pc-windows-msvc | |
| path: ./rust/target/release/ | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| cache: maven | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v2 | |
| with: | |
| version: "23.x" | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build with Maven | |
| run: | | |
| MAVEN_OPTS="-Xmx4000m" mvn -q -B package --file pom.xml -Pcross-build -DskipTests -Dmaven.test.skip=true | |
| - name: Upload Package | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: maven-package-upload | |
| path: | | |
| lakesoul-spark/target/lakesoul-spark-*.jar | |
| lakesoul-flink/target/lakesoul-flink-*.jar | |
| lakesoul-presto/target/lakesoul-presto-*.jar | |
| retention-days: 3 | |
| if-no-files-found: error |