build-test-graph.yml

# Copyright 2020-2023 Alibaba Group Holding Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Vineyard Graph CI

on:
  push:
    branches:
      - main
    paths:
      - 'modules/graph/**'
  pull_request:
    branches:
      - main
    paths:
      - 'modules/graph/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  CMAKE_C_COMPILER_LAUNCHER: ccache
  CMAKE_CXX_COMPILER_LAUNCHER: ccache

jobs:
  ci:
    runs-on: ${{ matrix.os }}
    if: ${{ github.repository == 'v6d-io/v6d' }}
    strategy:
      matrix:
        os: [ubuntu-20.04]
        malloc: [dlmalloc]
        metadata: [etcd]
    env:
      RUNNER_ARGS: "--meta=${{ matrix.metadata }}"
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Generate Summary for Submodules
        run: |
          git submodule > git-modules.txt
          cat git-modules.txt

      - name: Cache for ccache
        uses: actions/cache@v3
        with:
          path: ~/.ccache
          key: ${{ runner.os }}-${{ matrix.malloc }}-${{ matrix.metadata }}-ccache-graph-${{ hashFiles('**/git-modules.txt') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.malloc }}-${{ matrix.metadata }}-ccache-graph-

      - name: Install Dependencies for Linux
        if: runner.os == 'Linux'
        run: |
          export PATH=/usr/lib/ccache:$PATH

          sudo apt update -y
          sudo apt install -y ca-certificates \
                              ccache \
                              cmake \
                              doxygen \
                              fuse3 \
                              libboost-all-dev \
                              libcurl4-openssl-dev \
                              libfuse3-dev \
                              libgflags-dev \
                              libgoogle-glog-dev \
                              libgmock-dev \
                              libgrpc-dev \
                              libgrpc++-dev \
                              libkrb5-dev \
                              libmpich-dev \
                              libprotobuf-dev \
                              librdkafka-dev \
                              libgsasl7-dev \
                              librdkafka-dev \
                              libssl-dev \
                              libunwind-dev \
                              libuv1-dev \
                              libxml2-dev \
                              libz-dev \
                              lsb-release \
                              redis-server \
                              pandoc \
                              protobuf-compiler-grpc \
                              python3-pip \
                              uuid-dev \
                              wget

          # install apache-arrow
          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
          sudo apt update
          sudo apt install -y libarrow-dev=14.0.1-1 \
                              libarrow-dataset-dev=14.0.1-1 \
                              libarrow-acero-dev=14.0.1-1 \
                              libarrow-flight-dev=14.0.1-1 \
                              libgandiva-dev=14.0.1-1 \
                              libparquet-dev=14.0.1-1

          # install clang-format
          sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-1d7ec53d/clang-format-11_linux-amd64 --output /usr/bin/clang-format
          sudo chmod +x /usr/bin/clang-format

          # avoid possible permission errors
          ccache -o cache_dir=~/.ccache
          sudo mkdir -p ~/.ccache
          sudo chmod -R a+wrx ~/.ccache

          # install python packages for codegen, and io adaptors
          sudo pip3 install -U "Pygments>=2.4.1"
          sudo pip3 install -r requirements-setup.txt -r requirements.txt -r requirements-dev.txt

      - name: Setup tmate session
        if: false
        uses: mxschmitt/action-tmate@v3

      - name: CMake
        run: |
          export PATH=/usr/lib/ccache:$PATH

          mkdir build
          cd build
          cmake .. -DCMAKE_BUILD_TYPE=Debug \
                   -DBUILD_SHARED_LIBS=ON \
                   -DUSE_ASAN=ON \
                   -DWITH_ALLOCATOR=${{ matrix.malloc }} \
                   -DBUILD_VINEYARD_COVERAGE=ON \
                   -DBUILD_VINEYARD_PYTHON_BINDINGS=OFF \
                   -DBUILD_VINEYARD_BASIC=ON \
                   -DBUILD_VINEYARD_IO=ON \
                   -DBUILD_VINEYARD_IO_KAFKA=ON \
                   -DBUILD_VINEYARD_MALLOC=OFF \
                   -DBUILD_VINEYARD_HOSSEINMOEIN_DATAFRAME=OFF \
                   -DBUILD_VINEYARD_FUSE=OFF \
                   -DBUILD_VINEYARD_FUSE_PARQUET=OFF \
                   -DBUILD_VINEYARD_GRAPH=ON \
                   -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON \
                   -DBUILD_VINEYARD_TESTS_ALL=ON

          if [ "${{ matrix.metadata }}" == "redis" ]; then
            cmake .. -DBUILD_VINEYARD_SERVER_REDIS=ON
          fi

      - name: Cpp format and lint
        run: |
          cd build

          function prepend() { while read line; do echo "${1}${line}"; done; }

          make vineyard_clformat
          GIT_DIFF=$(git diff --ignore-submodules)
          if [[ -n $GIT_DIFF ]]; then
              echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
              echo "| clang-format failures found!"
              echo "|"
              echo "$GIT_DIFF" | prepend "| "
              echo "|"
              echo "| Run: "
              echo "|"
              echo "|    make vineyard_clformat"
              echo "|"
              echo "| to fix this error."
              echo "|"
              echo "| Ensure you are working with clang-format-11, which can be obtained from"
              echo "|"
              echo "|    https://github.com/muttleyxd/clang-tools-static-binaries/releases "
              echo "|"
              echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
              exit -1
          fi

          function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; }

          ec make vineyard_cpplint
          if [[ "$ec" != "0" ]]; then
              echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
              echo "| cpplint failures found! Run: "
              echo "|"
              echo "|    make vineyard_cpplint"
              echo "|"
              echo "| to fix this error."
              echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
              exit -1
          fi

      - name: Build
        run: |
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64

          pushd build
          make vineyardd vineyard_graph_tests -j`nproc`
          make ccache-stats
          popd

      - name: Get test data
        run: |
          git clone https://github.com/GraphScope/gstest.git --depth=1
          git clone https://github.com/GraphScope/gar-test.git --depth=1

      - name: Setup tmate session
        if: false
        uses: mxschmitt/action-tmate@v3

      - name: Run graph tests
        run: |
          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu

          export VINEYARD_DATA_DIR=`pwd`/gstest
          export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

          rm -rf default.etcd
          rm -rf /dev/shm/etcd*
          python3 test/runner.py $RUNNER_ARGS --with-graph

      - name: Upload Coverage
        if: runner.os == 'Linux'
        uses: codecov/codecov-action@v3
        with:
          verbose: true