Skip to content

GraphAr: Support load and write subgraph to graphar with selected vertices/edges and properties #91

GraphAr: Support load and write subgraph to graphar with selected vertices/edges and properties

GraphAr: Support load and write subgraph to graphar with selected vertices/edges and properties #91

# Copyright 2020-2023 Alibaba Group Holding Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Vineyard Graph CI
on:
push:
branches:
- main
paths:
- 'modules/graph/**'
pull_request:
branches:
- main
paths:
- 'modules/graph/**'
workflow_dispatch:
concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
env:
CMAKE_C_COMPILER_LAUNCHER: ccache
CMAKE_CXX_COMPILER_LAUNCHER: ccache
jobs:
ci:
runs-on: ${{ matrix.os }}
if: ${{ github.repository == 'v6d-io/v6d' }}
strategy:
matrix:
os: [ubuntu-20.04]
malloc: [dlmalloc]
metadata: [etcd]
env:
RUNNER_ARGS: "--meta=${{ matrix.metadata }}"
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Generate Summary for Submodules
run: |
git submodule > git-modules.txt
cat git-modules.txt
- name: Cache for ccache
uses: actions/cache@v3
with:
path: ~/.ccache
key: ${{ runner.os }}-${{ matrix.malloc }}-${{ matrix.metadata }}-ccache-graph-${{ hashFiles('**/git-modules.txt') }}
restore-keys: |
${{ runner.os }}-${{ matrix.malloc }}-${{ matrix.metadata }}-ccache-graph-
- name: Install Dependencies for Linux
if: runner.os == 'Linux'
run: |
export PATH=/usr/lib/ccache:$PATH
sudo apt update -y
sudo apt install -y ca-certificates \
ccache \
cmake \
doxygen \
fuse3 \
libboost-all-dev \
libcurl4-openssl-dev \
libfuse3-dev \
libgflags-dev \
libgoogle-glog-dev \
libgmock-dev \
libgrpc-dev \
libgrpc++-dev \
libkrb5-dev \
libmpich-dev \
libprotobuf-dev \
librdkafka-dev \
libgsasl7-dev \
librdkafka-dev \
libssl-dev \
libunwind-dev \
libuv1-dev \
libxml2-dev \
libz-dev \
lsb-release \
redis-server \
pandoc \
protobuf-compiler-grpc \
python3-pip \
uuid-dev \
wget
# install apache-arrow
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt update
sudo apt install -y libarrow-dev=14.0.1-1 \
libarrow-dataset-dev=14.0.1-1 \
libarrow-acero-dev=14.0.1-1 \
libarrow-flight-dev=14.0.1-1 \
libgandiva-dev=14.0.1-1 \
libparquet-dev=14.0.1-1
# install clang-format
sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-1d7ec53d/clang-format-11_linux-amd64 --output /usr/bin/clang-format
sudo chmod +x /usr/bin/clang-format
# avoid possible permission errors
ccache -o cache_dir=~/.ccache
sudo mkdir -p ~/.ccache
sudo chmod -R a+wrx ~/.ccache
# install python packages for codegen, and io adaptors
sudo pip3 install -U "Pygments>=2.4.1"
sudo pip3 install -r requirements-setup.txt -r requirements.txt -r requirements-dev.txt
- name: Setup tmate session
if: false
uses: mxschmitt/action-tmate@v3
- name: CMake
run: |
export PATH=/usr/lib/ccache:$PATH
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Debug \
-DBUILD_SHARED_LIBS=ON \
-DUSE_ASAN=ON \
-DWITH_ALLOCATOR=${{ matrix.malloc }} \
-DBUILD_VINEYARD_COVERAGE=ON \
-DBUILD_VINEYARD_PYTHON_BINDINGS=OFF \
-DBUILD_VINEYARD_BASIC=ON \
-DBUILD_VINEYARD_IO=ON \
-DBUILD_VINEYARD_IO_KAFKA=ON \
-DBUILD_VINEYARD_MALLOC=OFF \
-DBUILD_VINEYARD_HOSSEINMOEIN_DATAFRAME=OFF \
-DBUILD_VINEYARD_FUSE=OFF \
-DBUILD_VINEYARD_FUSE_PARQUET=OFF \
-DBUILD_VINEYARD_GRAPH=ON \
-DBUILD_VINEYARD_GRAPH_WITH_GAR=ON \
-DBUILD_VINEYARD_TESTS_ALL=ON
if [ "${{ matrix.metadata }}" == "redis" ]; then
cmake .. -DBUILD_VINEYARD_SERVER_REDIS=ON
fi
- name: Cpp format and lint
run: |
cd build
function prepend() { while read line; do echo "${1}${line}"; done; }
make vineyard_clformat
GIT_DIFF=$(git diff --ignore-submodules)
if [[ -n $GIT_DIFF ]]; then
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "| clang-format failures found!"
echo "|"
echo "$GIT_DIFF" | prepend "| "
echo "|"
echo "| Run: "
echo "|"
echo "| make vineyard_clformat"
echo "|"
echo "| to fix this error."
echo "|"
echo "| Ensure you are working with clang-format-11, which can be obtained from"
echo "|"
echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases "
echo "|"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
exit -1
fi
function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; }
ec make vineyard_cpplint
if [[ "$ec" != "0" ]]; then
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "| cpplint failures found! Run: "
echo "|"
echo "| make vineyard_cpplint"
echo "|"
echo "| to fix this error."
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
exit -1
fi
- name: Build
run: |
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64
pushd build
make vineyardd vineyard_graph_tests -j`nproc`
make ccache-stats
popd
- name: Get test data
run: |
git clone https://github.com/GraphScope/gstest.git --depth=1
git clone https://github.com/GraphScope/gar-test.git --depth=1
- name: Setup tmate session
if: false
uses: mxschmitt/action-tmate@v3
- name: Run graph tests
run: |
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
export VINEYARD_DATA_DIR=`pwd`/gstest
export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"
rm -rf default.etcd
rm -rf /dev/shm/etcd*
python3 test/runner.py $RUNNER_ARGS --with-graph
- name: Upload Coverage
if: runner.os == 'Linux'
uses: codecov/codecov-action@v3
with:
verbose: true