Merge branch 'build' of github.com:Zhanghyi/graphlearn-for-pytorch in…

…to build

CMakeLists.txt

@@ -16,6 +16,9 @@ set(GLT_THIRD_PARTY_DIR ${GLT_ROOT}/third_party)
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${GLT_BUILT_DIR}/bin)
 set(LIBRARY_OUTPUT_PATH ${GLT_BUILT_DIR}/lib)
 
+set(CXX11_ABI_TRUE_FLAG "_GLIBCXX_USE_CXX11_ABI=1")
+set(CXX11_ABI_FALSE_FLAG "_GLIBCXX_USE_CXX11_ABI=0")
+
 if(DEBUG)
   set(GLT_MODE_FLAGS -g)
   set(CMAKE_BUILD_TYPE Debug)
@@ -33,8 +36,7 @@ set(GLT_CXX_FLAGS
   -msse4.1
   -Wno-attributes
   -Wno-deprecated-declarations
-  -Werror=return-type
-  -D_GLIBCXX_USE_CXX11_ABI=0)
+  -Werror=return-type)
 
 # Link to Python when building
 find_package(PythonInterp REQUIRED)
@@ -79,72 +81,91 @@ if(WITH_CUDA)
   add_definitions(-DWITH_CUDA)
 endif()
 
-# Link to Vineyard if enabled
-if(WITH_VINEYARD)
-  add_definitions(-DWITH_VINEYARD)
-  find_package(vineyard REQUIRED)
-endif()
-
-# Build library
-add_library(graphlearn_torch SHARED)
-
 function(target_source_tree target)
   file(GLOB_RECURSE SRCS ${ARGN})
   target_sources(${target} PRIVATE ${SRCS})
 endfunction()
 
-target_source_tree(graphlearn_torch
-  ${GLT_CSRC_DIR}/*.cc
-  PROPERTIES COMPILE_FLAGS ${GLT_CXX_FLAGS})
-target_source_tree(graphlearn_torch
-  ${GLT_CSRC_DIR}/cpu/*.cc
-  PROPERTIES COMPILE_FLAGS ${GLT_CXX_FLAGS})
-if(WITH_CUDA)
-  target_source_tree(graphlearn_torch
-    ${GLT_CSRC_DIR}/cuda/*.cu)
-endif()
+# Build graphlearn_torch_vineyard library
+function(add_library_graphlearn_torch_vineyard)
+  set(GLT_V6D_CSRC_DIR ${GLT_ROOT}/graphlearn_torch/v6d)
+  find_package(vineyard REQUIRED)
+  add_library(graphlearn_torch_vineyard SHARED)
+  target_source_tree(graphlearn_torch_vineyard 
+    ${GLT_V6D_CSRC_DIR}/*.cc
+    PROPERTIES COMPILE_FLAGS ${GLT_CXX_FLAGS})
+  target_include_directories(graphlearn_torch_vineyard
+    PUBLIC ${GLT_ROOT})
+  target_include_directories(graphlearn_torch_vineyard
+    PRIVATE ${VINEYARD_INCLUDE_DIRS})
 
-target_include_directories(graphlearn_torch
-  PUBLIC ${GLT_ROOT})
-target_link_libraries(graphlearn_torch
-  PUBLIC Python3::Python ${TORCH_LIBRARIES})
+  target_link_libraries(graphlearn_torch_vineyard
+    PUBLIC Python3::Python ${TORCH_LIBRARIES})
+  target_link_libraries(graphlearn_torch_vineyard
+    PUBLIC ${VINEYARD_LIBRARIES})
+  target_compile_definitions(graphlearn_torch_vineyard PRIVATE CXX11_ABI_FALSE_FLAG)
+endfunction()
 
-if(WITH_CUDA)
-  target_set_cuda_options(graphlearn_torch)
+if(WITH_VINEYARD)
+  add_library_graphlearn_torch_vineyard()
 endif()
 
-if(WITH_VINEYARD)
+function(add_library_graphlearn_torch)
+  add_library(graphlearn_torch SHARED)
+  target_source_tree(graphlearn_torch
+    ${GLT_CSRC_DIR}/*.cc
+    PROPERTIES COMPILE_FLAGS ${GLT_CXX_FLAGS})
+  target_source_tree(graphlearn_torch
+    ${GLT_CSRC_DIR}/cpu/*.cc
+    PROPERTIES COMPILE_FLAGS ${GLT_CXX_FLAGS})
+  if(WITH_CUDA)
+    target_source_tree(graphlearn_torch
+      ${GLT_CSRC_DIR}/cuda/*.cu)
+  endif()
   target_include_directories(graphlearn_torch
-    PRIVATE ${VINEYARD_INCLUDE_DIRS})
+    PUBLIC ${GLT_ROOT})
+  message("TORCH_LIBRARIES: ${TORCH_LIBRARIES}")
   target_link_libraries(graphlearn_torch
-    PUBLIC ${VINEYARD_LIBRARIES})
-endif()
+    PUBLIC Python3::Python ${TORCH_LIBRARIES})
+  target_compile_definitions(graphlearn_torch PRIVATE CXX11_ABI_FALSE_FLAG)
+
+  if(WITH_CUDA)
+    target_set_cuda_options(graphlearn_torch)
+  endif()
+endfunction()
+
+add_library_graphlearn_torch()
 
 # Build tests
 if(BUILD_TESTS)
   set (GTest_INSTALL_DIR ${GLT_THIRD_PARTY_DIR}/googletest/build)
   find_package(GTest REQUIRED PATHS ${GTest_INSTALL_DIR})
 
-  function(glt_add_test target)
+  function(add_tests target library)
     add_executable(${target} ${ARGN})
-    add_dependencies(${target} graphlearn_torch)
+    add_dependencies(${target} ${library})
     target_link_libraries(${target}
-      PUBLIC graphlearn_torch GTest::gtest GTest::gtest_main)
+      PUBLIC ${library} GTest::gtest GTest::gtest_main)
     if(WITH_CUDA)
       target_set_cuda_options(${target})
     endif()
+    target_compile_definitions(${target} PRIVATE CXX11_ABI_FALSE_FLAG)
   endfunction()
 
   if(WITH_CUDA)
     file(GLOB GLT_TEST_FILES ${GLT_CTEST_DIR}/test_*.cu)
   endif()
 
-  if (WITH_VINEYARD)
-    file(GLOB GLT_TEST_FILES ${GLT_CTEST_DIR}/test_vineyard.cc)
-  endif()
-
   foreach(t ${GLT_TEST_FILES})
     get_filename_component(name ${t} NAME_WE)
-    glt_add_test(${name} ${t})
+    add_tests(${name} graphlearn_torch ${t})
   endforeach()
+  if (WITH_VINEYARD)
+    file(GLOB GLT_V6D_TEST_FILES ${GLT_CTEST_DIR}/test_vineyard.cc)
+    foreach(t ${GLT_V6D_TEST_FILES})
+      get_filename_component(name ${t} NAME_WE)
+      add_tests(${name} graphlearn_torch_vineyard ${t})
+    endforeach()
+  endif()
+
 endif()

graphlearn_torch/python/data/vineyard_utils.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-from .. import py_graphlearn_torch as pywrap
+from .. import  py_graphlearn_torch_vineyard as pywrap
 
 
 def vineyard_to_csr(sock, fid, v_label_name, e_label_name, edge_dir, haseid=0):

graphlearn_torch/python/py_export.cc → graphlearn_torch/python/py_export_glt.cc

@@ -31,7 +31,7 @@ limitations under the License.
 #include "graphlearn_torch/include/sampler.h"
 #include "graphlearn_torch/include/stitch_sample_results.h"
 #include "graphlearn_torch/include/types.h"
-#include "graphlearn_torch/include/vineyard_utils.h"
+#include "graphlearn_torch/v6d/vineyard_utils.h"
 
 #ifdef WITH_CUDA
 #include "graphlearn_torch/csrc/cuda/inducer.cuh"
@@ -49,11 +49,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
   m.def("cpu_stitch_sample_results", &CPUStitchSampleResults);
 #ifdef WITH_CUDA
   m.def("cuda_stitch_sample_results", &CUDAStitchSampleResults);
-#endif
-#ifdef WITH_VINEYARD
-  m.def("vineyard_to_csr", &ToCSR);
-  m.def("load_vertex_feature_from_vineyard", &LoadVertexFeatures);
-  m.def("load_edge_feature_from_vineyard", &LoadEdgeFeatures);
 #endif
   py::enum_<GraphMode>(m, "GraphMode")
     .value("DMA", GraphMode::DMA)

graphlearn_torch/python/py_export_v6d.cc

@@ -0,0 +1,34 @@
+/* Copyright 2022 Alibaba Group Holding Limited. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <Python.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <pybind11/stl_bind.h>
+
+#include <torch/extension.h>
+
+#include "graphlearn_torch/v6d/vineyard_utils.h"
+
+namespace py = pybind11;
+
+using namespace graphlearn_torch::vineyard_utils;
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+  m.doc() = "Python bindings for vineyard utils C++ frontend";
+  m.def("vineyard_to_csr", &ToCSR);
+  m.def("load_vertex_feature_from_vineyard", &LoadVertexFeatures);
+  m.def("load_edge_feature_from_vineyard", &LoadEdgeFeatures);
+}

graphlearn_torch/python/utils/build.py

@@ -2,12 +2,63 @@
 
 from torch.utils.cpp_extension import CppExtension
 
+def glt_v6d_ext_module(
+  name: str,
+  root_path: str,
+):
+  include_dirs = []
+  library_dirs = []
+  libraries = []
+  extra_cxx_flags = []
+
+  include_dirs.append(root_path)
+  # We assume that glt_v6d is built in graphscope environment
+  include_dirs.append('/usr/lib/x86_64-linux-gnu/openmpi/include')
+  include_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/include' + '/vineyard')
+  include_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/include' + '/vineyard/contrib')
+  include_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/include')
+
+  library_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/lib')
+  library_dirs.append('/usr/local/lib')
+
+  libraries.append('pthread')
+  libraries.append('mpi')
+  libraries.append('glog')
+  libraries.append('vineyard_basic')
+  libraries.append('vineyard_client')
+  libraries.append('vineyard_graph')
+  libraries.append('vineyard_io')
+
+  extra_cxx_flags.append('-D_GLIBCXX_USE_CXX11_ABI=0')
+  extra_cxx_flags.append('-std=c++17')
+
+  sources = [os.path.join(root_path, 'graphlearn_torch/python/py_export_v6d.cc')]
+
+  import glob
+  sources += glob.glob(
+    os.path.join(root_path, 'graphlearn_torch/v6d/**.cc'), recursive=True
+  )
+  extra_link_args = []
+  define_macros = [('WITH_VINEYARD', 'ON')]
+  undef_macros = []
+  return CppExtension(
+    name,
+    sources,
+    extra_link_args=extra_link_args,
+    include_dirs=include_dirs,
+    library_dirs=library_dirs,
+    libraries=libraries,
+    extra_compile_args={
+      'cxx': extra_cxx_flags,
+    },
+    define_macros=define_macros,
+    undef_macros=undef_macros,
+  )
 
-def ext_module(
+def glt_ext_module(
   name: str,
   root_path: str,
   with_cuda: bool = False,
-  with_vineyard: bool = True,
   release: bool = False
 ):
   include_dirs = []
@@ -22,38 +73,11 @@ def ext_module(
   if with_cuda:
     include_dirs.append('/usr/local/cuda' + '/include')
     library_dirs.append('/usr/local/cuda' + 'lib64')
-  if with_vineyard:
-
-    # in graphscope environment
-    include_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/include' + '/vineyard')
-    include_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/include' + '/vineyard/contrib')
-    include_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/include')
-    # include_dirs.append('/usr/lib/x86_64-linux-gnu/openmpi/include')
-
-    include_dirs.append('/usr/local/include')
-    include_dirs.append('/usr/local/include' + '/glog')
-    include_dirs.append('/usr/local/include' + '/gflags')
-    include_dirs.append('/usr/local/include' + '/vineyard')
-    include_dirs.append('/usr/local/include' + '/vineyard/contrib')
-    include_dirs.append('/home/pai/include')
-
-    library_dirs.append(os.environ['GRAPHSCOPE_HOME'] + '/lib')
-
-    library_dirs.append('/usr/local/lib')
-    library_dirs.append('/home/pai/lib')
-
-    libraries.append('pthread')
-    libraries.append('mpi')
-    libraries.append('glog')
-    libraries.append('vineyard_basic')
-    libraries.append('vineyard_client')
-    libraries.append('vineyard_graph')
-    libraries.append('vineyard_io')
 
   extra_cxx_flags.append('-D_GLIBCXX_USE_CXX11_ABI=0')
   extra_cxx_flags.append('-std=c++17')
 
-  sources = [os.path.join(root_path, 'graphlearn_torch/python/py_export.cc')]
+  sources = [os.path.join(root_path, 'graphlearn_torch/python/py_export_glt.cc')]
 
   import glob
   sources += glob.glob(
@@ -65,11 +89,6 @@ def ext_module(
       os.path.join(root_path, 'graphlearn_torch/csrc/**/**.cu'), recursive=True
     )
 
-  if with_vineyard:
-    define_macros.append(('WITH_VINEYARD', 'ON'))
-  else:
-    undef_macros.append(('WITH_VINEYARD'))
-
   if with_cuda:
     define_macros.append(('WITH_CUDA', 'ON'))
   else:

graphlearn_torch/csrc/cpu/vineyard_utils.cc → graphlearn_torch/v6d/vineyard_utils.cc

@@ -13,9 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifdef WITH_VINEYARD
-
-#include "graphlearn_torch/include/vineyard_utils.h"
+#include "graphlearn_torch/v6d/vineyard_utils.h"
 
 #include <vineyard/client/client.h>
 #include <vineyard/graph/fragment/arrow_fragment.h>
@@ -24,6 +22,9 @@ limitations under the License.
 
 #include "glog/logging.h"
 
+namespace graphlearn_torch {
+namespace vineyard_utils {
+
 using GraphType = vineyard::ArrowFragment<vineyard::property_graph_types::OID_TYPE,
                                           vineyard::property_graph_types::VID_TYPE>;
 
@@ -260,4 +261,5 @@ torch::Tensor LoadEdgeFeatures(
   return feat;
 }
 
-#endif // WITH_VINEYARD
+} // namespace vineyard_utils
+}  // namespace graphlearn_torch

graphlearn_torch/include/vineyard_utils.h → graphlearn_torch/v6d/vineyard_utils.h

@@ -13,13 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifdef WITH_VINEYARD
-
 #include <torch/extension.h>
-
-#include "graphlearn_torch/include/graph.h"
-#include "graphlearn_torch/include/common.h"
-
+namespace graphlearn_torch {
+namespace vineyard_utils {
 
 std::tuple<torch::Tensor, torch::Tensor, torch::Tensor> ToCSR(
   const std::string& ipc_socket, const std::string& object_id_str,
@@ -36,4 +32,5 @@ torch::Tensor LoadEdgeFeatures(
   const std::string& e_label_name, std::vector<std::string>& ecols
 );
 
-#endif // WITH_VINEYARD
+} // namespace vineyard_utils
+}  // namespace graphlearn_torch