Batch encoding/decoding (with SIMD) of varint CSR (#1403)

What do these changes do? ------------------------- - add back the offset array for correct degree - use v8dec/nec for varint encoding - implements batch decoding in CompactNbr Part of #1373 Signed-off-by: Tao He <[email protected]>
v6d-io · May 30, 2023 · 2c0957b · 2c0957b
1 parent e895216
commit 2c0957b
Show file tree

Hide file tree

Showing 13 changed files with 212 additions and 179 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -50,3 +50,6 @@
  path = modules/graph/thirdparty/libgrape-lite
  url = https://github.com/alibaba/libgrape-lite.git
  shallow = true
+[submodule "modules/graph/thirdparty/powturbo"]
+ path = modules/graph/thirdparty/powturbo
+ url = https://github.com/powturbo/TurboPFor-Integer-Compression.git
diff --git a/modules/graph/CMakeLists.txt b/modules/graph/CMakeLists.txt
@@ -41,20 +41,32 @@ add_custom_target(vineyard_graph_java_gen
 
 add_dependencies(vineyard_graph_gen vineyard_basic_gen)
 
+# add powturbo library
+set(POWTURBO_SRC_FILES
+ ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/powturbo/lib/v8.c
+)
+add_library(powturbo-objects OBJECT ${POWTURBO_SRC_FILES})
+set(powturbo-target-objects $<TARGET_OBJECTS:powturbo-objects>)
+target_compile_options(powturbo-objects PRIVATE -Wp,-w -O3 -fPIC -falign-loops -fstrict-aliasing)
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+ target_compile_options(powturbo-objects PRIVATE -march=corei7-avx -mtune=corei7-avx)
+else()
+ target_compile_options(powturbo-objects PRIVATE "-D_NAVX2")
+endif()
+
 file(GLOB_RECURSE GRAPH_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}" "fragment/*.cc"
  "loader/*.cc"
  "utils/*.cc"
  "vertex_map/*.cc"
  "writer/*.cc"
 )
 
-add_library(vineyard_graph ${GRAPH_SRC_FILES})
+add_library(vineyard_graph ${GRAPH_SRC_FILES} ${powturbo-target-objects})
 target_add_debuginfo(vineyard_graph)
 target_compile_options(vineyard_graph PUBLIC "-fopenmp")
 target_link_options(vineyard_graph PUBLIC "-fopenmp")
-target_include_directories(vineyard_graph PUBLIC
- ${MPI_CXX_INCLUDE_PATH}
-)
+target_include_directories(vineyard_graph PUBLIC ${MPI_CXX_INCLUDE_PATH})
+target_compile_options(vineyard_graph PUBLIC "-DVARINT_ENCODING_BATCH_SIZE=16") # for varint batching
 
 find_package(Boost COMPONENTS leaf)
 if(Boost_LEAF_FOUND)
@@ -69,7 +81,6 @@ else()
  DESTINATION ${CMAKE_INSTALL_PREFIX}/include/vineyard/contrib
  PATTERN "*.hpp"
  )
-
  target_include_directories(vineyard_graph PUBLIC
  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/boost-leaf/include>
  $<INSTALL_INTERFACE:include/vineyard/contrib>
@@ -95,6 +106,16 @@ else()
  )
 endif()
 
+# install headers for powturbo
+target_include_directories(vineyard_graph PUBLIC
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/thirdparty>
+ $<INSTALL_INTERFACE:include>
+)
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/powturbo/include"
+ DESTINATION ${CMAKE_INSTALL_PREFIX}/include/vineyard/contrib/powturbo
+ PATTERN "*.h"
+)
+
 if(BUILD_VINEYARD_GRAPH_WITH_GAR)
  target_compile_definitions(vineyard_graph PUBLIC -DENABLE_GAR)
  find_package(gar QUIET)

diff --git a/modules/graph/fragment/arrow_fragment.h b/modules/graph/fragment/arrow_fragment.h
@@ -118,6 +118,8 @@ class BasicArrowFragmentBuilder
  compact_ie_lists_, compact_oe_lists_;
  std::vector<std::vector<std::shared_ptr<FixedInt64Builder>>>
  ie_offsets_lists_, oe_offsets_lists_;
+ std::vector<std::vector<std::shared_ptr<FixedInt64Builder>>>
+ ie_boffsets_lists_, oe_boffsets_lists_;
 
  std::shared_ptr<vertex_map_t> vm_ptr_;
  IdParser<vid_t> vid_parser_;

diff --git a/modules/graph/fragment/arrow_fragment.vineyard-mod b/modules/graph/fragment/arrow_fragment.vineyard-mod
@@ -92,7 +92,7 @@ class [[vineyard]] ArrowFragment
  using nbr_t = property_graph_utils::Nbr<vid_t, eid_t>;
  using nbr_unit_t = property_graph_utils::NbrUnit<vid_t, eid_t>;
  using adj_list_t = property_graph_utils::AdjList<vid_t, eid_t>;
- using compact_adj_list_t = property_graph_utils::EncodedAdjList<vid_t, eid_t>;
+ using compact_adj_list_t = property_graph_utils::CompactAdjList<vid_t, eid_t>;
  using raw_adj_list_t = property_graph_utils::RawAdjList<vid_t, eid_t>;
  using vertex_map_t = VERTEX_MAP_T;
  using vertex_t = grape::Vertex<vid_t>;
@@ -461,10 +461,12 @@ class [[vineyard]] ArrowFragment
  label_id_t v_label = vid_parser_.GetLabelId(vid);
  int64_t v_offset = vid_parser_.GetOffset(vid);
  const int64_t* offset_array = ie_offsets_ptr_lists_[v_label][e_label];
+ const int64_t* boffset_array = ie_boffsets_ptr_lists_[v_label][e_label];
  const uint8_t* ptr = compact_ie_ptr_lists_[v_label][e_label];
- return compact_adj_list_t(ptr, offset_array[v_offset],
- offset_array[v_offset + 1],
- flatten_edge_tables_columns_[e_label]);
+ return compact_adj_list_t(
+ ptr, boffset_array[v_offset], boffset_array[v_offset + 1],
+ offset_array[v_offset + 1] - offset_array[v_offset],
+ flatten_edge_tables_columns_[e_label]);
  }
 
  template <bool COMPACT_ = COMPACT>
@@ -499,10 +501,12 @@ class [[vineyard]] ArrowFragment
  label_id_t v_label = vid_parser_.GetLabelId(vid);
  int64_t v_offset = vid_parser_.GetOffset(vid);
  const int64_t* offset_array = oe_offsets_ptr_lists_[v_label][e_label];
+ const int64_t* boffset_array = oe_boffsets_ptr_lists_[v_label][e_label];
  const uint8_t* ptr = compact_oe_ptr_lists_[v_label][e_label];
- return compact_adj_list_t(ptr, offset_array[v_offset],
- offset_array[v_offset + 1],
- flatten_edge_tables_columns_[e_label]);
+ return compact_adj_list_t(
+ ptr, boffset_array[v_offset], boffset_array[v_offset + 1],
+ offset_array[v_offset + 1] - offset_array[v_offset],
+ flatten_edge_tables_columns_[e_label]);
  }
 
  template <bool COMPACT_ = COMPACT>
@@ -522,15 +526,13 @@ class [[vineyard]] ArrowFragment
  * later.
  */
 
- template <bool COMPACT_ = COMPACT>
- inline const typename std::enable_if<!COMPACT_, int64_t*>::type
- GetIncomingOffsetArray(label_id_t v_label, label_id_t e_label) const {
+ inline const int64_t* GetIncomingOffsetArray(label_id_t v_label,
+ label_id_t e_label) const {
  return ie_offsets_ptr_lists_[v_label][e_label];
  }
 
- template <bool COMPACT_ = COMPACT>
- inline typename std::enable_if<!COMPACT_, const int64_t*>::type
- GetOutgoingOffsetArray(label_id_t v_label, label_id_t e_label) const {
+ inline const int64_t* GetOutgoingOffsetArray(label_id_t v_label,
+ label_id_t e_label) const {
  return oe_offsets_ptr_lists_[v_label][e_label];
  }
 
@@ -760,6 +762,10 @@ class [[vineyard]] ArrowFragment
  oe_offsets_lists_;
  std::vector<std::vector<const int64_t*>> ie_offsets_ptr_lists_,
  oe_offsets_ptr_lists_;
+ [[shared]] List<List<std::shared_ptr<Int64Array>>> ie_boffsets_lists_,
+ oe_boffsets_lists_;
+ std::vector<std::vector<const int64_t*>> ie_boffsets_ptr_lists_,
+ oe_boffsets_ptr_lists_;
 
  std::vector<std::vector<std::vector<fid_t>>> idst_, odst_, iodst_;
  std::vector<std::vector<std::vector<fid_t*>>> idoffset_, odoffset_,

diff --git a/modules/graph/fragment/arrow_fragment_builder_impl.h b/modules/graph/fragment/arrow_fragment_builder_impl.h
@@ -1108,13 +1108,15 @@ BasicArrowFragmentBuilder<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::Build(
  if (this->directed_) {
  if (this->compact_edges_) {
  Base::compact_ie_lists_.resize(this->vertex_label_num_);
+ Base::ie_boffsets_lists_.resize(this->vertex_label_num_);
  } else {
  Base::ie_lists_.resize(this->vertex_label_num_);
  }
  Base::ie_offsets_lists_.resize(this->vertex_label_num_);
  }
  if (this->compact_edges_) {
  Base::compact_oe_lists_.resize(this->vertex_label_num_);
+ Base::oe_boffsets_lists_.resize(this->vertex_label_num_);
  } else {
  Base::oe_lists_.resize(this->vertex_label_num_);
  }
@@ -1124,13 +1126,15 @@ BasicArrowFragmentBuilder<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::Build(
  if (this->directed_) {
  if (this->compact_edges_) {
  Base::compact_ie_lists_[i].resize(this->edge_label_num_);
+ Base::ie_boffsets_lists_[i].resize(this->edge_label_num_);
  } else {
  Base::ie_lists_[i].resize(this->edge_label_num_);
  }
  Base::ie_offsets_lists_[i].resize(this->edge_label_num_);
  }
  if (this->compact_edges_) {
  Base::compact_oe_lists_[i].resize(this->edge_label_num_);
+ Base::oe_boffsets_lists_[i].resize(this->edge_label_num_);
  } else {
  Base::oe_lists_[i].resize(this->edge_label_num_);
  }
@@ -1143,6 +1147,8 @@ BasicArrowFragmentBuilder<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::Build(
  if (this->compact_edges_) {
  RETURN_ON_ERROR(compact_ie_lists_[i][j]->Seal(*client, object));
  this->set_compact_ie_lists_(i, j, object);
+ RETURN_ON_ERROR(ie_boffsets_lists_[i][j]->Seal(*client, object));
+ this->set_ie_boffsets_lists_(i, j, object);
  } else {
  RETURN_ON_ERROR(ie_lists_[i][j]->Seal(*client, object));
  this->set_ie_lists_(i, j, object);
@@ -1153,6 +1159,8 @@ BasicArrowFragmentBuilder<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::Build(
  if (this->compact_edges_) {
  RETURN_ON_ERROR(compact_oe_lists_[i][j]->Seal(*client, object));
  this->set_compact_oe_lists_(i, j, object);
+ RETURN_ON_ERROR(oe_boffsets_lists_[i][j]->Seal(*client, object));
+ this->set_oe_boffsets_lists_(i, j, object);
  } else {
  RETURN_ON_ERROR(oe_lists_[i][j]->Seal(*client, object));
  this->set_oe_lists_(i, j, object);
@@ -1348,7 +1356,8 @@ BasicArrowFragmentBuilder<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initEdges(
  varint_encoding_edges(client_, this->directed_, this->vertex_label_num_,
  this->edge_label_num_, ie_lists_, oe_lists_,
  compact_ie_lists_, compact_oe_lists_,
- ie_offsets_lists_, oe_offsets_lists_, concurrency);
+ ie_offsets_lists_, oe_offsets_lists_,
+ ie_boffsets_lists_, oe_boffsets_lists_, concurrency);
  }
  return {};
 }

diff --git a/modules/graph/fragment/arrow_fragment_impl.h b/modules/graph/fragment/arrow_fragment_impl.h
@@ -529,6 +529,7 @@ void ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initPointers() {
 
  if (this->compact_edges_) {
  compact_oe_ptr_lists_.resize(vertex_label_num_);
+ oe_boffsets_ptr_lists_.resize(vertex_label_num_);
  } else {
  oe_ptr_lists_.resize(vertex_label_num_);
  }
@@ -539,6 +540,7 @@ void ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initPointers() {
 
  if (this->compact_edges_) {
  compact_oe_ptr_lists_[i].resize(edge_label_num_);
+ oe_boffsets_ptr_lists_[i].resize(edge_label_num_);
  } else {
  oe_ptr_lists_[i].resize(edge_label_num_);
  }
@@ -556,6 +558,8 @@ void ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initPointers() {
  if (this->compact_edges_) {
  compact_oe_ptr_lists_[i][j] = reinterpret_cast<const uint8_t*>(
  compact_oe_lists_[i][j]->GetArray()->raw_values());
+ oe_boffsets_ptr_lists_[i][j] =
+ oe_boffsets_lists_[i][j]->GetArray()->raw_values();
  } else {
  oe_ptr_lists_[i][j] = reinterpret_cast<const nbr_unit_t*>(
  oe_lists_[i][j]->GetArray()->raw_values());
@@ -568,13 +572,15 @@ void ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initPointers() {
  if (directed_) {
  if (this->compact_edges_) {
  compact_ie_ptr_lists_.resize(vertex_label_num_);
+ ie_boffsets_ptr_lists_.resize(vertex_label_num_);
  } else {
  ie_ptr_lists_.resize(vertex_label_num_);
  }
  ie_offsets_ptr_lists_.resize(vertex_label_num_);
  for (label_id_t i = 0; i < vertex_label_num_; ++i) {
  if (this->compact_edges_) {
  compact_ie_ptr_lists_[i].resize(edge_label_num_);
+ ie_boffsets_ptr_lists_[i].resize(edge_label_num_);
  } else {
  ie_ptr_lists_[i].resize(edge_label_num_);
  }
@@ -584,6 +590,8 @@ void ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initPointers() {
  if (this->compact_edges_) {
  compact_ie_ptr_lists_[i][j] = reinterpret_cast<const uint8_t*>(
  compact_ie_lists_[i][j]->GetArray()->raw_values());
+ ie_boffsets_ptr_lists_[i][j] =
+ ie_boffsets_lists_[i][j]->GetArray()->raw_values();
  } else {
  ie_ptr_lists_[i][j] = reinterpret_cast<const nbr_unit_t*>(
  ie_lists_[i][j]->GetArray()->raw_values());
@@ -595,6 +603,7 @@ void ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, COMPACT>::initPointers() {
  } else {
  if (this->compact_edges_) {
  compact_ie_ptr_lists_ = compact_oe_ptr_lists_;
+ ie_boffsets_ptr_lists_ = oe_boffsets_ptr_lists_;
  } else {
  ie_ptr_lists_ = oe_ptr_lists_;
  }