refactor(dist): remove OpenMP dependency (#18)

* refactor(dist): remove OpenMP dependency in sparse alltoall

* fix(dist): crash in distributed contraction code when executing in hybrid mode due to non-parallelized section

* tests(dist): add hybryd endtoend test

* style(tests): add empty lines after namespace

* refactor(dist): remove remaining OpenMP dependencies

* ci: no longer install OpenMP

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
         build-mode: [Release]
     steps:
       - name: Install dependencies
-        run: brew install llvm libomp tbb open-mpi google-sparsehash
+        run: brew install llvm tbb open-mpi google-sparsehash
       - name: Checkout HEAD
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
@@ -63,7 +63,7 @@ jobs:
         build-mode: [Release]
     steps:
       - name: Install dependencies
-        run: sudo apt-get install -y libtbb-dev libhwloc-dev mpi libopenmpi-dev libomp-dev libsparsehash-dev
+        run: sudo apt-get install -y libtbb-dev libhwloc-dev mpi libopenmpi-dev libsparsehash-dev
       - name: Checkout HEAD
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:

apps/benchmarks/dist_block_clustering_benchmark.cc

@@ -12,7 +12,6 @@
 #include <fstream>
 
 #include <mpi.h>
-#include <omp.h>
 
 #include "kaminpar-dist/context.h"
 #include "kaminpar-dist/context_io.h"
@@ -56,7 +55,6 @@ int main(int argc, char *argv[]) {
   CLI11_PARSE(app, argc, argv);
 
   tbb::global_control gc(tbb::global_control::max_allowed_parallelism, ctx.parallel.num_threads);
-  omp_set_num_threads(ctx.parallel.num_threads);
 
   auto wrapper = load_partitioned_graph(graph_filename, partition_filename);
   auto &graph = *wrapper.graph;

apps/benchmarks/dist_coarsening_benchmark.cc

@@ -10,7 +10,6 @@
 // clang-format on
 
 #include <mpi.h>
-#include <omp.h>
 
 #include "kaminpar-dist/coarsening/coarsener.h"
 #include "kaminpar-dist/context.h"
@@ -49,7 +48,6 @@ int main(int argc, char *argv[]) {
   CLI11_PARSE(app, argc, argv);
 
   tbb::global_control gc(tbb::global_control::max_allowed_parallelism, ctx.parallel.num_threads);
-  omp_set_num_threads(ctx.parallel.num_threads);
 
   auto wrapper = load_graph(graph_filename);
   auto &graph = *wrapper.graph;

apps/benchmarks/dist_coloring_benchmark.cc

@@ -10,7 +10,6 @@
 // clang-format on
 
 #include <mpi.h>
-#include <omp.h>
 #include <tbb/enumerable_thread_specific.h>
 #include <tbb/parallel_for.h>
 
@@ -38,7 +37,6 @@ int main(int argc, char *argv[]) {
   CLI11_PARSE(app, argc, argv);
 
   tbb::global_control gc(tbb::global_control::max_allowed_parallelism, ctx.parallel.num_threads);
-  omp_set_num_threads(ctx.parallel.num_threads);
 
   auto wrapper = load_graph(graph_filename);
   auto &graph = *wrapper.graph;

apps/benchmarks/dist_contraction_benchmark.cc

@@ -10,7 +10,6 @@
 // clang-format on
 
 #include <mpi.h>
-#include <omp.h>
 
 #include "kaminpar-dist/coarsening/contraction/global_cluster_contraction.h"
 #include "kaminpar-dist/context.h"
@@ -42,7 +41,6 @@ int main(int argc, char *argv[]) {
   CLI11_PARSE(app, argc, argv);
 
   tbb::global_control gc(tbb::global_control::max_allowed_parallelism, ctx.parallel.num_threads);
-  omp_set_num_threads(ctx.parallel.num_threads);
 
   auto wrapper = load_graph(graph_filename);
   auto &graph = *wrapper.graph;

apps/benchmarks/dist_refinement_benchmark.cc

@@ -12,7 +12,6 @@
 #include <fstream>
 
 #include <mpi.h>
-#include <omp.h>
 
 #include "kaminpar-dist/context.h"
 #include "kaminpar-dist/context_io.h"
@@ -62,7 +61,6 @@ int main(int argc, char *argv[]) {
   }
 
   tbb::global_control gc(tbb::global_control::max_allowed_parallelism, ctx.parallel.num_threads);
-  omp_set_num_threads(ctx.parallel.num_threads);
 
   auto wrapper = load_partitioned_graph(graph_filename, partition_filename);
   auto &graph = *wrapper.graph;

kaminpar-dist/datastructures/growt.h

@@ -10,6 +10,8 @@
 #include <type_traits>
 
 #include <tbb/enumerable_thread_specific.h>
+#include <tbb/parallel_for.h>
+#include <tbb/task_arena.h>
 
 #include "kaminpar-dist/dkaminpar.h"
 
@@ -59,8 +61,7 @@ using StaticGhostNodeMapping = typename ::growt::
 template <typename Map, typename Lambda> void pfor_map(Map &map, Lambda &&lambda) {
   std::atomic_size_t counter = 0;
 
-#pragma omp parallel default(none) shared(map, counter, lambda)
-  {
+  tbb::parallel_for<int>(0, tbb::this_task_arena::max_concurrency(), [&](const int) {
     const std::size_t capacity = map.capacity();
     std::size_t cur_block = counter.fetch_add(4096);
 
@@ -71,14 +72,13 @@ template <typename Map, typename Lambda> void pfor_map(Map &map, Lambda &&lambda
       }
       cur_block = counter.fetch_add(4096);
     }
-  }
+  });
 }
 
 template <typename Handles, typename Lambda> void pfor_handles(Handles &handles, Lambda &&lambda) {
   std::atomic_size_t counter = 0;
 
-#pragma omp parallel default(none) shared(handles, counter, lambda)
-  {
+  tbb::parallel_for<int>(0, tbb::this_task_arena::max_concurrency(), [&](const int) {
     auto &handle = handles.local();
     const std::size_t capacity = handle.capacity();
     std::size_t cur_block = counter.fetch_add(4096);
@@ -90,6 +90,6 @@ template <typename Handles, typename Lambda> void pfor_handles(Handles &handles,
       }
       cur_block = counter.fetch_add(4096);
     }
-  }
+  });
 }
 } // namespace kaminpar::dist::growt

kaminpar-dist/dkaminpar.cc

@@ -11,7 +11,6 @@
 #include <utility>
 
 #include <mpi.h>
-#include <omp.h>
 #include <tbb/global_control.h>
 #include <tbb/parallel_invoke.h>
 
@@ -157,7 +156,6 @@ dKaMinPar::dKaMinPar(MPI_Comm comm, const int num_threads, const Context ctx)
       _num_threads(num_threads),
       _ctx(ctx),
       _gc(tbb::global_control::max_allowed_parallelism, num_threads) {
-  omp_set_num_threads(num_threads);
 #ifdef KAMINPAR_ENABLE_TIMERS
   GLOBAL_TIMER.reset();
 #endif // KAMINPAR_ENABLE_TIMERS