Merge pull request #14 from chrisbarber/master

Implement multithreaded `push_many` on wt_string
xxsds · May 8, 2019 · 829fb43 · 829fb43
2 parents 7492fc6 + 5b7c90b
commit 829fb43
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 1 deletion.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,7 @@
 cmake_minimum_required(VERSION 2.6)
 
+option(USE_OPENMP "Enable multi-threading" OFF)
+
 # Set a default build type if none was specified
 if(NOT CMAKE_BUILD_TYPE)
   message(STATUS "Setting build type to 'Debug' as none was specified.")
@@ -36,6 +38,12 @@ message("Building in ${CMAKE_BUILD_TYPE} mode")
 
 set(CMAKE_CXX_FLAGS "--std=c++11")
 
+if(XXSDS_DYN_MULTI_THREADED)
+  find_package(OpenMP REQUIRED)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+endif(XXSDS_DYN_MULTI_THREADED)
+
 set(CMAKE_CXX_FLAGS_DEBUG "-O0 -ggdb -g -p")
 set(CMAKE_CXX_FLAGS_RELEASE "-ggdb -Ofast -fstrict-aliasing -DNDEBUG -march=native")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -ggdb -Ofast -fstrict-aliasing -march=native")

diff --git a/include/internal/alphabet_encoder.hpp b/include/internal/alphabet_encoder.hpp
@@ -181,9 +181,18 @@ class alphabet_encoder{
 	 * alphabet size
 	 */
 	uint64_t size() const {
-        return enc_type==fixed ? 1ull<<log_sigma : sigma;
+           return enc_type==fixed ? 1ull<<log_sigma : sigma;
 	}
 
+        set<char_type> keys() const {
+           set<char_type> keys;
+           for(char_type c = 0; c < size(); ++c) {
+              if (char_exists(c) || enc_type==fixed)
+                 keys.insert(c);
+           }
+           return keys;
+        }
+
 	/*
 	 * Total number of bits allocated in RAM for this structure
 	 *

diff --git a/include/internal/includes.hpp b/include/internal/includes.hpp
@@ -17,6 +17,7 @@
 #include <string>
 #include <iostream>
 #include <set>
+#include <map>
 #include <vector>
 #include <fstream>
 #include <sstream>

diff --git a/include/internal/packed_vector.hpp b/include/internal/packed_vector.hpp
@@ -459,6 +459,11 @@ namespace dyn{
 
       void insert(uint64_t i, uint64_t x){
 
+         if(i==size()){
+            push_back(x);
+            return;
+         }
+
 	 if(bitsize(x)>width_){
 
 	    //auto vec = to_vector(i,x);

diff --git a/include/internal/wt_string.hpp b/include/internal/wt_string.hpp
@@ -24,6 +24,7 @@
 
 #include "includes.hpp"
 #include "alphabet_encoder.hpp"
+#include <thread>
 
 namespace dyn{
 
@@ -166,6 +167,25 @@ namespace dyn{
 
       }
 
+      template<class Vector>
+      void push_many(const Vector& values) {
+         for (ulint i = 0; i < values.size(); ++i) {
+            auto c = values[i];
+            if(!ae.char_exists(c))
+               ae.encode(c);
+         }
+
+         map<char_type, vector<bool>> path_to_leaf;
+         for(char_type c : ae.keys()) {
+            path_to_leaf[c] = ae.encode(c);
+         }
+
+         #pragma omp parallel
+         #pragma omp master
+         root.push_many(std::move(path_to_leaf), values);
+         n += values.size();
+      }
+
       void push_front(char_type c){
 
 	 insert(0,c);
@@ -441,6 +461,79 @@ namespace dyn{
 
 	 }
 
+    template<class Vector>
+    void push_many(map<char_type, vector<bool>>&& Bs,
+                   const Vector& values,
+                   ulint j=0,
+                   ulint offset=0) {
+
+        if(Bs.size()==1){
+            //this node must be a leaf
+            assert(bv.size()==0);
+
+            auto c = Bs.begin()->first;
+            assert(j==Bs[c].size());
+
+            if(is_leaf()){
+                //if it's already marked as leaf, check
+                //that the label is correct
+                assert(c==label());
+            }else{
+                //else, mark node as leaf
+                make_leaf(c);
+            }
+            return;
+        }
+
+        assert(not is_leaf());
+
+        bool task_started_0 = false;
+        bool task_started_1 = false;
+
+        for(ulint idx = offset; idx < values.size(); ++idx) {
+            char_type c = values[idx];
+            auto it = Bs.find(c);
+            if (it == Bs.end())
+                continue;
+
+            bool b = it->second[j];
+
+            bv.push_back(b);
+
+            if(b && !task_started_1){
+                task_started_1 = true;
+
+                if(not has_child1())
+                    child1_ = new node(this);
+
+                map<char_type, vector<bool>> new_Bs;
+                for_each(Bs.begin(), Bs.end(), [&new_Bs,j](const auto &pair) {
+                    if (pair.second[j])
+                        new_Bs.insert(pair);
+                });
+                #pragma omp task
+                child1_->push_many(std::move(new_Bs), values, j+1, idx);
+            }
+
+            if (!b && !task_started_0){
+                task_started_0 = true;
+
+                if(not has_child0())
+                    child0_ = new node(this);
+
+                map<char_type, vector<bool>> new_Bs;
+                for_each(Bs.begin(), Bs.end(), [&new_Bs,j](const auto &pair) {
+                    if (!pair.second[j])
+                        new_Bs.insert(pair);
+                });
+                #pragma omp task
+                child0_->push_many(std::move(new_Bs), values, j+1, idx);
+            }
+        }
+
+        #pragma omp taskwait
+    }
+
 	 /*
 	  * remove code B[j,...,B.size()-1] from position i. This code is associated
 	  * with character c