Skip to content

Commit

Permalink
update ddm utiliyy
Browse files Browse the repository at this point in the history
  • Loading branch information
@PierreMarchand20
PierreMarchand20 committed Nov 29, 2023
1 parent 2976587 commit ca6582c
Show file tree
Hide file tree
Showing 12 changed files with 569 additions and 265 deletions.
2 changes: 0 additions & 2 deletions include/htool/hmatrix/tree_builder/tree_builder.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ class HMatrixTreeBuilder {
int m_maxblocksize{1000000};
int m_minsourcedepth{0};
int m_mintargetdepth{0};
bool m_delay_dense_computation{false};
int m_reqrank{-1};
int m_target_partition_number{-1};

Expand Down Expand Up @@ -129,7 +128,6 @@ class HMatrixTreeBuilder {
void set_maximal_block_size(int maxblock_size) { m_maxblocksize = maxblock_size; }
void set_minimal_source_depth(int minimal_source_depth) { m_minsourcedepth = minimal_source_depth; }
void set_minimal_target_depth(int minimal_target_depth) { m_mintargetdepth = minimal_target_depth; }
void set_delay_dense_computation(bool delay_dense_computation) { m_delay_dense_computation = delay_dense_computation; }
void set_dense_blocks_generator(std::shared_ptr<VirtualDenseBlocksGenerator<CoefficientPrecision>> dense_blocks_generator) { m_dense_blocks_generator = dense_blocks_generator; }
};

Expand Down
118 changes: 88 additions & 30 deletions include/htool/solvers/utility.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,37 @@

namespace htool {

class LocalNumberingBuilder {
public:
std::vector<int> local_to_global_numbering;
std::vector<std::vector<int>> intersections;
LocalNumberingBuilder(const std::vector<int> &ovr_subdomain_to_global, const std::vector<int> &cluster_to_ovr_subdomain, const std::vector<std::vector<int>> &input_intersections) : local_to_global_numbering(ovr_subdomain_to_global.size()), intersections(input_intersections.size()) {

// Renumbering for the overlap
int local_size_with_overlap = ovr_subdomain_to_global.size();
std::vector<int> renum(local_size_with_overlap, -1);

for (int i = 0; i < cluster_to_ovr_subdomain.size(); i++) {
renum[cluster_to_ovr_subdomain[i]] = i;
local_to_global_numbering[i] = ovr_subdomain_to_global[cluster_to_ovr_subdomain[i]];
}
int count = cluster_to_ovr_subdomain.size();
for (int i = 0; i < local_size_with_overlap; i++) {
if (renum[i] == -1) {
renum[i] = count;
local_to_global_numbering[count++] = ovr_subdomain_to_global[i];
}
}

for (int i = 0; i < input_intersections.size(); i++) {
intersections[i].resize(input_intersections[i].size());
for (int j = 0; j < intersections[i].size(); j++) {
intersections[i][j] = renum[input_intersections[i][j]];
}
}
}
};

template <typename CoefficientPrecision, typename CoordinatePrecision>
class DefaultSolverBuilder {
private:
Expand All @@ -25,41 +56,22 @@ class DefaultSolverBuilder {
};

template <typename CoefficientPrecision, typename CoordinatePrecision>
class DefaultDDMSolverBuilder {
class DefaultDDMSolverBuilderAddingOverlap {
private:
LocalNumberingBuilder m_local_numbering;

public:
const std::vector<int> &local_to_global_numbering;

private:
std::function<Matrix<CoefficientPrecision>(DistributedOperator<CoefficientPrecision> &, const HMatrix<CoefficientPrecision, CoordinatePrecision> *, const VirtualGeneratorWithPermutation<CoefficientPrecision> &, const std::vector<int> &, const std::vector<int> &, const std::vector<int> &, const std::vector<std::vector<int>> &)> initialize_diagonal_block = [this](DistributedOperator<CoefficientPrecision> &distributed_operator0, const HMatrix<CoefficientPrecision, CoordinatePrecision> *block_diagonal_hmatrix0, const VirtualGeneratorWithPermutation<CoefficientPrecision> &generator0, const std::vector<int> &ovr_subdomain_to_global0, const std::vector<int> &cluster_to_ovr_subdomain0, const std::vector<int> &neighbors0, const std::vector<std::vector<int>> &input_intersections0) {
int local_size_with_overlap = ovr_subdomain_to_global0.size();
std::function<Matrix<CoefficientPrecision>(DistributedOperator<CoefficientPrecision> &, const HMatrix<CoefficientPrecision, CoordinatePrecision> *, const VirtualGeneratorWithPermutation<CoefficientPrecision> &)> initialize_diagonal_block = [this](DistributedOperator<CoefficientPrecision> &distributed_operator0, const HMatrix<CoefficientPrecision, CoordinatePrecision> *block_diagonal_hmatrix0, const VirtualGeneratorWithPermutation<CoefficientPrecision> &generator0) {
int local_size_with_overlap = local_to_global_numbering.size();
int local_size_without_overlap = block_diagonal_hmatrix0->get_target_cluster().get_size();
Matrix<CoefficientPrecision> block_diagonal_dense_matrix_with_overlap(local_size_with_overlap, local_size_with_overlap), block_diagonal_dense_matrix_without_overlap(local_size_without_overlap, local_size_without_overlap);

// Diagonal block without overlap
copy_to_dense(*block_diagonal_hmatrix0, block_diagonal_dense_matrix_without_overlap.data());

// Renumbering for the overlap
std::vector<int> renum(local_size_with_overlap, -1);
local_to_global_numbering.resize(local_size_with_overlap);

for (int i = 0; i < cluster_to_ovr_subdomain0.size(); i++) {
renum[cluster_to_ovr_subdomain0[i]] = i;
local_to_global_numbering[i] = ovr_subdomain_to_global0[cluster_to_ovr_subdomain0[i]];
}
int count = cluster_to_ovr_subdomain0.size();
// std::cout << count << std::endl;
for (int i = 0; i < local_size_with_overlap; i++) {
if (renum[i] == -1) {
renum[i] = count;
local_to_global_numbering[count++] = ovr_subdomain_to_global0[i];
}
}

m_intersections.resize(neighbors0.size());
for (int i = 0; i < neighbors0.size(); i++) {
m_intersections[i].resize(input_intersections0[i].size());
for (int j = 0; j < m_intersections[i].size(); j++) {
m_intersections[i][j] = renum[input_intersections0[i][j]];
}
}

// Assemble block diagonal dense matrix with overlap
for (int j = 0; j < local_size_without_overlap; j++) {
std::copy_n(block_diagonal_dense_matrix_without_overlap.data() + j * local_size_without_overlap, local_size_without_overlap, block_diagonal_dense_matrix_with_overlap.data() + j * local_size_with_overlap);
Expand Down Expand Up @@ -93,16 +105,62 @@ class DefaultDDMSolverBuilder {
};
std::vector<std::vector<int>> m_intersections;

private:
Matrix<CoefficientPrecision> m_block_diagonal_dense_matrix;

public:
std::vector<int> local_to_global_numbering;
DDM<CoefficientPrecision> solver;

DefaultDDMSolverBuilderAddingOverlap(DistributedOperator<CoefficientPrecision> &distributed_operator, const HMatrix<CoefficientPrecision, CoordinatePrecision> *block_diagonal_hmatrix, const VirtualGeneratorWithPermutation<CoefficientPrecision> &generator, const std::vector<int> &ovr_subdomain_to_global, const std::vector<int> &cluster_to_ovr_subdomain, const std::vector<int> &neighbors, const std::vector<std::vector<int>> &intersections) : m_local_numbering(ovr_subdomain_to_global, cluster_to_ovr_subdomain, intersections), local_to_global_numbering(m_local_numbering.local_to_global_numbering), m_block_diagonal_dense_matrix(initialize_diagonal_block(distributed_operator, block_diagonal_hmatrix, generator)), solver(distributed_operator, m_block_diagonal_dense_matrix, neighbors, m_local_numbering.intersections) {}
};

template <typename CoefficientPrecision, typename CoordinatePrecision>
class DefaultDDMSolverBuilder {
private:
std::function<Matrix<CoefficientPrecision>(const HMatrix<CoefficientPrecision, CoordinatePrecision> &)> initialize_diagonal_block = [](const HMatrix<CoefficientPrecision, CoordinatePrecision> &block_diagonal_hmatrix0) {
int local_size_with_overlap = block_diagonal_hmatrix0.get_target_cluster().get_size();
Matrix<CoefficientPrecision> block_diagonal_dense_matrix_with_overlap(local_size_with_overlap, local_size_with_overlap);
Matrix<CoefficientPrecision> permuted_block_diagonal_dense_matrix_with_overlap(local_size_with_overlap, local_size_with_overlap);
// Diagonal block without overlap
copy_to_dense(block_diagonal_hmatrix0, block_diagonal_dense_matrix_with_overlap.data());

int nr = block_diagonal_hmatrix0.get_target_cluster().get_size();
int nc = block_diagonal_hmatrix0.get_source_cluster().get_size();
auto &target_permutation = block_diagonal_hmatrix0.get_target_cluster().get_permutation();

if (block_diagonal_hmatrix0.get_symmetry() == 'N') {
for (int i = 0; i < nr; i++) {
for (int j = 0; j < nc; j++) {

permuted_block_diagonal_dense_matrix_with_overlap(target_permutation[i], target_permutation[j]) = block_diagonal_dense_matrix_with_overlap(i, j);
}
}
} else {
int index_i, index_j;
for (int i = 0; i < nr; i++) {
for (int j = 0; j <= i; j++) {
if (target_permutation[i] < target_permutation[j]) {
index_i = target_permutation[j];
index_j = target_permutation[i];
} else {
index_i = target_permutation[i];
index_j = target_permutation[j];
}
permuted_block_diagonal_dense_matrix_with_overlap(index_i, index_j) = block_diagonal_dense_matrix_with_overlap(i, j);
}
}
}

return permuted_block_diagonal_dense_matrix_with_overlap;
};

private:
Matrix<CoefficientPrecision> m_block_diagonal_dense_matrix;

public:
DDM<CoefficientPrecision> solver;

DefaultDDMSolverBuilder(DistributedOperator<CoefficientPrecision> &distributed_operator, const HMatrix<CoefficientPrecision, CoordinatePrecision> *block_diagonal_hmatrix, const VirtualGeneratorWithPermutation<CoefficientPrecision> &generator, const std::vector<int> &ovr_subdomain_to_global, const std::vector<int> &cluster_to_ovr_subdomain, const std::vector<int> &neighbors, const std::vector<std::vector<int>> &intersections) : m_intersections(intersections), local_to_global_numbering(ovr_subdomain_to_global.size()), m_block_diagonal_dense_matrix(initialize_diagonal_block(distributed_operator, block_diagonal_hmatrix, generator, ovr_subdomain_to_global, cluster_to_ovr_subdomain, neighbors, intersections)), solver(distributed_operator, m_block_diagonal_dense_matrix, neighbors, m_intersections) {}
DefaultDDMSolverBuilder(DistributedOperator<CoefficientPrecision> &distributed_operator, const HMatrix<CoefficientPrecision, CoordinatePrecision> &block_diagonal_hmatrix, const std::vector<int> &neighbors, const std::vector<std::vector<int>> &intersections) : m_block_diagonal_dense_matrix(initialize_diagonal_block(block_diagonal_hmatrix)), solver(distributed_operator, m_block_diagonal_dense_matrix, neighbors, intersections) {}
};

} // namespace htool
Expand Down
17 changes: 17 additions & 0 deletions include/htool/testing/generator_test.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,23 @@ class GeneratorFromMatrix : public VirtualGeneratorWithPermutation<T> {
}
};

template <typename T>
class LocalGeneratorFromMatrix : public VirtualGeneratorWithPermutation<T> {
const Matrix<T> &m_A;
const std::vector<int> &m_target_local_to_global_numbering;
const std::vector<int> &m_source_local_to_global_numbering;

public:
explicit LocalGeneratorFromMatrix(const Matrix<T> &A, const std::vector<int> &target_permutation, const std::vector<int> &source_permutation, const std::vector<int> &target_local_to_global_numbering, const std::vector<int> &source_local_to_global_numbering) : VirtualGeneratorWithPermutation<T>(target_permutation.data(), source_permutation.data()), m_A(A), m_target_local_to_global_numbering(target_local_to_global_numbering), m_source_local_to_global_numbering(source_local_to_global_numbering) {}

void copy_submatrix_from_user_numbering(int M, int N, const int *const rows, const int *const cols, T *ptr) const override {
for (int i = 0; i < M; i++) {
for (int j = 0; j < N; j++) {
ptr[i + M * j] = m_A(m_target_local_to_global_numbering[rows[i]], m_source_local_to_global_numbering[cols[j]]);
}
}
}
};
} // namespace htool

#endif
53 changes: 13 additions & 40 deletions tests/functional_tests/solvers/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,47 +6,20 @@ include(FetchContent)
FetchContent_Declare(
data_test_repository
GIT_REPOSITORY "https://github.com/PierreMarchand20/htool_generate_data_test"
GIT_TAG origin/main)
FetchContent_GetProperties(data_test_repository)
if(NOT data_test_repository_POPULATED)
FetchContent_Populate(data_test_repository)
endif()
GIT_TAG origin/main
GIT_SHALLOW TRUE
GIT_SUBMODULES_RECURSE FALSE
GIT_SUBMODULES "")

set(Test_solver_non_symmetric_ARGS ${data_test_repository_SOURCE_DIR}/data/output_non_sym/)
set(Test_solver_symmetric_ARGS ${data_test_repository_SOURCE_DIR}/data/output_sym/)
message(${Test_solver_symmetric_ARGS})
add_executable(Test_solver_ddm_non_symmetric_single_rhs test_solver_ddm_non_symmetric_single_rhs.cpp)
target_link_libraries(Test_solver_ddm_non_symmetric_single_rhs htool)
add_dependencies(build-tests-solvers Test_solver_ddm_non_symmetric_single_rhs)
FetchContent_MakeAvailable(data_test_repository)

add_test(NAME Test_solver_ddm_non_symmetric_single_rhs_1 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_single_rhs ${Test_solver_non_symmetric_ARGS})
add_test(NAME Test_solver_ddm_non_symmetric_single_rhs_2 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_single_rhs ${Test_solver_non_symmetric_ARGS})
add_test(NAME Test_solver_ddm_non_symmetric_single_rhs_3 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_single_rhs ${Test_solver_non_symmetric_ARGS})
add_test(NAME Test_solver_ddm_non_symmetric_single_rhs_4 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_single_rhs ${Test_solver_non_symmetric_ARGS})
set(Test_solver_ARGS ${data_test_repository_SOURCE_DIR}/data/)

add_executable(Test_solver_ddm_symmetric_single_rhs test_solver_ddm_symmetric_single_rhs.cpp)
target_link_libraries(Test_solver_ddm_symmetric_single_rhs htool)
add_dependencies(build-tests-solvers Test_solver_ddm_symmetric_single_rhs)
add_executable(Test_solver test_solver.cpp)
target_link_libraries(Test_solver htool)
add_dependencies(build-tests-solvers Test_solver)

add_test(NAME Test_solver_ddm_symmetric_single_rhs_1 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_single_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_ddm_symmetric_single_rhs_2 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_single_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_ddm_symmetric_single_rhs_3 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_single_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_ddm_symmetric_single_rhs_4 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_single_rhs ${Test_solver_symmetric_ARGS})

add_executable(Test_solver_ddm_non_symmetric_multi_rhs test_solver_ddm_non_symmetric_multi_rhs.cpp)
target_link_libraries(Test_solver_ddm_non_symmetric_multi_rhs htool)
add_dependencies(build-tests-solvers Test_solver_ddm_non_symmetric_multi_rhs)

add_test(NAME Test_solver_ddm_non_symmetric_multi_rhs_1 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_multi_rhs ${Test_solver_non_symmetric_ARGS})
add_test(NAME Test_solver_ddm_non_symmetric_multi_rhs_2 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_multi_rhs ${Test_solver_non_symmetric_ARGS})
add_test(NAME Test_solver_ddm_non_symmetric_multi_rhs_3 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_multi_rhs ${Test_solver_non_symmetric_ARGS})
add_test(NAME Test_solver_ddm_non_symmetric_multi_rhs_4 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_non_symmetric_multi_rhs ${Test_solver_non_symmetric_ARGS})

add_executable(Test_solver_ddm_symmetric_multi_rhs test_solver_ddm_symmetric_multi_rhs.cpp)
target_link_libraries(Test_solver_ddm_symmetric_multi_rhs htool)
add_dependencies(build-tests-solvers Test_solver_ddm_symmetric_multi_rhs)

add_test(NAME Test_solver_ddm_symmetric_multi_rhs_1 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_multi_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_ddm_symmetric_multi_rhs_2 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_multi_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_ddm_symmetric_multi_rhs_3 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_multi_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_ddm_symmetric_multi_rhs_4 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver_ddm_symmetric_multi_rhs ${Test_solver_symmetric_ARGS})
add_test(NAME Test_solver_1 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver ${Test_solver_ARGS})
add_test(NAME Test_solver_2 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver ${Test_solver_ARGS})
add_test(NAME Test_solver_3 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 3 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver ${Test_solver_ARGS})
add_test(NAME Test_solver_4 COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/Test_solver ${Test_solver_ARGS})
38 changes: 38 additions & 0 deletions tests/functional_tests/solvers/test_solver.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#include "test_solver_ddm.hpp"
#include "test_solver_ddm_adding_overlap.hpp"
#include "test_solver_wo_overlap.hpp"

int main(int argc, char *argv[]) {
// Initialize the MPI environment
MPI_Init(&argc, &argv);

// Input file
if (argc < 2) { // argc should be 5 or more for correct execution
// We print argv[0] assuming it is the program name
cout << "usage: " << argv[0] << " datapath\n"; // LCOV_EXCL_LINE
return 1; // LCOV_EXCL_LINE
}
string datapath = argv[1];

bool is_error = false;

for (auto nb_rhs : {1, 10}) {
for (auto symmetry : {'S'}) {
std::string datapath_final = symmetry == 'S' ? datapath + "/output_sym/" : datapath + "/output_non_sym/";

is_error = is_error || test_solver_wo_overlap(argc, argv, nb_rhs, symmetry, datapath_final);
is_error = is_error || test_solver_ddm_adding_overlap(argc, argv, nb_rhs, symmetry, datapath_final);
is_error = is_error || test_solver_ddm(argc, argv, nb_rhs, symmetry, datapath_final);
}
}

// test = test || test_solver_ddm(argc, argv, 1, 'S', true);

// Finalize the MPI environment.
MPI_Finalize();

if (is_error) {
return 1;
}
return 0;
}
Loading

0 comments on commit ca6582c

Please sign in to comment.