From 483dabc1a354c7011fc205e110b377d1cf146cf9 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 30 Aug 2024 09:12:13 +0200 Subject: [PATCH 1/7] Add basic support for indirect mesh access --- src/common.cpp | 2 +- src/modes.cpp | 30 ++++++++++++++++++++++++++---- src/utilities.cpp | 45 +++++++++++++++++++++++++++++++++++++++++++++ src/utilities.hpp | 2 ++ 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/common.cpp b/src/common.cpp index 2645df2d..0042d630 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -22,7 +22,7 @@ OptionMap getOptions(int argc, char *argv[]) "precice-aste-run will look for timeseries as well as distributed meshes (e.g. from preCICE exports) " "automatically and load them if required.")( "output", po::value(), - "Output file name.")("vector", po::bool_switch(), "Distinguish between vector valued data and scalar data")("verbose,v", po::bool_switch(), "Enable verbose output")("all,a", po::bool_switch(), "Enable output from secondary ranks"); + "Output file name.")("vector", po::bool_switch(), "Distinguish between vector valued data and scalar data")("verbose,v", po::bool_switch(), "Enable verbose output")("all,a", po::bool_switch(), "Enable output from secondary ranks")("indirect-read,ir", po::value(), "Instead of the conventional mapping in preCICE, make use of an indirect mapping, where values are read in batches of values."); po::variables_map vm; diff --git a/src/modes.cpp b/src/modes.cpp index 97973ac6..66920add 100644 --- a/src/modes.cpp +++ b/src/modes.cpp @@ -145,6 +145,12 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap const bool isVector = options["vector"].as(); const std::string preciceConfig = options["precice-config"].as(); + constexpr int conventionalReadData = -1; + // potentially make use of a indirect mapping using a prescribed batch size + const int batchSize = options.count("indirect-read") + ? options["indirect-read"].as() + : conventionalReadData; + addLogIdentity(participantName, context.rank); ASTE_INFO << "ASTE Running in mapping test mode"; @@ -185,7 +191,11 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap asteConfiguration.participantName = "B"; asteConfiguration.preciceConfigFilename = preciceConfig; aste::asteInterface asteInterface; - asteInterface.meshName = "B-Mesh"; + if (batchSize == conventionalReadData) { + asteInterface.meshName = "B-Mesh"; + } else { + asteInterface.meshName = "A-Mesh"; + } asteInterface.meshFilePrefix = meshname; asteInterface.meshes = aste::BaseName(meshname).findAll(context); @@ -211,7 +221,14 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap asteInterface.meshes.front().loadMesh(asteInterface.mesh, dim, requireConnectivity); asteInterface.meshes.front().loadData(asteInterface.mesh); ASTE_INFO << "The loaded mesh " << asteInterface.meshes.front().filename() << " contains: " << asteInterface.mesh.summary(); - auto vertexIDs = aste::setupMesh(preciceInterface, asteInterface.mesh, asteInterface.meshName); + + std::vector vertexIDs; + std::vector coordinates; + if (batchSize == conventionalReadData) { + vertexIDs = aste::setupMesh(preciceInterface, asteInterface.mesh, asteInterface.meshName); + } else { + coordinates = aste::setupDirectMeshAccess(preciceInterface, asteInterface.mesh, asteInterface.meshName, context); + } ASTE_DEBUG << "Mesh setup completed on Rank " << context.rank; if (preciceInterface.requiresInitialData()) { @@ -262,8 +279,13 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap for (auto &asteInterface : asteConfiguration.asteInterfaces) { for (auto &meshdata : asteInterface.mesh.meshdata) { if (meshdata.type == aste::datatype::READ) { - meshdata.dataVector.resize(vertexIDs.size() * meshdata.numcomp); - preciceInterface.readData(asteInterface.meshName, "Data", vertexIDs, dt, meshdata.dataVector); + if (batchSize == conventionalReadData) { + meshdata.dataVector.resize(vertexIDs.size() * meshdata.numcomp); + preciceInterface.readData(asteInterface.meshName, "Data", vertexIDs, dt, meshdata.dataVector); + } else { + meshdata.dataVector.resize((coordinates.size() / preciceInterface.getMeshDimensions(meshname)) * meshdata.numcomp); + preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinates, dt, meshdata.dataVector); + } ASTE_DEBUG << "Data read: " << asteInterface.mesh.previewData(meshdata); } } diff --git a/src/utilities.cpp b/src/utilities.cpp index 9e1d65cc..a10306d9 100644 --- a/src/utilities.cpp +++ b/src/utilities.cpp @@ -72,6 +72,51 @@ void aste::setupEdgeIDs(precice::Participant &interface, const aste::Mesh &mesh, } } +std::vector aste::setupDirectMeshAccess(precice::Participant &interface, const aste::Mesh &mesh, const std::string &meshName, const aste::ExecutionContext &exec) +{ + ASTE_DEBUG << "Setting up direct access for: " << meshName; + ASTE_DEBUG << "Setting up coordinates vector for mesh..."; + + const auto dim = interface.getMeshDimensions(meshName); + const auto nvertices = mesh.positions.size(); + std::vector coordinates(dim * nvertices); + for (unsigned long i = 0; i < nvertices; ++i) { + const auto &pos = mesh.positions[i]; + assert(pos.size() == static_cast(dim)); + std::copy(pos.begin(), pos.end(), &coordinates[i * dim]); + } + + ASTE_DEBUG << "Computing the (rank-local) bounding box:"; + + std::vector bb(2 * dim); + + // for parallel runs, we define a proper bounding box + if (exec.isParallel()) { + // Initialize min and max values + for (int i = 0; i < dim; ++i) { + bb[2 * i] = std::numeric_limits::max(); // min values + bb[2 * i + 1] = std::numeric_limits::lowest(); // max values + } + + // Iterate through the coordinates + for (size_t i = 0; i < coordinates.size(); i += dim) { + for (int j = 0; j < dim; ++j) { + double coord = coordinates[i + j]; + bb[2 * j] = std::min(coord - 1e-12, bb[2 * j]); // Update min + bb[2 * j + 1] = std::max(coord + 1e-12, bb[2 * j + 1]); // Update max + } + } + } else { + // for serial runs, we just take everything (to account for potential discretization imbalances) + for (int i = 0; i < dim; ++i) { + bb[2 * i] = std::numeric_limits::lowest(); // min values + bb[2 * i + 1] = std::numeric_limits::max(); // max values + } + } + interface.setMeshAccessRegion(meshName, bb); + return coordinates; +} + std::vector aste::setupMesh(precice::Participant &interface, const aste::Mesh &mesh, const std::string &meshName) { auto tstart = std::chrono::steady_clock::now(); diff --git a/src/utilities.hpp b/src/utilities.hpp index f2578f55..23389b2d 100644 --- a/src/utilities.hpp +++ b/src/utilities.hpp @@ -60,4 +60,6 @@ void setupEdgeIDs(precice::Participant &interface, const aste::Mesh &mesh, const * @return std::vector a vector of vertexIDs in preCICE */ std::vector setupMesh(precice::Participant &interface, const aste::Mesh &mesh, const std::string &meshName); + +std::vector setupDirectMeshAccess(precice::Participant &interface, const aste::Mesh &mesh, const std::string &meshName, const aste::ExecutionContext &exec); } // namespace aste From bdb2a65c68da4ff99e9e2a8a87ffb4decacbf334 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 30 Aug 2024 09:21:08 +0200 Subject: [PATCH 2/7] Add a test case for indirect access --- CMakeLists.txt | 2 +- examples/nn/precice-config.xml | 3 ++ examples/nn_indirect_access/clean.sh | 8 +++ .../nn_indirect_access/precice-config.xml | 51 +++++++++++++++++++ examples/nn_indirect_access/run.sh | 12 +++++ src/modes.cpp | 2 +- 6 files changed, 76 insertions(+), 2 deletions(-) create mode 100755 examples/nn_indirect_access/clean.sh create mode 100644 examples/nn_indirect_access/precice-config.xml create mode 100755 examples/nn_indirect_access/run.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a881184..bfb45a35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,7 +120,7 @@ add_test(NAME read_write_test COMMAND test-precice-aste WORKING_DIRECTORY "${CM # Detect and register examples as tests -set(_examples lci_2d lci_3d nn nng_scalar nng_vector mapping_tester replay_mode) +set(_examples lci_2d lci_3d nn nn_indirect_access nng_scalar nng_vector mapping_tester replay_mode) foreach(example IN LISTS _examples) add_test(NAME aste.example.${example}.setup diff --git a/examples/nn/precice-config.xml b/examples/nn/precice-config.xml index 58a057c3..926adaf5 100644 --- a/examples/nn/precice-config.xml +++ b/examples/nn/precice-config.xml @@ -26,6 +26,9 @@ + diff --git a/examples/nn_indirect_access/clean.sh b/examples/nn_indirect_access/clean.sh new file mode 100755 index 00000000..5893def4 --- /dev/null +++ b/examples/nn_indirect_access/clean.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e -x + +rm -f -r precice-run +rm -f -r precice-profiling +rm -f *.log +rm -f map*.vtk +rm -f fine_mesh_*.vtk diff --git a/examples/nn_indirect_access/precice-config.xml b/examples/nn_indirect_access/precice-config.xml new file mode 100644 index 00000000..dd2b7fe8 --- /dev/null +++ b/examples/nn_indirect_access/precice-config.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/nn_indirect_access/run.sh b/examples/nn_indirect_access/run.sh new file mode 100755 index 00000000..38accc44 --- /dev/null +++ b/examples/nn_indirect_access/run.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e -x + +# Calculate franke function on fine mesh +precice-aste-evaluate -m ../fine_mesh.vtk -f "franke3d" -d "Franke Function" -o "fine_mesh_nn.vtk" + +# Map from the finer mesh to coarser mesh +precice-aste-run -v -p A --mesh fine_mesh_nn --data "Franke Function" & +precice-aste-run -v -p B --mesh ../coarse_mesh --output map_nn --data "InterpolatedData" --indirect-read 0 + +# Calculate statistics +precice-aste-evaluate -m map_nn.vtk -f "franke3d" -d difference --diffdata "InterpolatedData" --diff diff --git a/src/modes.cpp b/src/modes.cpp index 66920add..210e4132 100644 --- a/src/modes.cpp +++ b/src/modes.cpp @@ -283,7 +283,7 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap meshdata.dataVector.resize(vertexIDs.size() * meshdata.numcomp); preciceInterface.readData(asteInterface.meshName, "Data", vertexIDs, dt, meshdata.dataVector); } else { - meshdata.dataVector.resize((coordinates.size() / preciceInterface.getMeshDimensions(meshname)) * meshdata.numcomp); + meshdata.dataVector.resize((coordinates.size() / preciceInterface.getMeshDimensions(asteInterface.meshName)) * meshdata.numcomp); preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinates, dt, meshdata.dataVector); } ASTE_DEBUG << "Data read: " << asteInterface.mesh.previewData(meshdata); From 573bbb316d3c59529e75e8f32ae3edd58d124467 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 30 Aug 2024 10:36:15 +0200 Subject: [PATCH 3/7] Implement a batch-wise processing of data --- src/modes.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/modes.cpp b/src/modes.cpp index 210e4132..2b4a55b5 100644 --- a/src/modes.cpp +++ b/src/modes.cpp @@ -150,6 +150,9 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap const int batchSize = options.count("indirect-read") ? options["indirect-read"].as() : conventionalReadData; + if (batchSize < -1) { + ASTE_ERROR << "Batch size needs to be greater or equal to zero."; + } addLogIdentity(participantName, context.rank); ASTE_INFO << "ASTE Running in mapping test mode"; @@ -283,8 +286,23 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap meshdata.dataVector.resize(vertexIDs.size() * meshdata.numcomp); preciceInterface.readData(asteInterface.meshName, "Data", vertexIDs, dt, meshdata.dataVector); } else { - meshdata.dataVector.resize((coordinates.size() / preciceInterface.getMeshDimensions(asteInterface.meshName)) * meshdata.numcomp); - preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinates, dt, meshdata.dataVector); + int dim = preciceInterface.getMeshDimensions(asteInterface.meshName); + int nVertices = coordinates.size() / dim; + meshdata.dataVector.resize(nVertices * meshdata.numcomp); + + for (int i = 0; i < nVertices; i += batchSize) { + int batchEnd = std::min(i + batchSize, nVertices); + int coordStart = i * dim; + int coordEnd = batchEnd * dim; + int dataStart = i * meshdata.numcomp; + int dataEnd = batchEnd * meshdata.numcomp; + + ::precice::span coordinateBatch(coordinates.data() + coordStart, coordEnd - coordStart); + ::precice::span dataBatch(meshdata.dataVector.data() + dataStart, dataEnd - dataStart); + + // Call the API function with the current batch + preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinateBatch, dt, dataBatch); + } } ASTE_DEBUG << "Data read: " << asteInterface.mesh.previewData(meshdata); } From c4946334e7409097d4f7a0214aa92d73cf61c55a Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 30 Aug 2024 10:46:27 +0200 Subject: [PATCH 4/7] Special treatment for batch size of zero --- src/modes.cpp | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/modes.cpp b/src/modes.cpp index 2b4a55b5..0b459246 100644 --- a/src/modes.cpp +++ b/src/modes.cpp @@ -290,18 +290,23 @@ void aste::runMapperMode(const aste::ExecutionContext &context, const OptionMap int nVertices = coordinates.size() / dim; meshdata.dataVector.resize(nVertices * meshdata.numcomp); - for (int i = 0; i < nVertices; i += batchSize) { - int batchEnd = std::min(i + batchSize, nVertices); - int coordStart = i * dim; - int coordEnd = batchEnd * dim; - int dataStart = i * meshdata.numcomp; - int dataEnd = batchEnd * meshdata.numcomp; - - ::precice::span coordinateBatch(coordinates.data() + coordStart, coordEnd - coordStart); - ::precice::span dataBatch(meshdata.dataVector.data() + dataStart, dataEnd - dataStart); - - // Call the API function with the current batch - preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinateBatch, dt, dataBatch); + // pass everything in one go + if (batchSize == 0) { + preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinates, dt, meshdata.dataVector); + } else { // pass in batches + for (int i = 0; i < nVertices; i += batchSize) { + int availableVertices = std::min(i + batchSize, nVertices); + int coordStart = i * dim; + int coordEnd = availableVertices * dim; + int dataStart = i * meshdata.numcomp; + int dataEnd = availableVertices * meshdata.numcomp; + + ::precice::span coordinateBatch(coordinates.data() + coordStart, coordEnd - coordStart); + ::precice::span dataBatch(meshdata.dataVector.data() + dataStart, dataEnd - dataStart); + + // Call the API function with the current batch + preciceInterface.mapAndreadData(asteInterface.meshName, "Data", coordinateBatch, dt, dataBatch); + } } } ASTE_DEBUG << "Data read: " << asteInterface.mesh.previewData(meshdata); From dec89811ad5533a8c448c93d42d0829c8cff137f Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 30 Aug 2024 10:50:10 +0200 Subject: [PATCH 5/7] Update clean.sh file --- examples/nn_indirect_access/clean.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/nn_indirect_access/clean.sh b/examples/nn_indirect_access/clean.sh index 5893def4..e2feb93d 100755 --- a/examples/nn_indirect_access/clean.sh +++ b/examples/nn_indirect_access/clean.sh @@ -3,6 +3,8 @@ set -e -x rm -f -r precice-run rm -f -r precice-profiling +rm -f profiling.json +rm -f trace.json rm -f *.log rm -f map*.vtk rm -f fine_mesh_*.vtk From a6f6968a71012e0e2365a8bfe6dbb7090175668a Mon Sep 17 00:00:00 2001 From: David Schneider Date: Fri, 30 Aug 2024 15:24:05 +0200 Subject: [PATCH 6/7] Remove B-Mesh from direct access --- examples/nn_indirect_access/precice-config.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/nn_indirect_access/precice-config.xml b/examples/nn_indirect_access/precice-config.xml index dd2b7fe8..901569b7 100644 --- a/examples/nn_indirect_access/precice-config.xml +++ b/examples/nn_indirect_access/precice-config.xml @@ -17,10 +17,6 @@ - - - - From 0db004d40267cdb2e388222b661912f904521d05 Mon Sep 17 00:00:00 2001 From: David Schneider Date: Mon, 2 Sep 2024 11:05:12 +0200 Subject: [PATCH 7/7] Take batch size for mapping tester into account --- examples/mapping_tester/setup-test.json | 4 ++++ tools/mapping-tester/config-template.xml | 13 +++++++++---- tools/mapping-tester/generate.py | 12 ++++++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/examples/mapping_tester/setup-test.json b/examples/mapping_tester/setup-test.json index 27a346cd..af49d539 100644 --- a/examples/mapping_tester/setup-test.json +++ b/examples/mapping_tester/setup-test.json @@ -35,6 +35,10 @@ }, "np": { "kind": "nearest-projection" + }, + "nn": { + "kind": "nearest-neighbor", + "batch-size": "5" } } }, diff --git a/tools/mapping-tester/config-template.xml b/tools/mapping-tester/config-template.xml index 83263438..ea2442d7 100644 --- a/tools/mapping-tester/config-template.xml +++ b/tools/mapping-tester/config-template.xml @@ -15,10 +15,12 @@ - + + {% if mapping.get('batch-size') == "-1" %} + {% endif %} @@ -41,13 +43,16 @@ + {% if mapping.get('batch-size') == "-1" %} - + {% endif %} + + {% if mapping.constraint == "consistent" %} - + - + {% if mapping.kind.startswith("rbf") %} {% if mapping.executor %} diff --git a/tools/mapping-tester/generate.py b/tools/mapping-tester/generate.py index 7c09ddba..457aaeb8 100755 --- a/tools/mapping-tester/generate.py +++ b/tools/mapping-tester/generate.py @@ -53,6 +53,7 @@ def generateCases(setup): "executoroptions": mapping.get( "executor-options", "" ), + "batch-size": mapping.get("batch-size", "-1"), }, "A": { "ranks": ranksA, @@ -178,8 +179,15 @@ def createRunScript(outdir, path, case): os.path.join(outdir, "meshes", bmesh, str(branks), bmesh), path ) mapped_data_name = case["function"] + "(mapped)" - bcmd = 'env time -f %M -a -o memory-B.log precice-aste-run -v -a -p B --data "{}" --mesh {} --output mapped || kill 0 &'.format( - mapped_data_name, bmeshLocation + + # Handle batch_size condition + batch_size_flag = "" + if case["mapping"]["batch-size"] != "-1": + batch_size_flag = "--indirect-read {}".format(case["mapping"]["batch-size"]) + + # Generate runner script for participant B + bcmd = 'env time -f %M -a -o memory-B.log precice-aste-run -v -a -p B --data "{}" --mesh {} --output mapped {} || kill 0 &'.format( + mapped_data_name, bmeshLocation, batch_size_flag ) if branks > 1: bcmd = "mpirun -n {} $ASTE_B_MPIARGS {}".format(branks, bcmd)