From 7ce867ea4868afac3b44b7fac86f0768a86b0f4b Mon Sep 17 00:00:00 2001 From: Sam Reeve <6740307+streeve@users.noreply.github.com> Date: Mon, 25 Sep 2023 17:19:06 -0400 Subject: [PATCH] Enable variable dimension (2d/3d) --- examples/crack_branching.cpp | 6 +- examples/elastic_wave.cpp | 4 +- examples/kalthoff_winkler.cpp | 6 +- src/CMakeLists.txt | 8 +- src/CabanaPD_Input.cpp | 140 ---------------------------------- src/CabanaPD_Input.hpp | 83 ++++++++++++++++++-- src/CabanaPD_Particles.hpp | 92 +++++++++++++--------- src/CabanaPD_Solver.hpp | 45 ++++++----- 8 files changed, 167 insertions(+), 217 deletions(-) delete mode 100644 src/CabanaPD_Input.cpp diff --git a/examples/crack_branching.cpp b/examples/crack_branching.cpp index e0c80731..356e22bc 100644 --- a/examples/crack_branching.cpp +++ b/examples/crack_branching.cpp @@ -78,8 +78,8 @@ int main( int argc, char* argv[] ) using model_type = CabanaPD::ForceModel; model_type force_model( delta, K, G0 ); - CabanaPD::Inputs inputs( num_cell, low_corner, high_corner, t_final, dt, - output_frequency ); + CabanaPD::Inputs<3> inputs( num_cell, low_corner, high_corner, t_final, + dt, output_frequency ); inputs.read_args( argc, argv ); // Create particles from mesh. @@ -98,7 +98,7 @@ int main( int argc, char* argv[] ) auto nofail = particles->sliceNoFail(); // Relying on uniform grid here. - double dy = particles->dy; + double dy = particles->dx[1]; double b0 = 2e6 / dy; // Pa CabanaPD::RegionBoundary plane1( low_x, high_x, low_y - dy, low_y + dy, diff --git a/examples/elastic_wave.cpp b/examples/elastic_wave.cpp index 58af9524..b86bffca 100644 --- a/examples/elastic_wave.cpp +++ b/examples/elastic_wave.cpp @@ -50,8 +50,8 @@ int main( int argc, char* argv[] ) CabanaPD::ForceModel; model_type force_model( delta, K, G ); - CabanaPD::Inputs inputs( num_cell, low_corner, high_corner, t_final, dt, - output_frequency ); + CabanaPD::Inputs<3> inputs( num_cell, low_corner, high_corner, t_final, + dt, output_frequency ); inputs.read_args( argc, argv ); // Create particles from mesh. diff --git a/examples/kalthoff_winkler.cpp b/examples/kalthoff_winkler.cpp index 9b2e6fda..4315b55c 100644 --- a/examples/kalthoff_winkler.cpp +++ b/examples/kalthoff_winkler.cpp @@ -81,8 +81,8 @@ int main( int argc, char* argv[] ) // using model_type = // CabanaPD::ForceModel; // model_type force_model( delta, K, G, G0 ); - CabanaPD::Inputs inputs( num_cell, low_corner, high_corner, t_final, dt, - output_frequency ); + CabanaPD::Inputs<3> inputs( num_cell, low_corner, high_corner, t_final, + dt, output_frequency ); inputs.read_args( argc, argv ); // Create particles from mesh. @@ -100,7 +100,7 @@ int main( int argc, char* argv[] ) auto f = particles->sliceForce(); auto rho = particles->sliceDensity(); - double dx = particles->dx; + double dx = particles->dx[0]; double x_bc = -0.5 * height; CabanaPD::RegionBoundary plane( diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4079b57e..5a72ec8c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,17 +2,15 @@ configure_file(CabanaPD_config.hpp.cmakein CabanaPD_config.hpp @ONLY) file(GLOB HEADERS GLOB *.hpp) -file(GLOB SOURCES *.cpp) - install(FILES ${HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CabanaPD_config.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -add_library(CabanaPD ${SOURCES}) +add_library(CabanaPD INTERFACE) -target_include_directories(CabanaPD PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} +target_include_directories(CabanaPD INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} $ $) -target_link_libraries(CabanaPD Cabana::cabanacore Cabana::Cajita) +target_link_libraries(CabanaPD INTERFACE Cabana::cabanacore Cabana::Cajita) install(TARGETS CabanaPD DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/src/CabanaPD_Input.cpp b/src/CabanaPD_Input.cpp deleted file mode 100644 index 4c9f4c22..00000000 --- a/src/CabanaPD_Input.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/**************************************************************************** - * Copyright (c) 2022-2023 by Oak Ridge National Laboratory * - * All rights reserved. * - * * - * This file is part of CabanaPD. CabanaPD is distributed under a * - * BSD 3-clause license. For the licensing terms see the LICENSE file in * - * the top-level directory. * - * * - * SPDX-License-Identifier: BSD-3-Clause * - ****************************************************************************/ - -/**************************************************************************** - * Copyright (c) 2018-2021 by the Cabana authors * - * All rights reserved. * - * * - * This file is part of the Cabana library. Cabana is distributed under a * - * BSD 3-clause license. For the licensing terms see the LICENSE file in * - * the top-level directory. * - * * - * SPDX-License-Identifier: BSD-3-Clause * - ****************************************************************************/ - -//************************************************************************ -// ExaMiniMD v. 1.0 -// Copyright (2018) National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). -// -// Under the terms of Contract DE-NA-0003525 with NTESS, the U.S. Government -// retains certain rights in this software. -// -// ExaMiniMD is licensed under 3-clause BSD terms of use: Redistribution and -// use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the contributors -// may be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -// IN NO EVENT SHALL NTESS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -// IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. -// -//************************************************************************ - -#include - -#include - -#include -#include - -namespace CabanaPD -{ -// FIXME: hardcoded. -Inputs::Inputs( const std::array nc, std::array lc, - std::array hc, const double t_f, const double dt, - const int of ) - : num_cells( nc ) - , low_corner( lc ) - , high_corner( hc ) - , final_time( t_f ) - , timestep( dt ) - , output_frequency( of ) -{ - num_steps = final_time / timestep; -} - -Inputs::~Inputs() {} - -void Inputs::read_args( int argc, char* argv[] ) -{ - for ( int i = 1; i < argc; i++ ) - { - // Help command. - if ( ( strcmp( argv[i], "-h" ) == 0 ) || - ( strcmp( argv[i], "--help" ) == 0 ) ) - { - if ( print_rank() ) - { - log( std::cout, "CabanaPD\n", "Options:" ); - log( std::cout, - " -o [FILE] (OR)\n" - " --output-file [FILE]: Provide output file name\n" ); - log( std::cout, - " -e [FILE] (OR)\n" - " --error-file [FILE]: Provide error file name\n" ); - /* Not yet enabled. - log( - std::cout, - " --device-type [TYPE]: Kokkos device type to run ", - "with\n", - " (SERIAL, PTHREAD, OPENMP, " - "CUDA, HIP)" ); - */ - } - } - // Output file names. - else if ( ( strcmp( argv[i], "-o" ) == 0 ) || - ( strcmp( argv[i], "--output-file" ) == 0 ) ) - { - output_file = argv[i + 1]; - ++i; - } - else if ( ( strcmp( argv[i], "-e" ) == 0 ) || - ( strcmp( argv[i], "--error-file" ) == 0 ) ) - { - error_file = argv[i + 1]; - ++i; - } - - // Kokkos device type. - else if ( ( strcmp( argv[i], "--device-type" ) == 0 ) ) - { - device_type = argv[i + 1]; - ++i; - } - - else if ( ( strstr( argv[i], "--kokkos-" ) == NULL ) ) - { - log_err( std::cout, "Unknown command line argument: ", argv[i] ); - } - } -} - -} // namespace CabanaPD diff --git a/src/CabanaPD_Input.hpp b/src/CabanaPD_Input.hpp index 1eea581a..b7be304f 100644 --- a/src/CabanaPD_Input.hpp +++ b/src/CabanaPD_Input.hpp @@ -14,8 +14,11 @@ #include +#include + namespace CabanaPD { +template class Inputs { public: @@ -23,9 +26,9 @@ class Inputs std::string error_file = "cabanaPD.err"; std::string device_type = "SERIAL"; - std::array num_cells; - std::array low_corner; - std::array high_corner; + std::array num_cells; + std::array low_corner; + std::array high_corner; std::size_t num_steps; double final_time; @@ -34,11 +37,75 @@ class Inputs bool half_neigh = false; - Inputs( const std::array nc, std::array lc, - std::array hc, const double t_f, const double dt, - const int output_freq ); - ~Inputs(); - void read_args( int argc, char* argv[] ); + Inputs( const std::array nc, std::array lc, + std::array hc, const double t_f, const double dt, + const int of ) + : num_cells( nc ) + , low_corner( lc ) + , high_corner( hc ) + , final_time( t_f ) + , timestep( dt ) + , output_frequency( of ) + { + num_steps = final_time / timestep; + } + ~Inputs(){}; + + void read_args( int argc, char* argv[] ) + { + for ( int i = 1; i < argc; i++ ) + { + // Help command. + if ( ( strcmp( argv[i], "-h" ) == 0 ) || + ( strcmp( argv[i], "--help" ) == 0 ) ) + { + if ( print_rank() ) + { + log( std::cout, "CabanaPD\n", "Options:" ); + log( std::cout, " -o [FILE] (OR)\n" + " --output-file [FILE]: Provide output " + "file name\n" ); + log( + std::cout, + " -e [FILE] (OR)\n" + " --error-file [FILE]: Provide error file name\n" ); + /* Not yet enabled. + log( + std::cout, + " --device-type [TYPE]: Kokkos device type to run + ", "with\n", " (SERIAL, + PTHREAD, OPENMP, " "CUDA, HIP)" ); + */ + } + } + // Output file names. + else if ( ( strcmp( argv[i], "-o" ) == 0 ) || + ( strcmp( argv[i], "--output-file" ) == 0 ) ) + { + output_file = argv[i + 1]; + ++i; + } + else if ( ( strcmp( argv[i], "-e" ) == 0 ) || + ( strcmp( argv[i], "--error-file" ) == 0 ) ) + { + error_file = argv[i + 1]; + ++i; + } + + // Kokkos device type. + else if ( ( strcmp( argv[i], "--device-type" ) == 0 ) ) + { + device_type = argv[i + 1]; + ++i; + } + + else if ( ( strstr( argv[i], "--kokkos-" ) == NULL ) ) + { + log_err( std::cout, + "Unknown command line argument: ", argv[i] ); + } + } + } }; } // namespace CabanaPD diff --git a/src/CabanaPD_Particles.hpp b/src/CabanaPD_Particles.hpp index 353eba7b..fa27ed1e 100644 --- a/src/CabanaPD_Particles.hpp +++ b/src/CabanaPD_Particles.hpp @@ -120,61 +120,68 @@ class Particles int n_types = 1; // Simulation total domain. - std::array global_mesh_ext; + std::array global_mesh_ext; // Simulation sub domain (single MPI rank). - std::array local_mesh_ext; - std::array local_mesh_lo; - std::array local_mesh_hi; - std::array ghost_mesh_lo; - std::array ghost_mesh_hi; - std::shared_ptr>> local_grid; - double dx; - double dy; - double dz; + std::array local_mesh_ext; + std::array local_mesh_lo; + std::array local_mesh_hi; + std::array ghost_mesh_lo; + std::array ghost_mesh_hi; + std::shared_ptr>> + local_grid; + std::array dx; int halo_width; // Default constructor. Particles() { - global_mesh_ext = { 0.0, 0.0, 0.0 }; - local_mesh_lo = { 0.0, 0.0, 0.0 }; - local_mesh_hi = { 0.0, 0.0, 0.0 }; - ghost_mesh_lo = { 0.0, 0.0, 0.0 }; - ghost_mesh_hi = { 0.0, 0.0, 0.0 }; - local_mesh_ext = { 0.0, 0.0, 0.0 }; - + for ( int d = 0; d < dim; d++ ) + { + global_mesh_ext[d] = 0.0; + local_mesh_lo[d] = 0.0; + local_mesh_hi[d] = 0.0; + ghost_mesh_lo[d] = 0.0; + ghost_mesh_hi[d] = 0.0; + local_mesh_ext[d] = 0.0; + } resize( 0, 0 ); } // Constructor which initializes particles on regular grid. template - Particles( const ExecSpace& exec_space, std::array low_corner, - std::array high_corner, - const std::array num_cells, const int max_halo_width ) + Particles( const ExecSpace& exec_space, std::array low_corner, + std::array high_corner, + const std::array num_cells, const int max_halo_width ) : halo_width( max_halo_width ) { createDomain( low_corner, high_corner, num_cells ); createParticles( exec_space ); } - void createDomain( std::array low_corner, - std::array high_corner, - const std::array num_cells ) + void createDomain( std::array low_corner, + std::array high_corner, + const std::array num_cells ) { + for ( int d = 0; d < dim; d++ ) + std::cout << low_corner[d] << " " << high_corner[d] << " " + << num_cells[d] << std::endl; + // Create the MPI partitions. - Cajita::DimBlockPartitioner<3> partitioner; + Cajita::DimBlockPartitioner partitioner; // Create global mesh of MPI partitions. auto global_mesh = Cajita::createUniformGlobalMesh( low_corner, high_corner, num_cells ); - for ( int d = 0; d < 3; d++ ) + std::array is_periodic; + for ( int d = 0; d < dim; d++ ) + { global_mesh_ext[d] = global_mesh->extent( d ); - + is_periodic[d] = false; + } // Create the global grid. - std::array is_periodic = { false, false, false }; auto global_grid = Cajita::createGlobalGrid( MPI_COMM_WORLD, global_mesh, is_periodic, partitioner ); @@ -182,7 +189,7 @@ class Particles local_grid = Cajita::createLocalGrid( global_grid, halo_width ); auto local_mesh = Cajita::createLocalMesh( *local_grid ); - for ( int d = 0; d < 3; d++ ) + for ( int d = 0; d < dim; d++ ) { local_mesh_lo[d] = local_mesh.lowCorner( Cajita::Own(), d ); local_mesh_hi[d] = local_mesh.highCorner( Cajita::Own(), d ); @@ -192,10 +199,12 @@ class Particles } // Uniform mesh spacing. - int zero[3] = { 0, 0, 0 }; - dx = local_mesh.measure( Cajita::Edge(), zero ); - dy = local_mesh.measure( Cajita::Edge(), zero ); - dz = local_mesh.measure( Cajita::Edge(), zero ); + int zero[dim]; + for ( int d = 0; d < dim; d++ ) + zero[d] = 0.0; + dx[0] = local_mesh.measure( Cajita::Edge(), zero ); + dx[1] = local_mesh.measure( Cajita::Edge(), zero ); + dz( local_mesh, zero ); } template @@ -396,6 +405,17 @@ class Particles friend class Comm; protected: + template + std::enable_if_t<3 == Dim, void> dz( const LocalMeshType local_mesh, + const int zero[dim] ) + { + dx[2] = local_mesh.measure( Cajita::Edge(), zero ); + } + template + std::enable_if_t<2 == Dim, void> dz( const LocalMeshType, const int[dim] ) + { + } + aosoa_x_type _aosoa_x; aosoa_u_type _aosoa_u; aosoa_f_type _aosoa_f; @@ -445,8 +465,6 @@ class Particles using base_type::local_mesh_lo; using base_type::dx; - using base_type::dy; - using base_type::dz; using base_type::local_grid; using base_type::halo_width; @@ -461,9 +479,9 @@ class Particles // Constructor which initializes particles on regular grid. template - Particles( const ExecSpace& exec_space, std::array low_corner, - std::array high_corner, - const std::array num_cells, const int max_halo_width ) + Particles( const ExecSpace& exec_space, std::array low_corner, + std::array high_corner, + const std::array num_cells, const int max_halo_width ) : base_type( exec_space, low_corner, high_corner, num_cells, max_halo_width ) { diff --git a/src/CabanaPD_Solver.hpp b/src/CabanaPD_Solver.hpp index 15746079..f05ce487 100644 --- a/src/CabanaPD_Solver.hpp +++ b/src/CabanaPD_Solver.hpp @@ -89,7 +89,8 @@ class SolverBase virtual void run() = 0; }; -template +template class SolverElastic { public: @@ -106,11 +107,13 @@ class SolverElastic Cabana::VerletList; using neigh_iter_tag = Cabana::SerialOpTag; + using input_type = InputType; - SolverElastic( Inputs _inputs, std::shared_ptr _particles, + SolverElastic( input_type _inputs, + std::shared_ptr _particles, force_model_type force_model ) : particles( _particles ) - , inputs( std::make_shared( _inputs ) ) + , inputs( std::make_shared( _inputs ) ) { force_time = 0; integrate_time = 0; @@ -305,7 +308,7 @@ class SolverElastic protected: std::shared_ptr particles; - std::shared_ptr inputs; + std::shared_ptr inputs; std::shared_ptr comm; std::shared_ptr integrator; std::shared_ptr force; @@ -327,13 +330,14 @@ class SolverElastic bool print; }; -template +template class SolverFracture - : public SolverElastic + : public SolverElastic { public: - using base_type = SolverElastic; + using base_type = + SolverElastic; using exec_space = typename base_type::exec_space; using memory_space = typename base_type::memory_space; @@ -346,8 +350,10 @@ class SolverFracture using neigh_iter_tag = Cabana::SerialOpTag; using bc_type = BoundaryCondition; using prenotch_type = PrenotchType; + using input_type = typename base_type::input_type; - SolverFracture( Inputs _inputs, std::shared_ptr _particles, + SolverFracture( input_type _inputs, + std::shared_ptr _particles, force_model_type force_model, bc_type bc, prenotch_type prenotch ) : base_type( _inputs, _particles, force_model ) @@ -486,25 +492,26 @@ class SolverFracture using base_type::print; }; -template -auto createSolverElastic( Inputs inputs, +template +auto createSolverElastic( Inputs inputs, std::shared_ptr particles, ForceModel model ) { return std::make_shared< - SolverElastic>( inputs, particles, - model ); + SolverElastic, ParticleType, ForceModel>>( + inputs, particles, model ); } -template -auto createSolverFracture( Inputs inputs, +template +auto createSolverFracture( Inputs inputs, std::shared_ptr particles, ForceModel model, BCType bc, PrenotchType prenotch ) { - return std::make_shared>( - inputs, particles, model, bc, prenotch ); + return std::make_shared< + SolverFracture, ParticleType, ForceModel, + BCType, PrenotchType>>( inputs, particles, model, bc, + prenotch ); } /*