From 0fe4f428110b70f59da9bb5a332477d82b660d14 Mon Sep 17 00:00:00 2001 From: rcclay Date: Tue, 12 Sep 2023 09:14:19 -0600 Subject: [PATCH 001/168] First pass at making QMCCostFunctionBatched complex friendly. Variance component not checked, but guessed --- .../WFOpt/QMCCostFunctionBatched.cpp | 62 ++++++++++--------- src/QMCDrivers/WFOpt/QMCCostFunctionBatched.h | 2 +- .../tests/test_QMCCostFunctionBatched.cpp | 2 +- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.cpp b/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.cpp index 56bdce8f980..5a8f8309204 100644 --- a/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.cpp +++ b/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.cpp @@ -112,16 +112,18 @@ void QMCCostFunctionBatched::GradCost(std::vector& PGradient, ltz = false; Return_rt delE = std::pow(std::abs(eloc_new - EtargetEff), PowerE); Return_rt ddelE = PowerE * std::pow(std::abs(eloc_new - EtargetEff), PowerE - 1); - const Return_rt* Dsaved = DerivRecords_[iw]; + const Return_t* Dsaved = DerivRecords_[iw]; const Return_rt* HDsaved = HDerivRecords_[iw]; for (int pm = 0; pm < NumOptimizables; pm++) { - EDtotals_w[pm] += weight * (HDsaved[pm] + 2.0 * Dsaved[pm] * delta_l); + //From Toulouse J. Chem. Phys. 126, 084102 (2007), this is H_0j+H_j0, which are independent + //estimates of 1/2 the energy gradient g. So g1+g2 is an estimate of g. + EDtotals_w[pm] += weight * (HDsaved[pm] + 2.0 * std::real(Dsaved[pm]) * delta_l); URV[pm] += 2.0 * (eloc_new * HDsaved[pm] - curAvg * HD_avg[pm]); if (ltz) - EDtotals[pm] += weight * (2.0 * Dsaved[pm] * (delE - delE_bar) + ddelE * HDsaved[pm]); + EDtotals[pm] += weight * (2.0 * std::real(Dsaved[pm]) * (delE - delE_bar) + ddelE * HDsaved[pm]); else - EDtotals[pm] += weight * (2.0 * Dsaved[pm] * (delE - delE_bar) - ddelE * HDsaved[pm]); + EDtotals[pm] += weight * (2.0 * std::real(Dsaved[pm]) * (delE - delE_bar) - ddelE * HDsaved[pm]); } } } @@ -137,12 +139,12 @@ void QMCCostFunctionBatched::GradCost(std::vector& PGradient, Return_rt eloc_new = saved[ENERGY_NEW]; Return_rt delta_l = (eloc_new - curAvg_w); Return_rt sigma_l = delta_l * delta_l; - const Return_rt* Dsaved = DerivRecords_[iw]; + const Return_t* Dsaved = DerivRecords_[iw]; const Return_rt* HDsaved = HDerivRecords_[iw]; for (int pm = 0; pm < NumOptimizables; pm++) { E2Dtotals_w[pm] += - weight * 2.0 * (Dsaved[pm] * (sigma_l - curVar_w) + delta_l * (HDsaved[pm] - EDtotals_w[pm])); + weight * 2.0 * (std::real(Dsaved[pm]) * (sigma_l - curVar_w) + delta_l * (HDsaved[pm] - EDtotals_w[pm])); } } } @@ -272,7 +274,7 @@ void QMCCostFunctionBatched::checkConfigurations(EngineHandle& handle) auto evalOptConfig = [](int crowd_id, UPtrVector& opt_crowds, const std::vector& samples_per_crowd_offsets, const std::vector& walkers_per_crowd, std::vector& gradPsi, std::vector& lapPsi, - Matrix& RecordsOnNode, Matrix& DerivRecords, + Matrix& RecordsOnNode, Matrix& DerivRecords, Matrix& HDerivRecords, const SampleStack& samples, opt_variables_type& optVars, bool needGrads, EngineHandle& handle) { CostFunctionCrowdData& opt_data = *opt_crowds[crowd_id]; @@ -351,7 +353,9 @@ void QMCCostFunctionBatched::checkConfigurations(EngineHandle& handle) const int is = base_sample_index + ib; for (int j = 0; j < nparams; j++) { - DerivRecords[is][j] = std::real(dlogpsi_array[ib][j]); + //dlogpsi is in general complex if psi is complex. + DerivRecords[is][j] = dlogpsi_array[ib][j]; + //but E_L and d E_L/dc are real if c is real. HDerivRecords[is][j] = std::real(dhpsioverpsi_array[ib][j]); } RecordsOnNode[is][LOGPSI_FIXED] = opt_data.get_log_psi_fixed()[ib]; @@ -484,7 +488,7 @@ QMCCostFunctionBatched::EffectiveWeight QMCCostFunctionBatched::correlatedSampli auto evalOptCorrelated = [](int crowd_id, UPtrVector& opt_crowds, const std::vector& samples_per_crowd_offsets, const std::vector& walkers_per_crowd, std::vector& gradPsi, - std::vector& lapPsi, Matrix& RecordsOnNode, Matrix& DerivRecords, + std::vector& lapPsi, Matrix& RecordsOnNode, Matrix& DerivRecords, Matrix& HDerivRecords, const SampleStack& samples, const opt_variables_type& optVars, bool compute_all_from_scratch, Return_rt vmc_or_dmc, bool needGrad) { CostFunctionCrowdData& opt_data = *opt_crowds[crowd_id]; @@ -589,7 +593,9 @@ QMCCostFunctionBatched::EffectiveWeight QMCCostFunctionBatched::correlatedSampli { if (optVars.recompute(j)) { - DerivRecords[is][j] = std::real(dlogpsi_array[ib][j]); + //In general, dlogpsi is complex. + DerivRecords[is][j] = dlogpsi_array[ib][j]; + //However, E_L is always real, and so d E_L/dc is real, provided c is real. HDerivRecords[is][j] = std::real(dhpsioverpsi_array[ib][j]); } } @@ -716,14 +722,14 @@ QMCCostFunctionBatched::Return_rt QMCCostFunctionBatched::fillOverlapHamiltonian RealType H2_avg = 1.0 / (curAvg_w * curAvg_w); // RealType H2_avg = 1.0/std::sqrt(curAvg_w*curAvg_w*curAvg2_w); RealType V_avg = curAvg2_w - curAvg_w * curAvg_w; - std::vector D_avg(getNumParams(), 0.0); + std::vector D_avg(getNumParams(), 0.0); Return_rt wgtinv = 1.0 / SumValue[SUM_WGT]; for (int iw = 0; iw < rank_local_num_samples_; iw++) { const Return_rt* restrict saved = RecordsOnNode_[iw]; Return_rt weight = saved[REWEIGHT] * wgtinv; - const Return_rt* Dsaved = DerivRecords_[iw]; + const Return_t* Dsaved = DerivRecords_[iw]; for (int pm = 0; pm < getNumParams(); pm++) { D_avg[pm] += Dsaved[pm] * weight; @@ -737,7 +743,7 @@ QMCCostFunctionBatched::Return_rt QMCCostFunctionBatched::fillOverlapHamiltonian const Return_rt* restrict saved = RecordsOnNode_[iw]; Return_rt weight = saved[REWEIGHT] * wgtinv; Return_rt eloc_new = saved[ENERGY_NEW]; - const Return_rt* Dsaved = DerivRecords_[iw]; + const Return_t* Dsaved = DerivRecords_[iw]; const Return_rt* HDsaved = HDerivRecords_[iw]; size_t opt_num_crowds = walkers_per_crowd_.size(); @@ -745,38 +751,38 @@ QMCCostFunctionBatched::Return_rt QMCCostFunctionBatched::fillOverlapHamiltonian FairDivide(getNumParams(), opt_num_crowds, params_per_crowd); - auto constructMatrices = [](int crowd_id, std::vector& crowd_ranges, int numParams, const Return_rt* Dsaved, + auto constructMatrices = [](int crowd_id, std::vector& crowd_ranges, int numParams, const Return_t* Dsaved, const Return_rt* HDsaved, Return_rt weight, Return_rt eloc_new, RealType H2_avg, - RealType V_avg, std::vector& D_avg, RealType b1, RealType b2, + RealType V_avg, std::vector& D_avg, RealType b1, RealType b2, RealType curAvg_w, Matrix& Left, Matrix& Right) { int local_pm_start = crowd_ranges[crowd_id]; int local_pm_end = crowd_ranges[crowd_id + 1]; for (int pm = local_pm_start; pm < local_pm_end; pm++) { - Return_rt wfe = (HDsaved[pm] + (Dsaved[pm] - D_avg[pm]) * eloc_new) * weight; - Return_rt wfd = (Dsaved[pm] - D_avg[pm]) * weight; - Return_rt vterm = + Return_t wfe = (HDsaved[pm] + (Dsaved[pm] - D_avg[pm]) * eloc_new) * weight; + Return_t wfd = (Dsaved[pm] - D_avg[pm]) * weight; + Return_t vterm = HDsaved[pm] * (eloc_new - curAvg_w) + (Dsaved[pm] - D_avg[pm]) * eloc_new * (eloc_new - 2.0 * curAvg_w); // H2 - Right(0, pm + 1) += b1 * H2_avg * vterm * weight; - Right(pm + 1, 0) += b1 * H2_avg * vterm * weight; + Right(0, pm + 1) += b1 * H2_avg * std::real(vterm) * weight; + Right(pm + 1, 0) += b1 * H2_avg * std::real(vterm) * weight; // Variance - Left(0, pm + 1) += b2 * vterm * weight; - Left(pm + 1, 0) += b2 * vterm * weight; + Left(0, pm + 1) += b2 * std::real(vterm) * weight; + Left(pm + 1, 0) += b2 * std::real(vterm) * weight; // Hamiltonian - Left(0, pm + 1) += (1 - b2) * wfe; - Left(pm + 1, 0) += (1 - b2) * wfd * eloc_new; + Left(0, pm + 1) += (1 - b2) * std::real(wfe); + Left(pm + 1, 0) += (1 - b2) * std::real(wfd) * eloc_new; for (int pm2 = 0; pm2 < numParams; pm2++) { // Hamiltonian - Left(pm + 1, pm2 + 1) += (1 - b2) * wfd * (HDsaved[pm2] + (Dsaved[pm2] - D_avg[pm2]) * eloc_new); + Left(pm + 1, pm2 + 1) += std::real((1 - b2) * std::conj(wfd) * (HDsaved[pm2] + (Dsaved[pm2] - D_avg[pm2]) * eloc_new)); // Overlap - RealType ovlij = wfd * (Dsaved[pm2] - D_avg[pm2]); + RealType ovlij = std::real(std::conj(wfd) * (Dsaved[pm2] - D_avg[pm2])); Right(pm + 1, pm2 + 1) += ovlij; // Variance - RealType varij = weight * (HDsaved[pm] - 2.0 * (Dsaved[pm] - D_avg[pm]) * eloc_new) * - (HDsaved[pm2] - 2.0 * (Dsaved[pm2] - D_avg[pm2]) * eloc_new); + RealType varij = weight * std::real((HDsaved[pm] - 2.0 * std::conj(Dsaved[pm] - D_avg[pm]) * eloc_new) * + (HDsaved[pm2] - 2.0 * (Dsaved[pm2] - D_avg[pm2]) * eloc_new)); Left(pm + 1, pm2 + 1) += b2 * (varij + V_avg * ovlij); // H2 Right(pm + 1, pm2 + 1) += b1 * H2_avg * varij; diff --git a/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.h b/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.h index b9440a6bfbe..14709d9bbb8 100644 --- a/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.h +++ b/src/QMCDrivers/WFOpt/QMCCostFunctionBatched.h @@ -71,7 +71,7 @@ class QMCCostFunctionBatched : public QMCCostFunctionBase, public QMCTraits /** Temp derivative properties and Hderivative properties of all the walkers */ - Matrix DerivRecords_; + Matrix DerivRecords_; Matrix HDerivRecords_; EffectiveWeight correlatedSampling(bool needGrad = true) override; diff --git a/src/QMCDrivers/tests/test_QMCCostFunctionBatched.cpp b/src/QMCDrivers/tests/test_QMCCostFunctionBatched.cpp index 65864e5f43e..a38d02d585e 100644 --- a/src/QMCDrivers/tests/test_QMCCostFunctionBatched.cpp +++ b/src/QMCDrivers/tests/test_QMCCostFunctionBatched.cpp @@ -68,7 +68,7 @@ class LinearMethodTestSupport std::vector& getSumValue() { return costFn.SumValue; } Matrix& getRecordsOnNode() { return costFn.RecordsOnNode_; } - Matrix& getDerivRecords() { return costFn.DerivRecords_; } + Matrix& getDerivRecords() { return costFn.DerivRecords_; } Matrix& getHDerivRecords() { return costFn.HDerivRecords_; } void set_samples_and_param(int nsamples, int nparam) From b4e7febac075eafaa01bbe5f7d6b139acc3f61e0 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Mon, 9 Oct 2023 21:56:19 -0500 Subject: [PATCH 002/168] Fix wrong PhaseDiff. --- src/QMCWaveFunctions/TrialWaveFunction.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/QMCWaveFunctions/TrialWaveFunction.cpp b/src/QMCWaveFunctions/TrialWaveFunction.cpp index 8d3465d68e3..e7d091d5dc3 100644 --- a/src/QMCWaveFunctions/TrialWaveFunction.cpp +++ b/src/QMCWaveFunctions/TrialWaveFunction.cpp @@ -479,7 +479,7 @@ void TrialWaveFunction::mw_calcRatio(const RefVectorWithLeader Date: Mon, 9 Oct 2023 21:58:14 -0500 Subject: [PATCH 003/168] Fix another source of NaN in DMCBatched. prob[iw] = std::norm(ratios[iw]) * std::exp(log_gb[iw] - log_gf[iw]); Although ratios[iw] is zero, log_gb[iw] can be NaN due to dependency on NaN gradients. Thus mark reject early. --- src/QMCDrivers/DMC/DMCBatched.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/QMCDrivers/DMC/DMCBatched.cpp b/src/QMCDrivers/DMC/DMCBatched.cpp index e1cbaf326dd..693452860cb 100644 --- a/src/QMCDrivers/DMC/DMCBatched.cpp +++ b/src/QMCDrivers/DMC/DMCBatched.cpp @@ -33,6 +33,7 @@ namespace qmcplusplus { using std::placeholders::_1; using WP = WalkerProperties::Indexes; +using PsiValueType = TrialWaveFunction::PsiValueType; /** Constructor maintains proper ownership of input parameters * @@ -118,7 +119,7 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, //This generates an entire steps worth of deltas. makeGaussRandomWithEngine(walker_deltas, step_context.get_random_gen()); - std::vector ratios(num_walkers, TrialWaveFunction::PsiValueType(0.0)); + std::vector ratios(num_walkers, PsiValueType(0.0)); std::vector log_gf(num_walkers, 0.0); std::vector log_gb(num_walkers, 0.0); std::vector prob(num_walkers, 0.0); @@ -194,8 +195,8 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, computeLogGreensFunction(drifts_reverse, taus, log_gb); - auto checkPhaseChanged = [&sft](const TrialWaveFunction& twf, int& is_reject) { - if (sft.branch_engine.phaseChanged(twf.getPhaseDiff())) + auto checkPhaseChanged = [&sft](const PsiValueType& ratio, int& is_reject) { + if (ratio == PsiValueType(0) || sft.branch_engine.phaseChanged(std::arg(ratio))) is_reject = 1; else is_reject = 0; @@ -205,7 +206,7 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, std::vector rejects(num_walkers); // instead of std::vector for (int iw = 0; iw < num_walkers; ++iw) { - checkPhaseChanged(walker_twfs[iw], rejects[iw]); + checkPhaseChanged(ratios[iw], rejects[iw]); //This is just convenient to do here rr_proposed[iw] += rr[iw]; } From 24018cce645d347c12ff770741d79fce4667976e Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Mon, 9 Oct 2023 22:30:10 -0500 Subject: [PATCH 004/168] More NaN traps. --- src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp | 2 ++ src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp b/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp index 8570dbed352..b49f294a3c4 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp @@ -281,6 +281,8 @@ void DiracDeterminant::mw_ratioGrad(const RefVectorWithLeader void DiracDeterminant::acceptMove(ParticleSet& P, int iat, bool safe_to_delay) { + if (curRatio == PsiValueType(0)) + throw std::runtime_error("DiracDeterminant::acceptMove curRatio is zero! Report a bug.\n"); ScopedTimer local_timer(UpdateTimer); const int WorkingIndex = iat - FirstIndex; assert(WorkingIndex >= 0); diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp index 7c658dd43b2..9132a77ed10 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp @@ -444,6 +444,8 @@ typename DiracDeterminantBatched::PsiValue DiracDeterminantBatched void DiracDeterminantBatched::acceptMove(ParticleSet& P, int iat, bool safe_to_delay) { + if (curRatio == PsiValueType(0)) + throw std::runtime_error("DiracDeterminant::acceptMove curRatio is zero! Report a bug.\n"); const int WorkingIndex = iat - FirstIndex; log_value_ += convertValueToLog(curRatio); { @@ -496,6 +498,8 @@ void DiracDeterminantBatched::mw_accept_rejectMove( { psiM_g_dev_ptr_list[count] = det.psiM_vgl.device_data() + psiM_vgl.capacity() + NumOrbitals * WorkingIndex * DIM; psiM_l_dev_ptr_list[count] = det.psiM_vgl.device_data() + psiM_vgl.capacity() * 4 + NumOrbitals * WorkingIndex; + if (det.curRatio == PsiValueType(0)) + throw std::runtime_error("DiracDeterminant::mw_accept_rejectMove det.curRatio is zero! Report a bug.\n"); det.log_value_ += convertValueToLog(det.curRatio); count++; } From 9e2ef2c9aa142ed3c94f79476ab4ffb674741ff5 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Mon, 9 Oct 2023 22:55:09 -0500 Subject: [PATCH 005/168] Formatting. --- src/QMCDrivers/DMC/DMCBatched.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/QMCDrivers/DMC/DMCBatched.cpp b/src/QMCDrivers/DMC/DMCBatched.cpp index 693452860cb..f71ef2eb71d 100644 --- a/src/QMCDrivers/DMC/DMCBatched.cpp +++ b/src/QMCDrivers/DMC/DMCBatched.cpp @@ -32,7 +32,7 @@ namespace qmcplusplus { using std::placeholders::_1; -using WP = WalkerProperties::Indexes; +using WP = WalkerProperties::Indexes; using PsiValueType = TrialWaveFunction::PsiValueType; /** Constructor maintains proper ownership of input parameters @@ -217,7 +217,6 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, isAccepted.clear(); for (int iw = 0; iw < num_walkers; ++iw) - { if ((!rejects[iw]) && prob[iw] >= std::numeric_limits::epsilon() && step_context.get_random_gen()() < prob[iw]) { @@ -230,7 +229,6 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, crowd.incReject(); isAccepted.push_back(false); } - } twf_dispatcher.flex_accept_rejectMove(walker_twfs, walker_elecs, iat, isAccepted, true); From cc55cc2e0365d4ce123bcf9524ccabf3e2a8cc70 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 10 Oct 2023 16:28:32 -0500 Subject: [PATCH 006/168] atomic resource collection --- src/QMCWaveFunctions/BasisSetBase.h | 16 +++++ src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 23 ++++++- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h | 2 + src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 64 +++++++++++++++++-- .../LCAO/SoaLocalizedBasisSet.cpp | 50 +++++++++++++++ .../LCAO/SoaLocalizedBasisSet.h | 18 ++++++ 6 files changed, 168 insertions(+), 5 deletions(-) diff --git a/src/QMCWaveFunctions/BasisSetBase.h b/src/QMCWaveFunctions/BasisSetBase.h index db0b95af4e7..e4f4384825e 100644 --- a/src/QMCWaveFunctions/BasisSetBase.h +++ b/src/QMCWaveFunctions/BasisSetBase.h @@ -176,6 +176,22 @@ struct SoaBasisSetBase /// Determine which orbitals are S-type. Used for cusp correction. virtual void queryOrbitalsForSType(const std::vector& corrCenter, std::vector& is_s_orbital) const {} + + /** initialize a shared resource and hand it to collection + */ + virtual void createResource(ResourceCollection& collection) const {} + + /** acquire a shared resource from collection + */ + virtual void acquireResource(ResourceCollection& collection, + const RefVectorWithLeader& bset_list) const + {} + + /** return a shared resource to collection + */ + virtual void releaseResource(ResourceCollection& collection, + const RefVectorWithLeader& bset_list) const + {} }; } // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index ac394cb4d36..ae4be6ea4d0 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -109,13 +109,18 @@ void LCAOrbitalSet::checkObject() const void LCAOrbitalSet::createResource(ResourceCollection& collection) const { + myBasisSet->createResource(collection); + auto resource_index = collection.addResource(std::make_unique()); } void LCAOrbitalSet::acquireResource(ResourceCollection& collection, const RefVectorWithLeader& spo_list) const { assert(this == &spo_list.getLeader()); - auto& spo_leader = spo_list.getCastedLeader(); + auto& spo_leader = spo_list.getCastedLeader(); + + spo_leader.myBasisSet->acquireResource(collection, extractBasRefList(spo_list)); + spo_leader.mw_mem_handle_ = collection.lendResource(); } @@ -123,9 +128,25 @@ void LCAOrbitalSet::releaseResource(ResourceCollection& collection, const RefVec { assert(this == &spo_list.getLeader()); auto& spo_leader = spo_list.getCastedLeader(); + + spo_leader.myBasisSet->releaseResource(collection, extractBasRefList(spo_list)); + collection.takebackResource(spo_leader.mw_mem_handle_); } +RefVectorWithLeader LCAOrbitalSet::extractBasRefList( + const RefVectorWithLeader& spo_list) const +{ + auto& spo_leader = spo_list.getCastedLeader(); + RefVectorWithLeader bas_list(*spo_leader.myBasisSet); + bas_list.reserve(spo_list.size()); + for (size_t iw = 0; iw < spo_list.size(); iw++) + { + auto& spo_i = spo_list.getCastedElement(iw); + bas_list.push_back(*spo_i.myBasisSet); + } + return bas_list; +} std::unique_ptr LCAOrbitalSet::makeClone() const { return std::make_unique(*this); } void LCAOrbitalSet::evaluateValue(const ParticleSet& P, int iat, ValueVector& psi) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h index 2e31e228c14..3b35597815e 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h @@ -308,6 +308,8 @@ struct LCAOrbitalSet : public SPOSet void mw_evaluateValueVPsImplGEMM(const RefVectorWithLeader& spo_list, const RefVectorWithLeader& vp_list, OffloadMWVArray& phi_v) const; + + RefVectorWithLeader extractBasRefList(const RefVectorWithLeader& spo_list) const; struct LCAOMultiWalkerMem; ResourceHandle mw_mem_handle_; /// timer for basis set diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index a1f0555f48e..8b02f499ce2 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -16,6 +16,7 @@ #include "CPU/math.hpp" #include "OptimizableObject.h" +#include namespace qmcplusplus { @@ -29,10 +30,15 @@ namespace qmcplusplus template struct SoaAtomicBasisSet { - using RadialOrbital_t = ROT; - using RealType = typename ROT::RealType; - using GridType = typename ROT::GridType; - using ValueType = typename QMCTraits::ValueType; + using RadialOrbital_t = ROT; + using RealType = typename ROT::RealType; + using GridType = typename ROT::GridType; + using ValueType = typename QMCTraits::ValueType; + using OffloadNelecVGLPBCArray = Array>; // [VGL, elec, PBC, Rnl/Ylm] + using OffloadNelecVPBCArray = Array>; // [elec, PBC, Rnl/Ylm/xyz] + using OffloadNelecPBCArray = Array>; // [elec, PBC] + using OffloadRPBCArray = Array>; // [xyz, PBC] + using OffloadVector = Vector>; ///size of the basis set int BasisSetSize; @@ -56,6 +62,8 @@ struct SoaAtomicBasisSet std::vector RnlID; ///temporary storage VectorSoaContainer tempS; + struct SoaAtomicBSetMultiWalkerMem; + ResourceHandle mw_mem_handle_; ///the constructor explicit SoaAtomicBasisSet(int lmax, bool addsignforM = false) : Ylm(lmax, addsignforM) {} @@ -656,6 +664,54 @@ struct SoaAtomicBasisSet } } } + void createResource(ResourceCollection& collection) const + { + // Ylm.createResource(collection); + // MultiRnl.createResource(collection); + auto resource_index = collection.addResource(std::make_unique()); + } + + void acquireResource(ResourceCollection& collection, const RefVectorWithLeader& atom_bs_list) const + { + assert(this == &atom_bs_list.getLeader()); + // auto& atom_bs_leader = atom_bs_list.getCastedLeader(); + // auto& atom_bs_leader = atom_bs_list.getCastedLeader(); + // SoaAtomicBasisSet& atom_bs_leader = atom_bs_list.getCastedLeader(); + // const SoaAtomicBasisSet& atom_bs_leader = atom_bs_list.getCastedLeader(); + // const auto ylm_list(extractYlmRefList(atom_bs_list)); + auto& atom_bs_leader = atom_bs_list.template getCastedLeader(); + atom_bs_leader.mw_mem_handle_ = collection.lendResource(); + } + + void releaseResource(ResourceCollection& collection, const RefVectorWithLeader& atom_bs_list) const + { + assert(this == &atom_bs_list.getLeader()); + // auto& atom_bs_leader = atom_bs_list.getCastedLeader(); + // const SoaAtomicBasisSet& atom_bs_leader = atom_bs_list.getCastedLeader(); + auto& atom_bs_leader = atom_bs_list.template getCastedLeader(); + collection.takebackResource(atom_bs_leader.mw_mem_handle_); + } + + struct SoaAtomicBSetMultiWalkerMem : public Resource + { + SoaAtomicBSetMultiWalkerMem() : Resource("SoaAtomicBasisSet") {} + + SoaAtomicBSetMultiWalkerMem(const SoaAtomicBSetMultiWalkerMem&) : SoaAtomicBSetMultiWalkerMem() {} + + std::unique_ptr makeClone() const override + { + return std::make_unique(*this); + } + + OffloadNelecVPBCArray ylm_v; // [Nelec][PBC][NYlm] + OffloadNelecVPBCArray rnl_v; // [Nelec][PBC][NRnl] + OffloadNelecVGLPBCArray ylm_vgl; // [5][Nelec][PBC][NYlm] + OffloadNelecVGLPBCArray rnl_vgl; // [5][Nelec][PBC][NRnl] + OffloadRPBCArray dr_pbc; // [PBC][xyz] + OffloadNelecVPBCArray dr; // [Nelec][PBC][xyz] + OffloadNelecPBCArray r; // [Nelec][PBC] + OffloadVector correctphase; // [Nelec] + }; }; } // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index 6942b770a41..a090c881ac3 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -21,6 +21,56 @@ namespace qmcplusplus { + +template +void SoaLocalizedBasisSet::createResource(ResourceCollection& collection) const +{ + for (int i = 0; i < LOBasisSet.size(); i++) + LOBasisSet[i]->createResource(collection); +} +template +void SoaLocalizedBasisSet::acquireResource(ResourceCollection& collection, + const RefVectorWithLeader>& bs_list) const +{ + auto& loc_bs_leader = bs_list.template getCastedLeader>(); + auto& atom_bs_leader = loc_bs_leader.LOBasisSet; + const int num_ctr = loc_bs_leader.LOBasisSet.size(); + for (int i = 0; i < num_ctr; i++) + { + const auto atom_bs_list(extractLOBasisRefList(bs_list, i)); + atom_bs_leader[i]->acquireResource(collection, atom_bs_list); + } +} +template +void SoaLocalizedBasisSet::releaseResource(ResourceCollection& collection, + const RefVectorWithLeader>& bs_list) const +{ + auto& loc_bs_leader = bs_list.template getCastedLeader>(); + auto& atom_bs_leader = loc_bs_leader.LOBasisSet; + const int num_ctr = loc_bs_leader.LOBasisSet.size(); + for (int i = 0; i < num_ctr; i++) + { + const auto atom_bs_list(extractLOBasisRefList(bs_list, i)); + atom_bs_leader[i]->releaseResource(collection, atom_bs_list); + } +} +template +RefVectorWithLeader SoaLocalizedBasisSet::extractLOBasisRefList( + const RefVectorWithLeader>& bs_list, + int id) +{ + auto& bs_leader = bs_list.template getCastedLeader>(); + RefVectorWithLeader atom_bs_list(*bs_leader.LOBasisSet[id]); + atom_bs_list.reserve(bs_list.size()); + for (size_t iw = 0; iw < bs_list.size(); iw++) + { + auto& bs_i = bs_list.template getCastedElement>(iw); + atom_bs_list.push_back(*bs_i.LOBasisSet[id]); + } + return atom_bs_list; +} + + template SoaLocalizedBasisSet::SoaLocalizedBasisSet(ParticleSet& ions, ParticleSet& els) : ions_(ions), diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 9c352dbd14f..374af859a7d 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -172,6 +172,24 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase * @param aos a set of Centered Atomic Orbitals */ void add(int icenter, std::unique_ptr aos); + + + /** initialize a shared resource and hand it to collection + */ + void createResource(ResourceCollection& collection) const override; + + /** acquire a shared resource from collection + */ + void acquireResource(ResourceCollection& collection, + const RefVectorWithLeader>& spo_list) const override; + + /** return a shared resource to collection + */ + void releaseResource(ResourceCollection& collection, + const RefVectorWithLeader>& spo_list) const override; + + static RefVectorWithLeader extractLOBasisRefList(const RefVectorWithLeader>& bs_list, + int id); }; } // namespace qmcplusplus #endif From acac56a1880f8890aa527fc92315b09c0970ba4b Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 10 Oct 2023 16:44:46 -0500 Subject: [PATCH 007/168] remove old comments --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 8b02f499ce2..236e3c26cb1 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -666,19 +666,12 @@ struct SoaAtomicBasisSet } void createResource(ResourceCollection& collection) const { - // Ylm.createResource(collection); - // MultiRnl.createResource(collection); auto resource_index = collection.addResource(std::make_unique()); } void acquireResource(ResourceCollection& collection, const RefVectorWithLeader& atom_bs_list) const { assert(this == &atom_bs_list.getLeader()); - // auto& atom_bs_leader = atom_bs_list.getCastedLeader(); - // auto& atom_bs_leader = atom_bs_list.getCastedLeader(); - // SoaAtomicBasisSet& atom_bs_leader = atom_bs_list.getCastedLeader(); - // const SoaAtomicBasisSet& atom_bs_leader = atom_bs_list.getCastedLeader(); - // const auto ylm_list(extractYlmRefList(atom_bs_list)); auto& atom_bs_leader = atom_bs_list.template getCastedLeader(); atom_bs_leader.mw_mem_handle_ = collection.lendResource(); } @@ -686,8 +679,6 @@ struct SoaAtomicBasisSet void releaseResource(ResourceCollection& collection, const RefVectorWithLeader& atom_bs_list) const { assert(this == &atom_bs_list.getLeader()); - // auto& atom_bs_leader = atom_bs_list.getCastedLeader(); - // const SoaAtomicBasisSet& atom_bs_leader = atom_bs_list.getCastedLeader(); auto& atom_bs_leader = atom_bs_list.template getCastedLeader(); collection.takebackResource(atom_bs_leader.mw_mem_handle_); } From 0d185ba9cefe41e0721b9232634314d1fc0a7560 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 10 Oct 2023 16:48:46 -0500 Subject: [PATCH 008/168] rename offload arrays --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 236e3c26cb1..3ca609ac550 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -30,15 +30,14 @@ namespace qmcplusplus template struct SoaAtomicBasisSet { - using RadialOrbital_t = ROT; - using RealType = typename ROT::RealType; - using GridType = typename ROT::GridType; - using ValueType = typename QMCTraits::ValueType; - using OffloadNelecVGLPBCArray = Array>; // [VGL, elec, PBC, Rnl/Ylm] - using OffloadNelecVPBCArray = Array>; // [elec, PBC, Rnl/Ylm/xyz] - using OffloadNelecPBCArray = Array>; // [elec, PBC] - using OffloadRPBCArray = Array>; // [xyz, PBC] - using OffloadVector = Vector>; + using RadialOrbital_t = ROT; + using RealType = typename ROT::RealType; + using GridType = typename ROT::GridType; + using ValueType = typename QMCTraits::ValueType; + using OffloadArray4 = Array>; + using OffloadArray3 = Array>; + using OffloadArray2 = Array>; + using OffloadVector = Vector>; ///size of the basis set int BasisSetSize; @@ -694,14 +693,14 @@ struct SoaAtomicBasisSet return std::make_unique(*this); } - OffloadNelecVPBCArray ylm_v; // [Nelec][PBC][NYlm] - OffloadNelecVPBCArray rnl_v; // [Nelec][PBC][NRnl] - OffloadNelecVGLPBCArray ylm_vgl; // [5][Nelec][PBC][NYlm] - OffloadNelecVGLPBCArray rnl_vgl; // [5][Nelec][PBC][NRnl] - OffloadRPBCArray dr_pbc; // [PBC][xyz] - OffloadNelecVPBCArray dr; // [Nelec][PBC][xyz] - OffloadNelecPBCArray r; // [Nelec][PBC] - OffloadVector correctphase; // [Nelec] + OffloadArray3 ylm_v; // [Nelec][PBC][NYlm] + OffloadArray3 rnl_v; // [Nelec][PBC][NRnl] + OffloadArray4 ylm_vgl; // [5][Nelec][PBC][NYlm] + OffloadArray4 rnl_vgl; // [5][Nelec][PBC][NRnl] + OffloadArray2 dr_pbc; // [PBC][xyz] + OffloadArray3 dr; // [Nelec][PBC][xyz] + OffloadArray2 r; // [Nelec][PBC] + OffloadVector correctphase; // [Nelec] }; }; From 5aa68eb8a2c7964af1894d51008b2b6aade0ba91 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 10 Oct 2023 16:52:21 -0500 Subject: [PATCH 009/168] rename basis_lists --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 8 ++++---- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index ae4be6ea4d0..c326529fca5 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -138,14 +138,14 @@ RefVectorWithLeader LCAOrbitalSet::extractBa const RefVectorWithLeader& spo_list) const { auto& spo_leader = spo_list.getCastedLeader(); - RefVectorWithLeader bas_list(*spo_leader.myBasisSet); - bas_list.reserve(spo_list.size()); + RefVectorWithLeader basis_list(*spo_leader.myBasisSet); + basis_list.reserve(spo_list.size()); for (size_t iw = 0; iw < spo_list.size(); iw++) { auto& spo_i = spo_list.getCastedElement(iw); - bas_list.push_back(*spo_i.myBasisSet); + basis_list.push_back(*spo_i.myBasisSet); } - return bas_list; + return basis_list; } std::unique_ptr LCAOrbitalSet::makeClone() const { return std::make_unique(*this); } diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 3ca609ac550..07196043aaa 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -668,18 +668,20 @@ struct SoaAtomicBasisSet auto resource_index = collection.addResource(std::make_unique()); } - void acquireResource(ResourceCollection& collection, const RefVectorWithLeader& atom_bs_list) const + void acquireResource(ResourceCollection& collection, + const RefVectorWithLeader& atom_basis_list) const { - assert(this == &atom_bs_list.getLeader()); - auto& atom_bs_leader = atom_bs_list.template getCastedLeader(); - atom_bs_leader.mw_mem_handle_ = collection.lendResource(); + assert(this == &atom_basis_list.getLeader()); + auto& atom_basis_leader = atom_basis_list.template getCastedLeader(); + atom_basis_leader.mw_mem_handle_ = collection.lendResource(); } - void releaseResource(ResourceCollection& collection, const RefVectorWithLeader& atom_bs_list) const + void releaseResource(ResourceCollection& collection, + const RefVectorWithLeader& atom_basis_list) const { - assert(this == &atom_bs_list.getLeader()); - auto& atom_bs_leader = atom_bs_list.template getCastedLeader(); - collection.takebackResource(atom_bs_leader.mw_mem_handle_); + assert(this == &atom_basis_list.getLeader()); + auto& atom_basis_leader = atom_basis_list.template getCastedLeader(); + collection.takebackResource(atom_basis_leader.mw_mem_handle_); } struct SoaAtomicBSetMultiWalkerMem : public Resource From 811aa33b1538c1e1a7a8fd13a3eb426348f8ca00 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 10 Oct 2023 16:54:12 -0500 Subject: [PATCH 010/168] rename variables --- .../LCAO/SoaLocalizedBasisSet.cpp | 38 ++++++++++--------- .../LCAO/SoaLocalizedBasisSet.h | 2 +- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index a090c881ac3..d9c261d3bd1 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -29,45 +29,47 @@ void SoaLocalizedBasisSet::createResource(ResourceCollection& collect LOBasisSet[i]->createResource(collection); } template -void SoaLocalizedBasisSet::acquireResource(ResourceCollection& collection, - const RefVectorWithLeader>& bs_list) const +void SoaLocalizedBasisSet::acquireResource( + ResourceCollection& collection, + const RefVectorWithLeader>& basis_list) const { - auto& loc_bs_leader = bs_list.template getCastedLeader>(); + auto& loc_bs_leader = basis_list.template getCastedLeader>(); auto& atom_bs_leader = loc_bs_leader.LOBasisSet; const int num_ctr = loc_bs_leader.LOBasisSet.size(); for (int i = 0; i < num_ctr; i++) { - const auto atom_bs_list(extractLOBasisRefList(bs_list, i)); - atom_bs_leader[i]->acquireResource(collection, atom_bs_list); + const auto atom_basis_list(extractLOBasisRefList(basis_list, i)); + atom_bs_leader[i]->acquireResource(collection, atom_basis_list); } } template -void SoaLocalizedBasisSet::releaseResource(ResourceCollection& collection, - const RefVectorWithLeader>& bs_list) const +void SoaLocalizedBasisSet::releaseResource( + ResourceCollection& collection, + const RefVectorWithLeader>& basis_list) const { - auto& loc_bs_leader = bs_list.template getCastedLeader>(); + auto& loc_bs_leader = basis_list.template getCastedLeader>(); auto& atom_bs_leader = loc_bs_leader.LOBasisSet; const int num_ctr = loc_bs_leader.LOBasisSet.size(); for (int i = 0; i < num_ctr; i++) { - const auto atom_bs_list(extractLOBasisRefList(bs_list, i)); - atom_bs_leader[i]->releaseResource(collection, atom_bs_list); + const auto atom_basis_list(extractLOBasisRefList(basis_list, i)); + atom_bs_leader[i]->releaseResource(collection, atom_basis_list); } } template RefVectorWithLeader SoaLocalizedBasisSet::extractLOBasisRefList( - const RefVectorWithLeader>& bs_list, + const RefVectorWithLeader>& basis_list, int id) { - auto& bs_leader = bs_list.template getCastedLeader>(); - RefVectorWithLeader atom_bs_list(*bs_leader.LOBasisSet[id]); - atom_bs_list.reserve(bs_list.size()); - for (size_t iw = 0; iw < bs_list.size(); iw++) + auto& bs_leader = basis_list.template getCastedLeader>(); + RefVectorWithLeader atom_basis_list(*bs_leader.LOBasisSet[id]); + atom_basis_list.reserve(basis_list.size()); + for (size_t iw = 0; iw < basis_list.size(); iw++) { - auto& bs_i = bs_list.template getCastedElement>(iw); - atom_bs_list.push_back(*bs_i.LOBasisSet[id]); + auto& bs_i = basis_list.template getCastedElement>(iw); + atom_basis_list.push_back(*bs_i.LOBasisSet[id]); } - return atom_bs_list; + return atom_basis_list; } diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 374af859a7d..20d868c38bf 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -188,7 +188,7 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase void releaseResource(ResourceCollection& collection, const RefVectorWithLeader>& spo_list) const override; - static RefVectorWithLeader extractLOBasisRefList(const RefVectorWithLeader>& bs_list, + static RefVectorWithLeader extractLOBasisRefList(const RefVectorWithLeader>& basis_list, int id); }; } // namespace qmcplusplus From ec256026fe60f1929ca9459a66d514d03338e5dd Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Mon, 9 Oct 2023 15:06:24 -0500 Subject: [PATCH 011/168] Consolidate QMCHamiltonian code paths. --- src/QMCHamiltonians/QMCHamiltonian.cpp | 96 ++++++-------------------- src/QMCHamiltonians/QMCHamiltonian.h | 6 +- 2 files changed, 25 insertions(+), 77 deletions(-) diff --git a/src/QMCHamiltonians/QMCHamiltonian.cpp b/src/QMCHamiltonians/QMCHamiltonian.cpp index 52ef19ec51d..5f1b6b30f1f 100644 --- a/src/QMCHamiltonians/QMCHamiltonian.cpp +++ b/src/QMCHamiltonians/QMCHamiltonian.cpp @@ -543,25 +543,13 @@ QMCHamiltonian::FullPrecRealType QMCHamiltonian::evaluate(ParticleSet& P) for (int i = 0; i < H.size(); ++i) { ScopedTimer h_timer(my_timers_[i]); - const auto LocalEnergyComponent = H[i]->evaluate(P); - if (qmcplusplus::isnan(LocalEnergyComponent)) - { - std::ostringstream msg; - msg << "QMCHamiltonian::evaluate component " << H[i]->getName() << " returns NaN." << std::endl; - P.print(msg); - throw std::runtime_error(msg.str()); - } - LocalEnergy += LocalEnergyComponent; - H[i]->setObservables(Observables); + H[i]->evaluate(P); + updateComponent(*H[i], *this, P); #if !defined(REMOVE_TRACEMANAGER) H[i]->collectScalarTraces(); #endif - H[i]->setParticlePropertyList(P.PropertyList, myIndex); } - KineticEnergy = H[0]->getValue(); - P.PropertyList[WP::LOCALENERGY] = LocalEnergy; - P.PropertyList[WP::LOCALPOTENTIAL] = LocalEnergy - KineticEnergy; - // auxHevaluate(P); + updateKinetic(*this, P); return LocalEnergy; } @@ -572,34 +560,22 @@ QMCHamiltonian::FullPrecRealType QMCHamiltonian::evaluateDeterministic(ParticleS for (int i = 0; i < H.size(); ++i) { ScopedTimer h_timer(my_timers_[i]); - const auto LocalEnergyComponent = H[i]->evaluateDeterministic(P); - if (qmcplusplus::isnan(LocalEnergyComponent)) - { - std::ostringstream msg; - msg << "QMCHamiltonian::evaluateDeterministic component " << H[i]->getName() << " returns NaN." << std::endl; - P.print(msg); - throw std::runtime_error(msg.str()); - } - LocalEnergy += LocalEnergyComponent; - H[i]->setObservables(Observables); + H[i]->evaluateDeterministic(P); + updateComponent(*H[i], *this, P); #if !defined(REMOVE_TRACEMANAGER) H[i]->collectScalarTraces(); #endif - H[i]->setParticlePropertyList(P.PropertyList, myIndex); } - KineticEnergy = H[0]->getValue(); - P.PropertyList[WP::LOCALENERGY] = LocalEnergy; - P.PropertyList[WP::LOCALPOTENTIAL] = LocalEnergy - KineticEnergy; - // auxHevaluate(P); + updateKinetic(*this, P); return LocalEnergy; } -void QMCHamiltonian::updateNonKinetic(OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset) +void QMCHamiltonian::updateComponent(OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset) { // It's much better to be able to see where this is coming from. It is caught just fine. if (qmcplusplus::isnan(op.getValue())) { std::ostringstream msg; - msg << "QMCHamiltonian::updateNonKinetic component " << op.getName() << " returns NaN." << std::endl; + msg << "QMCHamiltonian::updateComponent component " << op.getName() << " returns NaN." << std::endl; pset.print(msg); throw std::runtime_error(msg.str()); } @@ -609,9 +585,9 @@ void QMCHamiltonian::updateNonKinetic(OperatorBase& op, QMCHamiltonian& ham, Par op.setParticlePropertyList(pset.PropertyList, ham.myIndex); } -void QMCHamiltonian::updateKinetic(OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset) +void QMCHamiltonian::updateKinetic(QMCHamiltonian& ham, ParticleSet& pset) { - ham.KineticEnergy = op.getValue(); + ham.KineticEnergy = ham.H[0]->getValue(); pset.PropertyList[WP::LOCALENERGY] = ham.LocalEnergy; pset.PropertyList[WP::LOCALPOTENTIAL] = ham.LocalEnergy - ham.KineticEnergy; } @@ -641,7 +617,7 @@ std::vector QMCHamiltonian::mw_evaluate( else ham_leader.H[kinetic_index]->mw_evaluate(HC_list, wf_list, p_list); for (int iw = 0; iw < ham_list.size(); iw++) - updateNonKinetic(HC_list[iw], ham_list[iw], p_list[iw]); + updateComponent(HC_list[iw], ham_list[iw], p_list[iw]); } for (int i_ham_op = 1; i_ham_op < num_ham_operators; ++i_ham_op) @@ -649,21 +625,6 @@ std::vector QMCHamiltonian::mw_evaluate( ScopedTimer h_timer(ham_leader.my_timers_[i_ham_op]); const auto HC_list(extract_HC_list(ham_list, i_ham_op)); - // This comment badly breaks the 1st rule of comments.. - // // This lambda accomplishes two things - // // 1. It makes clear T& and not std::reference_wrapper is desired removing need for gets. - // // 2. [] captures nothing insuring that we know these updates only depend on the three object involved. - // auto updateNonKinetic = [](OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset) { - // // both hamiltonian and operatorbase should have operator<< overides - // if (qmcplusplus::isnan(op.Value)) - // APP_ABORT("QMCHamiltonian::evaluate component " + op.myName + " returns NaN\n"); - - // // The following is a ridiculous breach of encapsulation. - // ham.LocalEnergy += op.Value; - // op.setObservables(ham.Observables); - // op.setParticlePropertyList(pset.PropertyList, ham.myIndex); - // }; - if (ham_leader.mw_res_handle_.getResource().potential_listeners_.size() > 0) ham_leader.H[i_ham_op]->mw_evaluatePerParticle(HC_list, wf_list, p_list, ham_leader.mw_res_handle_.getResource().potential_listeners_, @@ -671,17 +632,11 @@ std::vector QMCHamiltonian::mw_evaluate( else ham_leader.H[i_ham_op]->mw_evaluate(HC_list, wf_list, p_list); for (int iw = 0; iw < ham_list.size(); iw++) - updateNonKinetic(HC_list[iw], ham_list[iw], p_list[iw]); + updateComponent(HC_list[iw], ham_list[iw], p_list[iw]); } - // auto updateKinetic = [](OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset) { - // ham.KineticEnergy = op.Value; - // pset.PropertyList[WP::LOCALENERGY] = ham.LocalEnergy; - // pset.PropertyList[LOCALPOTENTIAL] = ham.LocalEnergy - ham.KineticEnergy; - // }; - const auto HC_list(extract_HC_list(ham_list, 0)); for (int iw = 0; iw < ham_list.size(); iw++) - updateKinetic(HC_list[iw], ham_list[iw], p_list[iw]); + updateKinetic(ham_list[iw], p_list[iw]); std::vector local_energies(ham_list.size(), 0.0); for (int iw = 0; iw < ham_list.size(); ++iw) @@ -738,14 +693,11 @@ std::vector QMCHamiltonian::mw_evaluateValueAn ham_leader.H[i_ham_op]->mw_evaluateWithParameterDerivatives(HC_list, p_list, optvars, dlogpsi, dhpsioverpsi); for (int iw = 0; iw < ham_list.size(); iw++) - updateNonKinetic(HC_list[iw], ham_list[iw], p_list[iw]); + updateComponent(HC_list[iw], ham_list[iw], p_list[iw]); } for (int iw = 0; iw < ham_list.size(); iw++) - { - const auto HC_list(extract_HC_list(ham_list, 0)); - updateKinetic(HC_list[iw], ham_list[iw], p_list[iw]); - } + updateKinetic(ham_list[iw], p_list[iw]); for (int iw = 0; iw < ham_list.size(); ++iw) local_energies[iw] = ham_list[iw].get_LocalEnergy(); @@ -847,16 +799,13 @@ QMCHamiltonian::FullPrecRealType QMCHamiltonian::evaluateWithToperator(ParticleS for (int i = 0; i < H.size(); ++i) { ScopedTimer h_timer(my_timers_[i]); - LocalEnergy += H[i]->evaluateWithToperator(P); - H[i]->setObservables(Observables); + H[i]->evaluateWithToperator(P); + updateComponent(*H[i], *this, P); #if !defined(REMOVE_TRACEMANAGER) H[i]->collectScalarTraces(); #endif } - KineticEnergy = H[0]->getValue(); - P.PropertyList[WP::LOCALENERGY] = LocalEnergy; - P.PropertyList[WP::LOCALPOTENTIAL] = LocalEnergy - KineticEnergy; - // auxHevaluate(P); + updateKinetic(*this, P); return LocalEnergy; } @@ -883,7 +832,7 @@ std::vector QMCHamiltonian::mw_evaluateWithTop else ham_leader.H[kinetic_index]->mw_evaluateWithToperator(HC_list, wf_list, p_list); for (int iw = 0; iw < ham_list.size(); iw++) - updateNonKinetic(HC_list[iw], ham_list[iw], p_list[iw]); + updateComponent(HC_list[iw], ham_list[iw], p_list[iw]); } for (int i_ham_op = 1; i_ham_op < num_ham_operators; ++i_ham_op) @@ -898,14 +847,11 @@ std::vector QMCHamiltonian::mw_evaluateWithTop else ham_leader.H[i_ham_op]->mw_evaluateWithToperator(HC_list, wf_list, p_list); for (int iw = 0; iw < ham_list.size(); ++iw) - updateNonKinetic(HC_list[iw], ham_list[iw], p_list[iw]); + updateComponent(HC_list[iw], ham_list[iw], p_list[iw]); } for (int iw = 0; iw < ham_list.size(); iw++) - { - const auto HC_list(extract_HC_list(ham_list, 0)); - updateKinetic(HC_list[iw], ham_list[iw], p_list[iw]); - } + updateKinetic(ham_list[iw], p_list[iw]); std::vector local_energies(ham_list.size()); for (int iw = 0; iw < ham_list.size(); ++iw) diff --git a/src/QMCHamiltonians/QMCHamiltonian.h b/src/QMCHamiltonians/QMCHamiltonian.h index 83c5e5601be..ec577a1b701 100644 --- a/src/QMCHamiltonians/QMCHamiltonian.h +++ b/src/QMCHamiltonians/QMCHamiltonian.h @@ -410,8 +410,10 @@ class QMCHamiltonian void setRandomGenerator(RandomBase* rng); - static void updateNonKinetic(OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset); - static void updateKinetic(OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset); + /// accumulate local energy and update Observables and PropertyList + static void updateComponent(OperatorBase& op, QMCHamiltonian& ham, ParticleSet& pset); + /// extract kinetic and potential energies. + static void updateKinetic(QMCHamiltonian& ham, ParticleSet& pset); /// initialize a shared resource and hand it to a collection void createResource(ResourceCollection& collection) const; From a8eab5121391e956230f9b13f66f3d2781bb107f Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Mon, 9 Oct 2023 22:49:50 -0500 Subject: [PATCH 012/168] Expand unit tests. --- src/QMCHamiltonians/tests/test_QMCHamiltonian.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/QMCHamiltonians/tests/test_QMCHamiltonian.cpp b/src/QMCHamiltonians/tests/test_QMCHamiltonian.cpp index 3ab2ac3a8ef..fc209ea3071 100644 --- a/src/QMCHamiltonians/tests/test_QMCHamiltonian.cpp +++ b/src/QMCHamiltonians/tests/test_QMCHamiltonian.cpp @@ -27,6 +27,7 @@ namespace qmcplusplus { using QMCT = QMCTraits; using Real = QMCT::RealType; +using WP = WalkerProperties::Indexes; constexpr bool generate_test_data = false; @@ -201,8 +202,12 @@ TEST_CASE("integrateListeners", "[hamiltonian]") ParticleSet::mw_update(p_list); - QMCHamiltonian::mw_evaluate(ham_list, twf_list, p_list); + auto energies = QMCHamiltonian::mw_evaluate(ham_list, twf_list, p_list); + CHECK(ham_list[0].getLocalEnergy() == energies[0]); + CHECK(ham_list[1].getLocalEnergy() == energies[1]); + CHECK(ham_list[0].getLocalEnergy() == p_list[0].PropertyList[WP::LOCALENERGY]); + CHECK(ham_list[1].getLocalEnergy() == p_list[1].PropertyList[WP::LOCALENERGY]); if constexpr (generate_test_data) { From 2c0776c9b26455c5b34e230c082155ded9c3f614 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 11 Oct 2023 12:21:25 -0500 Subject: [PATCH 013/168] rename vars and add documentation --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 8 +--- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h | 1 + src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 22 +++++---- .../LCAO/SoaLocalizedBasisSet.cpp | 48 +++++++++---------- .../LCAO/SoaLocalizedBasisSet.h | 13 +++-- 5 files changed, 47 insertions(+), 45 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index c326529fca5..7076b838c9b 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -137,14 +137,10 @@ void LCAOrbitalSet::releaseResource(ResourceCollection& collection, const RefVec RefVectorWithLeader LCAOrbitalSet::extractBasRefList( const RefVectorWithLeader& spo_list) const { - auto& spo_leader = spo_list.getCastedLeader(); - RefVectorWithLeader basis_list(*spo_leader.myBasisSet); + RefVectorWithLeader basis_list(*spo_list.getCastedLeader().myBasisSet); basis_list.reserve(spo_list.size()); for (size_t iw = 0; iw < spo_list.size(); iw++) - { - auto& spo_i = spo_list.getCastedElement(iw); - basis_list.push_back(*spo_i.myBasisSet); - } + basis_list.push_back(*spo_list.getCastedElement(iw).myBasisSet); return basis_list; } std::unique_ptr LCAOrbitalSet::makeClone() const { return std::make_unique(*this); } diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h index 3b35597815e..0b89c599493 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h @@ -309,6 +309,7 @@ struct LCAOrbitalSet : public SPOSet const RefVectorWithLeader& vp_list, OffloadMWVArray& phi_v) const; + /// helper function for extracting a list of basis sets from a list of LCAOrbitalSet RefVectorWithLeader extractBasRefList(const RefVectorWithLeader& spo_list) const; struct LCAOMultiWalkerMem; ResourceHandle mw_mem_handle_; diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 07196043aaa..0aebd13117d 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -36,9 +36,13 @@ struct SoaAtomicBasisSet using ValueType = typename QMCTraits::ValueType; using OffloadArray4 = Array>; using OffloadArray3 = Array>; - using OffloadArray2 = Array>; + using OffloadMatrix = Matrix>; using OffloadVector = Vector>; + /// multi walker shared memory buffer + struct SoaAtomicBSetMultiWalkerMem; + /// multi walker resource handle + ResourceHandle mw_mem_handle_; ///size of the basis set int BasisSetSize; ///Number of Cell images for the evaluation of the orbital with PBC. If No PBC, should be 0; @@ -61,8 +65,6 @@ struct SoaAtomicBasisSet std::vector RnlID; ///temporary storage VectorSoaContainer tempS; - struct SoaAtomicBSetMultiWalkerMem; - ResourceHandle mw_mem_handle_; ///the constructor explicit SoaAtomicBasisSet(int lmax, bool addsignforM = false) : Ylm(lmax, addsignforM) {} @@ -665,7 +667,7 @@ struct SoaAtomicBasisSet } void createResource(ResourceCollection& collection) const { - auto resource_index = collection.addResource(std::make_unique()); + collection.addResource(std::make_unique()); } void acquireResource(ResourceCollection& collection, @@ -695,14 +697,14 @@ struct SoaAtomicBasisSet return std::make_unique(*this); } - OffloadArray3 ylm_v; // [Nelec][PBC][NYlm] - OffloadArray3 rnl_v; // [Nelec][PBC][NRnl] OffloadArray4 ylm_vgl; // [5][Nelec][PBC][NYlm] OffloadArray4 rnl_vgl; // [5][Nelec][PBC][NRnl] - OffloadArray2 dr_pbc; // [PBC][xyz] - OffloadArray3 dr; // [Nelec][PBC][xyz] - OffloadArray2 r; // [Nelec][PBC] - OffloadVector correctphase; // [Nelec] + OffloadArray3 ylm_v; // [Nelec][PBC][NYlm] + OffloadArray3 rnl_v; // [Nelec][PBC][NRnl] + OffloadMatrix dr_pbc; // [PBC][xyz] translation vector for each image + OffloadArray3 dr; // [Nelec][PBC][xyz] ion->elec displacement for each image + OffloadMatrix r; // [Nelec][PBC] ion->elec distance for each image + OffloadVector correctphase; // [Nelec] overall phase }; }; diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index d9c261d3bd1..6a6c911267f 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -31,45 +31,43 @@ void SoaLocalizedBasisSet::createResource(ResourceCollection& collect template void SoaLocalizedBasisSet::acquireResource( ResourceCollection& collection, - const RefVectorWithLeader>& basis_list) const + const RefVectorWithLeader>& basisset_list) const { - auto& loc_bs_leader = basis_list.template getCastedLeader>(); - auto& atom_bs_leader = loc_bs_leader.LOBasisSet; - const int num_ctr = loc_bs_leader.LOBasisSet.size(); - for (int i = 0; i < num_ctr; i++) + // need to cast to SoaLocalizedBasisSet to access LOBasisSet (atomic basis) + auto& loc_basis_leader = basisset_list.template getCastedLeader>(); + auto& basisset_leader = loc_basis_leader.LOBasisSet; + for (int i = 0; i < basisset_leader.size(); i++) { - const auto atom_basis_list(extractLOBasisRefList(basis_list, i)); - atom_bs_leader[i]->acquireResource(collection, atom_basis_list); + const auto one_species_basis_list(extractOneSpeciesBasisRefList(basisset_list, i)); + basisset_leader[i]->acquireResource(collection, one_species_basis_list); } } template void SoaLocalizedBasisSet::releaseResource( ResourceCollection& collection, - const RefVectorWithLeader>& basis_list) const + const RefVectorWithLeader>& basisset_list) const { - auto& loc_bs_leader = basis_list.template getCastedLeader>(); - auto& atom_bs_leader = loc_bs_leader.LOBasisSet; - const int num_ctr = loc_bs_leader.LOBasisSet.size(); - for (int i = 0; i < num_ctr; i++) + // need to cast to SoaLocalizedBasisSet to access LOBasisSet (atomic basis) + auto& loc_basis_leader = basisset_list.template getCastedLeader>(); + auto& basisset_leader = loc_basis_leader.LOBasisSet; + for (int i = 0; i < basisset_leader.size(); i++) { - const auto atom_basis_list(extractLOBasisRefList(basis_list, i)); - atom_bs_leader[i]->releaseResource(collection, atom_basis_list); + const auto one_species_basis_list(extractOneSpeciesBasisRefList(basisset_list, i)); + basisset_leader[i]->releaseResource(collection, one_species_basis_list); } } template -RefVectorWithLeader SoaLocalizedBasisSet::extractLOBasisRefList( - const RefVectorWithLeader>& basis_list, +RefVectorWithLeader SoaLocalizedBasisSet::extractOneSpeciesBasisRefList( + const RefVectorWithLeader>& basisset_list, int id) { - auto& bs_leader = basis_list.template getCastedLeader>(); - RefVectorWithLeader atom_basis_list(*bs_leader.LOBasisSet[id]); - atom_basis_list.reserve(basis_list.size()); - for (size_t iw = 0; iw < basis_list.size(); iw++) - { - auto& bs_i = basis_list.template getCastedElement>(iw); - atom_basis_list.push_back(*bs_i.LOBasisSet[id]); - } - return atom_basis_list; + auto& loc_basis_leader = basisset_list.template getCastedLeader>(); + RefVectorWithLeader one_species_basis_list(*loc_basis_leader.LOBasisSet[id]); + one_species_basis_list.reserve(basisset_list.size()); + for (size_t iw = 0; iw < basisset_list.size(); iw++) + one_species_basis_list.push_back( + *basisset_list.template getCastedElement>(iw).LOBasisSet[id]); + return one_species_basis_list; } diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 20d868c38bf..86ecc385e12 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -181,15 +181,20 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase /** acquire a shared resource from collection */ void acquireResource(ResourceCollection& collection, - const RefVectorWithLeader>& spo_list) const override; + const RefVectorWithLeader>& basisset_list) const override; /** return a shared resource to collection */ void releaseResource(ResourceCollection& collection, - const RefVectorWithLeader>& spo_list) const override; + const RefVectorWithLeader>& basisset_list) const override; - static RefVectorWithLeader extractLOBasisRefList(const RefVectorWithLeader>& basis_list, - int id); + + /** helper function for extracting a list of atomic basis sets for a single species (indexed by `id`) + * from a list of basis sets + */ + static RefVectorWithLeader extractOneSpeciesBasisRefList( + const RefVectorWithLeader>& basisset_list, + int id); }; } // namespace qmcplusplus #endif From 3252d706210c9f58986bdc343e44e96bf4c8f15b Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 11 Oct 2023 12:26:24 -0500 Subject: [PATCH 014/168] minor cleanup --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 0aebd13117d..c5f110a5166 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -674,16 +674,15 @@ struct SoaAtomicBasisSet const RefVectorWithLeader& atom_basis_list) const { assert(this == &atom_basis_list.getLeader()); - auto& atom_basis_leader = atom_basis_list.template getCastedLeader(); - atom_basis_leader.mw_mem_handle_ = collection.lendResource(); + atom_basis_list.template getCastedLeader().mw_mem_handle_ = + collection.lendResource(); } void releaseResource(ResourceCollection& collection, const RefVectorWithLeader& atom_basis_list) const { assert(this == &atom_basis_list.getLeader()); - auto& atom_basis_leader = atom_basis_list.template getCastedLeader(); - collection.takebackResource(atom_basis_leader.mw_mem_handle_); + collection.takebackResource(atom_basis_list.template getCastedLeader().mw_mem_handle_); } struct SoaAtomicBSetMultiWalkerMem : public Resource From 7fb562e330136ef4776dad21ce9bbbff50d1d0f4 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 11 Oct 2023 13:03:17 -0500 Subject: [PATCH 015/168] extractBasRefList -> extractBasisRefList --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 6 +++--- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index 7076b838c9b..71e9ed3f6fb 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -119,7 +119,7 @@ void LCAOrbitalSet::acquireResource(ResourceCollection& collection, const RefVec assert(this == &spo_list.getLeader()); auto& spo_leader = spo_list.getCastedLeader(); - spo_leader.myBasisSet->acquireResource(collection, extractBasRefList(spo_list)); + spo_leader.myBasisSet->acquireResource(collection, extractBasisRefList(spo_list)); spo_leader.mw_mem_handle_ = collection.lendResource(); } @@ -129,12 +129,12 @@ void LCAOrbitalSet::releaseResource(ResourceCollection& collection, const RefVec assert(this == &spo_list.getLeader()); auto& spo_leader = spo_list.getCastedLeader(); - spo_leader.myBasisSet->releaseResource(collection, extractBasRefList(spo_list)); + spo_leader.myBasisSet->releaseResource(collection, extractBasisRefList(spo_list)); collection.takebackResource(spo_leader.mw_mem_handle_); } -RefVectorWithLeader LCAOrbitalSet::extractBasRefList( +RefVectorWithLeader LCAOrbitalSet::extractBasisRefList( const RefVectorWithLeader& spo_list) const { RefVectorWithLeader basis_list(*spo_list.getCastedLeader().myBasisSet); diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h index 0b89c599493..cf6706df952 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h @@ -310,7 +310,7 @@ struct LCAOrbitalSet : public SPOSet OffloadMWVArray& phi_v) const; /// helper function for extracting a list of basis sets from a list of LCAOrbitalSet - RefVectorWithLeader extractBasRefList(const RefVectorWithLeader& spo_list) const; + RefVectorWithLeader extractBasisRefList(const RefVectorWithLeader& spo_list) const; struct LCAOMultiWalkerMem; ResourceHandle mw_mem_handle_; /// timer for basis set From b41ed286127fbcab33e7f4aa7ef709bfb1789dd3 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 11 Oct 2023 13:23:53 -0500 Subject: [PATCH 016/168] fixed array names --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index c5f110a5166..69a562b2133 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -34,8 +34,8 @@ struct SoaAtomicBasisSet using RealType = typename ROT::RealType; using GridType = typename ROT::GridType; using ValueType = typename QMCTraits::ValueType; - using OffloadArray4 = Array>; - using OffloadArray3 = Array>; + using OffloadArray4D = Array>; + using OffloadArray3D = Array>; using OffloadMatrix = Matrix>; using OffloadVector = Vector>; @@ -696,12 +696,12 @@ struct SoaAtomicBasisSet return std::make_unique(*this); } - OffloadArray4 ylm_vgl; // [5][Nelec][PBC][NYlm] - OffloadArray4 rnl_vgl; // [5][Nelec][PBC][NRnl] - OffloadArray3 ylm_v; // [Nelec][PBC][NYlm] - OffloadArray3 rnl_v; // [Nelec][PBC][NRnl] + OffloadArray4D ylm_vgl; // [5][Nelec][PBC][NYlm] + OffloadArray4D rnl_vgl; // [5][Nelec][PBC][NRnl] + OffloadArray3D ylm_v; // [Nelec][PBC][NYlm] + OffloadArray3D rnl_v; // [Nelec][PBC][NRnl] OffloadMatrix dr_pbc; // [PBC][xyz] translation vector for each image - OffloadArray3 dr; // [Nelec][PBC][xyz] ion->elec displacement for each image + OffloadArray3D dr; // [Nelec][PBC][xyz] ion->elec displacement for each image OffloadMatrix r; // [Nelec][PBC] ion->elec distance for each image OffloadVector correctphase; // [Nelec] overall phase }; From 59a3e51580969d2db0e58d8b24d715dc98199bdb Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Wed, 11 Oct 2023 13:40:27 -0500 Subject: [PATCH 017/168] Repace get_LocalEnergy with getLocalEnergy. get_LocalEnergy returns in wrong precision. --- src/QMCHamiltonians/QMCHamiltonian.cpp | 6 +++--- src/QMCHamiltonians/QMCHamiltonian.h | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/QMCHamiltonians/QMCHamiltonian.cpp b/src/QMCHamiltonians/QMCHamiltonian.cpp index 5f1b6b30f1f..f058dd9c9cf 100644 --- a/src/QMCHamiltonians/QMCHamiltonian.cpp +++ b/src/QMCHamiltonians/QMCHamiltonian.cpp @@ -640,7 +640,7 @@ std::vector QMCHamiltonian::mw_evaluate( std::vector local_energies(ham_list.size(), 0.0); for (int iw = 0; iw < ham_list.size(); ++iw) - local_energies[iw] = ham_list[iw].get_LocalEnergy(); + local_energies[iw] = ham_list[iw].getLocalEnergy(); return local_energies; } @@ -700,7 +700,7 @@ std::vector QMCHamiltonian::mw_evaluateValueAn updateKinetic(ham_list[iw], p_list[iw]); for (int iw = 0; iw < ham_list.size(); ++iw) - local_energies[iw] = ham_list[iw].get_LocalEnergy(); + local_energies[iw] = ham_list[iw].getLocalEnergy(); } return local_energies; @@ -855,7 +855,7 @@ std::vector QMCHamiltonian::mw_evaluateWithTop std::vector local_energies(ham_list.size()); for (int iw = 0; iw < ham_list.size(); ++iw) - local_energies[iw] = ham_list[iw].get_LocalEnergy(); + local_energies[iw] = ham_list[iw].getLocalEnergy(); return local_energies; } diff --git a/src/QMCHamiltonians/QMCHamiltonian.h b/src/QMCHamiltonians/QMCHamiltonian.h index ec577a1b701..11dda4ac9c7 100644 --- a/src/QMCHamiltonians/QMCHamiltonian.h +++ b/src/QMCHamiltonians/QMCHamiltonian.h @@ -406,8 +406,6 @@ class QMCHamiltonian bool get(std::ostream& os) const; - RealType get_LocalEnergy() const { return LocalEnergy; } - void setRandomGenerator(RandomBase* rng); /// accumulate local energy and update Observables and PropertyList From f0a94d668b795a40e9264aa2bcc2d9f78cd225fd Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Wed, 11 Oct 2023 22:01:07 +0000 Subject: [PATCH 018/168] Fix mkl_sycl detection for oneapi 2024. --- CMake/FindMKL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMake/FindMKL.cmake b/CMake/FindMKL.cmake index 76e5b666240..a52b0b683c5 100644 --- a/CMake/FindMKL.cmake +++ b/CMake/FindMKL.cmake @@ -85,7 +85,7 @@ endif(HAVE_MKL) if(HAVE_MKL AND ENABLE_SYCL) find_library(MKL_SYCL mkl_sycl HINTS ${MKL_ROOT} $ENV{MKLROOT} $ENV{MKL_ROOT} $ENV{MKL_HOME} - PATH_SUFFIXES lib/intel64 + PATH_SUFFIXES lib/intel64 lib REQUIRED ) From fee8ab571e6401c4707c7736f2cc63af80d6bc13 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Thu, 12 Oct 2023 14:47:31 -0500 Subject: [PATCH 019/168] Type renaming Psi/LogValueType to Psi/LogValue --- src/QMCDrivers/DMC/DMCBatched.cpp | 10 +- src/QMCDrivers/QMCDriverNew.cpp | 4 +- src/QMCDrivers/QMCUpdateBase.cpp | 4 +- src/QMCDrivers/VMC/VMCBatched.cpp | 2 +- src/QMCDrivers/WaveFunctionTester.cpp | 12 +- src/QMCDrivers/WaveFunctionTester.h | 2 +- src/QMCWaveFunctions/AGPDeterminant.cpp | 10 +- src/QMCWaveFunctions/AGPDeterminant.h | 12 +- src/QMCWaveFunctions/ConstantOrbital.h | 14 +- src/QMCWaveFunctions/ExampleHeComponent.cpp | 16 +- src/QMCWaveFunctions/ExampleHeComponent.h | 12 +- .../Fermion/DiracDeterminant.cpp | 51 ++- .../Fermion/DiracDeterminant.h | 30 +- .../Fermion/DiracDeterminantBase.h | 2 +- .../Fermion/DiracDeterminantBatched.cpp | 4 +- .../Fermion/DiracDeterminantWithBackflow.cpp | 28 +- .../Fermion/DiracDeterminantWithBackflow.h | 14 +- .../Fermion/MultiDiracDeterminant.cpp | 6 +- .../Fermion/MultiDiracDeterminant.h | 24 +- .../Fermion/MultiSlaterDetTableMethod.cpp | 132 ++++---- .../Fermion/MultiSlaterDetTableMethod.h | 52 +-- src/QMCWaveFunctions/Fermion/SlaterDet.cpp | 28 +- src/QMCWaveFunctions/Fermion/SlaterDet.h | 28 +- .../Fermion/SlaterDetWithBackflow.cpp | 16 +- .../Fermion/SlaterDetWithBackflow.h | 16 +- .../Jastrow/CountingJastrow.h | 20 +- src/QMCWaveFunctions/Jastrow/J1OrbitalSoA.h | 24 +- src/QMCWaveFunctions/Jastrow/J1Spin.h | 24 +- src/QMCWaveFunctions/Jastrow/JeeIOrbitalSoA.h | 32 +- src/QMCWaveFunctions/Jastrow/RPAJastrow.cpp | 16 +- src/QMCWaveFunctions/Jastrow/RPAJastrow.h | 12 +- .../Jastrow/TwoBodyJastrow.cpp | 36 +-- src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.h | 24 +- .../Jastrow/kSpaceJastrow.cpp | 16 +- src/QMCWaveFunctions/Jastrow/kSpaceJastrow.h | 12 +- .../LatticeGaussianProduct.cpp | 26 +- src/QMCWaveFunctions/LatticeGaussianProduct.h | 12 +- src/QMCWaveFunctions/LinearOrbital.h | 12 +- src/QMCWaveFunctions/TWFFastDerivWrapper.cpp | 10 +- src/QMCWaveFunctions/TWFdispatcher.cpp | 8 +- src/QMCWaveFunctions/TWFdispatcher.h | 14 +- src/QMCWaveFunctions/TrialWaveFunction.cpp | 52 +-- src/QMCWaveFunctions/TrialWaveFunction.h | 10 +- .../WaveFunctionComponent.cpp | 24 +- src/QMCWaveFunctions/WaveFunctionComponent.h | 40 +-- .../tests/test_2d_jastrow.cpp | 37 +-- .../tests/test_DiracDeterminant.cpp | 62 ++-- .../tests/test_DiracDeterminantBatched.cpp | 70 ++--- .../tests/test_DiracMatrix.cpp | 20 +- src/QMCWaveFunctions/tests/test_J1Spin.cpp | 64 ++-- .../tests/test_J1_bspline.cpp | 8 +- .../tests/test_J2_bspline.cpp | 8 +- .../tests/test_TrialWaveFunction.cpp | 12 +- .../test_TrialWaveFunction_diamondC_2x1x1.cpp | 24 +- .../tests/test_counting_jastrow.cpp | 297 +++++++++--------- .../tests/test_example_he.cpp | 24 +- .../tests/test_lattice_gaussian.cpp | 6 +- .../tests/test_multi_slater_determinant.cpp | 32 +- .../tests/test_polynomial_eeI_jastrow.cpp | 14 +- 59 files changed, 812 insertions(+), 819 deletions(-) diff --git a/src/QMCDrivers/DMC/DMCBatched.cpp b/src/QMCDrivers/DMC/DMCBatched.cpp index f71ef2eb71d..b9ca7d0b483 100644 --- a/src/QMCDrivers/DMC/DMCBatched.cpp +++ b/src/QMCDrivers/DMC/DMCBatched.cpp @@ -32,8 +32,8 @@ namespace qmcplusplus { using std::placeholders::_1; -using WP = WalkerProperties::Indexes; -using PsiValueType = TrialWaveFunction::PsiValueType; +using WP = WalkerProperties::Indexes; +using PsiValue = TrialWaveFunction::PsiValue; /** Constructor maintains proper ownership of input parameters * @@ -119,7 +119,7 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, //This generates an entire steps worth of deltas. makeGaussRandomWithEngine(walker_deltas, step_context.get_random_gen()); - std::vector ratios(num_walkers, PsiValueType(0.0)); + std::vector ratios(num_walkers, PsiValue(0.0)); std::vector log_gf(num_walkers, 0.0); std::vector log_gb(num_walkers, 0.0); std::vector prob(num_walkers, 0.0); @@ -195,8 +195,8 @@ void DMCBatched::advanceWalkers(const StateForThread& sft, computeLogGreensFunction(drifts_reverse, taus, log_gb); - auto checkPhaseChanged = [&sft](const PsiValueType& ratio, int& is_reject) { - if (ratio == PsiValueType(0) || sft.branch_engine.phaseChanged(std::arg(ratio))) + auto checkPhaseChanged = [&sft](const PsiValue& ratio, int& is_reject) { + if (ratio == PsiValue(0) || sft.branch_engine.phaseChanged(std::arg(ratio))) is_reject = 1; else is_reject = 0; diff --git a/src/QMCDrivers/QMCDriverNew.cpp b/src/QMCDrivers/QMCDriverNew.cpp index 7ab97a1f1d3..181b6d79cfb 100644 --- a/src/QMCDrivers/QMCDriverNew.cpp +++ b/src/QMCDrivers/QMCDriverNew.cpp @@ -517,7 +517,7 @@ void QMCDriverNew::checkLogAndGL(Crowd& crowd, const std::string_view location) const RefVectorWithLeader walker_elecs(crowd.get_walker_elecs()[0], crowd.get_walker_elecs()); const RefVectorWithLeader walker_twfs(crowd.get_walker_twfs()[0], crowd.get_walker_twfs()); - std::vector log_values(walker_twfs.size()); + std::vector log_values(walker_twfs.size()); std::vector Gs; std::vector Ls; Gs.reserve(log_values.size()); @@ -545,7 +545,7 @@ void QMCDriverNew::checkLogAndGL(Crowd& crowd, const std::string_view location) { auto& ref_G = walker_twfs[iw].G; auto& ref_L = walker_twfs[iw].L; - TrialWaveFunction::LogValueType ref_log{walker_twfs[iw].getLogPsi(), walker_twfs[iw].getPhase()}; + TrialWaveFunction::LogValue ref_log{walker_twfs[iw].getLogPsi(), walker_twfs[iw].getPhase()}; if (std::abs(std::exp(log_values[iw]) - std::exp(ref_log)) > std::abs(std::exp(ref_log)) * threshold) { success = false; diff --git a/src/QMCDrivers/QMCUpdateBase.cpp b/src/QMCDrivers/QMCUpdateBase.cpp index fbc7cc14962..215bd37695a 100644 --- a/src/QMCDrivers/QMCUpdateBase.cpp +++ b/src/QMCDrivers/QMCUpdateBase.cpp @@ -277,7 +277,7 @@ QMCUpdateBase::RealType QMCUpdateBase::getNodeCorrection(const ParticleSet::Part void QMCUpdateBase::checkLogAndGL(ParticleSet& pset, TrialWaveFunction& twf, const std::string_view location) { bool success = true; - TrialWaveFunction::LogValueType log_value{twf.getLogPsi(), twf.getPhase()}; + TrialWaveFunction::LogValue log_value{twf.getLogPsi(), twf.getPhase()}; ParticleSet::ParticleGradient G_saved = twf.G; ParticleSet::ParticleLaplacian L_saved = twf.L; @@ -294,7 +294,7 @@ void QMCUpdateBase::checkLogAndGL(ParticleSet& pset, TrialWaveFunction& twf, con std::ostringstream msg; auto& ref_G = twf.G; auto& ref_L = twf.L; - TrialWaveFunction::LogValueType ref_log{twf.getLogPsi(), twf.getPhase()}; + TrialWaveFunction::LogValue ref_log{twf.getLogPsi(), twf.getPhase()}; if (std::abs(std::exp(log_value) - std::exp(ref_log)) > std::abs(std::exp(ref_log)) * threshold) { success = false; diff --git a/src/QMCDrivers/VMC/VMCBatched.cpp b/src/QMCDrivers/VMC/VMCBatched.cpp index 6d666668566..79edb231f68 100644 --- a/src/QMCDrivers/VMC/VMCBatched.cpp +++ b/src/QMCDrivers/VMC/VMCBatched.cpp @@ -83,7 +83,7 @@ void VMCBatched::advanceWalkers(const StateForThread& sft, constexpr RealType mhalf(-0.5); const bool use_drift = sft.vmcdrv_input.get_use_drift(); - std::vector ratios(num_walkers); + std::vector ratios(num_walkers); std::vector log_gf(num_walkers); std::vector log_gb(num_walkers); std::vector prob(num_walkers); diff --git a/src/QMCDrivers/WaveFunctionTester.cpp b/src/QMCDrivers/WaveFunctionTester.cpp index e1d2c490035..d87355427d9 100644 --- a/src/QMCDrivers/WaveFunctionTester.cpp +++ b/src/QMCDrivers/WaveFunctionTester.cpp @@ -313,7 +313,9 @@ class FiniteDifference : public QMCTraits FiniteDiff_LowOrder, // use simplest low-order formulas FiniteDiff_Richardson // use Richardson extrapolation }; - FiniteDifference(size_t ndim_in, FiniteDiffType fd_type = FiniteDiff_Richardson) : ndim(ndim_in), m_RichardsonSize(10), m_fd_type(fd_type) {} + FiniteDifference(size_t ndim_in, FiniteDiffType fd_type = FiniteDiff_Richardson) + : ndim(ndim_in), m_RichardsonSize(10), m_fd_type(fd_type) + {} const size_t ndim; int m_RichardsonSize; @@ -790,7 +792,7 @@ bool WaveFunctionTester::checkGradientAtConfiguration(MCWalkerConfiguration::Wal ParticleSet::ParticleLaplacian L(nat), tmpL(nat), L1(nat); - LogValueType logpsi1 = orb->evaluateLog(W, G, L); + LogValue logpsi1 = orb->evaluateLog(W, G, L); fail_log << "WaveFunctionComponent " << iorb << " " << orb->getClassName() << " log psi = " << logpsi1 << std::endl; @@ -804,7 +806,7 @@ bool WaveFunctionTester::checkGradientAtConfiguration(MCWalkerConfiguration::Wal ParticleSet::SingleParticlePos zeroR; W.makeMove(it->index, zeroR); - LogValueType logpsi0 = orb->evaluateLog(W, tmpG, tmpL); + LogValue logpsi0 = orb->evaluateLog(W, tmpG, tmpL); #if defined(QMC_COMPLEX) ValueType logpsi(logpsi0.real(), logpsi0.imag()); #else @@ -836,7 +838,7 @@ bool WaveFunctionTester::checkGradientAtConfiguration(MCWalkerConfiguration::Wal ParticleSet::ParticleGradient G(nat), tmpG(nat), G1(nat); ParticleSet::ParticleLaplacian L(nat), tmpL(nat), L1(nat); DiracDeterminantBase& det = *sd->Dets[isd]; - LogValueType logpsi2 = det.evaluateLog(W, G, L); // this won't work with backflow + LogValue logpsi2 = det.evaluateLog(W, G, L); // this won't work with backflow fail_log << " Slater Determiant " << isd << " (for particles " << det.getFirstIndex() << " to " << det.getLastIndex() << ") log psi = " << logpsi2 << std::endl; // Should really check the condition number on the matrix determinant. @@ -853,7 +855,7 @@ bool WaveFunctionTester::checkGradientAtConfiguration(MCWalkerConfiguration::Wal W.R[it->index] = it->r; W.update(); - LogValueType logpsi0 = det.evaluateLog(W, tmpG, tmpL); + LogValue logpsi0 = det.evaluateLog(W, tmpG, tmpL); #if defined(QMC_COMPLEX) ValueType logpsi(logpsi0.real(), logpsi0.imag()); #else diff --git a/src/QMCDrivers/WaveFunctionTester.h b/src/QMCDrivers/WaveFunctionTester.h index 46fe7fcdc75..096f6bbc652 100644 --- a/src/QMCDrivers/WaveFunctionTester.h +++ b/src/QMCDrivers/WaveFunctionTester.h @@ -43,7 +43,7 @@ class WaveFunctionTester : public QMCDriver { public: /// type definition - using LogValueType = WaveFunctionComponent::LogValueType; + using LogValue = WaveFunctionComponent::LogValue; /// Constructor. WaveFunctionTester(const ProjectData& project_data, diff --git a/src/QMCWaveFunctions/AGPDeterminant.cpp b/src/QMCWaveFunctions/AGPDeterminant.cpp index 4b6e6396b03..7230a4c4c2c 100644 --- a/src/QMCWaveFunctions/AGPDeterminant.cpp +++ b/src/QMCWaveFunctions/AGPDeterminant.cpp @@ -80,9 +80,9 @@ void AGPDeterminant::resize(int nup, int ndown) *contribution of the determinant to G(radient) and L(aplacian) *for local energy calculations. */ -AGPDeterminant::LogValueType AGPDeterminant::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +AGPDeterminant::LogValue AGPDeterminant::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { evaluateLogAndStore(P); G += myG; @@ -172,7 +172,7 @@ void AGPDeterminant::registerData(ParticleSet& P, WFBufferType& buf) //buf.add(myL.begin(), myL.end()); } -AGPDeterminant::LogValueType AGPDeterminant::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) +AGPDeterminant::LogValue AGPDeterminant::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { evaluateLogAndStore(P); P.G += myG; @@ -224,7 +224,7 @@ void AGPDeterminant::copyFromBuffer(ParticleSet& P, WFBufferType& buf) * @param P current configuration * @param iat the particle thas is being moved */ -AGPDeterminant::PsiValueType AGPDeterminant::ratio(ParticleSet& P, int iat) +AGPDeterminant::PsiValue AGPDeterminant::ratio(ParticleSet& P, int iat) { UpdateMode = ORB_PBYP_RATIO; //GeminalBasis->evaluate(P,iat); diff --git a/src/QMCWaveFunctions/AGPDeterminant.h b/src/QMCWaveFunctions/AGPDeterminant.h index d80d84c9c81..5aa23f9bcb6 100644 --- a/src/QMCWaveFunctions/AGPDeterminant.h +++ b/src/QMCWaveFunctions/AGPDeterminant.h @@ -53,7 +53,7 @@ class AGPDeterminant : public WaveFunctionComponent void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; @@ -61,7 +61,7 @@ class AGPDeterminant : public WaveFunctionComponent * @param P current configuration * @param iat the particle thas is being moved */ - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; void ratioUp(ParticleSet& P, int iat); @@ -87,9 +87,9 @@ class AGPDeterminant : public WaveFunctionComponent *contribution of the determinant to G(radient) and L(aplacian) *for local energy calculations. */ - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; std::unique_ptr makeClone(ParticleSet& tqp) const override; @@ -157,7 +157,7 @@ class AGPDeterminant : public WaveFunctionComponent IndexVector Pivot; ///current ratio - PsiValueType curRatio; + PsiValue curRatio; ///cummulate ratio for particle-by-particle update RealType cumRatio; ///address of dpsiU[0][0] diff --git a/src/QMCWaveFunctions/ConstantOrbital.h b/src/QMCWaveFunctions/ConstantOrbital.h index e7520418e14..2f88990aef7 100644 --- a/src/QMCWaveFunctions/ConstantOrbital.h +++ b/src/QMCWaveFunctions/ConstantOrbital.h @@ -20,15 +20,15 @@ namespace qmcplusplus class ConstantOrbital : public WaveFunctionComponent { public: - PsiValueType FakeGradRatio; + PsiValue FakeGradRatio; ConstantOrbital() : FakeGradRatio(1.0) {} std::string getClassName() const override { return "ConstantOrbital"; } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { G = 0.0; L = 0.0; @@ -39,15 +39,15 @@ class ConstantOrbital : public WaveFunctionComponent void restore(int iat) override {} - PsiValueType ratio(ParticleSet& P, int iat) override { return 1.0; } + PsiValue ratio(ParticleSet& P, int iat) override { return 1.0; } GradType evalGrad(ParticleSet& P, int iat) override { return GradType(0.0); } - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { return FakeGradRatio; } + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { return FakeGradRatio; } void registerData(ParticleSet& P, WFBufferType& buf) override {} - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { return 0.0; } + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { return 0.0; } void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override {} diff --git a/src/QMCWaveFunctions/ExampleHeComponent.cpp b/src/QMCWaveFunctions/ExampleHeComponent.cpp index f9844199ee1..b3e6b0fa1ca 100644 --- a/src/QMCWaveFunctions/ExampleHeComponent.cpp +++ b/src/QMCWaveFunctions/ExampleHeComponent.cpp @@ -62,9 +62,9 @@ bool ExampleHeComponent::put(xmlNodePtr cur) return true; } -ExampleHeComponent::LogValueType ExampleHeComponent::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +ExampleHeComponent::LogValue ExampleHeComponent::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { const auto& ee_table = P.getDistTableAA(my_table_ee_idx_); const auto& ee_dists = ee_table.getDistances(); @@ -111,7 +111,7 @@ ExampleHeComponent::LogValueType ExampleHeComponent::evaluateLog(const ParticleS return log_value_; } -ExampleHeComponent::PsiValueType ExampleHeComponent::ratio(ParticleSet& P, int iat) +ExampleHeComponent::PsiValue ExampleHeComponent::ratio(ParticleSet& P, int iat) { const auto& ee_table = P.getDistTableAA(my_table_ee_idx_); const auto& ee_dists = ee_table.getDistances(); @@ -134,7 +134,7 @@ ExampleHeComponent::PsiValueType ExampleHeComponent::ratio(ParticleSet& P, int i double log_v_old = -Z * (r_old)-u_old; double log_v_new = -Z * (r_new)-u_new; - return std::exp(static_cast(log_v_new - log_v_old)); + return std::exp(static_cast(log_v_new - log_v_old)); } ExampleHeComponent::GradType ExampleHeComponent::evalGrad(ParticleSet& P, int iat) @@ -159,7 +159,7 @@ ExampleHeComponent::GradType ExampleHeComponent::evalGrad(ParticleSet& P, int ia return Z * rhat + rhat12 * du; } -ExampleHeComponent::PsiValueType ExampleHeComponent::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +ExampleHeComponent::PsiValue ExampleHeComponent::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { const auto& ee_table = P.getDistTableAA(my_table_ee_idx_); const auto& ee_dists = ee_table.getDistances(); @@ -195,11 +195,11 @@ ExampleHeComponent::PsiValueType ExampleHeComponent::ratioGrad(ParticleSet& P, i double log_v_old = -Z * (r_old)-u_old; double log_v_new = -Z * (r_new)-u_new; - return std::exp(static_cast(log_v_new - log_v_old)); + return std::exp(static_cast(log_v_new - log_v_old)); } -ExampleHeComponent::LogValueType ExampleHeComponent::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) +ExampleHeComponent::LogValue ExampleHeComponent::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { return evaluateLog(P, P.G, P.L); } diff --git a/src/QMCWaveFunctions/ExampleHeComponent.h b/src/QMCWaveFunctions/ExampleHeComponent.h index 8e5cce23ff9..e8478dc44a0 100644 --- a/src/QMCWaveFunctions/ExampleHeComponent.h +++ b/src/QMCWaveFunctions/ExampleHeComponent.h @@ -42,19 +42,19 @@ class ExampleHeComponent : public WaveFunctionComponent, OptimizableObject void checkOutVariables(const OptVariablesType& active) override { my_vars_.getIndex(active); } void resetParametersExclusive(const OptVariablesType& active) override; - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; void acceptMove(ParticleSet& P, int iat, bool safe_to_delay = false) override {} void restore(int iat) override {} - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; GradType evalGrad(ParticleSet& P, int iat) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; void evaluateDerivatives(ParticleSet& P, const OptVariablesType& optvars, @@ -64,7 +64,7 @@ class ExampleHeComponent : public WaveFunctionComponent, OptimizableObject void registerData(ParticleSet& P, WFBufferType& buf) override {} - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override {} diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp b/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp index b49f294a3c4..1f0b097763a 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminant.cpp @@ -171,9 +171,9 @@ typename DiracDeterminant::GradType DiracDeterminant::evalGrad } template -typename DiracDeterminant::PsiValueType DiracDeterminant::ratioGrad(ParticleSet& P, - int iat, - GradType& grad_iat) +typename DiracDeterminant::PsiValue DiracDeterminant::ratioGrad(ParticleSet& P, + int iat, + GradType& grad_iat) { { ScopedTimer local_timer(SPOVGLTimer); @@ -183,8 +183,7 @@ typename DiracDeterminant::PsiValueType DiracDeterminant::rati } template -typename DiracDeterminant::PsiValueType DiracDeterminant::ratioGrad_compute(int iat, - GradType& grad_iat) +typename DiracDeterminant::PsiValue DiracDeterminant::ratioGrad_compute(int iat, GradType& grad_iat) { ScopedTimer local_timer(RatioTimer); @@ -201,16 +200,16 @@ typename DiracDeterminant::PsiValueType DiracDeterminant::rati updateEng.getInvRow(psiM, WorkingIndex, invRow); } curRatio = simd::dot(invRow.data(), psiV.data(), invRow.size()); - grad_iat += static_cast(static_cast(1.0) / curRatio) * + grad_iat += static_cast(static_cast(1.0) / curRatio) * simd::dot(invRow.data(), dpsiV.data(), invRow.size()); return curRatio; } template -typename DiracDeterminant::PsiValueType DiracDeterminant::ratioGradWithSpin(ParticleSet& P, - int iat, - GradType& grad_iat, - ComplexType& spingrad_iat) +typename DiracDeterminant::PsiValue DiracDeterminant::ratioGradWithSpin(ParticleSet& P, + int iat, + GradType& grad_iat, + ComplexType& spingrad_iat) { { ScopedTimer local_timer(SPOVGLTimer); @@ -231,10 +230,10 @@ typename DiracDeterminant::PsiValueType DiracDeterminant::rati updateEng.getInvRow(psiM, WorkingIndex, invRow); } curRatio = simd::dot(invRow.data(), psiV.data(), invRow.size()); - grad_iat += static_cast(static_cast(1.0) / curRatio) * + grad_iat += static_cast(static_cast(1.0) / curRatio) * simd::dot(invRow.data(), dpsiV.data(), invRow.size()); - spingrad_iat += static_cast(static_cast(1.0) / curRatio) * + spingrad_iat += static_cast(static_cast(1.0) / curRatio) * simd::dot(invRow.data(), dspin_psiV.data(), invRow.size()); } @@ -245,7 +244,7 @@ template void DiracDeterminant::mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const { { @@ -281,7 +280,7 @@ void DiracDeterminant::mw_ratioGrad(const RefVectorWithLeader void DiracDeterminant::acceptMove(ParticleSet& P, int iat, bool safe_to_delay) { - if (curRatio == PsiValueType(0)) + if (curRatio == PsiValue(0)) throw std::runtime_error("DiracDeterminant::acceptMove curRatio is zero! Report a bug.\n"); ScopedTimer local_timer(UpdateTimer); const int WorkingIndex = iat - FirstIndex; @@ -368,11 +367,10 @@ void DiracDeterminant::registerData(ParticleSet& P, WFBufferType& buf) } template -typename DiracDeterminant::LogValueType DiracDeterminant::evaluateGL( - const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch) +typename DiracDeterminant::LogValue DiracDeterminant::evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch) { if (fromscratch) evaluateLog(P, G, L); @@ -382,9 +380,9 @@ typename DiracDeterminant::LogValueType DiracDeterminant::eval } template -typename DiracDeterminant::LogValueType DiracDeterminant::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch) +typename DiracDeterminant::LogValue DiracDeterminant::updateBuffer(ParticleSet& P, + WFBufferType& buf, + bool fromscratch) { evaluateGL(P, P.G, P.L, fromscratch); { @@ -419,7 +417,7 @@ void DiracDeterminant::registerTWFFastDerivWrapper(const ParticleSet& P * @param iat the particle thas is being moved */ template -typename DiracDeterminant::PsiValueType DiracDeterminant::ratio(ParticleSet& P, int iat) +typename DiracDeterminant::PsiValue DiracDeterminant::ratio(ParticleSet& P, int iat) { UpdateMode = ORB_PBYP_RATIO; const int WorkingIndex = iat - FirstIndex; @@ -693,10 +691,9 @@ typename DiracDeterminant::GradType DiracDeterminant::evalGrad *for local energy calculations. */ template -typename DiracDeterminant::LogValueType DiracDeterminant::evaluateLog( - const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +typename DiracDeterminant::LogValue DiracDeterminant::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { recompute(P); diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminant.h b/src/QMCWaveFunctions/Fermion/DiracDeterminant.h index 5f8c67e2494..e94497d3fcd 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminant.h +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminant.h @@ -82,7 +82,7 @@ class DiracDeterminant : public DiracDeterminantBase void updateAfterSweep(const ParticleSet& P, ParticleSet::ParticleGradient& G, ParticleSet::ParticleLaplacian& L); - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; @@ -94,13 +94,13 @@ class DiracDeterminant : public DiracDeterminantBase * @param P current configuration * @param iat the particle thas is being moved */ - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; //Ye: TODO, good performance needs batched SPO evaluation. //void mw_calcRatio(const std::vector& wfc_list, // const std::vector& p_list, // int iat, - // std::vector& ratios) override; + // std::vector& ratios) override; /** compute multiple ratios for a particle move */ @@ -115,14 +115,14 @@ class DiracDeterminant : public DiracDeterminantBase std::vector& ratios, Matrix& dratios) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; - PsiValueType ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad) final; + PsiValue ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad) final; void mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const override; GradType evalGrad(ParticleSet& P, int iat) override; @@ -167,9 +167,9 @@ class DiracDeterminant : public DiracDeterminantBase void restore(int iat) override; ///evaluate log of a determinant for a particle set - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; //Ye: TODO, good performance needs batched SPO evaluation. //void mw_evaluateLog(const std::vector& wfc_list, @@ -179,10 +179,10 @@ class DiracDeterminant : public DiracDeterminantBase void recompute(const ParticleSet& P) override; - LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch) override; + LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch) override; void evaluateHessian(ParticleSet& P, HessVector& grad_grad_psi) override; @@ -252,7 +252,7 @@ class DiracDeterminant : public DiracDeterminantBase */ int invRow_id; - PsiValueType curRatio; + PsiValue curRatio; ValueType* FirstAddressOfdV; ValueType* LastAddressOfdV; @@ -270,7 +270,7 @@ class DiracDeterminant : public DiracDeterminantBase void resizeScratchObjectsForIonDerivs(); /// internal function computing ratio and gradients after computing the SPOs, used by ratioGrad. - PsiValueType ratioGrad_compute(int iat, GradType& grad_iat); + PsiValue ratioGrad_compute(int iat, GradType& grad_iat); }; extern template class DiracDeterminant<>; diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantBase.h b/src/QMCWaveFunctions/Fermion/DiracDeterminantBase.h index 5c9174c59ce..350bac5a68f 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantBase.h +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantBase.h @@ -147,7 +147,7 @@ class DiracDeterminantBase : public WaveFunctionComponent return std::unique_ptr(); } - PsiValueType ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad) override + PsiValue ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad) override { APP_ABORT(" DiracDeterminantBase::ratioGradWithSpin(): Implementation required\n"); return 0.0; diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp index 9132a77ed10..61c62335ff7 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantBatched.cpp @@ -444,7 +444,7 @@ typename DiracDeterminantBatched::PsiValue DiracDeterminantBatched void DiracDeterminantBatched::acceptMove(ParticleSet& P, int iat, bool safe_to_delay) { - if (curRatio == PsiValueType(0)) + if (curRatio == PsiValue(0)) throw std::runtime_error("DiracDeterminant::acceptMove curRatio is zero! Report a bug.\n"); const int WorkingIndex = iat - FirstIndex; log_value_ += convertValueToLog(curRatio); @@ -498,7 +498,7 @@ void DiracDeterminantBatched::mw_accept_rejectMove( { psiM_g_dev_ptr_list[count] = det.psiM_vgl.device_data() + psiM_vgl.capacity() + NumOrbitals * WorkingIndex * DIM; psiM_l_dev_ptr_list[count] = det.psiM_vgl.device_data() + psiM_vgl.capacity() * 4 + NumOrbitals * WorkingIndex; - if (det.curRatio == PsiValueType(0)) + if (det.curRatio == PsiValue(0)) throw std::runtime_error("DiracDeterminant::mw_accept_rejectMove det.curRatio is zero! Report a bug.\n"); det.log_value_ += convertValueToLog(det.curRatio); count++; diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.cpp b/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.cpp index bbd089fde3b..e4c6a116119 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.cpp +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.cpp @@ -138,9 +138,9 @@ void DiracDeterminantWithBackflow::registerData(ParticleSet& P, WFBufferType& bu buf.add(log_value_); } -DiracDeterminantWithBackflow::LogValueType DiracDeterminantWithBackflow::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch) +DiracDeterminantWithBackflow::LogValue DiracDeterminantWithBackflow::updateBuffer(ParticleSet& P, + WFBufferType& buf, + bool fromscratch) { // for now, always recalculate from scratch // enable from_scratch = true later @@ -187,7 +187,7 @@ void DiracDeterminantWithBackflow::copyFromBuffer(ParticleSet& P, WFBufferType& * @param P current configuration * @param iat the particle thas is being moved */ -DiracDeterminantWithBackflow::PsiValueType DiracDeterminantWithBackflow::ratio(ParticleSet& P, int iat) +DiracDeterminantWithBackflow::PsiValue DiracDeterminantWithBackflow::ratio(ParticleSet& P, int iat) { // FIX FIX FIX : code Woodbury formula psiM_temp = psiM; @@ -215,10 +215,10 @@ DiracDeterminantWithBackflow::PsiValueType DiracDeterminantWithBackflow::ratio(P psiMinv_temp = psiM_temp; // FIX FIX FIX : code Woodbury formula InverseTimer.start(); - LogValueType NewLog; + LogValue NewLog; InvertWithLog(psiMinv_temp.data(), NumPtcls, NumOrbitals, WorkSpace.data(), Pivot.data(), NewLog); InverseTimer.stop(); - return curRatio = LogToValue::convert(NewLog - log_value_); + return curRatio = LogToValue::convert(NewLog - log_value_); } void DiracDeterminantWithBackflow::evaluateRatiosAlltoOne(ParticleSet& P, std::vector& ratios) @@ -256,9 +256,9 @@ DiracDeterminantWithBackflow::GradType DiracDeterminantWithBackflow::evalGradSou return GradType(); } -DiracDeterminantWithBackflow::PsiValueType DiracDeterminantWithBackflow::ratioGrad(ParticleSet& P, - int iat, - GradType& grad_iat) +DiracDeterminantWithBackflow::PsiValue DiracDeterminantWithBackflow::ratioGrad(ParticleSet& P, + int iat, + GradType& grad_iat) { // FIX FIX FIX : code Woodbury formula psiM_temp = psiM; @@ -289,7 +289,7 @@ DiracDeterminantWithBackflow::PsiValueType DiracDeterminantWithBackflow::ratioGr psiMinv_temp = psiM_temp; // FIX FIX FIX : code Woodbury formula InverseTimer.start(); - LogValueType NewLog; + LogValue NewLog; InvertWithLog(psiMinv_temp.data(), NumPtcls, NumOrbitals, WorkSpace.data(), Pivot.data(), NewLog); InverseTimer.stop(); // update Fmatdiag_temp @@ -298,7 +298,7 @@ DiracDeterminantWithBackflow::PsiValueType DiracDeterminantWithBackflow::ratioGr Fmatdiag_temp[j] = simd::dot(psiMinv_temp[j], dpsiM_temp[j], NumOrbitals); grad_iat += dot(BFTrans_.Amat_temp(iat, FirstIndex + j), Fmatdiag_temp[j]); } - return curRatio = LogToValue::convert(NewLog - log_value_); + return curRatio = LogToValue::convert(NewLog - log_value_); } void DiracDeterminantWithBackflow::testL(ParticleSet& P) @@ -477,9 +477,9 @@ void DiracDeterminantWithBackflow::testL(ParticleSet& P) *contribution of the determinant to G(radient) and L(aplacian) *for local energy calculations. */ -DiracDeterminantWithBackflow::LogValueType DiracDeterminantWithBackflow::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +DiracDeterminantWithBackflow::LogValue DiracDeterminantWithBackflow::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { //testGG(P); //testL(P); diff --git a/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.h b/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.h index d7d65608991..5a2e743fb80 100644 --- a/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.h +++ b/src/QMCWaveFunctions/Fermion/DiracDeterminantWithBackflow.h @@ -80,7 +80,7 @@ class DiracDeterminantWithBackflow : public DiracDeterminantBase void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; @@ -88,11 +88,11 @@ class DiracDeterminantWithBackflow : public DiracDeterminantBase * @param P current configuration * @param iat the particle thas is being moved */ - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; void evaluateRatiosAlltoOne(ParticleSet& P, std::vector& ratios) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; GradType evalGrad(ParticleSet& P, int iat) override; GradType evalGradSource(ParticleSet& P, ParticleSet& source, int iat) override; @@ -110,9 +110,9 @@ class DiracDeterminantWithBackflow : public DiracDeterminantBase */ void restore(int iat) override; - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; /** cloning function * @param tqp target particleset @@ -189,7 +189,7 @@ class DiracDeterminantWithBackflow : public DiracDeterminantBase GradVector dpsiV; ValueVector d2psiV; - PsiValueType curRatio; + PsiValue curRatio; ParticleSet::SingleParticleValue* FirstAddressOfG; ParticleSet::SingleParticleValue* LastAddressOfG; ValueType* FirstAddressOfdV; diff --git a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp index 47c98a92716..48a43a7d93c 100644 --- a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp @@ -358,9 +358,7 @@ void MultiDiracDeterminant::evaluateForWalkerMoveWithSpin(const ParticleSet& P, } -MultiDiracDeterminant::LogValueType MultiDiracDeterminant::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch) +MultiDiracDeterminant::LogValue MultiDiracDeterminant::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { assert(P.isSpinor() == is_spinor_); if (is_spinor_) @@ -911,7 +909,7 @@ void MultiDiracDeterminant::evaluateDerivativesWF(ParticleSet& P, const opt_variables_type& optvars, Vector& dlogpsi, const MultiDiracDeterminant& pseudo_dn, - const PsiValueType& psiCurrent, + const PsiValue& psiCurrent, const std::vector& Coeff, const std::vector& C2node_up, const std::vector& C2node_dn) diff --git a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.h b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.h index f90b5f88f9d..4bc2c99f1c9 100644 --- a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.h +++ b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.h @@ -185,7 +185,7 @@ class MultiDiracDeterminant : public WaveFunctionComponent const opt_variables_type& optvars, Vector& dlogpsi, const MultiDiracDeterminant& pseudo_dn, - const PsiValueType& psiCurrent, + const PsiValue& psiCurrent, const std::vector& Coeff, const std::vector& C2node_up, const std::vector& C2node_dn); @@ -193,7 +193,7 @@ class MultiDiracDeterminant : public WaveFunctionComponent void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; @@ -222,10 +222,10 @@ class MultiDiracDeterminant : public WaveFunctionComponent * These functions should not be called. ***************************************************************************/ - PsiValueType ratio(ParticleSet& P, int iat) override + PsiValue ratio(ParticleSet& P, int iat) override { APP_ABORT(" MultiDiracDeterminant: This should not be called. \n"); - return PsiValueType(); + return PsiValue(); } GradType evalGrad(ParticleSet& P, int iat) override @@ -234,15 +234,15 @@ class MultiDiracDeterminant : public WaveFunctionComponent return GradType(); } - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { APP_ABORT(" MultiDiracDeterminant: This should not be called. \n"); - return PsiValueType(); + return PsiValue(); } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { APP_ABORT(" MultiDiracDeterminant: This should not be called. \n"); return 0.0; @@ -324,8 +324,8 @@ class MultiDiracDeterminant : public WaveFunctionComponent const Matrix& getSpinGrads() const { return spingrads; } const Matrix& getNewSpinGrads() const { return new_spingrads; } - PsiValueType getRefDetRatio() const { return static_cast(curRatio); } - LogValueType getLogValueRefDet() const { return log_value_ref_det_; } + PsiValue getRefDetRatio() const { return static_cast(curRatio); } + LogValue getLogValueRefDet() const { return log_value_ref_det_; } private: void mw_InverseUpdateByColumn(MultiDiracDetMultiWalkerResource& mw_res, @@ -573,7 +573,7 @@ class MultiDiracDeterminant : public WaveFunctionComponent /// new value of the reference determinant over the old value upon a proposed move ValueType curRatio; /// log value of the reference determinant - LogValueType log_value_ref_det_; + LogValue log_value_ref_det_; /// store determinant grads (old and new) Matrix grads, new_grads; /// store determinant lapls (old and new) diff --git a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp index 118da1b461b..809f7ce1ea0 100644 --- a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp @@ -108,9 +108,9 @@ std::unique_ptr MultiSlaterDetTableMethod::makeClone(Part * - evaluateLog(P,G,L,buf,fillbuffer) * Miguel's note: can this change over time??? I don't know yet */ -WaveFunctionComponent::LogValueType MultiSlaterDetTableMethod::evaluate_vgl_impl(const ParticleSet& P, - ParticleSet::ParticleGradient& g_tmp, - ParticleSet::ParticleLaplacian& l_tmp) +WaveFunctionComponent::LogValue MultiSlaterDetTableMethod::evaluate_vgl_impl(const ParticleSet& P, + ParticleSet::ParticleGradient& g_tmp, + ParticleSet::ParticleLaplacian& l_tmp) { const ValueType czero(0); psi_ratio_to_ref_det_ = czero; @@ -131,18 +131,18 @@ WaveFunctionComponent::LogValueType MultiSlaterDetTableMethod::evaluate_vgl_impl l_tmp[n] += C_otherDs[id][i] * Dets[id]->getLapls()(i, k); } - ValueType psiinv = static_cast(PsiValueType(1.0) / psi_ratio_to_ref_det_); + ValueType psiinv = static_cast(PsiValue(1.0) / psi_ratio_to_ref_det_); g_tmp *= psiinv; l_tmp *= psiinv; - LogValueType log_psi = convertValueToLog(psi_ratio_to_ref_det_); + LogValue log_psi = convertValueToLog(psi_ratio_to_ref_det_); for (size_t id = 0; id < Dets.size(); id++) log_psi += Dets[id]->getLogValueRefDet(); return log_psi; } -WaveFunctionComponent::LogValueType MultiSlaterDetTableMethod::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +WaveFunctionComponent::LogValue MultiSlaterDetTableMethod::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { ScopedTimer local_timer(EvaluateTimer); for (size_t id = 0; id < Dets.size(); id++) @@ -171,10 +171,10 @@ void MultiSlaterDetTableMethod::mw_evaluateLog(const RefVectorWithLeadergetNewRatiosToRefDet() : Dets[det_id]->getRatiosToRefDet(); const size_t noffset = Dets[det_id]->getFirstIndex(); - PsiValueType psi(0); + PsiValue psi(0); // enforce full precision reduction due to numerical sensitivity QTFull::GradType g_sum; for (size_t i = 0; i < Dets[det_id]->getNumDets(); i++) @@ -201,11 +201,11 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGrad_impl(Par return psi; } -WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGradWithSpin_impl(ParticleSet& P, - int iat, - bool newpos, - GradType& g_at, - ComplexType& sg_at) +WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::evalGradWithSpin_impl(ParticleSet& P, + int iat, + bool newpos, + GradType& g_at, + ComplexType& sg_at) { const int det_id = getDetID(iat); @@ -220,15 +220,15 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGradWithSpin_ const Matrix& spingrads = (newpos) ? Dets[det_id]->getNewSpinGrads() : Dets[det_id]->getSpinGrads(); const size_t noffset = Dets[det_id]->getFirstIndex(); - PsiValueType psi(0); + PsiValue psi(0); for (size_t i = 0; i < Dets[det_id]->getNumDets(); i++) { psi += detValues0[i] * C_otherDs[det_id][i]; g_at += C_otherDs[det_id][i] * grads(i, iat - noffset); sg_at += C_otherDs[det_id][i] * spingrads(i, iat - noffset); } - g_at *= PsiValueType(1.0) / psi; - sg_at *= PsiValueType(1.0) / psi; + g_at *= PsiValue(1.0) / psi; + sg_at *= PsiValue(1.0) / psi; return psi; } @@ -237,7 +237,7 @@ void MultiSlaterDetTableMethod::mw_evalGrad_impl(const RefVectorWithLeader& grad_now, - std::vector& psi_list) + std::vector& psi_list) { auto& det_leader = WFC_list.getCastedLeader(); const int det_id = det_leader.getDetID(iat); @@ -284,10 +284,10 @@ void MultiSlaterDetTableMethod::mw_evalGrad_impl(const RefVectorWithLeader(PsiValueType(1.0) / psi_list[iw]); + auto psi_inv = static_cast(PsiValue(1.0) / psi_list[iw]); grad_now[iw][0] = grad_now_list[iw * 3 + 0] * psi_inv; grad_now[iw][1] = grad_now_list[iw * 3 + 1] * psi_inv; grad_now[iw][2] = grad_now_list[iw * 3 + 2] * psi_inv; } } -WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGrad_impl_no_precompute(ParticleSet& P, - int iat, - bool newpos, - GradType& g_at) +WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::evalGrad_impl_no_precompute(ParticleSet& P, + int iat, + bool newpos, + GradType& g_at) { const int det_id = getDetID(iat); @@ -330,7 +330,7 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGrad_impl_no_ const ValueType* restrict cptr = C->data(); const size_t nc = C->size(); const size_t noffset = Dets[det_id]->getFirstIndex(); - PsiValueType psi(0); + PsiValue psi(0); for (size_t i = 0; i < nc; ++i) { const size_t d0 = det0[i]; @@ -341,15 +341,15 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGrad_impl_no_ psi += t * detValues0[d0]; g_at += t * grads(d0, iat - noffset); } - g_at *= PsiValueType(1.0) / psi; + g_at *= PsiValue(1.0) / psi; return psi; } -WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGradWithSpin_impl_no_precompute(ParticleSet& P, - int iat, - bool newpos, - GradType& g_at, - ComplexType& sg_at) +WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::evalGradWithSpin_impl_no_precompute(ParticleSet& P, + int iat, + bool newpos, + GradType& g_at, + ComplexType& sg_at) { const int det_id = getDetID(iat); @@ -365,7 +365,7 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGradWithSpin_ const ValueType* restrict cptr = C->data(); const size_t nc = C->size(); const size_t noffset = Dets[det_id]->getFirstIndex(); - PsiValueType psi(0); + PsiValue psi(0); for (size_t i = 0; i < nc; ++i) { const size_t d0 = det0[i]; @@ -377,8 +377,8 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::evalGradWithSpin_ g_at += t * grads(d0, iat - noffset); sg_at += t * spingrads(d0, iat - noffset); } - g_at *= PsiValueType(1.0) / psi; - sg_at *= PsiValueType(1.0) / psi; + g_at *= PsiValue(1.0) / psi; + sg_at *= PsiValue(1.0) / psi; return psi; } @@ -429,12 +429,12 @@ void MultiSlaterDetTableMethod::mw_evalGrad(const RefVectorWithLeader psi_list(nw, 0); + std::vector psi_list(nw, 0); mw_evalGrad_impl(WFC_list, P_list, iat, false, grad_now, psi_list); } -WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { ScopedTimer local_timer(RatioGradTimer); UpdateMode = ORB_PBYP_PARTIAL; @@ -451,10 +451,10 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::ratioGrad(Particl return curRatio; } -WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::ratioGradWithSpin(ParticleSet& P, - int iat, - GradType& grad_iat, - ComplexType& spingrad_iat) +WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::ratioGradWithSpin(ParticleSet& P, + int iat, + GradType& grad_iat, + ComplexType& spingrad_iat) { ScopedTimer local_timer(RatioGradTimer); UpdateMode = ORB_PBYP_PARTIAL; @@ -476,7 +476,7 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::ratioGradWithSpin void MultiSlaterDetTableMethod::mw_ratioGrad(const RefVectorWithLeader& WFC_list, const RefVectorWithLeader& P_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const { if (!use_pre_computing_) @@ -489,7 +489,7 @@ void MultiSlaterDetTableMethod::mw_ratioGrad(const RefVectorWithLeader psi_list(nw, 0); + std::vector psi_list(nw, 0); std::vector dummy; dummy.resize(nw); @@ -505,11 +505,11 @@ void MultiSlaterDetTableMethod::mw_ratioGrad(const RefVectorWithLeadergetNewRatiosToRefDet(); - PsiValueType psi = 0; + PsiValue psi = 0; if (use_pre_computing_) { // This function computes @@ -539,7 +539,7 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::computeRatio_NewM } // use ci_node for this routine only -WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::ratio(ParticleSet& P, int iat) +WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::ratio(ParticleSet& P, int iat) { ScopedTimer local_timer(RatioTimer); UpdateMode = ORB_PBYP_RATIO; @@ -555,7 +555,7 @@ WaveFunctionComponent::PsiValueType MultiSlaterDetTableMethod::ratio(ParticleSet void MultiSlaterDetTableMethod::mw_calcRatio(const RefVectorWithLeader& WFC_list, const RefVectorWithLeader& P_list, int iat, - std::vector& ratios) const + std::vector& ratios) const { if (!use_pre_computing_) { @@ -592,7 +592,7 @@ void MultiSlaterDetTableMethod::mw_calcRatio(const RefVectorWithLeadergetNewRatiosToRefDet().device_data(); } - std::vector psi_list(nw, 0); + std::vector psi_list(nw, 0); auto* psi_list_ptr = psi_list.data(); auto* C_otherDs_ptr_list_ptr = mw_res.C_otherDs_ptr_list.data(); auto* det_value_ptr_list_ptr = det_value_ptr_list.data(); @@ -602,7 +602,7 @@ void MultiSlaterDetTableMethod::mw_calcRatio(const RefVectorWithLeaderevaluateDetsForPtclMove(VP, iat, VP.refPtcl); const OffloadVector& detValues0 = Dets[det_id]->getNewRatiosToRefDet(); - PsiValueType psiNew = computeRatio_NewMultiDet_to_NewRefDet(det_id); - ratios[iat] = Dets[det_id]->getRefDetRatio() * psiNew / psi_ratio_to_ref_det_; + PsiValue psiNew = computeRatio_NewMultiDet_to_NewRefDet(det_id); + ratios[iat] = Dets[det_id]->getRefDetRatio() * psiNew / psi_ratio_to_ref_det_; } } @@ -687,9 +687,9 @@ void MultiSlaterDetTableMethod::registerData(ParticleSet& P, WFBufferType& buf) buf.add(psi_ratio_to_ref_det_); } -WaveFunctionComponent::LogValueType MultiSlaterDetTableMethod::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch) +WaveFunctionComponent::LogValue MultiSlaterDetTableMethod::updateBuffer(ParticleSet& P, + WFBufferType& buf, + bool fromscratch) { ScopedTimer local_timer(UpdateTimer); @@ -807,7 +807,7 @@ void MultiSlaterDetTableMethod::evaluateDerivatives(ParticleSet& P, // need to modify for CSF later on, right now assume Slater Det basis if (recalculate) { - ValueType psiinv = static_cast(PsiValueType(1.0) / psi_ratio_to_ref_det_); + ValueType psiinv = static_cast(PsiValue(1.0) / psi_ratio_to_ref_det_); laplSum.resize(Dets.size()); for (size_t id = 0; id < Dets.size(); id++) { @@ -990,7 +990,7 @@ void MultiSlaterDetTableMethod::evaluateDerivativesWF(ParticleSet& P, evaluateMultiDiracDeterminantDerivativesWF(P, optvars, dlogpsi); } -void MultiSlaterDetTableMethod::evaluateDerivativesMSD(const PsiValueType& multi_det_to_ref, +void MultiSlaterDetTableMethod::evaluateDerivativesMSD(const PsiValue& multi_det_to_ref, Vector& dlogpsi, int det_id) const { @@ -999,7 +999,7 @@ void MultiSlaterDetTableMethod::evaluateDerivativesMSD(const PsiValueType& multi if (det_id < 0) det_id = 0; - ValueType psiinv = static_cast(PsiValueType(1.0) / multi_det_to_ref); + ValueType psiinv = static_cast(PsiValue(1.0) / multi_det_to_ref); const auto& detValues0 = newpos ? Dets[det_id]->getNewRatiosToRefDet() : Dets[det_id]->getRatiosToRefDet(); if (csf_data_) // CSF @@ -1067,8 +1067,8 @@ void MultiSlaterDetTableMethod::evaluateDerivRatios(const VirtualParticleSet& VP const OffloadVector& detValues0 = Dets[det_id]->getNewRatiosToRefDet(); // calculate VP ratios - PsiValueType psiNew = computeRatio_NewMultiDet_to_NewRefDet(det_id); - ratios[iat] = Dets[det_id]->getRefDetRatio() * psiNew / psi_ratio_to_ref_det_; + PsiValue psiNew = computeRatio_NewMultiDet_to_NewRefDet(det_id); + ratios[iat] = Dets[det_id]->getRefDetRatio() * psiNew / psi_ratio_to_ref_det_; // calculate VP ratios derivatives if (recalculate) @@ -1175,7 +1175,7 @@ void MultiSlaterDetTableMethod::precomputeC_otherDs(const ParticleSet& P, int ig for (size_t i = 0; i < C->size(); i++) { // enforce full precision reduction on C_otherDs due to numerical sensitivity - PsiValueType product = (*C)[i]; + PsiValue product = (*C)[i]; for (size_t id = 0; id < Dets.size(); id++) if (id != ig) product *= Dets[id]->getRatiosToRefDet()[(*C2node)[id][i]]; diff --git a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.h b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.h index b51f7f6298c..f07a831c6bc 100644 --- a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.h +++ b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.h @@ -106,13 +106,13 @@ class MultiSlaterDetTableMethod : public WaveFunctionComponent, public Optimizab //builds orbital rotation parameters using MultiSlater member variables void buildOptVariables(); - LogValueType evaluate_vgl_impl(const ParticleSet& P, - ParticleSet::ParticleGradient& g_tmp, - ParticleSet::ParticleLaplacian& l_tmp); + LogValue evaluate_vgl_impl(const ParticleSet& P, + ParticleSet::ParticleGradient& g_tmp, + ParticleSet::ParticleLaplacian& l_tmp); - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; /* void mw_evaluateLog(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, @@ -137,18 +137,18 @@ class MultiSlaterDetTableMethod : public WaveFunctionComponent, public Optimizab void mw_ratioGrad(const RefVectorWithLeader& WFC_list, const RefVectorWithLeader& P_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const override; void mw_calcRatio(const RefVectorWithLeader& WFC_list, const RefVectorWithLeader& P_list, int iat, - std::vector& ratios) const override; + std::vector& ratios) const override; - PsiValueType ratio(ParticleSet& P, int iat) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; //ratioGradWithSpin, but includes tthe spin gradient info - PsiValueType ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) override; + PsiValue ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) override; void evaluateRatios(const VirtualParticleSet& VP, std::vector& ratios) override; @@ -169,7 +169,7 @@ class MultiSlaterDetTableMethod : public WaveFunctionComponent, public Optimizab bool safe_to_delay = false) const override; void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; void createResource(ResourceCollection& collection) const override; @@ -217,31 +217,31 @@ class MultiSlaterDetTableMethod : public WaveFunctionComponent, public Optimizab /** an implementation shared by evalGrad and ratioGrad. Use precomputed data * @param newpos to distinguish evalGrad(false) ratioGrad(true) */ - PsiValueType evalGrad_impl(ParticleSet& P, int iat, bool newpos, GradType& g_at); + PsiValue evalGrad_impl(ParticleSet& P, int iat, bool newpos, GradType& g_at); /// multi walker version of evalGrad_impl static void mw_evalGrad_impl(const RefVectorWithLeader& WFC_list, const RefVectorWithLeader& P_list, int iat, bool newpos, std::vector& grad_now, - std::vector& psi_list); + std::vector& psi_list); /** an implementation shared by evalGrad and ratioGrad. No use of precomputed data * @param newpos to distinguish evalGrad(false) ratioGrad(true) */ - PsiValueType evalGrad_impl_no_precompute(ParticleSet& P, int iat, bool newpos, GradType& g_at); + PsiValue evalGrad_impl_no_precompute(ParticleSet& P, int iat, bool newpos, GradType& g_at); //implemtation for evalGradWithSpin - PsiValueType evalGradWithSpin_impl(ParticleSet& P, int iat, bool newpos, GradType& g_at, ComplexType& sg_at); + PsiValue evalGradWithSpin_impl(ParticleSet& P, int iat, bool newpos, GradType& g_at, ComplexType& sg_at); //implemtation for evalGradWithSpin with no precomputation - PsiValueType evalGradWithSpin_impl_no_precompute(ParticleSet& P, - int iat, - bool newpos, - GradType& g_at, - ComplexType& sg_at); + PsiValue evalGradWithSpin_impl_no_precompute(ParticleSet& P, + int iat, + bool newpos, + GradType& g_at, + ComplexType& sg_at); // compute the new multi determinant to reference determinant ratio based on temporarycoordinates. - PsiValueType computeRatio_NewMultiDet_to_NewRefDet(int det_id) const; + PsiValue computeRatio_NewMultiDet_to_NewRefDet(int det_id) const; /** precompute C_otherDs for a given particle group * @param P a particle set @@ -263,7 +263,7 @@ class MultiSlaterDetTableMethod : public WaveFunctionComponent, public Optimizab * @param dlogpsi saved derivatives * @param det_id provide this argument to affect determinant group id for virtual moves */ - void evaluateDerivativesMSD(const PsiValueType& multi_det_to_ref, Vector& dlogpsi, int det_id = -1) const; + void evaluateDerivativesMSD(const PsiValue& multi_det_to_ref, Vector& dlogpsi, int det_id = -1) const; /// determinant collection std::vector> Dets; @@ -288,12 +288,12 @@ class MultiSlaterDetTableMethod : public WaveFunctionComponent, public Optimizab const bool use_pre_computing_; /// current psi over ref single det - PsiValueType psi_ratio_to_ref_det_; + PsiValue psi_ratio_to_ref_det_; /// new psi over new ref single det when one particle is moved - PsiValueType new_psi_ratio_to_new_ref_det_; + PsiValue new_psi_ratio_to_new_ref_det_; size_t ActiveSpin; - PsiValueType curRatio; + PsiValue curRatio; /// C_n x D^1_n x D^2_n ... D^3_n with one D removed. Summed by group. [spin, unique det id] //std::vector>> C_otherDs; diff --git a/src/QMCWaveFunctions/Fermion/SlaterDet.cpp b/src/QMCWaveFunctions/Fermion/SlaterDet.cpp index 8b20e48a2da..9bb3b042f28 100644 --- a/src/QMCWaveFunctions/Fermion/SlaterDet.cpp +++ b/src/QMCWaveFunctions/Fermion/SlaterDet.cpp @@ -22,7 +22,7 @@ namespace qmcplusplus { // for return types -using PsiValueType = WaveFunctionComponent::PsiValueType; +using PsiValue = WaveFunctionComponent::PsiValue; SlaterDet::SlaterDet(ParticleSet& targetPtcl, std::vector> dets, @@ -62,12 +62,12 @@ void SlaterDet::checkOutVariables(const opt_variables_type& active) myVars.getIndex(active); } -PsiValueType SlaterDet::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +PsiValue SlaterDet::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { return Dets[getDetID(iat)]->ratioGrad(P, iat, grad_iat); } -PsiValueType SlaterDet::ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) +PsiValue SlaterDet::ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) { return Dets[getDetID(iat)]->ratioGradWithSpin(P, iat, grad_iat, spingrad_iat); } @@ -75,7 +75,7 @@ PsiValueType SlaterDet::ratioGradWithSpin(ParticleSet& P, int iat, GradType& gra void SlaterDet::mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_now) const { const int det_id = getDetID(iat); @@ -96,9 +96,9 @@ void SlaterDet::evaluateDerivRatios(const VirtualParticleSet& VP, return Dets[getDetID(VP.refPtcl)]->evaluateDerivRatios(VP, optvars, ratios, dratios); } -SlaterDet::LogValueType SlaterDet::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +SlaterDet::LogValue SlaterDet::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { log_value_ = 0.0; for (int i = 0; i < Dets.size(); ++i) @@ -111,7 +111,7 @@ void SlaterDet::mw_evaluateLog(const RefVectorWithLeader& const RefVector& G_list, const RefVector& L_list) const { - constexpr LogValueType czero(0); + constexpr LogValue czero(0); for (int iw = 0; iw < wfc_list.size(); iw++) wfc_list.getCastedElement(iw).log_value_ = czero; @@ -125,10 +125,10 @@ void SlaterDet::mw_evaluateLog(const RefVectorWithLeader& } } -SlaterDet::LogValueType SlaterDet::evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool from_scratch) +SlaterDet::LogValue SlaterDet::evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool from_scratch) { log_value_ = 0.0; for (int i = 0; i < Dets.size(); ++i) @@ -142,7 +142,7 @@ void SlaterDet::mw_evaluateGL(const RefVectorWithLeader& const RefVector& L_list, bool fromscratch) const { - constexpr LogValueType czero(0); + constexpr LogValue czero(0); for (int iw = 0; iw < wfc_list.size(); iw++) wfc_list.getCastedElement(iw).log_value_ = czero; @@ -223,7 +223,7 @@ void SlaterDet::registerData(ParticleSet& P, WFBufferType& buf) DEBUG_PSIBUFFER(" SlaterDet::registerData ", buf.current()); } -SlaterDet::LogValueType SlaterDet::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) +SlaterDet::LogValue SlaterDet::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { DEBUG_PSIBUFFER(" SlaterDet::updateBuffer ", buf.current()); log_value_ = 0.0; diff --git a/src/QMCWaveFunctions/Fermion/SlaterDet.h b/src/QMCWaveFunctions/Fermion/SlaterDet.h index 98f7b01f14b..11d31edbef9 100644 --- a/src/QMCWaveFunctions/Fermion/SlaterDet.h +++ b/src/QMCWaveFunctions/Fermion/SlaterDet.h @@ -59,19 +59,19 @@ class SlaterDet : public WaveFunctionComponent void registerTWFFastDerivWrapper(const ParticleSet& P, TWFFastDerivWrapper& twf) const override; - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; void mw_evaluateLog(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, const RefVector& G_list, const RefVector& L_list) const override; - LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch) override; + LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch) override; void mw_evaluateGL(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, @@ -92,7 +92,7 @@ class SlaterDet : public WaveFunctionComponent void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; @@ -126,14 +126,14 @@ class SlaterDet : public WaveFunctionComponent } } - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; - PsiValueType ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) override; + PsiValue ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) override; void mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_now) const override; GradType evalGrad(ParticleSet& P, int iat) override { return Dets[getDetID(iat)]->evalGrad(P, iat); } @@ -189,7 +189,7 @@ class SlaterDet : public WaveFunctionComponent const std::vector& isAccepted, bool safe_to_delay = false) const override { - constexpr LogValueType czero(0); + constexpr LogValue czero(0); // This log_value_ is in the slater determinant, it's still around but not consistent anymore with the // sum of the log_values in its determinants. Caching the state seems like a bad call, but the wfc base class @@ -223,12 +223,12 @@ class SlaterDet : public WaveFunctionComponent Dets[i]->mw_completeUpdates(extract_DetRef_list(wfc_list, i)); } - inline PsiValueType ratio(ParticleSet& P, int iat) override { return Dets[getDetID(iat)]->ratio(P, iat); } + inline PsiValue ratio(ParticleSet& P, int iat) override { return Dets[getDetID(iat)]->ratio(P, iat); } void mw_calcRatio(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios) const override + std::vector& ratios) const override { const int det_id = getDetID(iat); Dets[det_id]->mw_calcRatio(extract_DetRef_list(wfc_list, det_id), p_list, iat, ratios); diff --git a/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.cpp b/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.cpp index 87e213e757f..fae7e4b2d31 100644 --- a/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.cpp +++ b/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.cpp @@ -50,9 +50,9 @@ void SlaterDetWithBackflow::evaluateRatiosAlltoOne(ParticleSet& P, std::vectorevaluateRatiosAlltoOne(P, ratios); } -SlaterDetWithBackflow::LogValueType SlaterDetWithBackflow::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +SlaterDetWithBackflow::LogValue SlaterDetWithBackflow::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { BFTrans->evaluate(P); log_value_ = 0.0; @@ -68,9 +68,7 @@ void SlaterDetWithBackflow::registerData(ParticleSet& P, WFBufferType& buf) Dets[i]->registerData(P, buf); } -SlaterDetWithBackflow::LogValueType SlaterDetWithBackflow::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch) +SlaterDetWithBackflow::LogValue SlaterDetWithBackflow::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { BFTrans->updateBuffer(P, buf, fromscratch); log_value_ = 0.0; @@ -122,9 +120,9 @@ void SlaterDetWithBackflow::testDerivGL(ParticleSet& P) L0.resize(P.getTotalNum()); L1.resize(P.getTotalNum()); L2.resize(P.getTotalNum()); - LogValueType psi1 = 1.0; - LogValueType psi2 = 1.0; - RealType dh = 0.00001; + LogValue psi1 = 1.0; + LogValue psi2 = 1.0; + RealType dh = 0.00001; for (int k = 0; k < Dets.size(); k++) { DiracDeterminantWithBackflow* Dets_ = dynamic_cast(Dets[k].get()); diff --git a/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.h b/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.h index ed30848ee50..b577fe9e2d2 100644 --- a/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.h +++ b/src/QMCWaveFunctions/Fermion/SlaterDetWithBackflow.h @@ -54,19 +54,19 @@ class SlaterDetWithBackflow : public WaveFunctionComponent } } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; - inline PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override + inline PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { BFTrans->evaluatePbyPWithGrad(P, iat); //BFTrans->evaluate(P); - PsiValueType psi = 1.0; + PsiValue psi = 1.0; for (int i = 0; i < Dets.size(); ++i) psi *= Dets[i]->ratioGrad(P, iat, grad_iat); return psi; @@ -111,11 +111,11 @@ class SlaterDetWithBackflow : public WaveFunctionComponent } - inline PsiValueType ratio(ParticleSet& P, int iat) override + inline PsiValue ratio(ParticleSet& P, int iat) override { BFTrans->evaluatePbyP(P, iat); //BFTrans->evaluate(P); - PsiValueType ratio = 1.0; + PsiValue ratio = 1.0; for (int i = 0; i < Dets.size(); ++i) ratio *= Dets[i]->ratio(P, iat); return ratio; diff --git a/src/QMCWaveFunctions/Jastrow/CountingJastrow.h b/src/QMCWaveFunctions/Jastrow/CountingJastrow.h index 9df5aaa21ca..eb0e10b8675 100644 --- a/src/QMCWaveFunctions/Jastrow/CountingJastrow.h +++ b/src/QMCWaveFunctions/Jastrow/CountingJastrow.h @@ -218,9 +218,9 @@ class CountingJastrow : public WaveFunctionComponent, public OptimizableObject } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { evaluateExponents(P); for (int i = 0; i < num_els; ++i) @@ -385,11 +385,11 @@ class CountingJastrow : public WaveFunctionComponent, public OptimizableObject return Jgrad[iat]; } - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { evaluateTempExponents(P, iat); grad_iat += Jgrad_t[iat]; - return std::exp(static_cast(Jval_t - Jval)); + return std::exp(static_cast(Jval_t - Jval)); } void acceptMove(ParticleSet& P, int iat, bool safe_to_delay = false) override @@ -415,15 +415,15 @@ class CountingJastrow : public WaveFunctionComponent, public OptimizableObject void restore(int iat) override { C->restore(iat); } - PsiValueType ratio(ParticleSet& P, int iat) override + PsiValue ratio(ParticleSet& P, int iat) override { evaluateTempExponents(P, iat); - return std::exp(static_cast(Jval_t - Jval)); + return std::exp(static_cast(Jval_t - Jval)); } void registerData(ParticleSet& P, WFBufferType& buf) override { - LogValueType logValue = evaluateLog(P, P.G, P.L); + LogValue logValue = evaluateLog(P, P.G, P.L); RealType* Jlap_begin = &Jlap[0]; RealType* Jlap_end = Jlap_begin + Jlap.size(); RealType* Jgrad_begin = &Jgrad[0][0]; @@ -435,9 +435,9 @@ class CountingJastrow : public WaveFunctionComponent, public OptimizableObject DEBUG_PSIBUFFER(" CountingJastrow::registerData", buf.current()); } - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { - LogValueType logValue = evaluateLog(P, P.G, P.L); + LogValue logValue = evaluateLog(P, P.G, P.L); RealType* Jlap_begin = &Jlap[0]; RealType* Jlap_end = Jlap_begin + Jlap.size(); RealType* Jgrad_begin = &Jgrad[0][0]; diff --git a/src/QMCWaveFunctions/Jastrow/J1OrbitalSoA.h b/src/QMCWaveFunctions/Jastrow/J1OrbitalSoA.h index 16ab2645b69..ed3e3068087 100644 --- a/src/QMCWaveFunctions/Jastrow/J1OrbitalSoA.h +++ b/src/QMCWaveFunctions/Jastrow/J1OrbitalSoA.h @@ -122,10 +122,10 @@ class J1OrbitalSoA : public WaveFunctionComponent return -simd::accumulate_n(Vat.data(), Nelec, QTFull::RealType()); } - inline LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch = false) override + inline LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch = false) override { return log_value_ = computeGL(G, L); } @@ -248,9 +248,9 @@ class J1OrbitalSoA : public WaveFunctionComponent } } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { recompute(P); return log_value_ = computeGL(G, L); @@ -285,11 +285,11 @@ class J1OrbitalSoA : public WaveFunctionComponent } } - PsiValueType ratio(ParticleSet& P, int iat) override + PsiValue ratio(ParticleSet& P, int iat) override { UpdateMode = ORB_PBYP_RATIO; curAt = computeU(P.getDistTableAB(myTableID).getTempDists()); - return std::exp(static_cast(Vat[iat] - curAt)); + return std::exp(static_cast(Vat[iat] - curAt)); } inline void evaluateRatios(const VirtualParticleSet& VP, std::vector& ratios) override @@ -440,7 +440,7 @@ class J1OrbitalSoA : public WaveFunctionComponent * * Using getTempDists(). curAt, curGrad and curLap are computed. */ - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { UpdateMode = ORB_PBYP_PARTIAL; @@ -448,7 +448,7 @@ class J1OrbitalSoA : public WaveFunctionComponent curLap = accumulateGL(dU.data(), d2U.data(), P.getDistTableAB(myTableID).getTempDispls(), curGrad); curAt = simd::accumulate_n(U.data(), Nions, valT()); grad_iat += curGrad; - return std::exp(static_cast(Vat[iat] - curAt)); + return std::exp(static_cast(Vat[iat] - curAt)); } /** Rejected move. Nothing to do */ @@ -490,7 +490,7 @@ class J1OrbitalSoA : public WaveFunctionComponent } } - inline LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override + inline LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { log_value_ = computeGL(P.G, P.L); buf.forward(Bytes_in_WFBuffer); diff --git a/src/QMCWaveFunctions/Jastrow/J1Spin.h b/src/QMCWaveFunctions/Jastrow/J1Spin.h index e94cf6d3566..c119a6b3716 100644 --- a/src/QMCWaveFunctions/Jastrow/J1Spin.h +++ b/src/QMCWaveFunctions/Jastrow/J1Spin.h @@ -168,9 +168,9 @@ struct J1Spin : public WaveFunctionComponent } } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { recompute(P); return log_value_ = computeGL(G, L); @@ -205,11 +205,11 @@ struct J1Spin : public WaveFunctionComponent } } - PsiValueType ratio(ParticleSet& P, int iat) override + PsiValue ratio(ParticleSet& P, int iat) override { UpdateMode = ORB_PBYP_RATIO; curAt = computeU(P, iat, P.getDistTableAB(myTableID).getTempDists()); - return std::exp(static_cast(Vat[iat] - curAt)); + return std::exp(static_cast(Vat[iat] - curAt)); } inline void evaluateRatios(const VirtualParticleSet& VP, std::vector& ratios) override @@ -377,10 +377,10 @@ struct J1Spin : public WaveFunctionComponent return -simd::accumulate_n(Vat.data(), Nelec, QTFull::RealType()); } - inline LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch = false) override + inline LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch = false) override { return log_value_ = computeGL(G, L); } @@ -440,7 +440,7 @@ struct J1Spin : public WaveFunctionComponent * * Using getTempDists(). curAt, curGrad and curLap are computed. */ - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { UpdateMode = ORB_PBYP_PARTIAL; @@ -448,7 +448,7 @@ struct J1Spin : public WaveFunctionComponent curLap = accumulateGL(dU.data(), d2U.data(), P.getDistTableAB(myTableID).getTempDispls(), curGrad); curAt = simd::accumulate_n(U.data(), Nions, valT()); grad_iat += curGrad; - return std::exp(static_cast(Vat[iat] - curAt)); + return std::exp(static_cast(Vat[iat] - curAt)); } /** Rejected move. Nothing to do */ @@ -490,7 +490,7 @@ struct J1Spin : public WaveFunctionComponent } } - inline LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override + inline LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { log_value_ = computeGL(P.G, P.L); buf.forward(Bytes_in_WFBuffer); diff --git a/src/QMCWaveFunctions/Jastrow/JeeIOrbitalSoA.h b/src/QMCWaveFunctions/Jastrow/JeeIOrbitalSoA.h index 3f39c98d4d1..8591cdeddd2 100644 --- a/src/QMCWaveFunctions/Jastrow/JeeIOrbitalSoA.h +++ b/src/QMCWaveFunctions/Jastrow/JeeIOrbitalSoA.h @@ -375,15 +375,15 @@ class JeeIOrbitalSoA : public WaveFunctionComponent } } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { recompute(P); return log_value_ = computeGL(G, L); } - PsiValueType ratio(ParticleSet& P, int iat) override + PsiValue ratio(ParticleSet& P, int iat) override { UpdateMode = ORB_PBYP_RATIO; @@ -391,7 +391,7 @@ class JeeIOrbitalSoA : public WaveFunctionComponent const auto& ee_table = P.getDistTableAA(ee_Table_ID_); cur_Uat = computeU(P, iat, P.GroupID[iat], eI_table.getTempDists(), ee_table.getTempDists(), ions_nearby_new); DiffVal = Uat[iat] - cur_Uat; - return std::exp(static_cast(DiffVal)); + return std::exp(static_cast(DiffVal)); } void evaluateRatios(const VirtualParticleSet& VP, std::vector& ratios) override @@ -435,7 +435,7 @@ class JeeIOrbitalSoA : public WaveFunctionComponent GradType evalGrad(ParticleSet& P, int iat) override { return GradType(dUat[iat]); } - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { UpdateMode = ORB_PBYP_PARTIAL; @@ -445,7 +445,7 @@ class JeeIOrbitalSoA : public WaveFunctionComponent ee_table.getTempDispls(), cur_Uat, cur_dUat, cur_d2Uat, newUk, newdUk, newd2Uk, ions_nearby_new); DiffVal = Uat[iat] - cur_Uat; grad_iat += cur_dUat; - return std::exp(static_cast(DiffVal)); + return std::exp(static_cast(DiffVal)); } inline void restore(int iat) override {} @@ -804,7 +804,7 @@ class JeeIOrbitalSoA : public WaveFunctionComponent } } - inline LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override + inline LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { log_value_ = computeGL(P.G, P.L); buf.forward(Bytes_in_WFBuffer); @@ -819,10 +819,10 @@ class JeeIOrbitalSoA : public WaveFunctionComponent build_compact_list(P); } - LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch = false) override + LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch = false) override { return log_value_ = computeGL(G, L); } @@ -1136,12 +1136,12 @@ class JeeIOrbitalSoA : public WaveFunctionComponent source.update(); P.update(); - LogValueType log_p = evaluateLog(P, tempG, tempL); + LogValue log_p = evaluateLog(P, tempG, tempL); source.R[isrc][iondim] = rI[iondim] - delta; source.update(); P.update(); - LogValueType log_m = evaluateLog(P, tempG, tempL); + LogValue log_m = evaluateLog(P, tempG, tempL); QTFull::RealType log_p_r(0.0), log_m_r(0.0); @@ -1191,12 +1191,12 @@ class JeeIOrbitalSoA : public WaveFunctionComponent source.update(); P.update(); - LogValueType log_p = evaluateLog(P, Gp, Lp); + LogValue log_p = evaluateLog(P, Gp, Lp); source.R[isrc][iondim] = rI[iondim] - delta; source.update(); P.update(); - LogValueType log_m = evaluateLog(P, Gm, Lm); + LogValue log_m = evaluateLog(P, Gm, Lm); QTFull::RealType log_p_r(0.0), log_m_r(0.0); log_p_r = log_p.real(); diff --git a/src/QMCWaveFunctions/Jastrow/RPAJastrow.cpp b/src/QMCWaveFunctions/Jastrow/RPAJastrow.cpp index 1f31b8ac50c..f18c00c28b1 100644 --- a/src/QMCWaveFunctions/Jastrow/RPAJastrow.cpp +++ b/src/QMCWaveFunctions/Jastrow/RPAJastrow.cpp @@ -206,9 +206,9 @@ void RPAJastrow::checkOutVariables(const opt_variables_type& active) ShortRangeRPA->checkOutVariables(active); } -RPAJastrow::LogValueType RPAJastrow::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +RPAJastrow::LogValue RPAJastrow::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { log_value_ = 0.0; for (int i = 0; i < Psi.size(); i++) @@ -216,12 +216,12 @@ RPAJastrow::LogValueType RPAJastrow::evaluateLog(const ParticleSet& P, return log_value_; } -RPAJastrow::PsiValueType RPAJastrow::ratio(ParticleSet& P, int iat) +RPAJastrow::PsiValue RPAJastrow::ratio(ParticleSet& P, int iat) { ValueType r(1.0); for (int i = 0; i < Psi.size(); i++) r *= Psi[i]->ratio(P, iat); - return static_cast(r); + return static_cast(r); } RPAJastrow::GradType RPAJastrow::evalGrad(ParticleSet& P, int iat) @@ -232,14 +232,14 @@ RPAJastrow::GradType RPAJastrow::evalGrad(ParticleSet& P, int iat) return grad; } -RPAJastrow::PsiValueType RPAJastrow::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +RPAJastrow::PsiValue RPAJastrow::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { ValueType r(1); for (int i = 0; i < Psi.size(); i++) { r *= Psi[i]->ratioGrad(P, iat, grad_iat); } - return static_cast(r); + return static_cast(r); } @@ -261,7 +261,7 @@ void RPAJastrow::registerData(ParticleSet& P, WFBufferType& buf) Psi[i]->registerData(P, buf); } -RPAJastrow::LogValueType RPAJastrow::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) +RPAJastrow::LogValue RPAJastrow::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { log_value_ = 0.0; for (int i = 0; i < Psi.size(); i++) diff --git a/src/QMCWaveFunctions/Jastrow/RPAJastrow.h b/src/QMCWaveFunctions/Jastrow/RPAJastrow.h index cbad1a2d7a4..30319a471cc 100644 --- a/src/QMCWaveFunctions/Jastrow/RPAJastrow.h +++ b/src/QMCWaveFunctions/Jastrow/RPAJastrow.h @@ -67,13 +67,13 @@ class RPAJastrow : public WaveFunctionComponent ShortRangeRPA->extractOptimizableObjectRefs(opt_obj_refs); } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; GradType evalGrad(ParticleSet& P, int iat) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; void acceptMove(ParticleSet& P, int iat, bool safe_to_delay = false) override; @@ -81,7 +81,7 @@ class RPAJastrow : public WaveFunctionComponent void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; diff --git a/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.cpp b/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.cpp index e9fbb6ecdf3..e21fe5e47d8 100644 --- a/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.cpp +++ b/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.cpp @@ -246,9 +246,9 @@ void TwoBodyJastrow::copyFromBuffer(ParticleSet& P, WFBufferType& buf) } template -typename TwoBodyJastrow::LogValueType TwoBodyJastrow::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch) +typename TwoBodyJastrow::LogValue TwoBodyJastrow::updateBuffer(ParticleSet& P, + WFBufferType& buf, + bool fromscratch) { log_value_ = computeGL(P.G, P.L); buf.forward(Bytes_in_WFBuffer); @@ -448,19 +448,19 @@ void TwoBodyJastrow::computeU3(const ParticleSet& P, } template -typename TwoBodyJastrow::PsiValueType TwoBodyJastrow::ratio(ParticleSet& P, int iat) +typename TwoBodyJastrow::PsiValue TwoBodyJastrow::ratio(ParticleSet& P, int iat) { //only ratio, ready to compute it again UpdateMode = ORB_PBYP_RATIO; cur_Uat = computeU(P, iat, P.getDistTableAA(my_table_ID_).getTempDists()); - return std::exp(static_cast(Uat[iat] - cur_Uat)); + return std::exp(static_cast(Uat[iat] - cur_Uat)); } template void TwoBodyJastrow::mw_calcRatio(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios) const + std::vector& ratios) const { if (!use_offload_) { @@ -489,7 +489,7 @@ void TwoBodyJastrow::mw_calcRatio(const RefVectorWithLeader>(iw); wfc.cur_Uat = mw_vgl[iw][0]; - ratios[iw] = std::exp(static_cast(wfc.Uat[iat] - wfc.cur_Uat)); + ratios[iw] = std::exp(static_cast(wfc.Uat[iat] - wfc.cur_Uat)); } } @@ -529,7 +529,7 @@ typename TwoBodyJastrow::GradType TwoBodyJastrow::evalGrad(ParticleSet& } template -typename TwoBodyJastrow::PsiValueType TwoBodyJastrow::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +typename TwoBodyJastrow::PsiValue TwoBodyJastrow::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { UpdateMode = ORB_PBYP_PARTIAL; @@ -537,14 +537,14 @@ typename TwoBodyJastrow::PsiValueType TwoBodyJastrow::ratioGrad(Particle cur_Uat = simd::accumulate_n(cur_u.data(), N, valT()); DiffVal = Uat[iat] - cur_Uat; grad_iat += accumulateG(cur_du.data(), P.getDistTableAA(my_table_ID_).getTempDispls()); - return std::exp(static_cast(DiffVal)); + return std::exp(static_cast(DiffVal)); } template void TwoBodyJastrow::mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const { if (!use_offload_) @@ -573,7 +573,7 @@ void TwoBodyJastrow::mw_ratioGrad(const RefVectorWithLeader>(iw); wfc.cur_Uat = mw_vgl[iw][0]; - ratios[iw] = std::exp(static_cast(wfc.Uat[iat] - wfc.cur_Uat)); + ratios[iw] = std::exp(static_cast(wfc.Uat[iat] - wfc.cur_Uat)); for (int idim = 0; idim < ndim; idim++) grad_new[iw][idim] += mw_vgl[iw][idim + 1]; } @@ -734,9 +734,9 @@ void TwoBodyJastrow::mw_recompute(const RefVectorWithLeader -typename TwoBodyJastrow::LogValueType TwoBodyJastrow::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +typename TwoBodyJastrow::LogValue TwoBodyJastrow::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { recompute(P); return log_value_ = computeGL(G, L); @@ -780,10 +780,10 @@ typename TwoBodyJastrow::QTFull::RealType TwoBodyJastrow::computeGL(Part } template -WaveFunctionComponent::LogValueType TwoBodyJastrow::evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch) +WaveFunctionComponent::LogValue TwoBodyJastrow::evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch) { return log_value_ = computeGL(G, L); } diff --git a/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.h b/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.h index 05222561b57..619ba16750b 100644 --- a/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.h +++ b/src/QMCWaveFunctions/Jastrow/TwoBodyJastrow.h @@ -178,9 +178,9 @@ class TwoBodyJastrow : public WaveFunctionComponent std::unique_ptr makeClone(ParticleSet& tqp) const override; - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; void mw_evaluateLog(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, const RefVector& G_list, @@ -194,11 +194,11 @@ class TwoBodyJastrow : public WaveFunctionComponent const RefVectorWithLeader& p_list, const std::vector& recompute) const override; - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; void mw_calcRatio(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios) const override; + std::vector& ratios) const override; void evaluateRatios(const VirtualParticleSet& VP, std::vector& ratios) override; void evaluateRatiosAlltoOne(ParticleSet& P, std::vector& ratios) override; @@ -209,11 +209,11 @@ class TwoBodyJastrow : public WaveFunctionComponent GradType evalGrad(ParticleSet& P, int iat) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; void mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const override; void acceptMove(ParticleSet& P, int iat, bool safe_to_delay = false) override; @@ -227,10 +227,10 @@ class TwoBodyJastrow : public WaveFunctionComponent /** compute G and L after the sweep */ - LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch = false) override; + LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch = false) override; void mw_evaluateGL(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, const RefVector& G_list, @@ -241,7 +241,7 @@ class TwoBodyJastrow : public WaveFunctionComponent void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override; inline RealType ChiesaKEcorrection() { return KEcorr = j2_ke_corr_helper.computeKEcorr(); } diff --git a/src/QMCWaveFunctions/Jastrow/kSpaceJastrow.cpp b/src/QMCWaveFunctions/Jastrow/kSpaceJastrow.cpp index 26f045bb2b8..4650d18d252 100644 --- a/src/QMCWaveFunctions/Jastrow/kSpaceJastrow.cpp +++ b/src/QMCWaveFunctions/Jastrow/kSpaceJastrow.cpp @@ -366,9 +366,9 @@ void kSpaceJastrow::setCoefficients(std::vector& oneBodyCoefs, std::ve // Evaluation functions // /////////////////////////////////////////////////////////////// -kSpaceJastrow::LogValueType kSpaceJastrow::evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) +kSpaceJastrow::LogValue kSpaceJastrow::evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) { RealType J1(0.0), J2(0.0); int N = P.getTotalNum(); @@ -481,7 +481,7 @@ kSpaceJastrow::GradType kSpaceJastrow::evalGrad(ParticleSet& P, int iat) return G; } -kSpaceJastrow::PsiValueType kSpaceJastrow::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +kSpaceJastrow::PsiValue kSpaceJastrow::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { ComplexType eye(0.0, 1.0); RealType J1new(0.0), J1old(0.0), J2new(0.0), J2old(0.0); @@ -527,13 +527,13 @@ kSpaceJastrow::PsiValueType kSpaceJastrow::ratioGrad(ParticleSet& P, int iat, Gr grad_iat += -Prefactor * 2.0 * TwoBodyGvecs[i] * TwoBodyCoefs[i] * imag(qmcplusplus::conj(TwoBody_rhoG[i]) * TwoBody_e2iGr_new[i]); } - return std::exp(static_cast(J1new + J2new - (J1old + J2old))); + return std::exp(static_cast(J1new + J2new - (J1old + J2old))); } /* evaluate the ratio with P.R[iat] * */ -kSpaceJastrow::PsiValueType kSpaceJastrow::ratio(ParticleSet& P, int iat) +kSpaceJastrow::PsiValue kSpaceJastrow::ratio(ParticleSet& P, int iat) { RealType J1new(0.0), J1old(0.0), J2new(0.0), J2old(0.0); const PosType &rnew(P.getActivePos()), &rold(P.R[iat]); @@ -567,7 +567,7 @@ kSpaceJastrow::PsiValueType kSpaceJastrow::ratio(ParticleSet& P, int iat) ComplexType rho_G = TwoBody_rhoG[i] + TwoBody_e2iGr_new[i] - TwoBody_e2iGr_old[i]; J2new += Prefactor * TwoBodyCoefs[i] * std::norm(rho_G); } - return std::exp(static_cast(J1new + J2new - (J1old + J2old))); + return std::exp(static_cast(J1new + J2new - (J1old + J2old))); } /** evaluate the ratio @@ -645,7 +645,7 @@ void kSpaceJastrow::registerData(ParticleSet& P, WFBufferType& buf) // return LogValue; } -kSpaceJastrow::LogValueType kSpaceJastrow::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) +kSpaceJastrow::LogValue kSpaceJastrow::updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) { log_value_ = evaluateLog(P, P.G, P.L); // for(int iat=0; iat(U[iat] - curVal)); + return std::exp(static_cast(U[iat] - curVal)); } @@ -107,7 +107,7 @@ GradType LatticeGaussianProduct::evalGrad(ParticleSet& P, int iat) } -PsiValueType LatticeGaussianProduct::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) +PsiValue LatticeGaussianProduct::ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) { const auto& d_table = P.getDistTableAB(myTableID); int icent = ParticleCenter[iat]; @@ -119,7 +119,7 @@ PsiValueType LatticeGaussianProduct::ratioGrad(ParticleSet& P, int iat, GradType curVal = a * newdist * newdist; curGrad = -2.0 * a * newdisp; grad_iat += curGrad; - return std::exp(static_cast(U[iat] - curVal)); + return std::exp(static_cast(U[iat] - curVal)); } void LatticeGaussianProduct::restore(int iat) {} @@ -171,9 +171,9 @@ void LatticeGaussianProduct::registerData(ParticleSet& P, WFBufferType& buf) buf.add(FirstAddressOfdU, LastAddressOfdU); } -LatticeGaussianProduct::LogValueType LatticeGaussianProduct::updateBuffer(ParticleSet& P, - WFBufferType& buf, - bool fromscratch = false) +LatticeGaussianProduct::LogValue LatticeGaussianProduct::updateBuffer(ParticleSet& P, + WFBufferType& buf, + bool fromscratch = false) { evaluateLogAndStore(P, P.G, P.L); buf.put(U.first_address(), U.last_address()); diff --git a/src/QMCWaveFunctions/LatticeGaussianProduct.h b/src/QMCWaveFunctions/LatticeGaussianProduct.h index 9aff77831de..d349b6815f9 100644 --- a/src/QMCWaveFunctions/LatticeGaussianProduct.h +++ b/src/QMCWaveFunctions/LatticeGaussianProduct.h @@ -48,11 +48,11 @@ class LatticeGaussianProduct : public WaveFunctionComponent std::string getClassName() const override { return "LatticeGaussianProduct"; } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override; + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override; - PsiValueType ratio(ParticleSet& P, int iat) override; + PsiValue ratio(ParticleSet& P, int iat) override; void acceptMove(ParticleSet& P, int iat, bool safe_to_delay = false) override; @@ -60,13 +60,13 @@ class LatticeGaussianProduct : public WaveFunctionComponent void registerData(ParticleSet& P, WFBufferType& buf) override; - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) override; + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch) override; void copyFromBuffer(ParticleSet& P, WFBufferType& buf) override; GradType evalGrad(ParticleSet& P, int iat) override; - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override; std::unique_ptr makeClone(ParticleSet& tqp) const override; diff --git a/src/QMCWaveFunctions/LinearOrbital.h b/src/QMCWaveFunctions/LinearOrbital.h index 48566edc06a..7d3b78c30d6 100644 --- a/src/QMCWaveFunctions/LinearOrbital.h +++ b/src/QMCWaveFunctions/LinearOrbital.h @@ -35,9 +35,9 @@ class LinearOrbital : public WaveFunctionComponent std::string getClassName() const override { return "LinearOrbital"; } - LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) override + LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) override { ValueType v = 0.0; for (int i = 0; i < P.R.size(); i++) @@ -57,15 +57,15 @@ class LinearOrbital : public WaveFunctionComponent void restore(int iat) override {} - PsiValueType ratio(ParticleSet& P, int iat) override { return 1.0; } + PsiValue ratio(ParticleSet& P, int iat) override { return 1.0; } GradType evalGrad(ParticleSet& P, int iat) override { return GradType(coeff); } - PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { return 1.0; } + PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat) override { return 1.0; } void registerData(ParticleSet& P, WFBufferType& buf) override {} - LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override + LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) override { return evaluateLog(P, P.G, P.L); } diff --git a/src/QMCWaveFunctions/TWFFastDerivWrapper.cpp b/src/QMCWaveFunctions/TWFFastDerivWrapper.cpp index 735c96ae850..c01f009d234 100644 --- a/src/QMCWaveFunctions/TWFFastDerivWrapper.cpp +++ b/src/QMCWaveFunctions/TWFFastDerivWrapper.cpp @@ -58,9 +58,9 @@ TWFFastDerivWrapper::RealType TWFFastDerivWrapper::evaluateJastrowVGL(const Part ParticleSet::ParticleGradient& G, ParticleSet::ParticleLaplacian& L) const { - WaveFunctionComponent::LogValueType logpsi = 0.0; - G = 0.0; - L = 0.0; + WaveFunctionComponent::LogValue logpsi = 0.0; + G = 0.0; + L = 0.0; for (int i = 0; i < jastrow_list_.size(); ++i) { logpsi += jastrow_list_[i]->evaluateLog(P, G, L); @@ -73,7 +73,7 @@ TWFFastDerivWrapper::RealType TWFFastDerivWrapper::evaluateJastrowRatio(Particle { //legacy calls are hit and miss with const. Remove const for index. int iel_(iel); - WaveFunctionComponent::PsiValueType r(1.0); + WaveFunctionComponent::PsiValue r(1.0); for (int i = 0; i < jastrow_list_.size(); ++i) { r *= jastrow_list_[i]->ratio(P, iel_); @@ -89,7 +89,7 @@ TWFFastDerivWrapper::RealType TWFFastDerivWrapper::calcJastrowRatioGrad(Particle GradType& grad) const { int iel_(iel); - WaveFunctionComponent::PsiValueType r(1.0); + WaveFunctionComponent::PsiValue r(1.0); for (int i = 0; i < jastrow_list_.size(); ++i) { r *= jastrow_list_[i]->ratioGrad(P, iel_, grad); diff --git a/src/QMCWaveFunctions/TWFdispatcher.cpp b/src/QMCWaveFunctions/TWFdispatcher.cpp index b76f949d84d..ec33cefe48a 100644 --- a/src/QMCWaveFunctions/TWFdispatcher.cpp +++ b/src/QMCWaveFunctions/TWFdispatcher.cpp @@ -45,7 +45,7 @@ void TWFdispatcher::flex_recompute(const RefVectorWithLeader& void TWFdispatcher::flex_calcRatio(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, ComputeType ct) const { assert(wf_list.size() == p_list.size()); @@ -97,7 +97,7 @@ template void TWFdispatcher::flex_calcRatioGrad(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads) const { assert(wf_list.size() == p_list.size()); @@ -180,13 +180,13 @@ template void TWFdispatcher::flex_evalGrad(const RefVector template void TWFdispatcher::flex_calcRatioGrad(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads) const; template void TWFdispatcher::flex_calcRatioGrad( const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads) const; } // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/TWFdispatcher.h b/src/QMCWaveFunctions/TWFdispatcher.h index 8a44d39be43..9d5a6de72bd 100644 --- a/src/QMCWaveFunctions/TWFdispatcher.h +++ b/src/QMCWaveFunctions/TWFdispatcher.h @@ -25,11 +25,11 @@ namespace qmcplusplus class TWFdispatcher { public: - using PsiValueType = TrialWaveFunction::PsiValueType; - using ComputeType = TrialWaveFunction::ComputeType; - using ValueType = TrialWaveFunction::ValueType; - using GradType = TrialWaveFunction::GradType; - using Complex = TrialWaveFunction::ComplexType; + using PsiValue = TrialWaveFunction::PsiValue; + using ComputeType = TrialWaveFunction::ComputeType; + using ValueType = TrialWaveFunction::ValueType; + using GradType = TrialWaveFunction::GradType; + using Complex = TrialWaveFunction::ComplexType; TWFdispatcher(bool use_batch); @@ -43,7 +43,7 @@ class TWFdispatcher void flex_calcRatio(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, ComputeType ct = ComputeType::ALL) const; void flex_prepareGroup(const RefVectorWithLeader& wf_list, @@ -60,7 +60,7 @@ class TWFdispatcher void flex_calcRatioGrad(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads) const; void flex_accept_rejectMove(const RefVectorWithLeader& wf_list, diff --git a/src/QMCWaveFunctions/TrialWaveFunction.cpp b/src/QMCWaveFunctions/TrialWaveFunction.cpp index e7d091d5dc3..3df1cf438e3 100644 --- a/src/QMCWaveFunctions/TrialWaveFunction.cpp +++ b/src/QMCWaveFunctions/TrialWaveFunction.cpp @@ -114,7 +114,7 @@ TrialWaveFunction::RealType TrialWaveFunction::evaluateLog(ParticleSet& P) ScopedTimer local_timer(TWF_timers_[RECOMPUTE_TIMER]); P.G = 0.0; P.L = 0.0; - LogValueType logpsi(0.0); + LogValue logpsi(0.0); for (int i = 0; i < Z.size(); ++i) { ScopedTimer z_timer(WFC_timers_[RECOMPUTE_TIMER + TIMER_SKIP * i]); @@ -223,7 +223,7 @@ TrialWaveFunction::RealType TrialWaveFunction::evaluateDeltaLog(ParticleSet& P, ScopedTimer local_timer(TWF_timers_[RECOMPUTE_TIMER]); P.G = 0.0; P.L = 0.0; - LogValueType logpsi(0.0); + LogValue logpsi(0.0); for (int i = 0; i < Z.size(); ++i) { ScopedTimer z_timer(WFC_timers_[RECOMPUTE_TIMER + TIMER_SKIP * i]); @@ -262,8 +262,8 @@ void TrialWaveFunction::evaluateDeltaLogSetup(ParticleSet& P, P.L = 0.0; fixedL = 0.0; fixedG = 0.0; - LogValueType logpsi_fixed(0.0); - LogValueType logpsi_opt(0.0); + LogValue logpsi_fixed(0.0); + LogValue logpsi_opt(0.0); for (int i = 0; i < Z.size(); ++i) { @@ -439,7 +439,7 @@ void TrialWaveFunction::evaluateHessian(ParticleSet& P, HessVector& grad_grad_ps TrialWaveFunction::ValueType TrialWaveFunction::calcRatio(ParticleSet& P, int iat, ComputeType ct) { ScopedTimer local_timer(TWF_timers_[V_TIMER]); - PsiValueType r(1.0); + PsiValue r(1.0); for (int i = 0; i < Z.size(); i++) if (ct == ComputeType::ALL || (Z[i]->isFermionic() && ct == ComputeType::FERMIONIC) || (!Z[i]->isFermionic() && ct == ComputeType::NONFERMIONIC)) @@ -453,19 +453,19 @@ TrialWaveFunction::ValueType TrialWaveFunction::calcRatio(ParticleSet& P, int ia void TrialWaveFunction::mw_calcRatio(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, ComputeType ct) { const int num_wf = wf_list.size(); ratios.resize(num_wf); - std::fill(ratios.begin(), ratios.end(), PsiValueType(1)); + std::fill(ratios.begin(), ratios.end(), PsiValue(1)); auto& wf_leader = wf_list.getLeader(); ScopedTimer local_timer(wf_leader.TWF_timers_[V_TIMER]); const int num_wfc = wf_leader.Z.size(); auto& wavefunction_components = wf_leader.Z; - std::vector ratios_z(num_wf); + std::vector ratios_z(num_wf); for (int i = 0; i < num_wfc; i++) { if (ct == ComputeType::ALL || (wavefunction_components[i]->isFermionic() && ct == ComputeType::FERMIONIC) || @@ -593,11 +593,11 @@ TrialWaveFunction::ValueType TrialWaveFunction::calcRatioGrad(ParticleSet& P, in { ScopedTimer local_timer(TWF_timers_[VGL_TIMER]); grad_iat = 0.0; - PsiValueType r(1.0); + PsiValue r(1.0); if (use_tasking_) { std::vector grad_components(Z.size(), GradType(0.0)); - std::vector ratio_components(Z.size(), 0.0); + std::vector ratio_components(Z.size(), 0.0); PRAGMA_OMP_TASKLOOP("omp taskloop default(shared)") for (int i = 0; i < Z.size(); ++i) { @@ -618,10 +618,10 @@ TrialWaveFunction::ValueType TrialWaveFunction::calcRatioGrad(ParticleSet& P, in r *= Z[i]->ratioGrad(P, iat, grad_iat); } - if(r != PsiValueType(0)) // grad_iat is meaningful only when r is strictly non-zero + if (r != PsiValue(0)) // grad_iat is meaningful only when r is strictly non-zero checkOneParticleGradientsNaN(iat, grad_iat, "TWF::calcRatioGrad"); - LogValueType logratio = convertValueToLog(r); - PhaseDiff = std::imag(logratio); + LogValue logratio = convertValueToLog(r); + PhaseDiff = std::imag(logratio); return static_cast(r); } @@ -633,7 +633,7 @@ TrialWaveFunction::ValueType TrialWaveFunction::calcRatioGradWithSpin(ParticleSe ScopedTimer local_timer(TWF_timers_[VGL_TIMER]); grad_iat = 0.0; spingrad_iat = 0.0; - PsiValueType r(1.0); + PsiValue r(1.0); for (int i = 0; i < Z.size(); ++i) { ScopedTimer z_timer(WFC_timers_[VGL_TIMER + TIMER_SKIP * i]); @@ -641,8 +641,8 @@ TrialWaveFunction::ValueType TrialWaveFunction::calcRatioGradWithSpin(ParticleSe } checkOneParticleGradientsNaN(iat, grad_iat, "TWF::calcRatioGradWithSpin"); - LogValueType logratio = convertValueToLog(r); - PhaseDiff = std::imag(logratio); + LogValue logratio = convertValueToLog(r); + PhaseDiff = std::imag(logratio); return static_cast(r); } @@ -650,12 +650,12 @@ template void TrialWaveFunction::mw_calcRatioGrad(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grad_new) { const int num_wf = wf_list.size(); ratios.resize(num_wf); - std::fill(ratios.begin(), ratios.end(), PsiValueType(1)); + std::fill(ratios.begin(), ratios.end(), PsiValue(1)); grad_new = TWFGrads(num_wf); auto& wf_leader = wf_list.getLeader(); @@ -665,7 +665,7 @@ void TrialWaveFunction::mw_calcRatioGrad(const RefVectorWithLeader> ratios_components(num_wfc, std::vector(wf_list.size())); + std::vector> ratios_components(num_wfc, std::vector(wf_list.size())); std::vector> grads_components(num_wfc, TWFGrads(num_wf)); PRAGMA_OMP_TASKLOOP("omp taskloop default(shared)") for (int i = 0; i < num_wfc; ++i) @@ -684,7 +684,7 @@ void TrialWaveFunction::mw_calcRatioGrad(const RefVectorWithLeader ratios_z(wf_list.size()); + std::vector ratios_z(wf_list.size()); for (int i = 0; i < num_wfc; ++i) { ScopedTimer z_timer(wf_leader.WFC_timers_[VGL_TIMER + TIMER_SKIP * i]); @@ -697,7 +697,7 @@ void TrialWaveFunction::mw_calcRatioGrad(const RefVectorWithLeader( const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads); template void TrialWaveFunction::mw_calcRatioGrad( const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads); } // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/TrialWaveFunction.h b/src/QMCWaveFunctions/TrialWaveFunction.h index 45597332e88..c27196f494a 100644 --- a/src/QMCWaveFunctions/TrialWaveFunction.h +++ b/src/QMCWaveFunctions/TrialWaveFunction.h @@ -72,8 +72,8 @@ class TrialWaveFunction using WFBufferType = WaveFunctionComponent::WFBufferType; using HessType = WaveFunctionComponent::HessType; using HessVector = WaveFunctionComponent::HessVector; - using LogValueType = WaveFunctionComponent::LogValueType; - using PsiValueType = WaveFunctionComponent::PsiValueType; + using LogValue = WaveFunctionComponent::LogValue; + using PsiValue = WaveFunctionComponent::PsiValue; using SPOMap = SPOSet::SPOMap; @@ -290,7 +290,7 @@ class TrialWaveFunction static void mw_calcRatio(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, ComputeType ct = ComputeType::ALL); /** compulte multiple ratios to handle non-local moves and other virtual moves @@ -352,7 +352,7 @@ class TrialWaveFunction static void mw_calcRatioGrad(const RefVectorWithLeader& wf_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grads); /** Prepare internal data for updating WFC correspond to a particle group @@ -412,7 +412,7 @@ class TrialWaveFunction /** compute gradients and laplacian of the TWF with respect to each particle. * See WaveFunctionComponent::evaluateGL for more detail */ - LogValueType evaluateGL(ParticleSet& P, bool fromscratch); + LogValue evaluateGL(ParticleSet& P, bool fromscratch); /* batched version of evaluateGL. */ static void mw_evaluateGL(const RefVectorWithLeader& wf_list, diff --git a/src/QMCWaveFunctions/WaveFunctionComponent.cpp b/src/QMCWaveFunctions/WaveFunctionComponent.cpp index 0853e9916f2..703810aabc2 100644 --- a/src/QMCWaveFunctions/WaveFunctionComponent.cpp +++ b/src/QMCWaveFunctions/WaveFunctionComponent.cpp @@ -20,7 +20,7 @@ namespace qmcplusplus { // for return types -using PsiValueType = WaveFunctionComponent::PsiValueType; +using PsiValue = WaveFunctionComponent::PsiValue; WaveFunctionComponent::WaveFunctionComponent(const std::string& obj_name) : UpdateMode(ORB_WALKER), Bytes_in_WFBuffer(0), my_name_(obj_name), log_value_(0.0) @@ -101,7 +101,7 @@ void WaveFunctionComponent::mw_evalGradWithSpin(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios) const + std::vector& ratios) const { assert(this == &wfc_list.getLeader()); for (int iw = 0; iw < wfc_list.size(); iw++) @@ -109,7 +109,7 @@ void WaveFunctionComponent::mw_calcRatio(const RefVectorWithLeader void WaveFunctionComponent::mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grad_new) const { if constexpr (CT == CoordsType::POS_SPIN) @@ -131,7 +131,7 @@ void WaveFunctionComponent::mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const { assert(this == &wfc_list.getLeader()); @@ -142,7 +142,7 @@ void WaveFunctionComponent::mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new, std::vector& spingrad_new) const { @@ -172,10 +172,10 @@ void WaveFunctionComponent::mw_completeUpdates(const RefVectorWithLeader( const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grad_new) const; template void WaveFunctionComponent::mw_ratioGrad( const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grad_new) const; } // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/WaveFunctionComponent.h b/src/QMCWaveFunctions/WaveFunctionComponent.h index 662f9532595..10e6c33bab3 100644 --- a/src/QMCWaveFunctions/WaveFunctionComponent.h +++ b/src/QMCWaveFunctions/WaveFunctionComponent.h @@ -79,9 +79,9 @@ class WaveFunctionComponent : public QMCTraits using HessVector = OrbitalSetTraits::HessVector; // the value type for log(psi) - using LogValueType = std::complex; + using LogValue = std::complex; // the value type for psi(r')/psi(r) - using PsiValueType = QTFull::ValueType; + using PsiValue = QTFull::ValueType; /** current update mode */ int UpdateMode; @@ -102,10 +102,10 @@ class WaveFunctionComponent : public QMCTraits * * There could be others. */ - LogValueType log_value_; + LogValue log_value_; public: - const LogValueType& get_log_value() const { return log_value_; } + const LogValue& get_log_value() const { return log_value_; } /// default constructor WaveFunctionComponent(const std::string& obj_name = ""); @@ -122,7 +122,7 @@ class WaveFunctionComponent : public QMCTraits virtual std::string getClassName() const = 0; ///assembles the full value - PsiValueType getValue() const { return LogToValue::convert(log_value_); } + PsiValue getValue() const { return LogToValue::convert(log_value_); } /** true, if this component is fermionic */ virtual bool isFermionic() const { return false; } @@ -145,9 +145,9 @@ class WaveFunctionComponent : public QMCTraits * Mainly for walker-by-walker move. The initial stage of particle-by-particle * move also uses this. causes complete state update in WFC's */ - virtual LogValueType evaluateLog(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L) = 0; + virtual LogValue evaluateLog(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L) = 0; /** evaluate from scratch the same type WaveFunctionComponent of multiple walkers * @param wfc_list the list of WaveFunctionComponent pointers of the same component in a walker batch @@ -272,7 +272,7 @@ class WaveFunctionComponent : public QMCTraits * @param iat the index of a particle * @param grad_iat Gradient for the active particle */ - virtual PsiValueType ratioGrad(ParticleSet& P, int iat, GradType& grad_iat); + virtual PsiValue ratioGrad(ParticleSet& P, int iat, GradType& grad_iat); /** evaluate the ratio of the new to old WaveFunctionComponent value and the new spin gradient * Default implementation assumes that WaveFunctionComponent does not explicitly depend on Spin. @@ -281,7 +281,7 @@ class WaveFunctionComponent : public QMCTraits * @param grad_iat realspace gradient for the active particle * @param spingrad_iat spin gradient for the active particle */ - virtual PsiValueType ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) + virtual PsiValue ratioGradWithSpin(ParticleSet& P, int iat, GradType& grad_iat, ComplexType& spingrad_iat) { return ratioGrad(P, iat, grad_iat); } @@ -290,7 +290,7 @@ class WaveFunctionComponent : public QMCTraits void mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, TWFGrads& grad_new) const; /** compute the ratio of the new to old WaveFunctionComponent value and the new gradient of multiple walkers @@ -303,7 +303,7 @@ class WaveFunctionComponent : public QMCTraits virtual void mw_ratioGrad(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new) const; /** a move for iat-th particle is accepted. Update the current content. @@ -351,7 +351,7 @@ class WaveFunctionComponent : public QMCTraits * * Specialized for particle-by-particle move */ - virtual PsiValueType ratio(ParticleSet& P, int iat) = 0; + virtual PsiValue ratio(ParticleSet& P, int iat) = 0; /** compute the ratio of the new to old WaveFunctionComponent value of multiple walkers * @param wfc_list the list of WaveFunctionComponent pointers of the same component in a walker batch @@ -362,7 +362,7 @@ class WaveFunctionComponent : public QMCTraits virtual void mw_calcRatio(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios) const; + std::vector& ratios) const; /** compute gradients and laplacian of the TWF with respect to each particle. * @param P particle set @@ -372,10 +372,10 @@ class WaveFunctionComponent : public QMCTraits * all the internal data are recomputed from scratch. * @return log(psi) */ - virtual LogValueType evaluateGL(const ParticleSet& P, - ParticleSet::ParticleGradient& G, - ParticleSet::ParticleLaplacian& L, - bool fromscratch); + virtual LogValue evaluateGL(const ParticleSet& P, + ParticleSet::ParticleGradient& G, + ParticleSet::ParticleLaplacian& L, + bool fromscratch); /** evaluate gradients and laplacian of the same type WaveFunctionComponent of multiple walkers * @param wfc_list the list of WaveFunctionComponent pointers of the same component in a walker batch @@ -406,7 +406,7 @@ class WaveFunctionComponent : public QMCTraits * pieces of wavefunction from scratch * @return log value of the wavefunction. */ - virtual LogValueType updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) = 0; + virtual LogValue updateBuffer(ParticleSet& P, WFBufferType& buf, bool fromscratch = false) = 0; /** For particle-by-particle move. Copy data or attach memory * from a walker buffer to the objects of this class. @@ -552,7 +552,7 @@ class WaveFunctionComponent : public QMCTraits virtual void mw_ratioGradWithSpin(const RefVectorWithLeader& wfc_list, const RefVectorWithLeader& p_list, int iat, - std::vector& ratios, + std::vector& ratios, std::vector& grad_new, std::vector& spingrad_new) const; }; diff --git a/src/QMCWaveFunctions/tests/test_2d_jastrow.cpp b/src/QMCWaveFunctions/tests/test_2d_jastrow.cpp index 1c8281e13eb..d85377fc67a 100644 --- a/src/QMCWaveFunctions/tests/test_2d_jastrow.cpp +++ b/src/QMCWaveFunctions/tests/test_2d_jastrow.cpp @@ -17,15 +17,15 @@ #include "QMCWaveFunctions/Jastrow/RadialJastrowBuilder.h" #include "QMCWaveFunctions/Jastrow/TwoBodyJastrow.h" -using std::real; using std::imag; +using std::real; namespace qmcplusplus { -using RealType = WaveFunctionComponent::RealType; -using PsiValueType = WaveFunctionComponent::PsiValueType; -using ValueType = QMCTraits::ValueType; // for real and mixed precision -using PosType = QMCTraits::PosType; +using RealType = WaveFunctionComponent::RealType; +using PsiValue = WaveFunctionComponent::PsiValue; +using ValueType = QMCTraits::ValueType; // for real and mixed precision +using PosType = QMCTraits::PosType; TEST_CASE("Jastrow 2D", "[wavefunction]") { @@ -40,7 +40,7 @@ TEST_CASE("Jastrow 2D", "[wavefunction]") // ParticleSet( SimulationCell( CrystalLattice ) ) CrystalLattice lattice; lattice.BoxBConds = true; - lattice.R.diagonal(70.89815403622065); + lattice.R.diagonal(70.89815403622065); lattice.ndim = 2; lattice.reset(); const SimulationCell cell(lattice); @@ -64,13 +64,14 @@ TEST_CASE("Jastrow 2D", "[wavefunction]") )"; - okay = doc.parseFromString(particle_text); + okay = doc.parseFromString(particle_text); REQUIRE(okay); root = doc.getRoot(); node = xmlFirstElementChild(root); XMLParticleParser parse_electrons(elec); parse_electrons.readXML(node); - int itab; elec.addTable(elec); + int itab; + elec.addTable(elec); elec.update(); // update distance tables const int nelec = elec.getTotalNum(); // read Jastrow component from xml text @@ -84,7 +85,7 @@ TEST_CASE("Jastrow 2D", "[wavefunction]") )"; - okay = doc.parseFromString(jastrow_text); + okay = doc.parseFromString(jastrow_text); REQUIRE(okay); root = doc.getRoot(); node = xmlFirstElementChild(root); @@ -105,14 +106,14 @@ TEST_CASE("Jastrow 2D", "[wavefunction]") CHECK(real(elec.G[2][1]) == Approx(0.009109497845)); CHECK(real(elec.G[3][0]) == Approx(0.04363441629)); CHECK(real(elec.G[3][1]) == Approx(-0.02087588652)); - for (int i=0;i lap_values = {-0.00916449, -0.0166369, -0.00351783, -0.0153977}; - for (int m=0;m ratios(nelec); j2->evaluateRatiosAlltoOne(elec, ratios); std::vector ratio_values = {1.46023, 1.46559, 0.444258, 1.90226}; - for (int i=0;iratio(elec, i); + elec.makeMove(i, newpos - elec.R[i]); + PsiValue rat1 = j2->ratio(elec, i); CHECK(real(rat1) == Approx(ratio_values[i])); elec.rejectMove(i); } @@ -181,11 +182,11 @@ TEST_CASE("Jastrow 2D", "[wavefunction]") app_log() << "param=" << iparam << " : " << dlogpsi[iparam] << " " << dhpsioverpsi[iparam] << std::endl; app_log() << std::endl; const std::vector dlogpsi_values = {0, 0, 0, 0, 0, 0, -1.521258e-04, -2.194165e-01, 0, 0, -2.779981e-02, -5.327999e-01, -9.356640e-01, -4.847466e-01, -1.945081e-02, -2.453297e-01}; - const std::vector dhpsi_values = {0, 0, 0, 0, 0, 0, 5.953288e-03, 8.618836e-03, 0, 0, 2.572195e-02, -5.048126e-02, -3.139861e-02, 2.197638e-02, 4.319522e-02, 3.512834e-02}; + const std::vector dhpsi_values = {0, 0, 0, 0, 0, 0, 5.953288e-03, 8.618836e-03, 0, 0, 2.572195e-02, -5.048126e-02, -3.139861e-02, 2.197638e-02, 4.319522e-02, 3.512834e-02}; for (int iparam = 0; iparam < NumOptimizables; iparam++) { CHECK(real(dlogpsi[iparam]) == Approx(dlogpsi_values[iparam])); CHECK(real(dhpsioverpsi[iparam]) == Approx(dhpsi_values[iparam])); } } -} +} // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/tests/test_DiracDeterminant.cpp b/src/QMCWaveFunctions/tests/test_DiracDeterminant.cpp index 40eb622a6f0..7a4de1225a7 100644 --- a/src/QMCWaveFunctions/tests/test_DiracDeterminant.cpp +++ b/src/QMCWaveFunctions/tests/test_DiracDeterminant.cpp @@ -28,12 +28,12 @@ using std::string; namespace qmcplusplus { -using RealType = QMCTraits::RealType; -using ValueType = QMCTraits::ValueType; -using ComplexType = QMCTraits::ComplexType; -using PosType = QMCTraits::PosType; -using LogValueType = std::complex; -using PsiValueType = QMCTraits::QTFull::ValueType; +using RealType = QMCTraits::RealType; +using ValueType = QMCTraits::ValueType; +using ComplexType = QMCTraits::ComplexType; +using PosType = QMCTraits::PosType; +using LogValue = std::complex; +using PsiValue = QMCTraits::QTFull::ValueType; template void check_matrix(Matrix& a, Matrix& b) @@ -83,8 +83,8 @@ void test_DiracDeterminant_first(const DetMatInvertor inverter_kind) check_matrix(ddb.psiM, b); ParticleSet::GradType grad; - PsiValueType det_ratio = ddb.ratioGrad(elec, 0, grad); - PsiValueType det_ratio1 = 0.178276269185; + PsiValue det_ratio = ddb.ratioGrad(elec, 0, grad); + PsiValue det_ratio1 = 0.178276269185; CHECK(det_ratio1 == ValueApprox(det_ratio)); ddb.acceptMove(elec, 0); @@ -113,7 +113,7 @@ void test_DiracDeterminant_first(const DetMatInvertor inverter_kind) CHECK(std::real(ratios[2]) == Approx(-1.3145695364)); elec.makeMove(0, newpos - elec.R[0]); - PsiValueType ratio_0 = ddb.ratio(elec, 0); + PsiValue ratio_0 = ddb.ratio(elec, 0); elec.rejectMove(0); CHECK(std::real(ratio_0) == Approx(-0.5343861437)); @@ -131,7 +131,7 @@ void test_DiracDeterminant_first(const DetMatInvertor inverter_kind) //test acceptMove elec.makeMove(1, newpos - elec.R[1]); - PsiValueType ratio_1 = ddb.ratio(elec, 1); + PsiValue ratio_1 = ddb.ratio(elec, 1); ddb.acceptMove(elec, 1); elec.acceptMove(1); @@ -219,13 +219,13 @@ void test_DiracDeterminant_second(const DetMatInvertor inverter_kind) } ParticleSet::GradType grad; - PsiValueType det_ratio = ddb.ratioGrad(elec, 0, grad); + PsiValue det_ratio = ddb.ratioGrad(elec, 0, grad); simd::transpose(a_update1.data(), a_update1.rows(), a_update1.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update1; + LogValue det_update1; dm.invert_transpose(scratchT, a_update1, det_update1); - PsiValueType det_ratio1 = LogToValue::convert(det_update1 - ddb.get_log_value()); + PsiValue det_ratio1 = LogToValue::convert(det_update1 - ddb.get_log_value()); #ifdef DUMP_INFO app_log() << "det 0 = " << std::exp(ddb.get_log_value()) << std::endl; app_log() << "det 1 = " << std::exp(det_update1) << std::endl; @@ -237,12 +237,12 @@ void test_DiracDeterminant_second(const DetMatInvertor inverter_kind) ddb.acceptMove(elec, 0); - PsiValueType det_ratio2 = ddb.ratioGrad(elec, 1, grad); - LogValueType det_update2; + PsiValue det_ratio2 = ddb.ratioGrad(elec, 1, grad); + LogValue det_update2; simd::transpose(a_update2.data(), a_update2.rows(), a_update2.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); dm.invert_transpose(scratchT, a_update2, det_update2); - PsiValueType det_ratio2_val = LogToValue::convert(det_update2 - det_update1); + PsiValue det_ratio2_val = LogToValue::convert(det_update2 - det_update1); #ifdef DUMP_INFO app_log() << "det 1 = " << std::exp(ddb.get_log_value()) << std::endl; app_log() << "det 2 = " << std::exp(det_update2) << std::endl; @@ -253,12 +253,12 @@ void test_DiracDeterminant_second(const DetMatInvertor inverter_kind) ddb.acceptMove(elec, 1); - PsiValueType det_ratio3 = ddb.ratioGrad(elec, 2, grad); - LogValueType det_update3; + PsiValue det_ratio3 = ddb.ratioGrad(elec, 2, grad); + LogValue det_update3; simd::transpose(a_update3.data(), a_update3.rows(), a_update3.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); dm.invert_transpose(scratchT, a_update3, det_update3); - PsiValueType det_ratio3_val = LogToValue::convert(det_update3 - det_update2); + PsiValue det_ratio3_val = LogToValue::convert(det_update3 - det_update2); #ifdef DUMP_INFO app_log() << "det 2 = " << std::exp(ddb.get_log_value()) << std::endl; app_log() << "det 3 = " << std::exp(det_update3) << std::endl; @@ -362,13 +362,13 @@ void test_DiracDeterminant_delayed_update(const DetMatInvertor inverter_kind) ParticleSet::GradType grad; - PsiValueType det_ratio = ddc.ratioGrad(elec, 0, grad); + PsiValue det_ratio = ddc.ratioGrad(elec, 0, grad); simd::transpose(a_update1.data(), a_update1.rows(), a_update1.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update1; + LogValue det_update1; dm.invert_transpose(scratchT, a_update1, det_update1); - PsiValueType det_ratio1 = LogToValue::convert(det_update1 - ddc.get_log_value()); + PsiValue det_ratio1 = LogToValue::convert(det_update1 - ddc.get_log_value()); #ifdef DUMP_INFO app_log() << "det 0 = " << std::exp(ddc.get_log_value()) << std::endl; app_log() << "det 1 = " << std::exp(det_update1) << std::endl; @@ -387,12 +387,12 @@ void test_DiracDeterminant_delayed_update(const DetMatInvertor inverter_kind) grad = ddc.evalGrad(elec, 1); - PsiValueType det_ratio2 = ddc.ratioGrad(elec, 1, grad); + PsiValue det_ratio2 = ddc.ratioGrad(elec, 1, grad); simd::transpose(a_update2.data(), a_update2.rows(), a_update2.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update2; + LogValue det_update2; dm.invert_transpose(scratchT, a_update2, det_update2); - PsiValueType det_ratio2_val = LogToValue::convert(det_update2 - det_update1); + PsiValue det_ratio2_val = LogToValue::convert(det_update2 - det_update1); #ifdef DUMP_INFO app_log() << "det 1 = " << std::exp(ddc.get_log_value()) << std::endl; app_log() << "det 2 = " << std::exp(det_update2) << std::endl; @@ -407,12 +407,12 @@ void test_DiracDeterminant_delayed_update(const DetMatInvertor inverter_kind) grad = ddc.evalGrad(elec, 2); - PsiValueType det_ratio3 = ddc.ratioGrad(elec, 2, grad); + PsiValue det_ratio3 = ddc.ratioGrad(elec, 2, grad); simd::transpose(a_update3.data(), a_update3.rows(), a_update3.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update3; + LogValue det_update3; dm.invert_transpose(scratchT, a_update3, det_update3); - PsiValueType det_ratio3_val = LogToValue::convert(det_update3 - det_update2); + PsiValue det_ratio3_val = LogToValue::convert(det_update3 - det_update2); #ifdef DUMP_INFO app_log() << "det 2 = " << std::exp(ddc.get_log_value()) << std::endl; app_log() << "det 3 = " << std::exp(det_update3) << std::endl; @@ -463,7 +463,7 @@ void test_DiracDeterminant_spinor_update(const DetMatInvertor inverter_kind) using ValueType = QMCTraits::ValueType; using PosType = QMCTraits::PosType; using GradType = QMCTraits::GradType; - using LogValueType = WaveFunctionComponent::LogValueType; + using LogValue = WaveFunctionComponent::LogValue; using ParticlePos = ParticleSet::ParticlePos; using ParticleGradient = ParticleSet::ParticleGradient; using ParticleLaplacian = ParticleSet::ParticleLaplacian; @@ -552,7 +552,7 @@ void test_DiracDeterminant_spinor_update(const DetMatInvertor inverter_kind) //In this section, we're going to test that values and various derivatives come out //correctly at the reference configuration. - LogValueType logref = dd.evaluateLog(elec_, G, L); + LogValue logref = dd.evaluateLog(elec_, G, L); CHECK(logref == ComplexApprox(ValueType(-1.1619939279564413, 0.8794794652468605))); CHECK(G[0][0] == ComplexApprox(ValueType(0.13416635, 0.2468612))); @@ -636,7 +636,7 @@ void test_DiracDeterminant_spinor_update(const DetMatInvertor inverter_kind) elec_.makeMoveAndCheckWithSpin(1, dr, ds); elec_.acceptMove(1); - LogValueType lognew(0.0); + LogValue lognew(0.0); G = 0.0; //evalauteLog += onto the G and L arguments. So we zero them out. L = 0.0; SG = 0.0; diff --git a/src/QMCWaveFunctions/tests/test_DiracDeterminantBatched.cpp b/src/QMCWaveFunctions/tests/test_DiracDeterminantBatched.cpp index 0516ad7ec63..4ce591df94f 100644 --- a/src/QMCWaveFunctions/tests/test_DiracDeterminantBatched.cpp +++ b/src/QMCWaveFunctions/tests/test_DiracDeterminantBatched.cpp @@ -25,13 +25,13 @@ namespace qmcplusplus { -using RealType = QMCTraits::RealType; -using ValueType = QMCTraits::ValueType; -using ComplexType = QMCTraits::ComplexType; -using PosType = QMCTraits::PosType; -using GradType = QMCTraits::GradType; -using LogValueType = std::complex; -using PsiValueType = QMCTraits::QTFull::ValueType; +using RealType = QMCTraits::RealType; +using ValueType = QMCTraits::ValueType; +using ComplexType = QMCTraits::ComplexType; +using PosType = QMCTraits::PosType; +using GradType = QMCTraits::GradType; +using LogValue = std::complex; +using PsiValue = QMCTraits::QTFull::ValueType; template void test_DiracDeterminantBatched_first() @@ -70,8 +70,8 @@ void test_DiracDeterminantBatched_first() CHECKED_ELSE(check.result) { FAIL(check.result_message); } ParticleSet::GradType grad; - PsiValueType det_ratio = ddb.ratioGrad(elec, 0, grad); - PsiValueType det_ratio1 = 0.178276269185; + PsiValue det_ratio = ddb.ratioGrad(elec, 0, grad); + PsiValue det_ratio1 = 0.178276269185; CHECK(det_ratio1 == ValueApprox(det_ratio)); ddb.acceptMove(elec, 0); @@ -101,7 +101,7 @@ void test_DiracDeterminantBatched_first() CHECK(std::real(ratios[2]) == Approx(-1.3145695364)); elec.makeMove(0, newpos - elec.R[0]); - PsiValueType ratio_0 = ddb.ratio(elec, 0); + PsiValue ratio_0 = ddb.ratio(elec, 0); elec.rejectMove(0); CHECK(std::real(ratio_0) == Approx(-0.5343861437)); @@ -119,7 +119,7 @@ void test_DiracDeterminantBatched_first() //test acceptMove elec.makeMove(1, newpos - elec.R[1]); - PsiValueType ratio_1 = ddb.ratio(elec, 1); + PsiValue ratio_1 = ddb.ratio(elec, 1); ddb.acceptMove(elec, 1); elec.acceptMove(1); @@ -201,13 +201,13 @@ void test_DiracDeterminantBatched_second() } ParticleSet::GradType grad; - PsiValueType det_ratio = ddb.ratioGrad(elec, 0, grad); + PsiValue det_ratio = ddb.ratioGrad(elec, 0, grad); simd::transpose(a_update1.data(), a_update1.rows(), a_update1.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update1; + LogValue det_update1; dm.invert_transpose(scratchT, a_update1, det_update1); - PsiValueType det_ratio1 = LogToValue::convert(det_update1 - ddb.get_log_value()); + PsiValue det_ratio1 = LogToValue::convert(det_update1 - ddb.get_log_value()); #ifdef DUMP_INFO app_log() << "det 0 = " << std::exp(ddb.get_log_value()) << std::endl; app_log() << "det 1 = " << std::exp(det_update1) << std::endl; @@ -219,12 +219,12 @@ void test_DiracDeterminantBatched_second() ddb.acceptMove(elec, 0); - PsiValueType det_ratio2 = ddb.ratioGrad(elec, 1, grad); - LogValueType det_update2; + PsiValue det_ratio2 = ddb.ratioGrad(elec, 1, grad); + LogValue det_update2; simd::transpose(a_update2.data(), a_update2.rows(), a_update2.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); dm.invert_transpose(scratchT, a_update2, det_update2); - PsiValueType det_ratio2_val = LogToValue::convert(det_update2 - det_update1); + PsiValue det_ratio2_val = LogToValue::convert(det_update2 - det_update1); #ifdef DUMP_INFO app_log() << "det 1 = " << std::exp(ddb.get_log_value()) << std::endl; app_log() << "det 2 = " << std::exp(det_update2) << std::endl; @@ -235,12 +235,12 @@ void test_DiracDeterminantBatched_second() ddb.acceptMove(elec, 1); - PsiValueType det_ratio3 = ddb.ratioGrad(elec, 2, grad); - LogValueType det_update3; + PsiValue det_ratio3 = ddb.ratioGrad(elec, 2, grad); + LogValue det_update3; simd::transpose(a_update3.data(), a_update3.rows(), a_update3.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); dm.invert_transpose(scratchT, a_update3, det_update3); - PsiValueType det_ratio3_val = LogToValue::convert(det_update3 - det_update2); + PsiValue det_ratio3_val = LogToValue::convert(det_update3 - det_update2); #ifdef DUMP_INFO app_log() << "det 2 = " << std::exp(ddb.get_log_value()) << std::endl; app_log() << "det 3 = " << std::exp(det_update3) << std::endl; @@ -338,13 +338,13 @@ void test_DiracDeterminantBatched_delayed_update(int delay_rank, DetMatInvertor ParticleSet::GradType grad; - PsiValueType det_ratio = ddc.ratioGrad(elec, 0, grad); + PsiValue det_ratio = ddc.ratioGrad(elec, 0, grad); simd::transpose(a_update1.data(), a_update1.rows(), a_update1.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update1; + LogValue det_update1; dm.invert_transpose(scratchT, a_update1, det_update1); - PsiValueType det_ratio1 = LogToValue::convert(det_update1 - ddc.get_log_value()); + PsiValue det_ratio1 = LogToValue::convert(det_update1 - ddc.get_log_value()); #ifdef DUMP_INFO app_log() << "det 0 = " << std::exp(ddc.get_log_value()) << std::endl; app_log() << "det 1 = " << std::exp(det_update1) << std::endl; @@ -364,12 +364,12 @@ void test_DiracDeterminantBatched_delayed_update(int delay_rank, DetMatInvertor grad = ddc.evalGrad(elec, 1); - PsiValueType det_ratio2 = ddc.ratioGrad(elec, 1, grad); + PsiValue det_ratio2 = ddc.ratioGrad(elec, 1, grad); simd::transpose(a_update2.data(), a_update2.rows(), a_update2.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update2; + LogValue det_update2; dm.invert_transpose(scratchT, a_update2, det_update2); - PsiValueType det_ratio2_val = LogToValue::convert(det_update2 - det_update1); + PsiValue det_ratio2_val = LogToValue::convert(det_update2 - det_update1); #ifdef DUMP_INFO app_log() << "det 1 = " << std::exp(ddc.get_log_value()) << std::endl; app_log() << "det 2 = " << std::exp(det_update2) << std::endl; @@ -384,12 +384,12 @@ void test_DiracDeterminantBatched_delayed_update(int delay_rank, DetMatInvertor grad = ddc.evalGrad(elec, 2); - PsiValueType det_ratio3 = ddc.ratioGrad(elec, 2, grad); + PsiValue det_ratio3 = ddc.ratioGrad(elec, 2, grad); simd::transpose(a_update3.data(), a_update3.rows(), a_update3.cols(), scratchT.data(), scratchT.rows(), scratchT.cols()); - LogValueType det_update3; + LogValue det_update3; dm.invert_transpose(scratchT, a_update3, det_update3); - PsiValueType det_ratio3_val = LogToValue::convert(det_update3 - det_update2); + PsiValue det_ratio3_val = LogToValue::convert(det_update3 - det_update2); #ifdef DUMP_INFO app_log() << "det 2 = " << std::exp(ddc.get_log_value()) << std::endl; app_log() << "det 3 = " << std::exp(det_update3) << std::endl; @@ -441,7 +441,7 @@ void test_DiracDeterminantBatched_delayed_update(int delay_rank, DetMatInvertor ParticleSet::mw_update(p_ref_list); ddc.mw_recompute(ddc_ref_list, p_ref_list, isAccepted); - std::vector ratios(2); + std::vector ratios(2); std::vector grad_new(2); ddc.mw_ratioGrad(ddc_ref_list, p_ref_list, 0, ratios, grad_new); @@ -591,7 +591,7 @@ void test_DiracDeterminantBatched_spinor_update(const int delay_rank, DetMatInve //In this section, we're going to test that values and various derivatives come out //correctly at the reference configuration. - LogValueType logref = dd.evaluateLog(elec_, G, L); + LogValue logref = dd.evaluateLog(elec_, G, L); CHECK(logref == ComplexApprox(ValueType(-1.1619939279564413, 0.8794794652468605))); CHECK(G[0][0] == ComplexApprox(ValueType(0.13416635, 0.2468612))); @@ -675,7 +675,7 @@ void test_DiracDeterminantBatched_spinor_update(const int delay_rank, DetMatInve elec_.makeMoveAndCheckWithSpin(1, dr, ds); elec_.acceptMove(1); - LogValueType lognew(0.0); + LogValue lognew(0.0); G = 0.0; //evalauteLog += onto the G and L arguments. So we zero them out. L = 0.0; SG = 0.0; @@ -727,7 +727,7 @@ void test_DiracDeterminantBatched_spinor_update(const int delay_rank, DetMatInve dd.mw_evaluateLog(dd_ref_list, p_ref_list, G_list, L_list); for (int iw = 0; iw < dd_ref_list.size(); iw++) { - PsiValueType ref = dd_ref_list[iw].getValue(); + PsiValue ref = dd_ref_list[iw].getValue(); CHECK(std::log(ref) == ComplexApprox(ValueType(-1.1619939279564413, 0.8794794652468605))); CHECK(G_list[iw].get()[0][0] == ComplexApprox(ValueType(0.13416635, 0.2468612))); CHECK(G_list[iw].get()[0][1] == ComplexApprox(ValueType(-1.1165475, 0.71497753))); @@ -750,7 +750,7 @@ void test_DiracDeterminantBatched_spinor_update(const int delay_rank, DetMatInve elec_.mw_makeMove(p_ref_list, 1, displs); //Check ratios and grads for both walkers for proposed move - std::vector ratios(2); + std::vector ratios(2); std::vector grads(2); std::vector spingrads(2); dd.mw_ratioGrad(dd_ref_list, p_ref_list, 1, ratios, grads); @@ -807,7 +807,7 @@ void test_DiracDeterminantBatched_spinor_update(const int delay_rank, DetMatInve dd.mw_evaluateLog(dd_ref_list, p_ref_list, G_list, L_list); for (int iw = 0; iw < dd_ref_list.size(); iw++) { - PsiValueType ref = dd_ref_list[iw].getValue(); + PsiValue ref = dd_ref_list[iw].getValue(); CHECK(std::log(ref) == ComplexApprox(ValueType(-0.41337396772929913, 1.4774106123071726))); CHECK(G_list[iw].get()[1][0] == ComplexApprox(ValueType(0.5496675534224996, -0.07968022499097227))); CHECK(G_list[iw].get()[1][1] == ComplexApprox(ValueType(0.4927399293808675, -0.29971549854643653))); diff --git a/src/QMCWaveFunctions/tests/test_DiracMatrix.cpp b/src/QMCWaveFunctions/tests/test_DiracMatrix.cpp index 643122c2fae..3b17a4b84d1 100644 --- a/src/QMCWaveFunctions/tests/test_DiracMatrix.cpp +++ b/src/QMCWaveFunctions/tests/test_DiracMatrix.cpp @@ -28,17 +28,17 @@ using std::string; namespace qmcplusplus { -using RealType = QMCTraits::RealType; -using ValueType = QMCTraits::ValueType; -using LogValueType = std::complex; +using RealType = QMCTraits::RealType; +using ValueType = QMCTraits::ValueType; +using LogValue = std::complex; using LogComplexApprox = Catch::Detail::LogComplexApprox; - + TEST_CASE("DiracMatrix_identity", "[wavefunction][fermion]") { DiracMatrix dm; Matrix m, m_invT; - LogValueType log_value; + LogValue log_value; m.resize(3, 3); m_invT.resize(3, 3); @@ -60,7 +60,7 @@ TEST_CASE("DiracMatrix_inverse", "[wavefunction][fermion]") DiracMatrix dm; Matrix a, a_T, a_inv; - LogValueType log_value; + LogValue log_value; a.resize(3, 3); a_T.resize(3, 3); a_inv.resize(3, 3); @@ -85,7 +85,7 @@ TEST_CASE("DiracMatrix_inverse_matching", "[wavefunction][fermion]") DiracMatrix dm; Matrix a, a_T, a_inv; - LogValueType log_value; + LogValue log_value; a.resize(4, 4); a_T.resize(4, 4); a_inv.resize(4, 4); @@ -139,7 +139,7 @@ TEST_CASE("DiracMatrix_inverse_matching_2", "[wavefunction][fermion]") DiracMatrix dm; Matrix a, a_T, a_inv; - LogValueType log_value; + LogValue log_value; a.resize(4, 4); a_T.resize(4, 4); a_inv.resize(4, 4); @@ -222,7 +222,7 @@ TEST_CASE("DiracMatrix_inverse_complex", "[wavefunction][fermion]") DiracMatrix> dm; Matrix> a, a_T, a_inv; - LogValueType log_value; + LogValue log_value; a.resize(4, 4); a_T.resize(4, 4); a_inv.resize(4, 4); @@ -319,7 +319,7 @@ TEST_CASE("DiracMatrix_update_row", "[wavefunction][fermion]") updateEng.resize(3, 1); Matrix a, a_T, a_inv; - LogValueType log_value; + LogValue log_value; a.resize(3, 3); a_T.resize(3, 3); a_inv.resize(3, 3); diff --git a/src/QMCWaveFunctions/tests/test_J1Spin.cpp b/src/QMCWaveFunctions/tests/test_J1Spin.cpp index a20f27718d5..5edd00264de 100644 --- a/src/QMCWaveFunctions/tests/test_J1Spin.cpp +++ b/src/QMCWaveFunctions/tests/test_J1Spin.cpp @@ -20,9 +20,9 @@ namespace qmcplusplus { -using RealType = WaveFunctionComponent::RealType; -using LogValueType = WaveFunctionComponent::LogValueType; -using ValueType = QMCTraits::ValueType; +using RealType = WaveFunctionComponent::RealType; +using LogValue = WaveFunctionComponent::LogValue; +using ValueType = QMCTraits::ValueType; TEST_CASE("J1 spin evaluate derivatives Jastrow", "[wavefunction]") { @@ -97,15 +97,15 @@ TEST_CASE("J1 spin evaluate derivatives Jastrow", "[wavefunction]") // check logs //evaluateLog += into G + L so reset - elec_.G = 0.0; - elec_.L = 0.0; - LogValueType log = twf_component_list[0]->evaluateLog(elec_, elec_.G, elec_.L); - LogValueType expected_log{-0.568775, 0.0}; + elec_.G = 0.0; + elec_.L = 0.0; + LogValue log = twf_component_list[0]->evaluateLog(elec_, elec_.G, elec_.L); + LogValue expected_log{-0.568775, 0.0}; CHECK(log == LogComplexApprox(expected_log)); //evaluateLog += into G + L so reset - elec_.G = 0.0; - elec_.L = 0.0; - LogValueType cloned_log = cloned_j1spin->evaluateLog(elec_, elec_.G, elec_.L); + elec_.G = 0.0; + elec_.L = 0.0; + LogValue cloned_log = cloned_j1spin->evaluateLog(elec_, elec_.G, elec_.L); CHECK(cloned_log == LogComplexApprox(expected_log)); // check derivatives @@ -142,38 +142,38 @@ TEST_CASE("J1 spin evaluate derivatives multiparticle Jastrow", "[wavefunction]" ions_.setName("ion0"); ptcl.addParticleSet(std::move(ions_uptr)); ions_.create({2}); - ions_.R[0] = {-1.0, 0.0, 0.0}; - ions_.R[1] = { 1.0, 0.0, 0.0}; - SpeciesSet& ispecies = ions_.getSpeciesSet(); + ions_.R[0] = {-1.0, 0.0, 0.0}; + ions_.R[1] = {1.0, 0.0, 0.0}; + SpeciesSet& ispecies = ions_.getSpeciesSet(); int BeIdx = ispecies.addSpecies("Be"); - int ichargeIdx = ispecies.addAttribute("charge"); + int ichargeIdx = ispecies.addAttribute("charge"); ispecies(ichargeIdx, BeIdx) = 4.0; elec_.setName("e"); ptcl.addParticleSet(std::move(elec_uptr)); elec_.create({4, 4, 1}); - elec_.R[0] = { 0.5, 0.5, 0.5}; - elec_.R[1] = {-0.5, 0.5, 0.5}; - elec_.R[2] = { 0.5, -0.5, 0.5}; - elec_.R[3] = { 0.5, 0.5, -0.5}; - elec_.R[4] = {-0.5, -0.5, 0.5}; - elec_.R[5] = { 0.5, -0.5, -0.5}; - elec_.R[6] = {-0.5, 0.5, -0.5}; + elec_.R[0] = {0.5, 0.5, 0.5}; + elec_.R[1] = {-0.5, 0.5, 0.5}; + elec_.R[2] = {0.5, -0.5, 0.5}; + elec_.R[3] = {0.5, 0.5, -0.5}; + elec_.R[4] = {-0.5, -0.5, 0.5}; + elec_.R[5] = {0.5, -0.5, -0.5}; + elec_.R[6] = {-0.5, 0.5, -0.5}; elec_.R[7] = {-0.5, -0.5, -0.5}; - elec_.R[8] = { 1.5, 1.5, 1.5}; + elec_.R[8] = {1.5, 1.5, 1.5}; SpeciesSet& tspecies = elec_.getSpeciesSet(); int upIdx = tspecies.addSpecies("u"); int downIdx = tspecies.addSpecies("d"); - int posIdx = tspecies.addSpecies("p"); + int posIdx = tspecies.addSpecies("p"); int massIdx = tspecies.addAttribute("mass"); int chargeIdx = tspecies.addAttribute("charge"); tspecies(massIdx, upIdx) = 1.0; tspecies(massIdx, downIdx) = 1.0; - tspecies(massIdx, posIdx) = 1.0; + tspecies(massIdx, posIdx) = 1.0; tspecies(chargeIdx, upIdx) = -1.0; tspecies(massIdx, downIdx) = -1.0; - tspecies(massIdx, posIdx) = 1.0; + tspecies(massIdx, posIdx) = 1.0; // Necessary to set mass elec_.resetGroups(); @@ -216,15 +216,15 @@ TEST_CASE("J1 spin evaluate derivatives multiparticle Jastrow", "[wavefunction]" // check logs //evaluateLog += into G + L so reset - elec_.G = 0.0; - elec_.L = 0.0; - LogValueType log = twf_component_list[0]->evaluateLog(elec_, elec_.G, elec_.L); - LogValueType expected_log{-3.58983, 0.0}; + elec_.G = 0.0; + elec_.L = 0.0; + LogValue log = twf_component_list[0]->evaluateLog(elec_, elec_.G, elec_.L); + LogValue expected_log{-3.58983, 0.0}; CHECK(log == LogComplexApprox(expected_log)); //evaluateLog += into G + L so reset - elec_.G = 0.0; - elec_.L = 0.0; - LogValueType cloned_log = cloned_j1spin->evaluateLog(elec_, elec_.G, elec_.L); + elec_.G = 0.0; + elec_.L = 0.0; + LogValue cloned_log = cloned_j1spin->evaluateLog(elec_, elec_.G, elec_.L); CHECK(cloned_log == LogComplexApprox(expected_log)); // check derivatives diff --git a/src/QMCWaveFunctions/tests/test_J1_bspline.cpp b/src/QMCWaveFunctions/tests/test_J1_bspline.cpp index 1f4701d5c76..1b69314b9fe 100644 --- a/src/QMCWaveFunctions/tests/test_J1_bspline.cpp +++ b/src/QMCWaveFunctions/tests/test_J1_bspline.cpp @@ -33,8 +33,8 @@ using std::string; namespace qmcplusplus { -using RealType = WaveFunctionComponent::RealType; -using PsiValueType = WaveFunctionComponent::PsiValueType; +using RealType = WaveFunctionComponent::RealType; +using PsiValue = WaveFunctionComponent::PsiValue; TEST_CASE("BSpline functor zero", "[wavefunction]") { @@ -288,14 +288,14 @@ void test_J1_spline(const DynamicCoordinateKind kind_selected) CHECK(std::real(ratios[1]) == Approx(1.0040884258)); elec_.makeMove(0, newpos - elec_.R[0]); - PsiValueType ratio_0 = j1->ratio(elec_, 0); + PsiValue ratio_0 = j1->ratio(elec_, 0); elec_.rejectMove(0); CHECK(std::real(ratio_0) == Approx(0.9819208747)); // test acceptMove results elec_.makeMove(1, newpos - elec_.R[1]); - PsiValueType ratio_1 = j1->ratio(elec_, 1); + PsiValue ratio_1 = j1->ratio(elec_, 1); j1->acceptMove(elec_, 1); elec_.acceptMove(1); diff --git a/src/QMCWaveFunctions/tests/test_J2_bspline.cpp b/src/QMCWaveFunctions/tests/test_J2_bspline.cpp index 9c035de0239..efd6c01fe83 100644 --- a/src/QMCWaveFunctions/tests/test_J2_bspline.cpp +++ b/src/QMCWaveFunctions/tests/test_J2_bspline.cpp @@ -32,8 +32,8 @@ using std::string; namespace qmcplusplus { -using RealType = WaveFunctionComponent::RealType; -using PsiValueType = WaveFunctionComponent::PsiValueType; +using RealType = WaveFunctionComponent::RealType; +using PsiValue = WaveFunctionComponent::PsiValue; TEST_CASE("BSpline builder Jastrow J2", "[wavefunction]") { @@ -221,7 +221,7 @@ TEST_CASE("BSpline builder Jastrow J2", "[wavefunction]") CHECK(std::real(ratios[1]) == Approx(0.9871985577)); elec_.makeMove(0, newpos - elec_.R[0]); - PsiValueType ratio_0 = j2->ratio(elec_, 0); + PsiValue ratio_0 = j2->ratio(elec_, 0); elec_.rejectMove(0); CHECK(std::real(ratio_0) == Approx(0.9522052017)); @@ -239,7 +239,7 @@ TEST_CASE("BSpline builder Jastrow J2", "[wavefunction]") //test acceptMove elec_.makeMove(1, newpos - elec_.R[1]); - PsiValueType ratio_1 = j2->ratio(elec_, 1); + PsiValue ratio_1 = j2->ratio(elec_, 1); j2->acceptMove(elec_, 1); elec_.acceptMove(1); diff --git a/src/QMCWaveFunctions/tests/test_TrialWaveFunction.cpp b/src/QMCWaveFunctions/tests/test_TrialWaveFunction.cpp index 091af357686..e0150004cc8 100644 --- a/src/QMCWaveFunctions/tests/test_TrialWaveFunction.cpp +++ b/src/QMCWaveFunctions/tests/test_TrialWaveFunction.cpp @@ -33,9 +33,9 @@ using DiracDet = DiracDeterminant>; #endif -using LogValueType = TrialWaveFunction::LogValueType; -using PsiValueType = TrialWaveFunction::PsiValueType; -using GradType = TrialWaveFunction::GradType; +using LogValue = TrialWaveFunction::LogValue; +using PsiValue = TrialWaveFunction::PsiValue; +using GradType = TrialWaveFunction::GradType; TEST_CASE("TrialWaveFunction_diamondC_1x1x1", "[wavefunction]") { @@ -285,12 +285,12 @@ TEST_CASE("TrialWaveFunction_diamondC_1x1x1", "[wavefunction]") displs = {delta_zero, delta}; ParticleSet::mw_makeMove(p_ref_list, moved_elec_id, displs); - std::vector ratios(2); + std::vector ratios(2); TrialWaveFunction::mw_calcRatio(wf_ref_list, p_ref_list, moved_elec_id, ratios); app_log() << "calcRatio " << std::setprecision(14) << ratios[0] << " " << ratios[1] << std::endl; #if defined(QMC_COMPLEX) - CHECK(ratios[0] == ComplexApprox(PsiValueType(1, 0))); - CHECK(ratios[1] == ComplexApprox(PsiValueType(1.6538214581548, 0.54849918598717))); + CHECK(ratios[0] == ComplexApprox(PsiValue(1, 0))); + CHECK(ratios[1] == ComplexApprox(PsiValue(1.6538214581548, 0.54849918598717))); #else CHECK(ratios[0] == Approx(1)); CHECK(ratios[1] == Approx(2.3055913093424)); diff --git a/src/QMCWaveFunctions/tests/test_TrialWaveFunction_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_TrialWaveFunction_diamondC_2x1x1.cpp index e36e2f59c26..558eed3ab3d 100644 --- a/src/QMCWaveFunctions/tests/test_TrialWaveFunction_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_TrialWaveFunction_diamondC_2x1x1.cpp @@ -30,9 +30,9 @@ namespace qmcplusplus { -using LogValueType = TrialWaveFunction::LogValueType; -using PsiValueType = TrialWaveFunction::PsiValueType; -using GradType = TrialWaveFunction::GradType; +using LogValue = TrialWaveFunction::LogValue; +using PsiValue = TrialWaveFunction::PsiValue; +using GradType = TrialWaveFunction::GradType; struct double_tag {}; @@ -345,13 +345,13 @@ void testTrialWaveFunction_diamondC_2x1x1(const int ndelay, const OffloadSwitche displs = {delta_zero, delta}; ParticleSet::mw_makeMove(p_ref_list, moved_elec_id, displs); - std::vector ratios(2); + std::vector ratios(2); TrialWaveFunction::mw_calcRatio(wf_ref_list, p_ref_list, moved_elec_id, ratios); app_log() << "mixed move calcRatio " << std::setprecision(14) << ratios[0] << " " << ratios[1] << std::endl; #if defined(QMC_COMPLEX) - CHECK(ratios[0] == ComplexApprox(PsiValueType(1, 0)).epsilon(5e-4)); - CHECK(ratios[1] == ComplexApprox(PsiValueType(0.12487384604679, 0)).epsilon(5e-5)); + CHECK(ratios[0] == ComplexApprox(PsiValue(1, 0)).epsilon(5e-4)); + CHECK(ratios[1] == ComplexApprox(PsiValue(0.12487384604679, 0)).epsilon(5e-5)); #else CHECK(ratios[0] == Approx(1).epsilon(5e-5)); #if defined(MIXED_PRECISION) @@ -509,16 +509,16 @@ void testTrialWaveFunction_diamondC_2x1x1(const int ndelay, const OffloadSwitche CHECK(det_up->getPsiMinv()[1][1] == Approx(45.51992500251).epsilon(1e-4)); #endif #endif // NDEBUG - std::vector log_values(wf_ref_list.size()); + std::vector log_values(wf_ref_list.size()); TrialWaveFunction::mw_evaluateGL(wf_ref_list, p_ref_list, false); for (int iw = 0; iw < log_values.size(); iw++) log_values[iw] = {wf_ref_list[iw].getLogPsi(), wf_ref_list[iw].getPhase()}; #if defined(QMC_COMPLEX) - CHECK(LogComplexApprox(log_values[0]) == LogValueType{-4.1148130068943, -6.2831779860047}); - CHECK(LogComplexApprox(log_values[1]) == LogValueType{-6.6269077659586, -3.1416312090662}); + CHECK(LogComplexApprox(log_values[0]) == LogValue{-4.1148130068943, -6.2831779860047}); + CHECK(LogComplexApprox(log_values[1]) == LogValue{-6.6269077659586, -3.1416312090662}); #else - CHECK(LogComplexApprox(log_values[0]) == LogValueType{-5.5011162672993, 9.4247779607694}); - CHECK(LogComplexApprox(log_values[1]) == LogValueType{-8.0131646238354, 6.2831853071796}); + CHECK(LogComplexApprox(log_values[0]) == LogValue{-5.5011162672993, 9.4247779607694}); + CHECK(LogComplexApprox(log_values[1]) == LogValue{-8.0131646238354, 6.2831853071796}); #endif // This test has 4 electrons but only 2 particle moves are attempted. @@ -527,7 +527,7 @@ void testTrialWaveFunction_diamondC_2x1x1(const int ndelay, const OffloadSwitche ParticleSet::mw_update(p_ref_list); TrialWaveFunction::mw_evaluateGL(wf_ref_list, p_ref_list, true); for (int iw = 0; iw < log_values.size(); iw++) - CHECK(LogComplexApprox(log_values[iw]) == LogValueType{wf_ref_list[iw].getLogPsi(), wf_ref_list[iw].getPhase()}); + CHECK(LogComplexApprox(log_values[iw]) == LogValue{wf_ref_list[iw].getLogPsi(), wf_ref_list[iw].getPhase()}); // test NLPP related APIs const int nknot = 3; diff --git a/src/QMCWaveFunctions/tests/test_counting_jastrow.cpp b/src/QMCWaveFunctions/tests/test_counting_jastrow.cpp index d53acbfd530..8e7e905e965 100644 --- a/src/QMCWaveFunctions/tests/test_counting_jastrow.cpp +++ b/src/QMCWaveFunctions/tests/test_counting_jastrow.cpp @@ -27,10 +27,10 @@ namespace qmcplusplus { // CountingGaussian unit tests -TEST_CASE("Gaussian Functor","[wavefunction]") +TEST_CASE("Gaussian Functor", "[wavefunction]") { - using RealType = QMCTraits::RealType; - using PosType = QMCTraits::PosType; + using RealType = QMCTraits::RealType; + using PosType = QMCTraits::PosType; using TensorType = QMCTraits::TensorType; CountingGaussian gf_abc("gf_abc"); @@ -48,16 +48,16 @@ TEST_CASE("Gaussian Functor","[wavefunction]") 0.0 )"; Libxml2Document doc_abc, doc_adk; - bool parse_abc = doc_abc.parseFromString(gaussian_xml_abc); - bool parse_adk = doc_adk.parseFromString(gaussian_xml_adk); + bool parse_abc = doc_abc.parseFromString(gaussian_xml_abc); + bool parse_adk = doc_adk.parseFromString(gaussian_xml_adk); xmlNodePtr root_abc = doc_abc.getRoot(); xmlNodePtr root_adk = doc_adk.getRoot(); - bool put_abc = gf_abc.put(root_abc); - bool put_adk = gf_adk.put(root_adk); - REQUIRE( (parse_abc && parse_adk && put_abc && put_adk) == true); + bool put_abc = gf_abc.put(root_abc); + bool put_adk = gf_adk.put(root_adk); + REQUIRE((parse_abc && parse_adk && put_abc && put_adk) == true); - // test points - PosType r1(1,0,0); + // test points + PosType r1(1, 0, 0); PosType r2(1.707106781186547, 0.5, -0.5); PosType r3(0.4714617144631338, 0.1499413068889379, 0.2932213074999387); @@ -76,63 +76,63 @@ TEST_CASE("Gaussian Functor","[wavefunction]") // value tests for ADK input gf_adk.evaluate(r1, fval, fgrad, flap); gf_adk.evaluateLog(r1, lval, lgrad, llap); - CHECK( fval == Approx(1) ); - CHECK( fgrad[0] == Approx(0) ); - CHECK( fgrad[1] == Approx(0) ); - CHECK( fgrad[2] == Approx(0) ); - CHECK( flap == Approx(-12) ); - CHECK( lval == Approx(0) ); - CHECK( lgrad[0] == Approx(0) ); - CHECK( lgrad[1] == Approx(0) ); - CHECK( lgrad[2] == Approx(0) ); - CHECK( llap == Approx(-12) ); + CHECK(fval == Approx(1)); + CHECK(fgrad[0] == Approx(0)); + CHECK(fgrad[1] == Approx(0)); + CHECK(fgrad[2] == Approx(0)); + CHECK(flap == Approx(-12)); + CHECK(lval == Approx(0)); + CHECK(lgrad[0] == Approx(0)); + CHECK(lgrad[1] == Approx(0)); + CHECK(lgrad[2] == Approx(0)); + CHECK(llap == Approx(-12)); // value tests for ABC input gf_abc.evaluate(r1, fval, fgrad, flap); gf_abc.evaluateLog(r1, lval, lgrad, llap); - CHECK( fval == Approx(1) ); - CHECK( fgrad[0] == Approx(0) ); - CHECK( fgrad[1] == Approx(0) ); - CHECK( fgrad[2] == Approx(0) ); - CHECK( flap == Approx(-12) ); - CHECK( lval == Approx(0) ); - CHECK( lgrad[0] == Approx(0) ); - CHECK( lgrad[1] == Approx(0) ); - CHECK( lgrad[2] == Approx(0) ); - CHECK( llap == Approx(-12) ); + CHECK(fval == Approx(1)); + CHECK(fgrad[0] == Approx(0)); + CHECK(fgrad[1] == Approx(0)); + CHECK(fgrad[2] == Approx(0)); + CHECK(flap == Approx(-12)); + CHECK(lval == Approx(0)); + CHECK(lgrad[0] == Approx(0)); + CHECK(lgrad[1] == Approx(0)); + CHECK(lgrad[2] == Approx(0)); + CHECK(llap == Approx(-12)); // evaluateDerivatives gf_abc.evaluateDerivatives(r3, dfval, dfgrad, dflap); - CHECK( dfval[0] == Approx(0.113120472934)); - CHECK( dfval[1] == Approx(0.071952529875)); - CHECK( dfval[2] == Approx(0.140708490047)); - CHECK( dfval[3] == Approx(0.011441709933)); - CHECK( dfval[4] == Approx(0.044750218821)); - CHECK( dfval[5] == Approx(0.043756180154)); - CHECK( dfval[6] == Approx(-0.47987130010)); - CHECK( dfval[7] == Approx(-0.15261584911)); - CHECK( dfval[8] == Approx(-0.29845157248)); - CHECK( dfval[9] == Approx(0.508918630484)); + CHECK(dfval[0] == Approx(0.113120472934)); + CHECK(dfval[1] == Approx(0.071952529875)); + CHECK(dfval[2] == Approx(0.140708490047)); + CHECK(dfval[3] == Approx(0.011441709933)); + CHECK(dfval[4] == Approx(0.044750218821)); + CHECK(dfval[5] == Approx(0.043756180154)); + CHECK(dfval[6] == Approx(-0.47987130010)); + CHECK(dfval[7] == Approx(-0.15261584911)); + CHECK(dfval[8] == Approx(-0.29845157248)); + CHECK(dfval[9] == Approx(0.508918630484)); // evaluateLogDerivatives gf_abc.evaluateLogDerivatives(r3, dfval, dfgrad, dflap); - CHECK( dfval[0] == Approx(0.222276148205)); - CHECK( dfval[1] == Approx(0.141383171229)); - CHECK( dfval[2] == Approx(0.276485240702)); - CHECK( dfval[3] == Approx(0.022482395511)); - CHECK( dfval[4] == Approx(0.087931972108)); - CHECK( dfval[5] == Approx(0.085978735172)); - CHECK( dfval[6] == Approx(-0.94292342892)); - CHECK( dfval[7] == Approx(-0.29988261377)); - CHECK( dfval[8] == Approx(-0.58644261500)); - CHECK( dfval[9] == Approx(1)); + CHECK(dfval[0] == Approx(0.222276148205)); + CHECK(dfval[1] == Approx(0.141383171229)); + CHECK(dfval[2] == Approx(0.276485240702)); + CHECK(dfval[3] == Approx(0.022482395511)); + CHECK(dfval[4] == Approx(0.087931972108)); + CHECK(dfval[5] == Approx(0.085978735172)); + CHECK(dfval[6] == Approx(-0.94292342892)); + CHECK(dfval[7] == Approx(-0.29988261377)); + CHECK(dfval[8] == Approx(-0.58644261500)); + CHECK(dfval[9] == Approx(1)); } -TEST_CASE("CountingJastrow","[wavefunction]") +TEST_CASE("CountingJastrow", "[wavefunction]") { - using PosType = QMCTraits::PosType; - using GradType = QMCTraits::GradType; - using RealType = QMCTraits::RealType; - using ValueType = QMCTraits::ValueType; + using PosType = QMCTraits::PosType; + using GradType = QMCTraits::GradType; + using RealType = QMCTraits::RealType; + using ValueType = QMCTraits::ValueType; using VariableSet = optimize::VariableSet; - using LogValueType = std::complex; + using LogValue = std::complex; Communicate* c = OHMMS::Controller; @@ -141,30 +141,28 @@ TEST_CASE("CountingJastrow","[wavefunction]") ParticleSet elec(simulation_cell); std::vector egroup(1); int num_els = 4; - egroup[0] = num_els; + egroup[0] = num_els; elec.setName("e"); elec.create(egroup); - PosType Re[] = { PosType(2.4601162537, 6.7476360528,-1.9073129953), - PosType(2.2585811248, 2.1282254384,0.051545776028), - PosType(0.84796873937,5.1735597110,0.84642416761), - PosType(3.1597337850, 5.1079432473,1.0545953717)}; - for(int i = 0; i < num_els; ++i) - for(int k = 0; k < 3; ++k) + PosType Re[] = {PosType(2.4601162537, 6.7476360528, -1.9073129953), + PosType(2.2585811248, 2.1282254384, 0.051545776028), + PosType(0.84796873937, 5.1735597110, 0.84642416761), + PosType(3.1597337850, 5.1079432473, 1.0545953717)}; + for (int i = 0; i < num_els; ++i) + for (int k = 0; k < 3; ++k) elec.R[i][k] = Re[i][k]; ParticleSet ion0(simulation_cell); std::vector igroup(1); int num_ion = 4; - igroup[0] = num_ion; + igroup[0] = num_ion; ion0.setName("ion0"); ion0.create(igroup); - PosType Ri[] = {PosType(2.61363352510301, 5.01928226905281, 0.0), - PosType(3.74709851167814, 3.70007145722224, 0.0), - PosType(6.11011670934565, 1.66504047681825, 0.0), - PosType(8.10584803421091, 7.78608266172472, 0.0)}; - - for(int i = 0; i < num_ion; ++i) - for(int k = 0; k < 3; ++k) + PosType Ri[] = {PosType(2.61363352510301, 5.01928226905281, 0.0), PosType(3.74709851167814, 3.70007145722224, 0.0), + PosType(6.11011670934565, 1.66504047681825, 0.0), PosType(8.10584803421091, 7.78608266172472, 0.0)}; + + for (int i = 0; i < num_ion; ++i) + for (int k = 0; k < 3; ++k) ion0.R[i][k] = Ri[i][k]; const char* cj_normgauss_xml = R"( @@ -200,7 +198,7 @@ TEST_CASE("CountingJastrow","[wavefunction]") // test put for normalized_gaussian Libxml2Document doc; bool parse_cj = doc.parseFromString(cj_normgauss_xml); - REQUIRE( parse_cj ); + REQUIRE(parse_cj); xmlNodePtr cj_root = doc.getRoot(); CountingJastrowBuilder cjb(c, elec); @@ -209,24 +207,24 @@ TEST_CASE("CountingJastrow","[wavefunction]") CountingJastrow* cj = dynamic_cast*>(cj_uptr.get()); // reference for evaluateLog, evalGrad - RealType Jval_exact = 7.8100074447e+00; + RealType Jval_exact = 7.8100074447e+00; PosType Jgrad_exact[] = {PosType(3.6845037054e-04, -4.2882992861e-04, 0), - PosType(2.2032083234e-02, -2.5647245917e-02, 0), - PosType(6.0625112202e-04, -7.0560012380e-04, 0), - PosType(1.0622511249e-01, -1.2363268199e-01, 0)}; + PosType(2.2032083234e-02, -2.5647245917e-02, 0), + PosType(6.0625112202e-04, -7.0560012380e-04, 0), + PosType(1.0622511249e-01, -1.2363268199e-01, 0)}; RealType Jlap_exact[] = {1.9649428566e-03, -1.1385706794e-01, 3.2312809658e-03, 4.0668060285e-01}; // test evaluateLog for cj - LogValueType logval = cj->evaluateLog(elec, elec.G, elec.L); - for(int i = 0; i < num_els; ++i) + LogValue logval = cj->evaluateLog(elec, elec.G, elec.L); + for (int i = 0; i < num_els; ++i) { - for(int k = 0; k < 3; ++k) - CHECK( Jgrad_exact[i][k] == Approx( std::real(elec.G[i][k])) ); - CHECK( Jlap_exact[i] == Approx( std::real(elec.L[i])) ); + for (int k = 0; k < 3; ++k) + CHECK(Jgrad_exact[i][k] == Approx(std::real(elec.G[i][k]))); + CHECK(Jlap_exact[i] == Approx(std::real(elec.L[i]))); } - CHECK( ComplexApprox(Jval_exact) == logval ); - + CHECK(ComplexApprox(Jval_exact) == logval); + // test automatic/minimal voronoi generator const char* cj_voronoi_xml = R"( @@ -242,7 +240,7 @@ TEST_CASE("CountingJastrow","[wavefunction]") // test put Libxml2Document doc2; bool parse_cjv = doc2.parseFromString(cj_voronoi_xml); - REQUIRE( parse_cjv ); + REQUIRE(parse_cjv); xmlNodePtr cjv_root = doc2.getRoot(); CountingJastrowBuilder cjvb(c, elec, ion0); @@ -251,55 +249,55 @@ TEST_CASE("CountingJastrow","[wavefunction]") auto cjv_uptr = cjvb.buildComponent(cjv_root); CountingJastrow* cjv = dynamic_cast*>(cjv_uptr.get()); - for(int i = 0; i < num_els; ++i) + for (int i = 0; i < num_els; ++i) { - for(int k = 0; k < 3; ++k) + for (int k = 0; k < 3; ++k) elec.G[i][k] = 0; elec.L[i] = 0; } logval = cjv->evaluateLog(elec, elec.G, elec.L); - for(int i = 0; i < num_els; ++i) + for (int i = 0; i < num_els; ++i) { - for(int k = 0; k < 3; ++k) - CHECK( Jgrad_exact[i][k] == Approx( std::real(elec.G[i][k])) ); - CHECK( Jlap_exact[i] == Approx( std::real(elec.L[i])) ); + for (int k = 0; k < 3; ++k) + CHECK(Jgrad_exact[i][k] == Approx(std::real(elec.G[i][k]))); + CHECK(Jlap_exact[i] == Approx(std::real(elec.L[i]))); } - CHECK( ComplexApprox(Jval_exact) == logval ); - + CHECK(ComplexApprox(Jval_exact) == logval); + // test evalGrad - for(int iat = 0; iat < num_els; ++iat) + for (int iat = 0; iat < num_els; ++iat) { GradType Jgrad_iat = cj->evalGrad(elec, iat); - for(int k = 0; k < 3; ++k) - CHECK( Jgrad_exact[iat][k] == Approx( std::real(Jgrad_iat[k]) )); + for (int k = 0; k < 3; ++k) + CHECK(Jgrad_exact[iat][k] == Approx(std::real(Jgrad_iat[k]))); } // reference for ratio, ratioGrad, acceptMove - PosType dr[] = {PosType(0.0984629815, 0.0144420719, 0.1334309321), + PosType dr[] = {PosType(0.0984629815, 0.0144420719, 0.1334309321), PosType(-0.1026409581, 0.2289767772, 0.490138058592), - PosType(0.03517477469, 0.2693941041, 0.16922043039), - PosType(0.3851116893, -0.1387760973, 0.1980082182)}; + PosType(0.03517477469, 0.2693941041, 0.16922043039), + PosType(0.3851116893, -0.1387760973, 0.1980082182)}; - RealType ratioval_exact[] = { 1.00003304765, 0.987624289443, 0.999873210738, 1.09014860194}; + RealType ratioval_exact[] = {1.00003304765, 0.987624289443, 0.999873210738, 1.09014860194}; PosType Jgrad_t_exact[] = {PosType(4.4329270315e-04, -5.1593699287e-04, 0), - PosType(4.8722465115e-02, -5.6707785196e-02, 0), - PosType(3.2691265772e-04, -3.8048525335e-04, 0), - PosType(2.0373800011e-01, -2.3712542045e-01, 0)}; + PosType(4.8722465115e-02, -5.6707785196e-02, 0), + PosType(3.2691265772e-04, -3.8048525335e-04, 0), + PosType(2.0373800011e-01, -2.3712542045e-01, 0)}; // test ratio, ratioGrad, acceptMove - for(int iat = 0; iat < num_els; ++iat) + for (int iat = 0; iat < num_els; ++iat) { - elec.makeMoveAndCheck(iat,dr[iat]); - - RealType ratioval = std::real( cj->ratio(elec, iat) ); - CHECK( ratioval_exact[iat] == Approx(std::real(ratioval)) ); + elec.makeMoveAndCheck(iat, dr[iat]); + + RealType ratioval = std::real(cj->ratio(elec, iat)); + CHECK(ratioval_exact[iat] == Approx(std::real(ratioval))); GradType grad_iat(0, 0, 0); - RealType gradratioval = std::real( cj->ratioGrad(elec, iat, grad_iat) ); - CHECK( ratioval_exact[iat] == Approx(gradratioval)); - for(int k = 0; k < 3; ++k) + RealType gradratioval = std::real(cj->ratioGrad(elec, iat, grad_iat)); + CHECK(ratioval_exact[iat] == Approx(gradratioval)); + for (int k = 0; k < 3; ++k) CHECK(Jgrad_t_exact[iat][k] == Approx(std::real(grad_iat[k]))); cj->acceptMove(elec, iat); @@ -307,40 +305,42 @@ TEST_CASE("CountingJastrow","[wavefunction]") #ifndef QMC_COMPLEX // setup and reference for evaluateDerivatives - PosType R2[] = { PosType( 4.3280064837, 2.4657709845, 6.3466520181e-01), - PosType( 9.7075155012, 7.2984775093, -8.1975111678e-01), - PosType( 5.7514912378, 5.2001615327, 6.6673589235e-01), - PosType( 8.3805220665, 7.9424368608, -3.5106422506e-02)}; - PosType G2[] = {PosType( 3.3480105792e-01, 2.1316369526e-01, -4.1812914940e-01), - PosType(-7.0561066397e-01, 1.7863210008e-01, 3.5677994771e-01), - PosType(-9.2302398033e-01, -5.0740272660e-01, -2.6078603626e-01), - PosType(-8.3764545416e-01, -4.9181684009e-01, 1.0675382607e-01)}; - for(int i = 0; i < num_els; ++i) - for(int k = 0; k < 3; ++k) + PosType R2[] = {PosType(4.3280064837, 2.4657709845, 6.3466520181e-01), + PosType(9.7075155012, 7.2984775093, -8.1975111678e-01), + PosType(5.7514912378, 5.2001615327, 6.6673589235e-01), + PosType(8.3805220665, 7.9424368608, -3.5106422506e-02)}; + PosType G2[] = {PosType(3.3480105792e-01, 2.1316369526e-01, -4.1812914940e-01), + PosType(-7.0561066397e-01, 1.7863210008e-01, 3.5677994771e-01), + PosType(-9.2302398033e-01, -5.0740272660e-01, -2.6078603626e-01), + PosType(-8.3764545416e-01, -4.9181684009e-01, 1.0675382607e-01)}; + for (int i = 0; i < num_els; ++i) + for (int k = 0; k < 3; ++k) { elec.R[i][k] = R2[i][k]; elec.G[i][k] = G2[i][k]; } - int num_derivs = 39; - RealType dlogpsi_exact[] = { - 7.0982172306e-04, 9.8329357367e-02, 6.6879065207e-03, 1.0670293004e-01, 3.4053136887e+00, - 4.6322726464e-01, 7.3906096412e+00, 1.5753284303e-02, 5.0267496641e-01,-1.3874695168e+00, - -2.6249136239e+00,-4.2223002567e+00,-3.0798330637e+00, 3.7905326800e+00, 8.4038996349e+00, - 2.2816901707e+00, 4.1911712810e+00,-9.3658177215e+00,-1.2434457046e+01, 1.6771424507e+00, - 2.3712452266e+00, 3.6980955070e+00, 2.5407601111e+00, 1.8976924460e-01,-1.0446470315e+00, - -1.2992491105e+00,-8.5624882767e-01, 9.8686287993e+00, 1.1847431541e+01,-2.5193792283e-01, - -3.0763224769e-01, 1.2429858182e-01, 1.3295440602e-02, 6.4178676394e-02, 1.2758462324e-01, - 7.5131761426e-02, 1.1629004831e-01, 3.9639061816e-01, 6.7088705514e-01}; - RealType dhpsioverpsi_exact[] = { - -1.6695881381e-02,-4.8902571790e-01,-1.2725397012e-01,-6.6714806635e-01, 6.9379259933e+00, - -4.8393983437e+00, 7.4947765640e+00,-8.8373306290e-01,-6.8244030879e+00, 7.9150085031e-01, - -1.4313255643e+00, 3.7225112718e+01, 1.7787191536e+01,-1.6672327906e+01,-4.1705496948e+01, - -9.9674671566e+00,-2.0150790757e+01, 1.1226368249e+02, 1.2744525474e+02,-1.5801247401e+01, - -1.3618595564e+01,-2.8161585388e+01,-1.4057266918e+01, 1.7626748997e+00, 7.8913447811e+00, - 9.2144952390e+00, 4.6068416473e+00,-9.3975889104e+01,-8.8298321426e+01, 1.5097063606e+01, - 1.8605794463e+01,-7.3647009565e+00,-5.9114663448e-01,-3.9243955679e+00,-7.8630886487e+00, - -4.4437106408e+00,-7.0313362338e+00,-2.3986142270e+01,-4.0724297500e+01}; + int num_derivs = 39; + RealType dlogpsi_exact[] = {7.0982172306e-04, 9.8329357367e-02, 6.6879065207e-03, 1.0670293004e-01, + 3.4053136887e+00, 4.6322726464e-01, 7.3906096412e+00, 1.5753284303e-02, + 5.0267496641e-01, -1.3874695168e+00, -2.6249136239e+00, -4.2223002567e+00, + -3.0798330637e+00, 3.7905326800e+00, 8.4038996349e+00, 2.2816901707e+00, + 4.1911712810e+00, -9.3658177215e+00, -1.2434457046e+01, 1.6771424507e+00, + 2.3712452266e+00, 3.6980955070e+00, 2.5407601111e+00, 1.8976924460e-01, + -1.0446470315e+00, -1.2992491105e+00, -8.5624882767e-01, 9.8686287993e+00, + 1.1847431541e+01, -2.5193792283e-01, -3.0763224769e-01, 1.2429858182e-01, + 1.3295440602e-02, 6.4178676394e-02, 1.2758462324e-01, 7.5131761426e-02, + 1.1629004831e-01, 3.9639061816e-01, 6.7088705514e-01}; + RealType dhpsioverpsi_exact[] = {-1.6695881381e-02, -4.8902571790e-01, -1.2725397012e-01, -6.6714806635e-01, + 6.9379259933e+00, -4.8393983437e+00, 7.4947765640e+00, -8.8373306290e-01, + -6.8244030879e+00, 7.9150085031e-01, -1.4313255643e+00, 3.7225112718e+01, + 1.7787191536e+01, -1.6672327906e+01, -4.1705496948e+01, -9.9674671566e+00, + -2.0150790757e+01, 1.1226368249e+02, 1.2744525474e+02, -1.5801247401e+01, + -1.3618595564e+01, -2.8161585388e+01, -1.4057266918e+01, 1.7626748997e+00, + 7.8913447811e+00, 9.2144952390e+00, 4.6068416473e+00, -9.3975889104e+01, + -8.8298321426e+01, 1.5097063606e+01, 1.8605794463e+01, -7.3647009565e+00, + -5.9114663448e-01, -3.9243955679e+00, -7.8630886487e+00, -4.4437106408e+00, + -7.0313362338e+00, -2.3986142270e+01, -4.0724297500e+01}; Vector dlogpsi; Vector dhpsioverpsi; dlogpsi.resize(num_derivs); @@ -359,15 +359,13 @@ TEST_CASE("CountingJastrow","[wavefunction]") // test evaluateDerivatives cj->evaluateDerivatives(elec, optVars, dlogpsi, dhpsioverpsi); - for(int p = 0; p < num_derivs; ++p) + for (int p = 0; p < num_derivs; ++p) { - CHECK ( dlogpsi_exact[p] == Approx(std::real(dlogpsi[p])).epsilon(1e-3) ); - CHECK ( dhpsioverpsi_exact[p] == Approx(std::real(dhpsioverpsi[p])).epsilon(1e-3) ); + CHECK(dlogpsi_exact[p] == Approx(std::real(dlogpsi[p])).epsilon(1e-3)); + CHECK(dhpsioverpsi_exact[p] == Approx(std::real(dhpsioverpsi[p])).epsilon(1e-3)); } - - // test makeClone auto cj2_uptr = cj->makeClone(elec); CountingJastrow* cj2 = dynamic_cast*>(cj2_uptr.get()); @@ -384,20 +382,19 @@ TEST_CASE("CountingJastrow","[wavefunction]") optVars2.print(std::cout); cj2->evaluateDerivatives(elec, optVars2, dlogpsi, dhpsioverpsi); - for(int p = 0; p < num_derivs; ++p) + for (int p = 0; p < num_derivs; ++p) { - CHECK ( dlogpsi_exact[p] == Approx(std::real(dlogpsi[p])).epsilon(1e-3) ); - CHECK ( dhpsioverpsi_exact[p] == Approx(std::real(dhpsioverpsi[p])).epsilon(1e-3) ); + CHECK(dlogpsi_exact[p] == Approx(std::real(dlogpsi[p])).epsilon(1e-3)); + CHECK(dhpsioverpsi_exact[p] == Approx(std::real(dhpsioverpsi[p])).epsilon(1e-3)); } // test resetParameters, recompute - for(int p = 0; p < num_derivs; ++p) + for (int p = 0; p < num_derivs; ++p) optVars[p] = 0; cj->resetParametersExclusive(optVars); cj->recompute(elec); - REQUIRE( cj->get_log_value() == LogValueType(0) ); + REQUIRE(cj->get_log_value() == LogValue(0)); #endif - } } //namespace qmcplusplus diff --git a/src/QMCWaveFunctions/tests/test_example_he.cpp b/src/QMCWaveFunctions/tests/test_example_he.cpp index 47511dad479..a8f980f63a3 100644 --- a/src/QMCWaveFunctions/tests/test_example_he.cpp +++ b/src/QMCWaveFunctions/tests/test_example_he.cpp @@ -22,10 +22,10 @@ namespace qmcplusplus { -using RealType = WaveFunctionComponent::RealType; -using ValueType = WaveFunctionComponent::ValueType; -using LogValueType = WaveFunctionComponent::LogValueType; -using PsiValueType = WaveFunctionComponent::PsiValueType; +using RealType = WaveFunctionComponent::RealType; +using ValueType = WaveFunctionComponent::ValueType; +using LogValue = WaveFunctionComponent::LogValue; +using PsiValue = WaveFunctionComponent::PsiValue; TEST_CASE("ExampleHe", "[wavefunction]") { @@ -99,7 +99,7 @@ TEST_CASE("ExampleHe", "[wavefunction]") all_lap.resize(nelec); // Set the base expectations for wavefunction value and derivatives - LogValueType logpsi = example_he->evaluateLog(elec, all_grad, all_lap); + LogValue logpsi = example_he->evaluateLog(elec, all_grad, all_lap); // Comparisons are performed at a single set of electron coordinates. This should be expanded. @@ -128,7 +128,7 @@ TEST_CASE("ExampleHe", "[wavefunction]") elec.makeMove(iat, zero_displ); - PsiValueType ratio = example_he->ratio(elec, iat); + PsiValue ratio = example_he->ratio(elec, iat); CHECK(std::real(ratio) == Approx(1.0)); ratio = example_he->ratioGrad(elec, iat, grad0); @@ -164,19 +164,19 @@ TEST_CASE("ExampleHe", "[wavefunction]") new_lap.resize(nelec); // wavefunction value and derivatives at new position - LogValueType new_logpsi = example_he->evaluateLog(elec, new_grad, new_lap); - elec.R[0] = oldpos; + LogValue new_logpsi = example_he->evaluateLog(elec, new_grad, new_lap); + elec.R[0] = oldpos; elec.update(); iat = 0; elec.makeMove(iat, displ); ratio = example_he->ratio(elec, iat); - CHECK(ValueApprox(ratio) == LogToValue::convert(new_logpsi - logpsi)); + CHECK(ValueApprox(ratio) == LogToValue::convert(new_logpsi - logpsi)); ratio = example_he->ratioGrad(elec, iat, grad0); - CHECK(ValueApprox(ratio) == LogToValue::convert(new_logpsi - logpsi)); + CHECK(ValueApprox(ratio) == LogToValue::convert(new_logpsi - logpsi)); CHECK(grad0[0] == ValueApprox(new_grad[0][0])); CHECK(grad0[1] == ValueApprox(new_grad[0][1])); @@ -205,13 +205,13 @@ TEST_CASE("ExampleHe", "[wavefunction]") grad_plus_h.resize(nelec); lap_plus_h.resize(nelec); - LogValueType logpsi_plus_h = example_he->evaluateLog(elec, grad_plus_h, lap_plus_h); + LogValue logpsi_plus_h = example_he->evaluateLog(elec, grad_plus_h, lap_plus_h); //phase change is not allowed in finite difference REQUIRE(std::imag(logpsi_plus_h) == std::imag(logpsi)); // Finite difference derivative approximation - LogValueType fd_logpsi = (logpsi_plus_h - logpsi) / LogValueType(h); + LogValue fd_logpsi = (logpsi_plus_h - logpsi) / LogValue(h); Vector dlogpsi(nparam); Vector dhpsioverpsi(nparam); diff --git a/src/QMCWaveFunctions/tests/test_lattice_gaussian.cpp b/src/QMCWaveFunctions/tests/test_lattice_gaussian.cpp index 3accba93c1b..3d13240dd15 100644 --- a/src/QMCWaveFunctions/tests/test_lattice_gaussian.cpp +++ b/src/QMCWaveFunctions/tests/test_lattice_gaussian.cpp @@ -28,8 +28,8 @@ using std::string; namespace qmcplusplus { -using RealType = QMCTraits::RealType; -using LogValueType = std::complex; +using RealType = QMCTraits::RealType; +using LogValue = std::complex; TEST_CASE("lattice gaussian", "[wavefunction]") { @@ -115,7 +115,7 @@ TEST_CASE("lattice gaussian", "[wavefunction]") ions.update(); elec.update(); - LogValueType logpsi = LGP->evaluateLog(elec, elec.G, elec.L); + LogValue logpsi = LGP->evaluateLog(elec, elec.G, elec.L); // check answer RealType r2 = Dot(elec.R, elec.R); double wfval = std::exp(-alpha * r2); diff --git a/src/QMCWaveFunctions/tests/test_multi_slater_determinant.cpp b/src/QMCWaveFunctions/tests/test_multi_slater_determinant.cpp index a52932ebf27..7bcda07113a 100644 --- a/src/QMCWaveFunctions/tests/test_multi_slater_determinant.cpp +++ b/src/QMCWaveFunctions/tests/test_multi_slater_determinant.cpp @@ -30,12 +30,12 @@ using std::string; namespace qmcplusplus { -using PosType = ParticleSet::PosType; -using RealType = ParticleSet::RealType; -using ValueType = ParticleSet::ValueType; -using GradType = ParticleSet::GradType; -using LogValueType = WaveFunctionComponent::LogValueType; -using PsiValueType = WaveFunctionComponent::PsiValueType; +using PosType = ParticleSet::PosType; +using RealType = ParticleSet::RealType; +using ValueType = ParticleSet::ValueType; +using GradType = ParticleSet::GradType; +using LogValue = WaveFunctionComponent::LogValue; +using PsiValue = WaveFunctionComponent::PsiValue; void test_LiH_msd(const std::string& spo_xml_string, const std::string& check_sponame, @@ -269,18 +269,18 @@ void test_LiH_msd(const std::string& spo_xml_string, CHECK(grad_old.grads_positions[1][2] == ValueApprox(-5.8209379274)); TWFGrads grad_new(2); - std::vector ratios(2); + std::vector ratios(2); ParticleSet::mw_makeMove(p_ref_list, moved_elec_id, displ); TrialWaveFunction::mw_calcRatio(wf_ref_list, p_ref_list, moved_elec_id, ratios); - CHECK(ratios[0] == ValueApprox(PsiValueType(-0.6181619459))); - CHECK(ratios[1] == ValueApprox(PsiValueType(1.6186330488))); + CHECK(ratios[0] == ValueApprox(PsiValue(-0.6181619459))); + CHECK(ratios[1] == ValueApprox(PsiValue(1.6186330488))); TrialWaveFunction::mw_calcRatioGrad(wf_ref_list, p_ref_list, moved_elec_id, ratios, grad_new); - CHECK(ratios[0] == ValueApprox(PsiValueType(-0.6181619459))); - CHECK(ratios[1] == ValueApprox(PsiValueType(1.6186330488))); + CHECK(ratios[0] == ValueApprox(PsiValue(-0.6181619459))); + CHECK(ratios[1] == ValueApprox(PsiValue(1.6186330488))); CHECK(grad_new.grads_positions[0][0] == ValueApprox(1.2418467899)); CHECK(grad_new.grads_positions[0][1] == ValueApprox(1.2425653495)); @@ -311,13 +311,13 @@ void test_LiH_msd(const std::string& spo_xml_string, ParticleSet::mw_makeMove(p_ref_list, moved_elec_id_next, displ); TrialWaveFunction::mw_calcRatio(wf_ref_list, p_ref_list, moved_elec_id_next, ratios); - CHECK(ratios[0] == ValueApprox(PsiValueType(2.1080036144))); - CHECK(ratios[1] == ValueApprox(PsiValueType(0.4947158435))); + CHECK(ratios[0] == ValueApprox(PsiValue(2.1080036144))); + CHECK(ratios[1] == ValueApprox(PsiValue(0.4947158435))); TrialWaveFunction::mw_calcRatioGrad(wf_ref_list, p_ref_list, moved_elec_id_next, ratios, grad_new); - CHECK(ratios[0] == ValueApprox(PsiValueType(2.1080036144))); - CHECK(ratios[1] == ValueApprox(PsiValueType(0.4947158435))); + CHECK(ratios[0] == ValueApprox(PsiValue(2.1080036144))); + CHECK(ratios[1] == ValueApprox(PsiValue(0.4947158435))); CHECK(grad_new.grads_positions[0][0] == ValueApprox(1.8412365668)); CHECK(grad_new.grads_positions[0][1] == ValueApprox(1.3736370007)); @@ -528,7 +528,7 @@ void test_Bi_msd(const std::string& spo_xml_string, displs.spins = {ds, ds}; ParticleSet::mw_makeMove(p_list, moved_elec_id, displs); - std::vector ratios(num_walkers); + std::vector ratios(num_walkers); TrialWaveFunction::mw_calcRatio(twf_list, p_list, moved_elec_id, ratios); for (int iw = 0; iw < num_walkers; iw++) CHECK(ValueType(std::abs(ratios[iw])) == ValueApprox(0.991503).epsilon(1e-4)); diff --git a/src/QMCWaveFunctions/tests/test_polynomial_eeI_jastrow.cpp b/src/QMCWaveFunctions/tests/test_polynomial_eeI_jastrow.cpp index 353a3b3f6f4..de4fb880793 100644 --- a/src/QMCWaveFunctions/tests/test_polynomial_eeI_jastrow.cpp +++ b/src/QMCWaveFunctions/tests/test_polynomial_eeI_jastrow.cpp @@ -30,9 +30,9 @@ using std::string; namespace qmcplusplus { -using RealType = WaveFunctionComponent::RealType; -using LogValueType = WaveFunctionComponent::LogValueType; -using PsiValueType = WaveFunctionComponent::PsiValueType; +using RealType = WaveFunctionComponent::RealType; +using LogValue = WaveFunctionComponent::LogValue; +using PsiValue = WaveFunctionComponent::PsiValue; TEST_CASE("PolynomialFunctor3D functor zero", "[wavefunction]") { @@ -125,19 +125,19 @@ void test_J3_polynomial3D(const DynamicCoordinateKind kind_selected) CHECK(std::real(ratios[3]) == Approx(0.7987703724)); elec_.makeMove(0, newpos - elec_.R[0]); - PsiValueType ratio_0 = j3->ratio(elec_, 0); + PsiValue ratio_0 = j3->ratio(elec_, 0); elec_.rejectMove(0); elec_.makeMove(1, newpos - elec_.R[1]); - PsiValueType ratio_1 = j3->ratio(elec_, 1); + PsiValue ratio_1 = j3->ratio(elec_, 1); elec_.rejectMove(1); elec_.makeMove(2, newpos - elec_.R[2]); - PsiValueType ratio_2 = j3->ratio(elec_, 2); + PsiValue ratio_2 = j3->ratio(elec_, 2); elec_.rejectMove(2); elec_.makeMove(3, newpos - elec_.R[3]); - PsiValueType ratio_3 = j3->ratio(elec_, 3); + PsiValue ratio_3 = j3->ratio(elec_, 3); elec_.rejectMove(3); CHECK(std::real(ratio_0) == Approx(0.8744938582)); From dccc14868955f8a34f29a4fb45b183f23bd91381 Mon Sep 17 00:00:00 2001 From: Mark Dewing Date: Fri, 13 Oct 2023 15:53:29 +0000 Subject: [PATCH 020/168] Workaround for assignment-after-reduction bug Loop index over nw must be 32 bits in size. Bug affects offload with NVidia. See https://github.com/QMCPACK/qmcpack/issues/4767 --- src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp | 8 +++++--- .../Fermion/MultiSlaterDetTableMethod.cpp | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp index 643e2204d04..9f8ee6c62f7 100644 --- a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp @@ -443,7 +443,7 @@ void MultiDiracDeterminant::mw_evaluateDetsForPtclMove(const RefVectorWithLeader PRAGMA_OFFLOAD("omp target teams distribute map(always, from:curRatio_list_ptr[:nw]) \ is_device_ptr(psiV_list_devptr, psiMinv_temp_list_devptr)") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) { ValueType c_ratio = 0.0; PRAGMA_OFFLOAD("omp parallel for reduction(+ : c_ratio)") @@ -780,9 +780,11 @@ void MultiDiracDeterminant::mw_evaluateDetsAndGradsForPtclMove( throw std::runtime_error("In MultiDiracDeterminant ompBLAS::copy_batched_offset failed."); + // Index of loop over nw must be 32 bit sized to avoid assignment-after-reduction offload bug + // See https://github.com/QMCPACK/qmcpack/issues/4767 PRAGMA_OFFLOAD("omp target teams distribute is_device_ptr(psiV_list_devptr, psiMinv_temp_list_devptr) \ map(always, from:curRatio_list_ptr[:nw])") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) { GradType ratioGradRef_local(0); PRAGMA_OFFLOAD("omp parallel for reduction(+ : ratioGradRef_local)") @@ -1048,7 +1050,7 @@ void MultiDiracDeterminant::mw_evaluateGrads(const RefVectorWithLeader Date: Fri, 13 Oct 2023 18:11:48 -0500 Subject: [PATCH 021/168] Additional zero/NaN protection in MSD. When the proposed configuration is on the nodal surface. Reference determinant curRatio goes to zero. This move should be rejected eventually by Monte Carlo. However, to make the code proceed, some NaN protection is needed. When Dets[det_id]->getRefDetRatio() is zero, new_psi_ratio_to_new_ref_det_ the sum of all the dets with respect to the ref one is NaN because it is the sum of all the 1/0. So keeping curRatio zero should be healthy. --- .../Fermion/MultiDiracDeterminant.2.cpp | 5 ++- .../Fermion/MultiDiracDeterminant.cpp | 2 ++ .../Fermion/MultiSlaterDetTableMethod.cpp | 32 +++++++++++++------ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp index 9f8ee6c62f7..dff4132ded6 100644 --- a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp @@ -515,9 +515,8 @@ void MultiDiracDeterminant::evaluateDetsForPtclMove(const ParticleSet& P, int ia psiMinv_temp = psiMinv; for (size_t i = 0; i < NumPtcls; i++) psiV_temp[i] = psiV[*(it++)]; - auto ratio_old_ref_det = DetRatioByColumn(psiMinv_temp, psiV_temp, WorkingIndex); - curRatio = ratio_old_ref_det; - InverseUpdateByColumn(psiMinv_temp, psiV_temp, workV1, workV2, WorkingIndex, ratio_old_ref_det); + curRatio = DetRatioByColumn(psiMinv_temp, psiV_temp, WorkingIndex); + InverseUpdateByColumn(psiMinv_temp, psiV_temp, workV1, workV2, WorkingIndex, curRatio); for (size_t i = 0; i < NumOrbitals; i++) TpsiM(i, WorkingIndex) = psiV[i]; } diff --git a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp index 48a43a7d93c..901feee4579 100644 --- a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.cpp @@ -413,6 +413,8 @@ void MultiDiracDeterminant::acceptMove(ParticleSet& P, int iat, bool safe_to_del const int WorkingIndex = iat - FirstIndex; assert(WorkingIndex >= 0 && WorkingIndex < LastIndex - FirstIndex); assert(P.isSpinor() == is_spinor_); + if (curRatio == ValueType(0)) + throw std::runtime_error("MultiDiracDeterminant::acceptMove curRatio is zero! Report a bug.\n"); log_value_ref_det_ += convertValueToLog(curRatio); curRatio = ValueType(1); switch (UpdateMode) diff --git a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp index d268cbf82a3..affa2ab6d96 100644 --- a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp @@ -446,7 +446,9 @@ WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::ratioGrad(ParticleSet new_psi_ratio_to_new_ref_det_ = evalGrad_impl_no_precompute(P, iat, true, dummy); const int det_id = getDetID(iat); - curRatio = Dets[det_id]->getRefDetRatio() * new_psi_ratio_to_new_ref_det_ / psi_ratio_to_ref_det_; + curRatio = Dets[det_id]->getRefDetRatio(); + if (curRatio != PsiValue(0)) + curRatio *= new_psi_ratio_to_new_ref_det_ / psi_ratio_to_ref_det_; grad_iat += dummy; return curRatio; } @@ -467,7 +469,9 @@ WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::ratioGradWithSpin(Par new_psi_ratio_to_new_ref_det_ = evalGradWithSpin_impl_no_precompute(P, iat, true, dummy, spindummy); const int det_id = getDetID(iat); - curRatio = Dets[det_id]->getRefDetRatio() * new_psi_ratio_to_new_ref_det_ / psi_ratio_to_ref_det_; + curRatio = Dets[det_id]->getRefDetRatio(); + if (curRatio != PsiValue(0)) + curRatio *= new_psi_ratio_to_new_ref_det_ / psi_ratio_to_ref_det_; grad_iat += dummy; spingrad_iat += spindummy; return curRatio; @@ -501,7 +505,10 @@ void MultiSlaterDetTableMethod::mw_ratioGrad(const RefVectorWithLeader(iw); det.new_psi_ratio_to_new_ref_det_ = psi_list[iw]; grad_new[iw] += dummy[iw]; - ratios[iw] = det.curRatio = det.Dets[det_id]->getRefDetRatio() * psi_list[iw] / det.psi_ratio_to_ref_det_; + det.curRatio = det.Dets[det_id]->getRefDetRatio(); + if (det.curRatio != PsiValue(0)) + det.curRatio *= psi_list[iw] / det.psi_ratio_to_ref_det_; + ratios[iw] = det.curRatio; } } @@ -548,7 +555,9 @@ WaveFunctionComponent::PsiValue MultiSlaterDetTableMethod::ratio(ParticleSet& P, Dets[det_id]->evaluateDetsForPtclMove(P, iat); new_psi_ratio_to_new_ref_det_ = computeRatio_NewMultiDet_to_NewRefDet(det_id); - curRatio = Dets[det_id]->getRefDetRatio() * new_psi_ratio_to_new_ref_det_ / psi_ratio_to_ref_det_; + curRatio = Dets[det_id]->getRefDetRatio(); + if (curRatio != PsiValue(0)) + curRatio *= new_psi_ratio_to_new_ref_det_ / psi_ratio_to_ref_det_; return curRatio; } @@ -605,9 +614,7 @@ void MultiSlaterDetTableMethod::mw_calcRatio(const RefVectorWithLeader(iw); det.new_psi_ratio_to_new_ref_det_ = psi_list[iw]; - ratios[iw] = det.curRatio = det.Dets[det_id]->getRefDetRatio() * psi_list[iw] / det.psi_ratio_to_ref_det_; + det.curRatio = det.Dets[det_id]->getRefDetRatio(); + if (det.curRatio != PsiValue(0)) + det.curRatio *= psi_list[iw] / det.psi_ratio_to_ref_det_; + ratios[iw] = det.curRatio; } } @@ -632,7 +642,9 @@ void MultiSlaterDetTableMethod::evaluateRatios(const VirtualParticleSet& VP, std const OffloadVector& detValues0 = Dets[det_id]->getNewRatiosToRefDet(); PsiValue psiNew = computeRatio_NewMultiDet_to_NewRefDet(det_id); - ratios[iat] = Dets[det_id]->getRefDetRatio() * psiNew / psi_ratio_to_ref_det_; + ratios[iat] = Dets[det_id]->getRefDetRatio(); + if (ratios[iat] != ValueType(0)) + ratios[iat] *= psiNew / psi_ratio_to_ref_det_; } } @@ -1068,7 +1080,9 @@ void MultiSlaterDetTableMethod::evaluateDerivRatios(const VirtualParticleSet& VP // calculate VP ratios PsiValue psiNew = computeRatio_NewMultiDet_to_NewRefDet(det_id); - ratios[iat] = Dets[det_id]->getRefDetRatio() * psiNew / psi_ratio_to_ref_det_; + ratios[iat] = Dets[det_id]->getRefDetRatio(); + if (ratios[iat] != ValueType(0)) + ratios[iat] *= psiNew / psi_ratio_to_ref_det_; // calculate VP ratios derivatives if (recalculate) From 805c2adc795b40abda5b16cebabbb98f3339f1a2 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Fri, 13 Oct 2023 22:26:01 -0500 Subject: [PATCH 022/168] Fix AFQMC compilation with CUDA 12.x --- .../Numerics/detail/CUDA/Kernels/strided_2Drange.hpp | 10 +++++----- .../Numerics/detail/CUDA/Kernels/strided_range.hpp | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_2Drange.hpp b/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_2Drange.hpp index 6a788cc4b4b..b21e2db14da 100644 --- a/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_2Drange.hpp +++ b/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_2Drange.hpp @@ -15,16 +15,16 @@ #ifndef AFQMC_STRIDED_2DRANGE_KERNELS_HPP #define AFQMC_STRIDED_2DRANGE_KERNELS_HPP -namespace kernels -{ -/* Note: Taken from thrust examples. - */ - #include #include #include #include +namespace kernels +{ +/* Note: Taken from thrust examples. + */ + template class strided_2Drange { diff --git a/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_range.hpp b/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_range.hpp index dc4f3475142..91f1a729b26 100644 --- a/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_range.hpp +++ b/src/AFQMC/Numerics/detail/CUDA/Kernels/strided_range.hpp @@ -15,16 +15,16 @@ #ifndef AFQMC_STRIDED_RANGE_KERNELS_HPP #define AFQMC_STRIDED_RANGE_KERNELS_HPP -namespace kernels -{ -/* Note: Taken from thrust examples. - */ - #include #include #include #include +namespace kernels +{ +/* Note: Taken from thrust examples. + */ + template class strided_range { From 0ea62b7cb6ccbb65c8d68807ddec0385a4c0242c Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Fri, 13 Oct 2023 19:11:47 -0500 Subject: [PATCH 023/168] Use 32bit integer for loop indices in offload region. --- src/Platforms/OMPTarget/ompBLAS.cpp | 38 ++++++------- src/QMCHamiltonians/CoulombPBCAA.cpp | 6 +- .../BsplineFactory/SplineC2COMPTarget.cpp | 4 +- .../BsplineFactory/SplineC2COMPTarget.h | 2 +- .../BsplineFactory/SplineC2ROMPTarget.cpp | 4 +- .../BsplineFactory/SplineC2ROMPTarget.h | 2 +- .../Fermion/MatrixDelayedUpdateCUDA.h | 4 +- .../Fermion/MultiDiracDeterminant.2.cpp | 55 ++++++++++--------- .../Fermion/MultiSlaterDetTableMethod.cpp | 4 +- 9 files changed, 60 insertions(+), 59 deletions(-) diff --git a/src/Platforms/OMPTarget/ompBLAS.cpp b/src/Platforms/OMPTarget/ompBLAS.cpp index 0dce77844d3..76c67b748d4 100644 --- a/src/Platforms/OMPTarget/ompBLAS.cpp +++ b/src/Platforms/OMPTarget/ompBLAS.cpp @@ -42,11 +42,11 @@ ompBLAS_status gemv_impl(ompBLAS_handle& handle, throw std::runtime_error("incx !=1 or incy != 1 are not implemented in ompBLAS::gemv_impl!"); PRAGMA_OFFLOAD("omp target teams distribute num_teams(n) is_device_ptr(A, x, y)") - for(size_t i = 0; i < n; i++) + for (uint32_t i = 0; i < n; i++) { T dot_sum(0); PRAGMA_OFFLOAD("omp parallel for simd reduction(+: dot_sum)") - for(size_t j = 0; j < m; j++) + for (uint32_t j = 0; j < m; j++) dot_sum += x[j] * A[i * lda + j]; if (beta == T(0)) y[i] = alpha * dot_sum; // protecting NaN from y @@ -61,11 +61,11 @@ ompBLAS_status gemv_impl(ompBLAS_handle& handle, throw std::runtime_error("incx !=1 or incy != 1 are not implemented in ompBLAS::gemv_impl!"); PRAGMA_OFFLOAD("omp target teams distribute num_teams(n) is_device_ptr(A, x, y)") - for (size_t i = 0; i < m; i++) + for (uint32_t i = 0; i < m; i++) { T dot_sum(0); PRAGMA_OFFLOAD("omp parallel for simd reduction(+: dot_sum)") - for (size_t j = 0; j < n; j++) + for (uint32_t j = 0; j < n; j++) dot_sum += x[j] * A[j * lda + i]; if (beta == T(0)) y[i] = alpha * dot_sum; // protecting NaN from y @@ -166,12 +166,12 @@ ompBLAS_status gemv_batched_impl(ompBLAS_handle& handle, throw std::runtime_error("incx !=1 or incy != 1 are not implemented in ompBLAS::gemv_batched_impl!"); PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(batch_count * n) is_device_ptr(A, x, y, alpha, beta)") - for (size_t ib = 0; ib < batch_count; ib++) - for (size_t i = 0; i < n; i++) + for (uint32_t ib = 0; ib < batch_count; ib++) + for (uint32_t i = 0; i < n; i++) { T dot_sum(0); PRAGMA_OFFLOAD("omp parallel for simd reduction(+: dot_sum)") - for (size_t j = 0; j < m; j++) + for (uint32_t j = 0; j < m; j++) dot_sum += x[ib][j] * A[ib][i * lda + j]; if (beta[ib] == T(0)) y[ib][i] = alpha[ib] * dot_sum; // protecting NaN from y @@ -186,12 +186,12 @@ ompBLAS_status gemv_batched_impl(ompBLAS_handle& handle, throw std::runtime_error("incx !=1 or incy != 1 are not implemented in ompBLAS::gemv_batched_impl!"); PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(batch_count * n) is_device_ptr(A, x, y, alpha, beta)") - for (size_t ib = 0; ib < batch_count; ib++) - for (size_t i = 0; i < m; i++) + for (uint32_t ib = 0; ib < batch_count; ib++) + for (uint32_t i = 0; i < m; i++) { T dot_sum(0); PRAGMA_OFFLOAD("omp parallel for simd reduction(+: dot_sum)") - for (size_t j = 0; j < n; j++) + for (uint32_t j = 0; j < n; j++) dot_sum += x[ib][j] * A[ib][j * lda + i]; if (beta[ib] == T(0)) y[ib][i] = alpha[ib] * dot_sum; // protecting NaN from y @@ -290,8 +290,8 @@ ompBLAS_status ger_impl(ompBLAS_handle& handle, //BLAS::ger(m, n, alpha, x, incx, y, incy, A, lda); PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(A, x, y)") - for(size_t i = 0; i < n; i++) - for(size_t j = 0; j < m; j++) + for (uint32_t i = 0; i < n; i++) + for (uint32_t j = 0; j < m; j++) A[i * lda + j] += alpha * x[j] * y[i]; return 0; } @@ -374,9 +374,9 @@ ompBLAS_status ger_batched_impl(ompBLAS_handle& handle, throw std::runtime_error("incx !=1 or incy != 1 are not implemented in ompBLAS::ger_batched_impl!"); PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(3) is_device_ptr(A, x, y, alpha)") - for(size_t ib = 0; ib < batch_count; ib++) - for(size_t i = 0; i < n; i++) - for(size_t j = 0; j < m; j++) + for (uint32_t ib = 0; ib < batch_count; ib++) + for (uint32_t i = 0; i < n; i++) + for (uint32_t j = 0; j < m; j++) A[ib][i * lda + j] += alpha[ib] * x[ib][j] * y[ib][i]; return 0; } @@ -456,8 +456,8 @@ ompBLAS_status copy_batched_impl(ompBLAS_handle& handle, if (batch_count == 0) return 0; PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(x, y)") - for (size_t ib = 0; ib < batch_count; ib++) - for (size_t i = 0; i < n; i++) + for (uint32_t ib = 0; ib < batch_count; ib++) + for (uint32_t i = 0; i < n; i++) y[ib][i * incy] = x[ib][i * incx]; return 0; } @@ -522,8 +522,8 @@ ompBLAS_status copy_batched_offset_impl(ompBLAS_handle& handle, if (batch_count == 0) return 0; PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(x, y)") - for (size_t ib = 0; ib < batch_count; ib++) - for (size_t i = 0; i < n; i++) + for (uint32_t ib = 0; ib < batch_count; ib++) + for (uint32_t i = 0; i < n; i++) y[ib][y_offset + i * incy] = x[ib][x_offset + i * incx]; return 0; } diff --git a/src/QMCHamiltonians/CoulombPBCAA.cpp b/src/QMCHamiltonians/CoulombPBCAA.cpp index a08a0066120..1604f6930d7 100644 --- a/src/QMCHamiltonians/CoulombPBCAA.cpp +++ b/src/QMCHamiltonians/CoulombPBCAA.cpp @@ -704,12 +704,12 @@ std::vector CoulombPBCAA::mw_evalSR_offload(const RefVec ScopedTimer offload_scope(caa_leader.offload_timer_); PRAGMA_OFFLOAD("omp target teams distribute num_teams(nw)") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) { mRealType SR = 0.0; PRAGMA_OFFLOAD("omp parallel for reduction(+ : SR)") - for (size_t jcol = 0; jcol < total_num; jcol++) - for (size_t irow = first; irow < last; irow++) + for (uint32_t jcol = 0; jcol < total_num; jcol++) + for (uint32_t irow = first; irow < last; irow++) { const RealType dist = mw_dist[num_padded * (irow - first + iw * this_chunk_size) + jcol]; if (irow == jcol || (irow * 2 + 1 == total_num && jcol > irow)) diff --git a/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.cpp b/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.cpp index 94d66116973..2db35258641 100644 --- a/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.cpp +++ b/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.cpp @@ -566,7 +566,7 @@ void SplineC2COMPTarget::mw_evaluateVGL(const RefVectorWithLeader& s const RefVector& d2psi_v_list) const { assert(this == &sa_list.getLeader()); - auto& phi_leader = sa_list.getCastedLeader>(); + auto& phi_leader = sa_list.getCastedLeader>(); auto& mw_mem = phi_leader.mw_mem_handle_.getResource(); auto& mw_pos_copy = mw_mem.mw_pos_copy; auto& mw_offload_scratch = mw_mem.mw_offload_scratch; @@ -720,7 +720,7 @@ void SplineC2COMPTarget::mw_evaluateVGLandDetRatioGrads(const RefVectorWithL ValueType ratio(0), grad_x(0), grad_y(0), grad_z(0); PRAGMA_OFFLOAD("omp parallel for reduction(+: ratio, grad_x, grad_y, grad_z)") - for (size_t j = first_cplx; j < last_cplx; j++) + for (int j = first_cplx; j < last_cplx; j++) { const size_t psiIndex = first_spo_local + j; diff --git a/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.h b/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.h index 40f6018c6a8..774c6461181 100644 --- a/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.h +++ b/src/QMCWaveFunctions/BsplineFactory/SplineC2COMPTarget.h @@ -197,7 +197,7 @@ class SplineC2COMPTarget : public BsplineSet PRAGMA_OFFLOAD("omp target update to(mKK_ptr[0:mKK->size()])") auto* myKcart_ptr = myKcart->data(); PRAGMA_OFFLOAD("omp target update to(myKcart_ptr[0:myKcart->capacity()*3])") - for (size_t i = 0; i < 9; i++) + for (uint32_t i = 0; i < 9; i++) { (*GGt_offload)[i] = GGt[i]; (*PrimLattice_G_offload)[i] = PrimLattice.G[i]; diff --git a/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.cpp b/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.cpp index ea262bf8cc2..18c2a913c2d 100644 --- a/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.cpp +++ b/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.cpp @@ -707,7 +707,7 @@ void SplineC2ROMPTarget::mw_evaluateVGL(const RefVectorWithLeader& s const RefVector& d2psi_v_list) const { assert(this == &sa_list.getLeader()); - auto& phi_leader = sa_list.getCastedLeader>(); + auto& phi_leader = sa_list.getCastedLeader>(); auto& mw_mem = phi_leader.mw_mem_handle_.getResource(); auto& mw_pos_copy = mw_mem.mw_pos_copy; auto& mw_offload_scratch = mw_mem.mw_offload_scratch; @@ -866,7 +866,7 @@ void SplineC2ROMPTarget::mw_evaluateVGLandDetRatioGrads(const RefVectorWithL omptarget::min(last_cplx + omptarget::min(nComplexBands_local, last_cplx), requested_orb_size); ValueType ratio(0), grad_x(0), grad_y(0), grad_z(0); PRAGMA_OFFLOAD("omp parallel for reduction(+: ratio, grad_x, grad_y, grad_z)") - for (size_t j = first_real; j < last_real; j++) + for (int j = first_real; j < last_real; j++) { out_phi[j] = psi[j]; out_dphi_x[j] = dpsi_x[j]; diff --git a/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.h b/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.h index 624e4ef9acc..e41741e33c4 100644 --- a/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.h +++ b/src/QMCWaveFunctions/BsplineFactory/SplineC2ROMPTarget.h @@ -202,7 +202,7 @@ class SplineC2ROMPTarget : public BsplineSet PRAGMA_OFFLOAD("omp target update to(mKK_ptr[0:mKK->size()])") auto* myKcart_ptr = myKcart->data(); PRAGMA_OFFLOAD("omp target update to(myKcart_ptr[0:myKcart->capacity()*3])") - for (size_t i = 0; i < 9; i++) + for (uint32_t i = 0; i < 9; i++) { (*GGt_offload)[i] = GGt[i]; (*PrimLattice_G_offload)[i] = PrimLattice.G[i]; diff --git a/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h b/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h index 7592da867f5..7d8bbba819b 100644 --- a/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h +++ b/src/QMCWaveFunctions/Fermion/MatrixDelayedUpdateCUDA.h @@ -786,9 +786,9 @@ class MatrixDelayedUpdateCUDA auto* temp_ptr = engine.get_psiMinv().data(); PRAGMA_OFFLOAD("omp target update from(temp_ptr[:psiMinv_.size()])") std::cout << "debug Ainv devi"; - for (size_t row = 0; row < psiMinv_.rows(); row++) + for (int row = 0; row < psiMinv_.rows(); row++) { - for (size_t col = 0; col < psiMinv_.cols(); col++) + for (int col = 0; col < psiMinv_.cols(); col++) std::cout << " " << row << " " << col << " " << engine.get_psiMinv()[row][col]; std::cout << std::endl; } diff --git a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp index dff4132ded6..2ec0e37a3bb 100644 --- a/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiDiracDeterminant.2.cpp @@ -127,14 +127,14 @@ void MultiDiracDeterminant::mw_buildTableMatrix_calculateRatios_impl( map(always, to: psiinv_list_ptr[:nw], psi_list_ptr[:nw]) \ map(always, to: ratios_list_ptr[:nw], table_matrix_list_ptr[:nw]) \ map(to:first[:npairs], second[:npairs])") - for (size_t iw = 0; iw < nw; iw++) - for (size_t i = 0; i < npairs; ++i) + for (uint32_t iw = 0; iw < nw; iw++) + for (uint32_t i = 0; i < npairs; ++i) { const int I = first[i]; const int J = second[i]; ValueType table_matrix_local = 0.0; - for (size_t ind = 0; ind < num; ind++) + for (uint32_t ind = 0; ind < num; ind++) table_matrix_local += psiinv_list_ptr[iw][I * nb_cols_psiinv + ind] * psi_list_ptr[iw][J * nb_cols_psi + ind]; table_matrix_list_ptr[iw][I * nb_cols_table_matrix + J] = table_matrix_local; @@ -295,8 +295,8 @@ void MultiDiracDeterminant::mw_buildTableMatrix_calculateGradRatios( PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(from:mw_grads_ptr[:mw_grads.size()]) \ map(always, to:WorkSpace_list_ptr[:nw])") - for (size_t iw = 0; iw < nw; iw++) - for (size_t count = 0; count < getNumDets; ++count) + for (uint32_t iw = 0; iw < nw; iw++) + for (uint32_t count = 0; count < getNumDets; ++count) mw_grads_ptr[(3 * iw + dx) * Grads_cols + count] = WorkSpace_list_ptr[iw][count]; } @@ -447,7 +447,7 @@ void MultiDiracDeterminant::mw_evaluateDetsForPtclMove(const RefVectorWithLeader { ValueType c_ratio = 0.0; PRAGMA_OFFLOAD("omp parallel for reduction(+ : c_ratio)") - for (size_t jc = 0; jc < NumPtcls; jc++) + for (uint32_t jc = 0; jc < NumPtcls; jc++) { const size_t J = confgListOccup_ptr[jc]; psiV_temp_list_ptr[iw][jc] = psiV_list_devptr[iw][J]; @@ -477,8 +477,8 @@ void MultiDiracDeterminant::mw_evaluateDetsForPtclMove(const RefVectorWithLeader // restore the modified column of TpsiM. PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(TpsiM_list_devptr) \ map(always, to:psiM_list_ptr[:nw])") - for (size_t iw = 0; iw < nw; iw++) - for (size_t i = 0; i < NumOrbitals; i++) + for (uint32_t iw = 0; iw < nw; iw++) + for (uint32_t i = 0; i < NumOrbitals; i++) TpsiM_list_devptr[iw][i * TpsiM_cols + WorkingIndex] = psiM_list_ptr[iw][i + psiM_cols * WorkingIndex]; } @@ -787,7 +787,7 @@ void MultiDiracDeterminant::mw_evaluateDetsAndGradsForPtclMove( { GradType ratioGradRef_local(0); PRAGMA_OFFLOAD("omp parallel for reduction(+ : ratioGradRef_local)") - for (size_t i = 0; i < NumPtcls; i++) + for (uint32_t i = 0; i < NumPtcls; i++) { const size_t J = confgListOccup_ptr[i]; psiV_temp_list_ptr[iw][i] = psiV_list_devptr[iw][J]; @@ -797,7 +797,7 @@ void MultiDiracDeterminant::mw_evaluateDetsAndGradsForPtclMove( ValueType c_ratio = 0.0; PRAGMA_OFFLOAD("omp parallel for reduction(+ : c_ratio)") - for (size_t jc = 0; jc < psiMinv_cols; jc += 1) + for (uint32_t jc = 0; jc < psiMinv_cols; jc += 1) { const size_t ic = jc * psiMinv_cols; c_ratio += (psiMinv_temp_list_devptr[iw] + WorkingIndex)[ic] * psiV_temp_list_ptr[iw][jc]; @@ -837,11 +837,11 @@ void MultiDiracDeterminant::mw_evaluateDetsAndGradsForPtclMove( throw std::runtime_error("In MultiDiracDeterminant ompBLAS::copy_batched_offset failed."); PRAGMA_OFFLOAD("omp target teams distribute map(to: ratioGradRef_list_ptr[:nw])") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) { inv_curRatio_list_ptr[iw] = ValueType(1) / ratioGradRef_list_ptr[iw][idim]; - for (size_t i = 0; i < NumPtcls; i++) + for (uint32_t i = 0; i < NumPtcls; i++) { const size_t J = confgListOccup_ptr[i]; psiV_temp_list_ptr[iw][i] = dpsiV_list_ptr[iw][J][idim]; @@ -855,10 +855,10 @@ void MultiDiracDeterminant::mw_evaluateDetsAndGradsForPtclMove( PRAGMA_OFFLOAD("omp target teams distribute map(to:dpsiV_list_ptr[:nw], curRatio_list_ptr[:nw]) \ map(always,from:det0_grad_list_ptr[:nw]) \ is_device_ptr(TpsiM_list_devptr)") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) { det0_grad_list_ptr[iw] = ratioGradRef_list_ptr[iw][idim] / curRatio_list_ptr[iw]; - for (size_t i = 0; i < NumOrbitals; i++) + for (uint32_t i = 0; i < NumOrbitals; i++) TpsiM_list_devptr[iw][i * TpsiM_num_cols + WorkingIndex] = dpsiV_list_ptr[iw][i][idim]; } @@ -872,10 +872,11 @@ void MultiDiracDeterminant::mw_evaluateDetsAndGradsForPtclMove( // restore the modified column of TpsiM. PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(TpsiM_list_devptr) \ map(always, to:psiM_list_ptr[:nw])") - for (size_t iw = 0; iw < nw; iw++) - for (size_t i = 0; i < NumOrbitals; i++) + for (uint32_t iw = 0; iw < nw; iw++) + for (uint32_t i = 0; i < NumOrbitals; i++) TpsiM_list_devptr[iw][i * TpsiM_num_cols + WorkingIndex] = psiM_list_ptr[iw][i + psiM_num_cols * WorkingIndex]; } + for (size_t iw = 0; iw < nw; iw++) { MultiDiracDeterminant& det = (det_list[iw]); @@ -1040,8 +1041,8 @@ void MultiDiracDeterminant::mw_evaluateGrads(const RefVectorWithLeader& const auto* det0_list_ptr = det0_list.data(); PRAGMA_OFFLOAD("omp target teams distribute parallel for map(always, to: det0_list_ptr[:nw])") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) ratios_list_ptr[iw][0] = det0_list_ptr[iw]; } @@ -1146,8 +1147,8 @@ void MultiDiracDeterminant::mw_updateRatios(const size_t det_offset, const auto* table_matrix_list_ptr = table_matrix_deviceptr_list.data(); PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2)") - for (size_t iw = 0; iw < nw; iw++) - for (size_t count = 0; count < ndet_ext; ++count) + for (uint32_t iw = 0; iw < nw; iw++) + for (uint32_t count = 0; count < ndet_ext; ++count) { size_t det_id = det_offset + count; ratios_list_ptr[iw][det_id] = sign_ptr[det_id] * ratios_list_ptr[iw][0] * @@ -1189,7 +1190,7 @@ void MultiDiracDeterminant::mw_InverseUpdateByColumn(MultiDiracDetMultiWalkerRes throw std::runtime_error("In MultiDiracDeterminant ompBLAS::gemv_batched failed."); PRAGMA_OFFLOAD("omp target teams distribute parallel for is_device_ptr(workV1_list_ptr, inv_curRatio_list_devptr)") - for (size_t iw = 0; iw < nw; iw++) + for (uint32_t iw = 0; iw < nw; iw++) workV1_list_ptr[iw][working_index] = cone - inv_curRatio_list_devptr[iw]; if (success != 0) throw std::runtime_error("In MultiDiracDeterminant ompBLAS::copy_batched_offset failed."); diff --git a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp index affa2ab6d96..fc0ab91bfcc 100644 --- a/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp +++ b/src/QMCWaveFunctions/Fermion/MultiSlaterDetTableMethod.cpp @@ -289,7 +289,7 @@ void MultiSlaterDetTableMethod::mw_evalGrad_impl(const RefVectorWithLeader Date: Fri, 15 Sep 2023 23:51:14 -0500 Subject: [PATCH 024/168] Step 1 in specializing and batching evalutions of AO functions for VirtualParticles --- src/Particle/VirtualParticleSet.cpp | 28 ++++++++++ src/Particle/VirtualParticleSet.h | 9 ++++ src/QMCWaveFunctions/BasisSetBase.h | 2 + .../LCAO/SoaLocalizedBasisSet.cpp | 51 +++++++++++++++++-- .../LCAO/SoaLocalizedBasisSet.h | 4 ++ 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/src/Particle/VirtualParticleSet.cpp b/src/Particle/VirtualParticleSet.cpp index b7f03870f36..3746a5c17b9 100644 --- a/src/Particle/VirtualParticleSet.cpp +++ b/src/Particle/VirtualParticleSet.cpp @@ -97,6 +97,34 @@ void VirtualParticleSet::releaseResource(ResourceCollection& collection, ParticleSet::releaseResource(collection, p_list); } + +const RefVectorWithLeader VirtualParticleSet::extractDTRefList_vp( + const RefVectorWithLeader& vp_list, + int id) +{ + RefVectorWithLeader dt_list(vp_list.getLeader().getDistTableAB(id)); + dt_list.reserve(vp_list.size()); + for (const VirtualParticleSet& vp : vp_list) + { + const auto& d_table = vp.getDistTableAB(id); + dt_list.push_back(d_table); + } + return dt_list; +} + + +const std::vector VirtualParticleSet::extractCoordsRefList_vp( + const RefVectorWithLeader& vp_list) +{ + std::vector coords_list; + for (const VirtualParticleSet& vp : vp_list) + for (int iat = 0; iat < vp.getTotalNum(); iat++) + coords_list.push_back(vp.activeR(iat)); + + return coords_list; +} + + /// move virtual particles to new postions and update distance tables void VirtualParticleSet::makeMoves(const ParticleSet& refp, int jel, diff --git a/src/Particle/VirtualParticleSet.h b/src/Particle/VirtualParticleSet.h index 670f5902d24..7161f88a7ee 100644 --- a/src/Particle/VirtualParticleSet.h +++ b/src/Particle/VirtualParticleSet.h @@ -95,6 +95,15 @@ class VirtualParticleSet : public ParticleSet bool sphere = false, int iat = -1); + inline size_t getTotalNum() const { return TotalNum; } + /**Extract list of Distance Tables + */ + static const RefVectorWithLeader extractDTRefList_vp( + const RefVectorWithLeader& vp_list, + int id); + + static const std::vector extractCoordsRefList_vp( + const RefVectorWithLeader& vp_list); /** move virtual particles to new postions and update distance tables * @param refp reference particle set * @param jel reference particle that all the VP moves from diff --git a/src/QMCWaveFunctions/BasisSetBase.h b/src/QMCWaveFunctions/BasisSetBase.h index e4f4384825e..60bb639b57e 100644 --- a/src/QMCWaveFunctions/BasisSetBase.h +++ b/src/QMCWaveFunctions/BasisSetBase.h @@ -172,6 +172,8 @@ struct SoaBasisSetBase vghgh_type& vghgh) = 0; virtual void evaluateV(const ParticleSet& P, int iat, value_type* restrict vals) = 0; + virtual void mw_evaluateV_mvp(const RefVectorWithLeader& vp_list, + OffloadMWVArray& vals) = 0; virtual bool is_S_orbital(int mo_idx, int ao_idx) { return false; } /// Determine which orbitals are S-type. Used for cusp correction. diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index 6a6c911267f..fc2c3c6896a 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -247,6 +247,51 @@ void SoaLocalizedBasisSet::evaluateVGHGH(const ParticleSet& P, int ia } } + +template +void SoaLocalizedBasisSet::mw_evaluateV_mvp(const RefVectorWithLeader& vp_list, + OffloadMWVArray& vals) +{ + const size_t nVPs = vals.size(0); + const auto& IonID(ions_.GroupID); + + auto& vps_leader = vp_list.getLeader(); + + + const auto dt_list(vps_leader.extractDTRefList_vp(vp_list, myTableIndex)); + const auto coordR_list(vps_leader.extractCoordsRefList_vp(vp_list)); + std::vector Tv_list; + + std::vector dist_list; + + size_t index = 0; + for (size_t iw = 0; iw < vp_list.size(); iw++) + for (int iat = 0; iat < vp_list[iw].getTotalNum(); iat++) + { + dist_list.push_back(dt_list[iw].getDisplRow(iat)); + //index++; + } + + + for (size_t iw = 0; iw < vp_list.size(); iw++) + for (int iat = 0; iat < vp_list[iw].getTotalNum(); iat++) + { + const auto& dist = dt_list[iw].getDistRow(iat); + const auto& displ = dt_list[iw].getDisplRow(iat); + + PosType Tv; + for (int c = 0; c < NumCenters; c++) + { + Tv[0] = (ions_.R[c][0] - coordR_list[index][0]) - displ[c][0]; + Tv[1] = (ions_.R[c][1] - coordR_list[index][1]) - displ[c][1]; + Tv[2] = (ions_.R[c][2] - coordR_list[index][2]) - displ[c][2]; + + LOBasisSet[IonID[c]]->evaluateV(vp_list[iw].getLattice(), dist[c], displ[c], + vals.data_at(index, 0) + BasisOffset[c], Tv); + } + index++; + } +} template void SoaLocalizedBasisSet::evaluateV(const ParticleSet& P, int iat, ORBT* restrict vals) { @@ -279,11 +324,7 @@ template void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLeader& vp_list, OffloadMWVArray& v) { - assert(BasisSetSize == v.size(1)); - size_t index = 0; - for (size_t iw = 0; iw < vp_list.size(); iw++) - for (int iat = 0; iat < vp_list[iw].getTotalNum(); iat++) - evaluateV(vp_list[iw], iat, v.data_at(index++, 0)); + mw_evaluateV_mvp(vp_list, v); } diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 86ecc385e12..c897262a1bf 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -159,6 +159,10 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase */ void evaluateV(const ParticleSet& P, int iat, ORBT* restrict vals) override; + /** same as evaluateV, but specializes in Virtual ParticleSet. + */ + void mw_evaluateV_mvp(const RefVectorWithLeader& vp_list, OffloadMWVArray& vals) override; + void evaluateGradSourceV(const ParticleSet& P, int iat, const ParticleSet& ions, int jion, vgl_type& vgl) override; void evaluateGradSourceVGL(const ParticleSet& P, From a59fc1524261ac4c92df947da2980ad3b9fedc6c Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 17 Oct 2023 13:53:55 -0500 Subject: [PATCH 025/168] remove unnecessary function --- src/QMCWaveFunctions/BasisSetBase.h | 2 -- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp | 11 ++--------- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h | 4 ---- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/QMCWaveFunctions/BasisSetBase.h b/src/QMCWaveFunctions/BasisSetBase.h index 60bb639b57e..e4f4384825e 100644 --- a/src/QMCWaveFunctions/BasisSetBase.h +++ b/src/QMCWaveFunctions/BasisSetBase.h @@ -172,8 +172,6 @@ struct SoaBasisSetBase vghgh_type& vghgh) = 0; virtual void evaluateV(const ParticleSet& P, int iat, value_type* restrict vals) = 0; - virtual void mw_evaluateV_mvp(const RefVectorWithLeader& vp_list, - OffloadMWVArray& vals) = 0; virtual bool is_S_orbital(int mo_idx, int ao_idx) { return false; } /// Determine which orbitals are S-type. Used for cusp correction. diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index fc2c3c6896a..9c60a5b5fc4 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -249,8 +249,8 @@ void SoaLocalizedBasisSet::evaluateVGHGH(const ParticleSet& P, int ia template -void SoaLocalizedBasisSet::mw_evaluateV_mvp(const RefVectorWithLeader& vp_list, - OffloadMWVArray& vals) +void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLeader& vp_list, + OffloadMWVArray& vals) { const size_t nVPs = vals.size(0); const auto& IonID(ions_.GroupID); @@ -320,13 +320,6 @@ void SoaLocalizedBasisSet::mw_evaluateValue(const RefVectorWithLeader evaluateV(P_list[iw], iat, v.data_at(iw, 0)); } -template -void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLeader& vp_list, - OffloadMWVArray& v) -{ - mw_evaluateV_mvp(vp_list, v); -} - template void SoaLocalizedBasisSet::evaluateGradSourceV(const ParticleSet& P, diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index c897262a1bf..86ecc385e12 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -159,10 +159,6 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase */ void evaluateV(const ParticleSet& P, int iat, ORBT* restrict vals) override; - /** same as evaluateV, but specializes in Virtual ParticleSet. - */ - void mw_evaluateV_mvp(const RefVectorWithLeader& vp_list, OffloadMWVArray& vals) override; - void evaluateGradSourceV(const ParticleSet& P, int iat, const ParticleSet& ions, int jion, vgl_type& vgl) override; void evaluateGradSourceVGL(const ParticleSet& P, From 6cd37617c2133e3ea0d0bb928c7ec6ee6efe7a30 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 17 Oct 2023 14:52:17 -0500 Subject: [PATCH 026/168] offload AO eval for DetRatios --- src/Numerics/SoaCartesianTensor.h | 148 ++++++++++++- src/Numerics/SoaSphericalTensor.h | 147 ++++++++++++- src/QMCWaveFunctions/BasisSetBase.h | 3 +- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 4 +- .../LCAO/MultiFunctorAdapter.h | 39 +++- .../LCAO/MultiQuinticSpline1D.h | 83 +++++++- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 196 +++++++++++++++++- .../LCAO/SoaLocalizedBasisSet.cpp | 62 ++++-- .../LCAO/SoaLocalizedBasisSet.h | 5 +- 9 files changed, 648 insertions(+), 39 deletions(-) diff --git a/src/Numerics/SoaCartesianTensor.h b/src/Numerics/SoaCartesianTensor.h index 21fa7f52bf1..f86fe813977 100644 --- a/src/Numerics/SoaCartesianTensor.h +++ b/src/Numerics/SoaCartesianTensor.h @@ -23,6 +23,8 @@ #include #include "OhmmsSoA/VectorSoaContainer.h" +#include "OhmmsPETE/OhmmsArray.h" +#include "OMPTarget/OffloadAlignedAllocators.hpp" namespace qmcplusplus { @@ -37,8 +39,10 @@ namespace qmcplusplus template struct SoaCartesianTensor { - using value_type = T; - using ggg_type = TinyVector, 3>; + using value_type = T; + using ggg_type = TinyVector, 3>; + using OffloadArray2D = Array>; + using OffloadArray3D = Array>; ///maximum angular momentum size_t Lmax; @@ -57,6 +61,7 @@ struct SoaCartesianTensor ///compute Ylm void evaluate_bare(T x, T y, T z, T* XYZ) const; + static void evaluate_bare_impl(T x, T y, T z, T* XYZ, const size_t Lmax_); ///compute Ylm inline void evaluateV(T x, T y, T z, T* XYZ) const @@ -77,6 +82,40 @@ struct SoaCartesianTensor ///compute Ylm inline void evaluateV(T x, T y, T z) { evaluateV(x, y, z, cXYZ.data(0)); } + /** + * @brief evaluate for multiple electrons and multiple pbc images + * + * @param [in] xyz electron positions [Nelec, Npbc, 3(x,y,z)] + * @param [out] XYZ Cartesian tensor elements [Nelec, Npbc, Nlm] + */ + inline void batched_evaluateV(OffloadArray3D& xyz, OffloadArray3D& XYZ) const + { + const size_t nElec = xyz.size(0); + const size_t Npbc = xyz.size(1); // number of PBC images + assert(xyz.size(2) == 3); // x,y,z + + assert(XYZ.size(0) == nElec); + assert(XYZ.size(1) == Npbc); + const size_t Nlm = XYZ.size(2); + + size_t nR = nElec * Npbc; // total number of positions to evaluate + + auto* xyz_devptr = xyz.device_data(); + auto* XYZ_devptr = XYZ.device_data(); + auto* NormFactor_ptr = NormFactor.data(); + + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:NormFactor_ptr[:Nlm]) \ + is_device_ptr(xyz_devptr, XYZ_devptr)") + for (size_t ir = 0; ir < nR; ir++) + { + evaluate_bare_impl(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], + XYZ_devptr + (ir * Nlm), Lmax); + for (int i = 0; i < Nlm; i++) + XYZ_devptr[ir * Nlm + i] *= NormFactor_ptr[i]; + } + } + + ///makes a table of \f$ r^l S_l^m \f$ and their gradients up to Lmax. void evaluateVGL(T x, T y, T z); @@ -140,6 +179,111 @@ SoaCartesianTensor::SoaCartesianTensor(const int l_max, bool addsign) : Lmax( } +PRAGMA_OFFLOAD("omp declare target") +template +void SoaCartesianTensor::evaluate_bare_impl(T x, T y, T z, T* restrict XYZ, size_t Lmax_) +{ + const T x2 = x * x, y2 = y * y, z2 = z * z; + const T x3 = x2 * x, y3 = y2 * y, z3 = z2 * z; + const T x4 = x3 * x, y4 = y3 * y, z4 = z3 * z; + const T x5 = x4 * x, y5 = y4 * y, z5 = z4 * z; + switch (Lmax_) + { + case 6: + XYZ[83] = x2 * y2 * z2; // X2Y2Z2 + XYZ[82] = x * y2 * z3; // Z3Y2X + XYZ[81] = x2 * y * z3; // Z3X2Y + XYZ[80] = x * y3 * z2; // Y3Z2X + XYZ[79] = x2 * y3 * z; // Y3X2Z + XYZ[78] = x3 * y * z2; // X3Z2Y + XYZ[77] = x3 * y2 * z; // X3Y2Z + XYZ[76] = y3 * z3; // Y3Z3 + XYZ[75] = x3 * z3; // X3Z3 + XYZ[74] = x3 * y3; // X3Y3 + XYZ[73] = x * y * z4; // Z4XY + XYZ[72] = x * y4 * z; // Y4XZ + XYZ[71] = x4 * y * z; // X4YZ + XYZ[70] = y2 * z4; // Z4Y2 + XYZ[69] = x2 * z4; // Z4X2 + XYZ[68] = y4 * z2; // Y4Z2 + XYZ[67] = x2 * y4; // Y4X2 + XYZ[66] = x4 * z2; // X4Z2 + XYZ[65] = x4 * y2; // X4Y2 + XYZ[64] = y * z * z4; // Z5Y + XYZ[63] = x * z * z4; // Z5X + XYZ[62] = y * y4 * z; // Y5Z + XYZ[61] = x * y * y4; // Y5X + XYZ[60] = x * x4 * z; // X5Z + XYZ[59] = x * x4 * y; // X5Y + XYZ[58] = z * z5; // Z6 + XYZ[57] = y * y5; // Y6 + XYZ[56] = x * x5; // X6 + case 5: + XYZ[55] = x * y2 * z2; // YYZZX + XYZ[54] = x2 * y * z2; // XXZZY + XYZ[53] = x2 * y2 * z; // XXYYZ + XYZ[52] = x * y * z3; // ZZZXY + XYZ[51] = x * y3 * z; // YYYXZ + XYZ[50] = x3 * y * z; // XXXYZ + XYZ[49] = y2 * z3; // ZZZYY + XYZ[48] = x2 * z3; // ZZZXX + XYZ[47] = y3 * z2; // YYYZZ + XYZ[46] = x2 * y3; // YYYXX + XYZ[45] = x3 * z2; // XXXZZ + XYZ[44] = x3 * y2; // XXXYY + XYZ[43] = y * z4; // ZZZZY + XYZ[42] = x * z4; // ZZZZX + XYZ[41] = y4 * z; // YYYYZ + XYZ[40] = x * y4; // YYYYX + XYZ[39] = x4 * z; // XXXXZ + XYZ[38] = x4 * y; // XXXXY + XYZ[37] = z * z4; // ZZZZZ + XYZ[36] = y * y4; // YYYYY + XYZ[35] = x * x4; // XXXXX + case 4: + XYZ[34] = x * y * z2; // ZZXY + XYZ[33] = x * y2 * z; // YYXZ + XYZ[32] = x2 * y * z; // XXYZ + XYZ[31] = y2 * z2; // YYZZ + XYZ[30] = x2 * z2; // XXZZ + XYZ[29] = x2 * y2; // XXYY + XYZ[28] = y * z3; // ZZZY + XYZ[27] = x * z3; // ZZZX + XYZ[26] = y3 * z; // YYYZ + XYZ[25] = x * y3; // YYYX + XYZ[24] = x3 * z; // XXXZ + XYZ[23] = x3 * y; // XXXY + XYZ[22] = z4; // ZZZZ + XYZ[21] = y4; // YYYY + XYZ[20] = x4; // XXXX + case 3: + XYZ[19] = x * y * z; // XYZ + XYZ[18] = y * z2; // ZZY + XYZ[17] = x * z2; // ZZX + XYZ[16] = y2 * z; // YYZ + XYZ[15] = x * y2; // YYX + XYZ[14] = x2 * z; // XXZ + XYZ[13] = x2 * y; // XXY + XYZ[12] = z3; // ZZZ + XYZ[11] = y3; // YYY + XYZ[10] = x3; // XXX + case 2: + XYZ[9] = y * z; // YZ + XYZ[8] = x * z; // XZ + XYZ[7] = x * y; // XY + XYZ[6] = z2; // ZZ + XYZ[5] = y2; // YY + XYZ[4] = x2; // XX + case 1: + XYZ[3] = z; // Z + XYZ[2] = y; // Y + XYZ[1] = x; // X + case 0: + XYZ[0] = 1; // S + } +} +PRAGMA_OFFLOAD("omp end declare target") + template void SoaCartesianTensor::evaluate_bare(T x, T y, T z, T* restrict XYZ) const { diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index 56c638b42e6..e3b0ad67c67 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -17,6 +17,8 @@ #include #include "OhmmsSoA/VectorSoaContainer.h" #include "OhmmsPETE/Tensor.h" +#include "OhmmsPETE/OhmmsArray.h" +#include "OMPTarget/OffloadAlignedAllocators.hpp" namespace qmcplusplus { @@ -37,6 +39,8 @@ namespace qmcplusplus template struct SoaSphericalTensor { + using OffloadArray2D = Array>; + using OffloadArray3D = Array>; ///maximum angular momentum for the center int Lmax; /// Normalization factors @@ -56,6 +60,7 @@ struct SoaSphericalTensor ///compute Ylm void evaluate_bare(T x, T y, T z, T* Ylm) const; + static void evaluate_bare_impl(T x, T y, T z, T* Ylm, const size_t Lmax_, const T* FacL, const T* FacLM); ///compute Ylm inline void evaluateV(T x, T y, T z, T* Ylm) const @@ -65,6 +70,43 @@ struct SoaSphericalTensor Ylm[i] *= NormFactor[i]; } + /** + * @brief evaluate V for multiple electrons and multiple pbc images + * + * @param [in] xyz electron positions [Nelec, Npbc, 3(x,y,z)] + * @param [out] Ylm Spherical tensor elements [Nelec, Npbc, Nlm] + */ + inline void batched_evaluateV(OffloadArray3D& xyz, OffloadArray3D& Ylm) const + { + const size_t nElec = xyz.size(0); + const size_t Npbc = xyz.size(1); // number of PBC images + assert(xyz.size(2) == 3); + + assert(Ylm.size(0) == nElec); + assert(Ylm.size(1) == Npbc); + const size_t Nlm = Ylm.size(2); + + size_t nR = nElec * Npbc; // total number of positions to evaluate + + auto* xyz_devptr = xyz.device_data(); + auto* Ylm_devptr = Ylm.device_data(); + auto* flm_ptr = FactorLM.data(); + auto* fl_ptr = FactorL.data(); + auto* NormFactor_ptr = NormFactor.data(); + + + PRAGMA_OFFLOAD("omp target teams distribute parallel for \ + map(always, to:flm_ptr[:Nlm], fl_ptr[:Lmax+1], NormFactor_ptr[:Nlm]) \ + is_device_ptr(xyz_devptr, Ylm_devptr)") + for (size_t ir = 0; ir < nR; ir++) + { + evaluate_bare_impl(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], + Ylm_devptr + (ir * Nlm), Lmax, fl_ptr, flm_ptr); + for (int i = 0; i < Nlm; i++) + Ylm_devptr[ir * Nlm + i] *= NormFactor_ptr[i]; + } + } + ///compute Ylm inline void evaluateV(T x, T y, T z) { @@ -84,7 +126,7 @@ struct SoaSphericalTensor void evaluateVGHGH(T x, T y, T z); ///returns the index/locator for (\f$l,m\f$) combo, \f$ l(l+1)+m \f$ - inline int index(int l, int m) const { return (l * (l + 1)) + m; } + static inline int index(int l, int m) { return (l * (l + 1)) + m; } /** return the starting address of the component * @@ -169,6 +211,109 @@ inline SoaSphericalTensor::SoaSphericalTensor(const int l_max, bool addsign) } } +PRAGMA_OFFLOAD("omp declare target") +template +inline void SoaSphericalTensor::evaluate_bare_impl(const T x, + const T y, + const T z, + T* restrict Ylm, + const size_t Lmax_, + const T* FactorL_ptr, + const T* FactorLM_ptr) +{ + constexpr T czero(0); + constexpr T cone(1); + const T pi = 4.0 * std::atan(1.0); + const T omega = 1.0 / std::sqrt(4.0 * pi); + constexpr T eps2 = std::numeric_limits::epsilon() * std::numeric_limits::epsilon(); + + /* Calculate r, cos(theta), sin(theta), cos(phi), sin(phi) from input + coordinates. Check here the coordinate singularity at cos(theta) = +-1. + This also takes care of r=0 case. */ + T cphi, sphi, ctheta; + T r2xy = x * x + y * y; + T r = std::sqrt(r2xy + z * z); + if (r2xy < eps2) + { + cphi = czero; + sphi = cone; + ctheta = (z < czero) ? -cone : cone; + } + else + { + ctheta = z / r; + //protect ctheta, when ctheta is slightly >1 or <-1 + if (ctheta > cone) + ctheta = cone; + if (ctheta < -cone) + ctheta = -cone; + T rxyi = cone / std::sqrt(r2xy); + cphi = x * rxyi; + sphi = y * rxyi; + } + T stheta = std::sqrt(cone - ctheta * ctheta); + /* Now to calculate the associated legendre functions P_lm from the + recursion relation from l=0 to Lmax. Conventions of J.D. Jackson, + Classical Electrodynamics are used. */ + Ylm[0] = cone; + // calculate P_ll and P_l,l-1 + T fac = cone; + int j = -1; + for (int l = 1; l <= Lmax_; l++) + { + j += 2; + fac *= -j * stheta; + int ll = index(l, l); + int l1 = index(l, l - 1); + int l2 = index(l - 1, l - 1); + Ylm[ll] = fac; + Ylm[l1] = j * ctheta * Ylm[l2]; + } + // Use recurence to get other plm's // + for (int m = 0; m < Lmax_ - 1; m++) + { + int j = 2 * m + 1; + for (int l = m + 2; l <= Lmax_; l++) + { + j += 2; + int lm = index(l, m); + int l1 = index(l - 1, m); + int l2 = index(l - 2, m); + Ylm[lm] = (ctheta * j * Ylm[l1] - (l + m - 1) * Ylm[l2]) / (l - m); + } + } + // Now to calculate r^l Y_lm. // + T sphim, cphim, temp; + Ylm[0] = omega; //1.0/sqrt(pi4); + T rpow = 1.0; + for (int l = 1; l <= Lmax_; l++) + { + rpow *= r; + //fac = rpow*sqrt(static_cast(2*l+1))*omega;//rpow*sqrt((2*l+1)/pi4); + //FactorL[l] = sqrt(2*l+1)/sqrt(4*pi) + fac = rpow * FactorL_ptr[l]; + int l0 = index(l, 0); + Ylm[l0] *= fac; + cphim = cone; + sphim = czero; + for (int m = 1; m <= l; m++) + { + temp = cphim * cphi - sphim * sphi; + sphim = sphim * cphi + cphim * sphi; + cphim = temp; + int lm = index(l, m); + fac *= FactorLM_ptr[lm]; + temp = fac * Ylm[lm]; + Ylm[lm] = temp * cphim; + lm = index(l, -m); + Ylm[lm] = temp * sphim; + } + } + //for (int i=0; i inline void SoaSphericalTensor::evaluate_bare(T x, T y, T z, T* restrict Ylm) const { diff --git a/src/QMCWaveFunctions/BasisSetBase.h b/src/QMCWaveFunctions/BasisSetBase.h index e4f4384825e..b08f1988e19 100644 --- a/src/QMCWaveFunctions/BasisSetBase.h +++ b/src/QMCWaveFunctions/BasisSetBase.h @@ -155,7 +155,8 @@ struct SoaBasisSetBase //Evaluates value for electron "iat". places it in a offload array for batched code. virtual void mw_evaluateValue(const RefVectorWithLeader& P_list, int iat, OffloadMWVArray& v) = 0; //Evaluates value for all the electrons of the virtual particles. places it in a offload array for batched code. - virtual void mw_evaluateValueVPs(const RefVectorWithLeader& vp_list, + virtual void mw_evaluateValueVPs(RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& vp_list, OffloadMWVArray& v) = 0; //Evaluates value, gradient, and Hessian for electron "iat". Parks them into a temporary data structure "vgh". virtual void evaluateVGH(const ParticleSet& P, int iat, vgh_type& vgh) = 0; diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index 71e9ed3f6fb..d4e9c6758d5 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -497,7 +497,9 @@ void LCAOrbitalSet::mw_evaluateValueVPsImplGEMM(const RefVectorWithLeadermw_evaluateValueVPs(vp_list, vp_basis_v_mw); + auto basis_list = spo_leader.extractBasisRefList(spo_list); + myBasisSet->mw_evaluateValueVPs(basis_list, vp_list, vp_basis_v_mw); + vp_basis_v_mw.updateFrom(); // TODO: remove after offloading rest of function if (Identity) { diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index 74df4c0b59d..e004335c644 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -31,9 +31,11 @@ namespace qmcplusplus template struct MultiFunctorAdapter { - using RealType = typename FN::real_type; - using GridType = LogGridLight; - using single_type = FN; + using RealType = typename FN::real_type; + using GridType = LogGridLight; + using single_type = FN; + using OffloadArray2D = Array>; + using OffloadArray3D = Array>; aligned_vector> Rnl; MultiFunctorAdapter() = default; @@ -57,6 +59,37 @@ struct MultiFunctorAdapter u[i] = Rnl[i]->f(r); } + inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const + { + r.updateFrom(); // TODO: remove after offload + const size_t nElec = r.size(0); + const size_t Nxyz = r.size(1); // number of PBC images + assert(nElec == u.size(0)); + assert(Nxyz == u.size(1)); + const size_t nRnl = u.size(2); // number of splines + const size_t nR = nElec * Nxyz; // total number of positions to evaluate + + auto* r_ptr = r.data(); + auto* u_ptr = u.data(); + + + for (size_t ir = 0; ir < nR; ir++) + { + if (r_ptr[ir] >= Rmax) + { + for (size_t i = 0, n = Rnl.size(); i < n; ++i) + u_ptr[ir * nRnl + i] = 0.0; + } + else + { + for (size_t i = 0, n = Rnl.size(); i < n; ++i) + u_ptr[ir * nRnl + i] = Rnl[i]->f(r_ptr[ir]); + } + } + + u.updateTo(); // TODO: remove after offload + } + inline void evaluate(RealType r, RealType* restrict u, RealType* restrict du, RealType* restrict d2u) { const RealType rinv = RealType(1) / r; diff --git a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h index a434f82f04b..582b9bc1fa2 100644 --- a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h +++ b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h @@ -27,7 +27,10 @@ #include "Numerics/OneDimGridBase.h" #include "Numerics/OneDimQuinticSpline.h" #include "OhmmsPETE/OhmmsMatrix.h" +#include "OhmmsPETE/OhmmsArray.h" #include "CPU/SIMD/aligned_allocator.hpp" +#include "OMPTarget/OffloadAlignedAllocators.hpp" +#include namespace qmcplusplus { @@ -37,6 +40,7 @@ struct LogGridLight T lower_bound; T upper_bound; T OneOverLogDelta; + double LogDelta; std::vector r_values; inline void set(T ri, T rf, int n) @@ -47,11 +51,20 @@ struct LogGridLight double log_ratio = std::log(ratio); double dlog_ratio = log_ratio / static_cast(n - 1); OneOverLogDelta = 1.0 / dlog_ratio; + LogDelta = dlog_ratio; r_values.resize(n); for (int i = 0; i < n; i++) r_values[i] = ri * std::exp(i * dlog_ratio); } + PRAGMA_OFFLOAD("omp declare target") + static inline T getCL(T r, int& loc, double OneOverLogDelta_, T lower_bound_, double dlog_ratio_) + { + loc = static_cast(std::log(r / lower_bound_) * OneOverLogDelta_); + return r - lower_bound_ * std::exp(loc * dlog_ratio_); + } + PRAGMA_OFFLOAD("omp end declare target") + inline int locate(T r) const { int loc = static_cast(std::log(r / lower_bound) * OneOverLogDelta); @@ -84,9 +97,11 @@ template class MultiQuinticSpline1D { public: - using RealType = T; - using GridType = OneDimGridBase; - using CoeffType = Matrix>; + using RealType = T; + using GridType = OneDimGridBase; + using CoeffType = Matrix>; + using OffloadArray2D = Array>; + using OffloadArray3D = Array>; private: ///number of splines @@ -137,6 +152,68 @@ class MultiQuinticSpline1D } } + inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, T Rmax) const + { + const size_t nElec = r.size(0); + const size_t Nxyz = r.size(1); // number of PBC images + assert(nElec == u.size(0)); + assert(Nxyz == u.size(1)); + const size_t nRnl = u.size(2); // number of splines + const size_t nR = nElec * Nxyz; // total number of positions to evaluate + + auto* first_deriv_ptr = first_deriv.data(); + + double OneOverLogDelta = myGrid.OneOverLogDelta; + T lower_bound = myGrid.lower_bound; + T dlog_ratio = myGrid.LogDelta; + + auto* r_devptr = r.device_data(); + auto* u_devptr = u.device_data(); + + auto* coeff_ptr = coeffs->data(); + const size_t nCols = coeffs->cols(); + const size_t coefsize = coeffs->size(); + + + PRAGMA_OFFLOAD("omp target teams distribute parallel for \ + map(always, to:first_deriv_ptr[:num_splines_], coeff_ptr[:coefsize]) \ + is_device_ptr(r_devptr, u_devptr)") + for (size_t ir = 0; ir < nR; ir++) + { + if (r_devptr[ir] >= Rmax) + { + for (size_t i = 0; i < num_splines_; ++i) + u_devptr[ir * nRnl + i] = 0.0; + } + else if (r_devptr[ir] < lower_bound) + { + const T dr = r_devptr[ir] - lower_bound; + const T* restrict a = coeff_ptr; + // const T* restrict a = (*coeffs)[0]; + for (size_t i = 0; i < num_splines_; ++i) + u_devptr[ir * nRnl + i] = a[i] + first_deriv_ptr[i] * dr; + } + else + { + int loc; + const auto cL = LogGridLight::getCL(r_devptr[ir], loc, OneOverLogDelta, lower_bound, dlog_ratio); + // const auto cL = myGrid.getCLForQuintic(r_list[ir], loc); + const size_t offset = loc * 6; + //coeffs is an OhmmsMatrix and [] is a row access operator + //returning a pointer to 'row' which is normal type pointer [] + // const T* restrict a = (*coeffs)[offset + 0]; + const T* restrict a = coeff_ptr + nCols * (offset + 0); + const T* restrict b = coeff_ptr + nCols * (offset + 1); + const T* restrict c = coeff_ptr + nCols * (offset + 2); + const T* restrict d = coeff_ptr + nCols * (offset + 3); + const T* restrict e = coeff_ptr + nCols * (offset + 4); + const T* restrict f = coeff_ptr + nCols * (offset + 5); + for (size_t i = 0; i < num_splines_; ++i) + u_devptr[ir * nRnl + i] = a[i] + cL * (b[i] + cL * (c[i] + cL * (d[i] + cL * (e[i] + cL * f[i])))); + } + } + } + inline void evaluate(T r, T* restrict u, T* restrict du, T* restrict d2u) const { if (r < myGrid.lower_bound) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 69a562b2133..3bb90498e46 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -36,7 +36,7 @@ struct SoaAtomicBasisSet using ValueType = typename QMCTraits::ValueType; using OffloadArray4D = Array>; using OffloadArray3D = Array>; - using OffloadMatrix = Matrix>; + using OffloadArray2D = Array>; using OffloadVector = Vector>; /// multi walker shared memory buffer @@ -65,10 +65,23 @@ struct SoaAtomicBasisSet std::vector RnlID; ///temporary storage VectorSoaContainer tempS; + NewTimer& ylm_timer_; + NewTimer& rnl_timer_; + NewTimer& pbc_timer_; + NewTimer& nelec_pbc_timer_; + NewTimer& phase_timer_; + NewTimer& psi_timer_; ///the constructor - explicit SoaAtomicBasisSet(int lmax, bool addsignforM = false) : Ylm(lmax, addsignforM) {} - + explicit SoaAtomicBasisSet(int lmax, bool addsignforM = false) + : Ylm(lmax, addsignforM), + ylm_timer_(createGlobalTimer("SoaAtomicBasisSet::Ylm", timer_level_fine)), + rnl_timer_(createGlobalTimer("SoaAtomicBasisSet::Rnl", timer_level_fine)), + pbc_timer_(createGlobalTimer("SoaAtomicBasisSet::pbc_images", timer_level_fine)), + nelec_pbc_timer_(createGlobalTimer("SoaAtomicBasisSet::nelec_pbc_images", timer_level_fine)), + phase_timer_(createGlobalTimer("SoaAtomicBasisSet::phase", timer_level_fine)), + psi_timer_(createGlobalTimer("SoaAtomicBasisSet::psi", timer_level_fine)) + {} void checkInVariables(opt_variables_type& active) { //for(size_t nl=0; nl + inline void mw_evaluateV(const RefVectorWithLeader& atom_bs_list, + const LAT& lattice, + Array>& psi, + const Vector>& displ_list, + const Vector>& Tv_list, + const size_t nElec, + const size_t nBasTot, + const size_t c, + const size_t BasisOffset, + const size_t NumCenters) + { + assert(this == &atom_bs_list.getLeader()); + auto& atom_bs_leader = atom_bs_list.template getCastedLeader>(); + /* + psi [nElec, nBasTot] (start at [0, BasisOffset]) + displ_list [3 * nElec * NumCenters] (start at [3*nElec*c]) + Tv_list [3 * nElec * NumCenters] + */ + //TODO: use QMCTraits::DIM instead of 3? + int Nx = PBCImages[0] + 1; + int Ny = PBCImages[1] + 1; + int Nz = PBCImages[2] + 1; + int Nyz = Ny * Nz; + int Nxyz = Nx * Nyz; + assert(psi.size(0) == nElec); + assert(psi.size(1) == nBasTot); + + + auto& ylm_v = atom_bs_leader.mw_mem_handle_.getResource().ylm_v; + auto& rnl_v = atom_bs_leader.mw_mem_handle_.getResource().rnl_v; + auto& dr = atom_bs_leader.mw_mem_handle_.getResource().dr; + auto& r = atom_bs_leader.mw_mem_handle_.getResource().r; + + size_t nRnl = RnlID.size(); + size_t nYlm = Ylm.size(); + + ylm_v.resize(nElec, Nxyz, nYlm); + rnl_v.resize(nElec, Nxyz, nRnl); + dr.resize(nElec, Nxyz, 3); + r.resize(nElec, Nxyz); + + // TODO: move these outside? + auto& dr_pbc = atom_bs_leader.mw_mem_handle_.getResource().dr_pbc; + auto& correctphase = atom_bs_leader.mw_mem_handle_.getResource().correctphase; + dr_pbc.resize(Nxyz, 3); + correctphase.resize(nElec); + + auto* dr_pbc_devptr = dr_pbc.device_data(); + auto* dr_new_devptr = dr.device_data(); + auto* r_new_devptr = r.device_data(); + + auto* correctphase_devptr = correctphase.device_data(); + + auto* Tv_list_devptr = Tv_list.device_data(); + auto* displ_list_devptr = displ_list.device_data(); + + auto* psi_devptr = psi.device_data(); + + // need to map Tensor vals to device + auto* latR_ptr = lattice.R.data(); + + // build dr_pbc: translation vectors to all images + // should just do this once and store it (like with phase) + { + ScopedTimer local(pbc_timer_); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(always, to:latR_ptr[:9]) \ + is_device_ptr(dr_pbc_devptr) ") + for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) + { + // i_xyz = k + Nz * (j + Ny * i) + auto div_k = std::div(i_xyz, Nz); + int k = div_k.rem; + int ij = div_k.quot; // ij = (j + Ny * i) + auto div_ij = std::div(ij, Ny); + int j = div_ij.rem; + int i = div_ij.quot; + int TransX = ((i % 2) * 2 - 1) * ((i + 1) / 2); + int TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); + int TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); + for (size_t i_dim = 0; i_dim < 3; i_dim++) + dr_pbc_devptr[i_dim + 3 * i_xyz] = + (TransX * latR_ptr[i_dim + 0 * 3] + TransY * latR_ptr[i_dim + 1 * 3] + TransZ * latR_ptr[i_dim + 2 * 3]); + } + } + + + { + ScopedTimer local_timer(phase_timer_); +#if not defined(QMC_COMPLEX) + + PRAGMA_OFFLOAD("omp target teams distribute parallel for is_device_ptr(correctphase_devptr) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + correctphase_devptr[i_e] = 1.0; + +#else + auto* SuperTwist_ptr = SuperTwist.data(); + + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(always, to:SuperTwist_ptr[:9]) \ + is_device_ptr(Tv_list_devptr, correctphase_devptr) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + { + //RealType phasearg = dot(3, SuperTwist.data(), 1, Tv_list.data() + 3 * i_e, 1); + RealType phasearg = 0; + for (size_t i_dim = 0; i_dim < 3; i_dim++) + phasearg += SuperTwist[i_dim] * Tv_list_devptr[i_dim + 3 * i_e]; + RealType s, c; + qmcplusplus::sincos(-phasearg, &s, &c); + correctphase_devptr[i_e] = ValueType(c, s); + } +#endif + } + + { + ScopedTimer local_timer(nelec_pbc_timer_); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) \ + is_device_ptr(dr_new_devptr, dr_pbc_devptr, r_new_devptr, displ_list_devptr) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + { + for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) + { + RealType tmp_r2 = 0.0; + for (size_t i_dim = 0; i_dim < 3; i_dim++) + { + dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = + -(displ_list_devptr[i_dim + 3 * (i_e + c * nElec)] + dr_pbc_devptr[i_dim + 3 * i_xyz]); + tmp_r2 += dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] * dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)]; + } + r_new_devptr[i_xyz + Nxyz * i_e] = std::sqrt(tmp_r2); + } + } + } + + + { + ScopedTimer local(rnl_timer_); + MultiRnl.batched_evaluate(r, rnl_v, Rmax); + } + // dr_new is [3 * Nxyz * nElec] realtype + { + ScopedTimer local(ylm_timer_); + //Ylm.mw_evaluateV(dr.data(), ylm_v.data(), Nxyz * nElec, nYlm); + Ylm.batched_evaluateV(dr, ylm_v); + } + ///Phase for PBC containing the phase for the nearest image displacement and the correction due to the Distance table. + auto* phase_fac_ptr = periodic_image_phase_factors.data(); + auto* LM_ptr = LM.data(); + auto* NL_ptr = NL.data(); + auto* psi_ptr = psi.data(); + + auto* ylm_devptr = ylm_v.device_data(); + auto* rnl_devptr = rnl_v.device_data(); + { + ScopedTimer local_timer(psi_timer_); + PRAGMA_OFFLOAD( + "omp target teams distribute parallel for collapse(2) map(always, to:phase_fac_ptr[:Nxyz], LM_ptr[:BasisSetSize], NL_ptr[:BasisSetSize]) \ + is_device_ptr(ylm_devptr, rnl_devptr, psi_devptr, correctphase_devptr) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + { + for (size_t ib = 0; ib < BasisSetSize; ++ib) + { + for (size_t i_xyz = 0; i_xyz < Nxyz; i_xyz++) + { + const ValueType Phase = phase_fac_ptr[i_xyz] * correctphase_devptr[i_e]; + psi_devptr[BasisOffset + ib + i_e * nBasTot] += ylm_devptr[(i_xyz + Nxyz * i_e) * nYlm + LM_ptr[ib]] * + rnl_devptr[(i_xyz + Nxyz * i_e) * nRnl + NL_ptr[ib]] * Phase; + } + } + } + } + } + void createResource(ResourceCollection& collection) const { collection.addResource(std::make_unique()); @@ -700,9 +886,9 @@ struct SoaAtomicBasisSet OffloadArray4D rnl_vgl; // [5][Nelec][PBC][NRnl] OffloadArray3D ylm_v; // [Nelec][PBC][NYlm] OffloadArray3D rnl_v; // [Nelec][PBC][NRnl] - OffloadMatrix dr_pbc; // [PBC][xyz] translation vector for each image + OffloadArray2D dr_pbc; // [PBC][xyz] translation vector for each image OffloadArray3D dr; // [Nelec][PBC][xyz] ion->elec displacement for each image - OffloadMatrix r; // [Nelec][PBC] ion->elec distance for each image + OffloadArray2D r; // [Nelec][PBC] ion->elec distance for each image OffloadVector correctphase; // [Nelec] overall phase }; }; diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index 9c60a5b5fc4..f92fc0513f8 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -249,10 +249,15 @@ void SoaLocalizedBasisSet::evaluateVGHGH(const ParticleSet& P, int ia template -void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLeader& vp_list, - OffloadMWVArray& vals) +void SoaLocalizedBasisSet::mw_evaluateValueVPs(RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& vp_list, + OffloadMWVArray& vp_basis_v) { - const size_t nVPs = vals.size(0); + assert(this == &basis_list.getLeader()); + auto& basis_leader = basis_list.template getCastedLeader>(); + + const size_t nVPs = vp_basis_v.size(0); + assert(vp_basis_v.size(1) == BasisSetSize); const auto& IonID(ions_.GroupID); auto& vps_leader = vp_list.getLeader(); @@ -260,37 +265,50 @@ void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLea const auto dt_list(vps_leader.extractDTRefList_vp(vp_list, myTableIndex)); const auto coordR_list(vps_leader.extractCoordsRefList_vp(vp_list)); - std::vector Tv_list; - std::vector dist_list; + // make these shared resource? PinnedDualAllocator? OffloadPinnedAllocator? + Vector> Tv_list; + Vector> displ_list_tr; + Tv_list.resize(3 * NumCenters * nVPs); + displ_list_tr.resize(3 * NumCenters * nVPs); + // TODO: need one more level of indirection for offload? + // need to index into walkers/vps, but need walker num for distance table size_t index = 0; for (size_t iw = 0; iw < vp_list.size(); iw++) for (int iat = 0; iat < vp_list[iw].getTotalNum(); iat++) { - dist_list.push_back(dt_list[iw].getDisplRow(iat)); - //index++; - } - - - for (size_t iw = 0; iw < vp_list.size(); iw++) - for (int iat = 0; iat < vp_list[iw].getTotalNum(); iat++) - { - const auto& dist = dt_list[iw].getDistRow(iat); const auto& displ = dt_list[iw].getDisplRow(iat); - - PosType Tv; for (int c = 0; c < NumCenters; c++) { - Tv[0] = (ions_.R[c][0] - coordR_list[index][0]) - displ[c][0]; - Tv[1] = (ions_.R[c][1] - coordR_list[index][1]) - displ[c][1]; - Tv[2] = (ions_.R[c][2] - coordR_list[index][2]) - displ[c][2]; - - LOBasisSet[IonID[c]]->evaluateV(vp_list[iw].getLattice(), dist[c], displ[c], - vals.data_at(index, 0) + BasisOffset[c], Tv); + for (size_t idim = 0; idim < 3; idim++) + { + Tv_list[idim + 3 * (index + c * nVPs)] = (ions_.R[c][idim] - coordR_list[index][idim]) - displ[c][idim]; + displ_list_tr[idim + 3 * (index + c * nVPs)] = displ[c][idim]; + } } index++; } +#if defined(QMC_COMPLEX) + Tv_list.updateTo(); +#endif + displ_list_tr.updateTo(); + + // set AO data to zero on device + auto* vp_basis_v_devptr = vp_basis_v.device_data(); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(vp_basis_v_devptr) ") + for (size_t i_vp = 0; i_vp < nVPs; i_vp++) + for (size_t ib = 0; ib < BasisSetSize; ++ib) + vp_basis_v_devptr[ib + i_vp * BasisSetSize] = 0; + + // TODO: group/sort centers by species? + for (int c = 0; c < NumCenters; c++) + { + auto one_species_basis_list = extractOneSpeciesBasisRefList(basis_list, IonID[c]); + LOBasisSet[IonID[c]]->mw_evaluateV(one_species_basis_list, vps_leader.getLattice(), vp_basis_v, displ_list_tr, + Tv_list, nVPs, BasisSetSize, c, BasisOffset[c], NumCenters); + } + // vp_basis_v.updateFrom(); } template void SoaLocalizedBasisSet::evaluateV(const ParticleSet& P, int iat, ORBT* restrict vals) diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 86ecc385e12..97136486b7a 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -117,10 +117,13 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase void mw_evaluateValue(const RefVectorWithLeader& P_list, int iat, OffloadMWVArray& v) override; /** compute V using packed array with all walkers ++ * @param basis_list list of basis sets (one for each walker) * @param vp_list list of quantum virtual particleset (one for each walker) * @param v Array(n_walkers, BasisSetSize) */ - void mw_evaluateValueVPs(const RefVectorWithLeader& vp_list, OffloadMWVArray& v) override; + void mw_evaluateValueVPs(RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& vp_list, + OffloadMWVArray& v) override; /** compute VGL using packed array with all walkers From 11d3edfa8e3231ea19245c0697a6d3da7c44d406 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Tue, 17 Oct 2023 15:19:36 -0500 Subject: [PATCH 027/168] fix typo --- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 97136486b7a..27377a6908e 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -117,7 +117,7 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase void mw_evaluateValue(const RefVectorWithLeader& P_list, int iat, OffloadMWVArray& v) override; /** compute V using packed array with all walkers -+ * @param basis_list list of basis sets (one for each walker) + * @param basis_list list of basis sets (one for each walker) * @param vp_list list of quantum virtual particleset (one for each walker) * @param v Array(n_walkers, BasisSetSize) */ From 40b49cb144e0bc2f2dc1a02d22aa4a7473167962 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 21:09:23 +0000 Subject: [PATCH 028/168] Bump urllib3 from 2.0.6 to 2.0.7 in /docs Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.6 to 2.0.7. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.0.6...2.0.7) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 1c9cc6c5860..68fd750be0e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -69,7 +69,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -urllib3==2.0.6 +urllib3==2.0.7 # via requests # The following packages are considered to be unsafe in a requirements file: From 8076a49904091145c987ccd17b4075df379b5d56 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 15:11:40 +0000 Subject: [PATCH 029/168] workaround for offload segfault on sunspot, the calls to .size() in the offload pragmas in SoaDistanceTableABOMPTarget::mw_evaluate were causing a segfault. using a separate variable for the size seems to fix the problem --- src/Particle/SoaDistanceTableABOMPTarget.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Particle/SoaDistanceTableABOMPTarget.h b/src/Particle/SoaDistanceTableABOMPTarget.h index 05dac1eaf13..8ef2ace4d1a 100644 --- a/src/Particle/SoaDistanceTableABOMPTarget.h +++ b/src/Particle/SoaDistanceTableABOMPTarget.h @@ -298,11 +298,14 @@ class SoaDistanceTableABOMPTarget : public DTD_BConds, public Distance auto* input_ptr = offload_input.data(); const int num_sources_local = num_sources_; + const size_t num_r_dr = mw_r_dr.size(); + const size_t num_offload_input = offload_input.size(); + { ScopedTimer offload(dt_leader.offload_timer_); PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(total_targets*num_teams) \ - map(always, to: input_ptr[:offload_input.size()]) \ - depend(out:r_dr_ptr[:mw_r_dr.size()]) nowait") + map(always, to: input_ptr[:num_offload_input]) \ + depend(out:r_dr_ptr[:num_r_dr]) nowait") for (int iat = 0; iat < total_targets; ++iat) for (int team_id = 0; team_id < num_teams; team_id++) { @@ -329,7 +332,7 @@ class SoaDistanceTableABOMPTarget : public DTD_BConds, public Distance if (!(modes_ & DTModes::MW_EVALUATE_RESULT_NO_TRANSFER_TO_HOST)) { PRAGMA_OFFLOAD( - "omp target update from(r_dr_ptr[:mw_r_dr.size()]) depend(inout:r_dr_ptr[:mw_r_dr.size()]) nowait") + "omp target update from(r_dr_ptr[:num_r_dr]) depend(inout:r_dr_ptr[:num_r_dr]) nowait") } // wait for computing and (optional) transferring back to host. // It can potentially be moved to ParticleSet to fuse multiple similar taskwait From 4695b83250d34aea1e81885d3184d102ce3f6416 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 15:43:00 +0000 Subject: [PATCH 030/168] added ompBLAS gemm, copy --- src/Platforms/OMPTarget/ompBLAS.cpp | 188 ++++++++++++++++++++++++++++ src/Platforms/OMPTarget/ompBLAS.hpp | 88 +++++++++++++ 2 files changed, 276 insertions(+) diff --git a/src/Platforms/OMPTarget/ompBLAS.cpp b/src/Platforms/OMPTarget/ompBLAS.cpp index 0dce77844d3..0446c4442f0 100644 --- a/src/Platforms/OMPTarget/ompBLAS.cpp +++ b/src/Platforms/OMPTarget/ompBLAS.cpp @@ -22,6 +22,141 @@ namespace qmcplusplus namespace ompBLAS { +template +ompBLAS_status gemm_impl(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const T alpha, + const T* const A, + const int lda, + const T* const B, + const int ldb, + const T beta, + T* const C, + const int ldc) +{ + if (transa == 'T' && transb == 'N') //A(ji) * B(jk) -> C(ik) + { + PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") + for (size_t m = 0; m < M; m++) + for (size_t n = 0; n < N; n++) + { + C[n * ldc + m] *= beta; + for (size_t k = 0; k < K; k++) + C[n * ldc + m] += alpha * A[lda * m + k] * B[ldb * n + k]; + } + } + else if (transa == 'T' && transb == 'T') + { + PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") + for (size_t m = 0; m < M; m++) + for (size_t n = 0; n < N; n++) + { + C[n * ldc + m] *= beta; + for (size_t k = 0; k < K; k++) + C[n * ldc + m] += alpha * A[lda * m + k] * B[ldb * k + n]; + } + } + else if (transa == 'N' && transb == 'T') + { + PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") + for (size_t m = 0; m < M; m++) + for (size_t n = 0; n < N; n++) + { + C[n * ldc + m] *= beta; + for (size_t k = 0; k < K; k++) + C[n * ldc + m] += alpha * A[lda * k + m] * B[ldb * k + n]; + } + } + else if (transa == 'N' && transb == 'N') + { + PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") + for (size_t m = 0; m < M; m++) + for (size_t n = 0; n < N; n++) + { + C[n * ldc + m] *= beta; + for (size_t k = 0; k < K; k++) + C[n * ldc + m] += alpha * A[lda * k + m] * B[ldb * n + k]; + } + } + + // #endif + return 0; +} + + +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const float alpha, + const float* const A, + const int lda, + const float* const B, + const int ldb, + const float beta, + float* const C, + const int ldc) +{ + return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); +} +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const double alpha, + const double* const A, + const int lda, + const double* const B, + const int ldb, + const double beta, + double* const C, + const int ldc) +{ + return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); +} +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const std::complex alpha, + const std::complex* const A, + const int lda, + const std::complex* const B, + const int ldb, + const std::complex beta, + std::complex* const C, + const int ldc) +{ + return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); +} +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const std::complex alpha, + const std::complex* const A, + const int lda, + const std::complex* const B, + const int ldb, + const std::complex beta, + std::complex* const C, + const int ldc) +{ + return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); +} + template ompBLAS_status gemv_impl(ompBLAS_handle& handle, const char trans, @@ -443,6 +578,59 @@ ompBLAS_status ger_batched(ompBLAS_handle& handle, } #endif +template +ompBLAS_status copy_impl(ompBLAS_handle& handle, + const int n, + const T* const x, + const int incx, + T* const y, + const int incy) +{ + PRAGMA_OFFLOAD("omp target teams distribute parallel for is_device_ptr(x, y)") + for (size_t i = 0; i < n; i++) + y[i * incy] = x[i * incx]; + return 0; +} + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const float* const x, + const int incx, + float* const y, + const int incy) +{ + return copy_impl(handle, n, x, incx, y, incy); +} + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const double* const x, + const int incx, + double* const y, + const int incy) +{ + return copy_impl(handle, n, x, incx, y, incy); +} + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const std::complex* const x, + const int incx, + std::complex* const y, + const int incy) +{ + return copy_impl(handle, n, x, incx, y, incy); +} + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const std::complex* const x, + const int incx, + std::complex* const y, + const int incy) +{ + return copy_impl(handle, n, x, incx, y, incy); +} template ompBLAS_status copy_batched_impl(ompBLAS_handle& handle, diff --git a/src/Platforms/OMPTarget/ompBLAS.hpp b/src/Platforms/OMPTarget/ompBLAS.hpp index 610ebf1062d..278db707867 100644 --- a/src/Platforms/OMPTarget/ompBLAS.hpp +++ b/src/Platforms/OMPTarget/ompBLAS.hpp @@ -29,6 +29,66 @@ namespace ompBLAS using ompBLAS_status = int; using ompBLAS_handle = int; +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const float alpha, + const float* const A, + const int lda, + const float* const B, + const int ldb, + const float beta, + float* const C, + const int ldc); + +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const double alpha, + const double* const A, + const int lda, + const double* const B, + const int ldb, + const double beta, + double* const C, + const int ldc); + +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const std::complex alpha, + const std::complex* const A, + const int lda, + const std::complex* const B, + const int ldb, + const std::complex beta, + std::complex* const C, + const int ldc); + +ompBLAS_status gemm(ompBLAS_handle& handle, + const char transa, + const char transb, + const int M, + const int N, + const int K, + const std::complex alpha, + const std::complex* const A, + const int lda, + const std::complex* const B, + const int ldb, + const std::complex beta, + std::complex* const C, + const int ldc); + ompBLAS_status gemv(ompBLAS_handle& handle, const char trans, const int m, @@ -229,6 +289,34 @@ ompBLAS_status ger_batched(ompBLAS_handle& handle, const int lda, const int batch_count); +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const float* const x, + const int incx, + float* const y, + const int incy); + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const double* const x, + const int incx, + double* const y, + const int incy); + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const std::complex* const x, + const int incx, + std::complex* const y, + const int incy); + +ompBLAS_status copy(ompBLAS_handle& handle, + const int n, + const std::complex* const x, + const int incx, + std::complex* const y, + const int incy); + /** * @brief copy device data from x to y * From 6feda31a2e52bd70512c335ca68e8f906edbe22e Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 15:44:13 +0000 Subject: [PATCH 031/168] offload VP AO->MO gemm for LCAO --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 37 +++++++++++++-------- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h | 6 ++-- src/QMCWaveFunctions/SPOSet.h | 3 +- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index d4e9c6758d5..c9eeca9e39b 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -13,6 +13,7 @@ #include "LCAOrbitalSet.h" #include "Numerics/MatrixOperators.h" #include "CPU/BLAS.hpp" +#include "OMPTarget/ompBLAS.hpp" #include namespace qmcplusplus @@ -79,7 +80,7 @@ void LCAOrbitalSet::setOrbitalSetSize(int norbs) Identity = false; OrbitalSetSize = norbs; - C = std::make_shared(OrbitalSetSize, BasisSetSize); + C = std::make_shared(OrbitalSetSize, BasisSetSize); Tempv.resize(OrbitalSetSize); Temphv.resize(OrbitalSetSize); Tempghv.resize(OrbitalSetSize); @@ -162,8 +163,8 @@ void LCAOrbitalSet::evaluateValue(const ParticleSet& P, int iat, ValueVector& ps } /** Find a better place for other user classes, Matrix should be padded as well */ -template -inline void Product_ABt(const VectorSoaContainer& A, const Matrix& B, VectorSoaContainer& C) +template +inline void Product_ABt(const VectorSoaContainer& A, const Matrix& B, VectorSoaContainer& C) { constexpr char transa = 't'; constexpr char transb = 'n'; @@ -499,24 +500,34 @@ void LCAOrbitalSet::mw_evaluateValueVPsImplGEMM(const RefVectorWithLeadermw_evaluateValueVPs(basis_list, vp_list, vp_basis_v_mw); - vp_basis_v_mw.updateFrom(); // TODO: remove after offloading rest of function + + auto* vp_basis_devptr = vp_basis_v_mw.device_data_at(0, 0); + auto* vp_phi_devptr = vp_phi_v.device_data_at(0, 0); + int dummy_handle = 0; + int success = 0; if (Identity) { - std::copy_n(vp_basis_v_mw.data_at(0, 0), OrbitalSetSize * nVPs, vp_phi_v.data_at(0, 0)); + success = ompBLAS::copy(dummy_handle, OrbitalSetSize * nVPs, vp_basis_devptr, 1, vp_phi_devptr, 1); + if (success != 0) + throw std::runtime_error("In LCAOrbitalSet::mw_evaluateValueVPsImplGEMM ompBLAS::copy failed."); } else { const size_t requested_orb_size = vp_phi_v.size(1); assert(requested_orb_size <= OrbitalSetSize); - ValueMatrix C_partial_view(C->data(), requested_orb_size, BasisSetSize); - BLAS::gemm('T', 'N', - requested_orb_size, // MOs - nVPs, // walkers * Virtual Particles - BasisSetSize, // AOs - 1, C_partial_view.data(), BasisSetSize, vp_basis_v_mw.data(), BasisSetSize, 0, vp_phi_v.data(), - requested_orb_size); - } + C->updateTo(); + auto* c_devptr = C->device_data(); + success = + ompBLAS::gemm(dummy_handle, 'T', 'N', + requested_orb_size, // MOs + nVPs, // walkers * Virtual Particles + BasisSetSize, // AOs + 1, c_devptr, BasisSetSize, vp_basis_devptr, BasisSetSize, 0, vp_phi_devptr, requested_orb_size); + if (success != 0) + throw std::runtime_error("In LCAOrbitalSet::mw_evaluateValueVPsImplGEMM ompBLAS::gemm failed."); + } + vp_phi_v.updateFrom(); // TODO: remove after offloading downstream functions } void LCAOrbitalSet::mw_evaluateValue(const RefVectorWithLeader& spo_list, const RefVectorWithLeader& P_list, diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h index cf6706df952..39e93e13cf1 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h @@ -38,7 +38,7 @@ struct LCAOrbitalSet : public SPOSet ///pointer to the basis set std::unique_ptr myBasisSet; /// pointer to matrix containing the coefficients - std::shared_ptr C; + std::shared_ptr C; /** constructor * @param bs pointer to the BasisSet @@ -55,7 +55,7 @@ struct LCAOrbitalSet : public SPOSet std::unique_ptr makeClone() const final; - void storeParamsBeforeRotation() final { C_copy = std::make_shared(*C); } + void storeParamsBeforeRotation() final { C_copy = std::make_shared(*C); } void applyRotation(const ValueMatrix& rot_mat, bool use_stored_copy) final; @@ -219,7 +219,7 @@ struct LCAOrbitalSet : public SPOSet ///number of Single-particle orbitals const IndexType BasisSetSize; /// a copy of the original C before orbital rotation is applied; - std::shared_ptr C_copy; + std::shared_ptr C_copy; ///true if C is an identity matrix bool Identity; diff --git a/src/QMCWaveFunctions/SPOSet.h b/src/QMCWaveFunctions/SPOSet.h index 4f7c8e81750..4465dec0223 100644 --- a/src/QMCWaveFunctions/SPOSet.h +++ b/src/QMCWaveFunctions/SPOSet.h @@ -59,7 +59,8 @@ class SPOSet : public QMCTraits using OffloadMWVGLArray = Array>; // [VGL, walker, Orbs] using OffloadMWVArray = Array>; // [walker, Orbs] template - using OffloadMatrix = Matrix>; + using OffloadMatrix = Matrix>; + using OffloadValueMatrix = OffloadMatrix; /** constructor */ SPOSet(const std::string& my_name); From 382779d2e0f4a62fd99721548aa6105e353139ff Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 15:46:13 +0000 Subject: [PATCH 032/168] move D2H transfer outside of MO eval function --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index c9eeca9e39b..67e122cd8ce 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -527,7 +527,6 @@ void LCAOrbitalSet::mw_evaluateValueVPsImplGEMM(const RefVectorWithLeader& spo_list, const RefVectorWithLeader& P_list, @@ -593,6 +592,7 @@ void LCAOrbitalSet::mw_evaluateDetRatios(const RefVectorWithLeader& spo_ vp_phi_v.resize(nVPs, requested_orb_size); mw_evaluateValueVPsImplGEMM(spo_list, vp_list, vp_phi_v); + vp_phi_v.updateFrom(); // TODO: remove after offloading rest of function ///To be computed on Device through new varuable mw_ratios_list, then copied to ratios_list on host. size_t index = 0; From 6a5d9444bb5c58929763df7e57ebd41e473e3a4a Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Wed, 18 Oct 2023 13:34:19 -0500 Subject: [PATCH 033/168] Avoid std::div in offload regions. --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 3bb90498e46..629c7964277 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -750,12 +750,10 @@ struct SoaAtomicBasisSet for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) { // i_xyz = k + Nz * (j + Ny * i) - auto div_k = std::div(i_xyz, Nz); - int k = div_k.rem; - int ij = div_k.quot; // ij = (j + Ny * i) - auto div_ij = std::div(ij, Ny); - int j = div_ij.rem; - int i = div_ij.quot; + int ij = i_xyz / Nz; + int k = i_xyz - ij * Nz; + int i = ij / Ny; + int j = ij - i * Ny; int TransX = ((i % 2) * 2 - 1) * ((i + 1) / 2); int TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); int TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); From 02621a466a0ed24955a9c587ccd51e8f189905e9 Mon Sep 17 00:00:00 2001 From: Ye Luo Date: Wed, 18 Oct 2023 13:36:24 -0500 Subject: [PATCH 034/168] Respect QMC_GPU_ARCHS as the single source of truth --- CMake/DetermineDeviceArchitectures.cmake | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CMake/DetermineDeviceArchitectures.cmake b/CMake/DetermineDeviceArchitectures.cmake index 5c5c573b47b..1b95d8604c3 100644 --- a/CMake/DetermineDeviceArchitectures.cmake +++ b/CMake/DetermineDeviceArchitectures.cmake @@ -63,6 +63,7 @@ function(verifyNVIDIAGPUconsistency) endfunction() # auto detect QMC_GPU_ARCHS if not set by user and GPU features are enabled. +# CMAKE_CUDA/HIP_ARCHITECTURES are used as hints if(NOT QMC_GPU_ARCHS AND ENABLE_CUDA) if(QMC_CUDA2HIP) detectAMDGPU() @@ -93,3 +94,13 @@ endif() set(QMC_GPU_ARCHS ${QMC_GPU_ARCHS} CACHE STRING "Accelerator device architectures" FORCE) + +# QMC_GPU_ARCHS is the single source of truth and thus overwrite CMAKE_CUDA/HIP_ARCHITECTURES +if(ENABLE_CUDA) + if(QMC_CUDA2HIP) + set(CMAKE_HIP_ARCHITECTURES ${QMC_GPU_ARCHS} CACHE STRING "HIP architectures" FORCE) + else() + string(REPLACE "sm_" "" CUDA_ARCH_NUMBERS "${QMC_GPU_ARCHS}") + set(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_NUMBERS} CACHE STRING "CUDA architectures" FORCE) + endif() +endif() From 14d165fd9aa4912a94dd1d5c38e0b8d3073b5bc4 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 13:39:21 -0500 Subject: [PATCH 035/168] fixed types in SoaAtomic --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 629c7964277..f33a1721fcb 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -34,9 +34,9 @@ struct SoaAtomicBasisSet using RealType = typename ROT::RealType; using GridType = typename ROT::GridType; using ValueType = typename QMCTraits::ValueType; - using OffloadArray4D = Array>; - using OffloadArray3D = Array>; - using OffloadArray2D = Array>; + using OffloadArray4D = Array>; + using OffloadArray3D = Array>; + using OffloadArray2D = Array>; using OffloadVector = Vector>; /// multi walker shared memory buffer From 834c70ab6c015c350dc761d8dc8df85088fc51d9 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 13:59:39 -0500 Subject: [PATCH 036/168] offload radial spline coeffs --- .../LCAO/MultiQuinticSpline1D.h | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h index 582b9bc1fa2..ab75a281e3c 100644 --- a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h +++ b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h @@ -99,7 +99,7 @@ class MultiQuinticSpline1D public: using RealType = T; using GridType = OneDimGridBase; - using CoeffType = Matrix>; + using CoeffType = Matrix>; using OffloadArray2D = Array>; using OffloadArray3D = Array>; @@ -170,13 +170,13 @@ class MultiQuinticSpline1D auto* r_devptr = r.device_data(); auto* u_devptr = u.device_data(); - auto* coeff_ptr = coeffs->data(); + auto* coeff_devptr = coeffs->device_data(); const size_t nCols = coeffs->cols(); - const size_t coefsize = coeffs->size(); + // const size_t coefsize = coeffs->size(); PRAGMA_OFFLOAD("omp target teams distribute parallel for \ - map(always, to:first_deriv_ptr[:num_splines_], coeff_ptr[:coefsize]) \ + map(always, to:first_deriv_ptr[:num_splines_]) \ is_device_ptr(r_devptr, u_devptr)") for (size_t ir = 0; ir < nR; ir++) { @@ -188,8 +188,7 @@ class MultiQuinticSpline1D else if (r_devptr[ir] < lower_bound) { const T dr = r_devptr[ir] - lower_bound; - const T* restrict a = coeff_ptr; - // const T* restrict a = (*coeffs)[0]; + const T* restrict a = coeff_devptr; for (size_t i = 0; i < num_splines_; ++i) u_devptr[ir * nRnl + i] = a[i] + first_deriv_ptr[i] * dr; } @@ -197,17 +196,13 @@ class MultiQuinticSpline1D { int loc; const auto cL = LogGridLight::getCL(r_devptr[ir], loc, OneOverLogDelta, lower_bound, dlog_ratio); - // const auto cL = myGrid.getCLForQuintic(r_list[ir], loc); const size_t offset = loc * 6; - //coeffs is an OhmmsMatrix and [] is a row access operator - //returning a pointer to 'row' which is normal type pointer [] - // const T* restrict a = (*coeffs)[offset + 0]; - const T* restrict a = coeff_ptr + nCols * (offset + 0); - const T* restrict b = coeff_ptr + nCols * (offset + 1); - const T* restrict c = coeff_ptr + nCols * (offset + 2); - const T* restrict d = coeff_ptr + nCols * (offset + 3); - const T* restrict e = coeff_ptr + nCols * (offset + 4); - const T* restrict f = coeff_ptr + nCols * (offset + 5); + const T* restrict a = coeff_devptr + nCols * (offset + 0); + const T* restrict b = coeff_devptr + nCols * (offset + 1); + const T* restrict c = coeff_devptr + nCols * (offset + 2); + const T* restrict d = coeff_devptr + nCols * (offset + 3); + const T* restrict e = coeff_devptr + nCols * (offset + 4); + const T* restrict f = coeff_devptr + nCols * (offset + 5); for (size_t i = 0; i < num_splines_; ++i) u_devptr[ir * nRnl + i] = a[i] + cL * (b[i] + cL * (c[i] + cL * (d[i] + cL * (e[i] + cL * f[i])))); } @@ -351,6 +346,7 @@ class MultiQuinticSpline1D out[(i * 6 + 5) * ncols + ispline] = static_cast(F[i]); } } + coeffs->updateTo(); // FIXME: this is overkill, probably better to just do once after all splines added } int getNumSplines() const { return num_splines_; } From 1ec4282f36b3fc33813fc708c9ed17c6bf42549b Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 14:03:41 -0500 Subject: [PATCH 037/168] offload spline first deriv --- .../LCAO/MultiQuinticSpline1D.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h index ab75a281e3c..126177fc849 100644 --- a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h +++ b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h @@ -116,7 +116,7 @@ class MultiQuinticSpline1D ///coeffs[6*spline_points][num_splines+padding] std::shared_ptr coeffs; - aligned_vector first_deriv; + Vector> first_deriv; public: MultiQuinticSpline1D() = default; @@ -170,14 +170,14 @@ class MultiQuinticSpline1D auto* r_devptr = r.device_data(); auto* u_devptr = u.device_data(); - auto* coeff_devptr = coeffs->device_data(); - const size_t nCols = coeffs->cols(); + auto* coeff_devptr = coeffs->device_data(); + auto* first_deriv_devptr = first_deriv.device_data(); + const size_t nCols = coeffs->cols(); // const size_t coefsize = coeffs->size(); PRAGMA_OFFLOAD("omp target teams distribute parallel for \ - map(always, to:first_deriv_ptr[:num_splines_]) \ - is_device_ptr(r_devptr, u_devptr)") + is_device_ptr(r_devptr, u_devptr, coeff_devptr, first_deriv_devptr)") for (size_t ir = 0; ir < nR; ir++) { if (r_devptr[ir] >= Rmax) @@ -187,15 +187,14 @@ class MultiQuinticSpline1D } else if (r_devptr[ir] < lower_bound) { - const T dr = r_devptr[ir] - lower_bound; - const T* restrict a = coeff_devptr; + const T dr = r_devptr[ir] - lower_bound; for (size_t i = 0; i < num_splines_; ++i) - u_devptr[ir * nRnl + i] = a[i] + first_deriv_ptr[i] * dr; + u_devptr[ir * nRnl + i] = coeff_devptr[i] + first_deriv_devptr[i] * dr; } else { int loc; - const auto cL = LogGridLight::getCL(r_devptr[ir], loc, OneOverLogDelta, lower_bound, dlog_ratio); + const auto cL = LogGridLight::getCL(r_devptr[ir], loc, OneOverLogDelta, lower_bound, dlog_ratio); const size_t offset = loc * 6; const T* restrict a = coeff_devptr + nCols * (offset + 0); const T* restrict b = coeff_devptr + nCols * (offset + 1); @@ -346,6 +345,7 @@ class MultiQuinticSpline1D out[(i * 6 + 5) * ncols + ispline] = static_cast(F[i]); } } + first_deriv.updateTo(); coeffs->updateTo(); // FIXME: this is overkill, probably better to just do once after all splines added } From df37ec5dbe69b70293c95f1690fad0e9bb4ca6a9 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 14:30:38 -0500 Subject: [PATCH 038/168] LCAO->MO gemm on host --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 30 +++++++-------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index 67e122cd8ce..cd7a028c0b8 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -13,7 +13,6 @@ #include "LCAOrbitalSet.h" #include "Numerics/MatrixOperators.h" #include "CPU/BLAS.hpp" -#include "OMPTarget/ompBLAS.hpp" #include namespace qmcplusplus @@ -500,32 +499,23 @@ void LCAOrbitalSet::mw_evaluateValueVPsImplGEMM(const RefVectorWithLeadermw_evaluateValueVPs(basis_list, vp_list, vp_basis_v_mw); - - auto* vp_basis_devptr = vp_basis_v_mw.device_data_at(0, 0); - auto* vp_phi_devptr = vp_phi_v.device_data_at(0, 0); - int dummy_handle = 0; - int success = 0; + vp_basis_v_mw.updateFrom(); // TODO: remove this when gemm is implemented if (Identity) { - success = ompBLAS::copy(dummy_handle, OrbitalSetSize * nVPs, vp_basis_devptr, 1, vp_phi_devptr, 1); - if (success != 0) - throw std::runtime_error("In LCAOrbitalSet::mw_evaluateValueVPsImplGEMM ompBLAS::copy failed."); + std::copy_n(vp_basis_v_mw.data_at(0, 0), OrbitalSetSize * nVPs, vp_phi_v.data_at(0, 0)); } else { const size_t requested_orb_size = vp_phi_v.size(1); assert(requested_orb_size <= OrbitalSetSize); - C->updateTo(); - auto* c_devptr = C->device_data(); - success = - ompBLAS::gemm(dummy_handle, 'T', 'N', - requested_orb_size, // MOs - nVPs, // walkers * Virtual Particles - BasisSetSize, // AOs - 1, c_devptr, BasisSetSize, vp_basis_devptr, BasisSetSize, 0, vp_phi_devptr, requested_orb_size); - if (success != 0) - throw std::runtime_error("In LCAOrbitalSet::mw_evaluateValueVPsImplGEMM ompBLAS::gemm failed."); + ValueMatrix C_partial_view(C->data(), requested_orb_size, BasisSetSize); + BLAS::gemm('T', 'N', + requested_orb_size, // MOs + nVPs, // walkers * Virtual Particles + BasisSetSize, // AOs + 1, C_partial_view.data(), BasisSetSize, vp_basis_v_mw.data(), BasisSetSize, 0, vp_phi_v.data(), + requested_orb_size); } } void LCAOrbitalSet::mw_evaluateValue(const RefVectorWithLeader& spo_list, @@ -592,9 +582,7 @@ void LCAOrbitalSet::mw_evaluateDetRatios(const RefVectorWithLeader& spo_ vp_phi_v.resize(nVPs, requested_orb_size); mw_evaluateValueVPsImplGEMM(spo_list, vp_list, vp_phi_v); - vp_phi_v.updateFrom(); // TODO: remove after offloading rest of function - ///To be computed on Device through new varuable mw_ratios_list, then copied to ratios_list on host. size_t index = 0; for (size_t iw = 0; iw < vp_list.size(); iw++) for (size_t iat = 0; iat < vp_list[iw].getTotalNum(); iat++) From 15d7a90ebc892296c9de3a960349211438112074 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 14:33:46 -0500 Subject: [PATCH 039/168] remove gemm, copy from ompBLAS --- src/Platforms/OMPTarget/ompBLAS.cpp | 189 ---------------------------- src/Platforms/OMPTarget/ompBLAS.hpp | 88 ------------- 2 files changed, 277 deletions(-) diff --git a/src/Platforms/OMPTarget/ompBLAS.cpp b/src/Platforms/OMPTarget/ompBLAS.cpp index 0446c4442f0..6c243b74ae2 100644 --- a/src/Platforms/OMPTarget/ompBLAS.cpp +++ b/src/Platforms/OMPTarget/ompBLAS.cpp @@ -22,141 +22,6 @@ namespace qmcplusplus namespace ompBLAS { -template -ompBLAS_status gemm_impl(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const T alpha, - const T* const A, - const int lda, - const T* const B, - const int ldb, - const T beta, - T* const C, - const int ldc) -{ - if (transa == 'T' && transb == 'N') //A(ji) * B(jk) -> C(ik) - { - PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") - for (size_t m = 0; m < M; m++) - for (size_t n = 0; n < N; n++) - { - C[n * ldc + m] *= beta; - for (size_t k = 0; k < K; k++) - C[n * ldc + m] += alpha * A[lda * m + k] * B[ldb * n + k]; - } - } - else if (transa == 'T' && transb == 'T') - { - PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") - for (size_t m = 0; m < M; m++) - for (size_t n = 0; n < N; n++) - { - C[n * ldc + m] *= beta; - for (size_t k = 0; k < K; k++) - C[n * ldc + m] += alpha * A[lda * m + k] * B[ldb * k + n]; - } - } - else if (transa == 'N' && transb == 'T') - { - PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") - for (size_t m = 0; m < M; m++) - for (size_t n = 0; n < N; n++) - { - C[n * ldc + m] *= beta; - for (size_t k = 0; k < K; k++) - C[n * ldc + m] += alpha * A[lda * k + m] * B[ldb * k + n]; - } - } - else if (transa == 'N' && transb == 'N') - { - PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(M * N) is_device_ptr(A, B, C)") - for (size_t m = 0; m < M; m++) - for (size_t n = 0; n < N; n++) - { - C[n * ldc + m] *= beta; - for (size_t k = 0; k < K; k++) - C[n * ldc + m] += alpha * A[lda * k + m] * B[ldb * n + k]; - } - } - - // #endif - return 0; -} - - -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const float alpha, - const float* const A, - const int lda, - const float* const B, - const int ldb, - const float beta, - float* const C, - const int ldc) -{ - return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); -} -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const double alpha, - const double* const A, - const int lda, - const double* const B, - const int ldb, - const double beta, - double* const C, - const int ldc) -{ - return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); -} -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const std::complex alpha, - const std::complex* const A, - const int lda, - const std::complex* const B, - const int ldb, - const std::complex beta, - std::complex* const C, - const int ldc) -{ - return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); -} -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const std::complex alpha, - const std::complex* const A, - const int lda, - const std::complex* const B, - const int ldb, - const std::complex beta, - std::complex* const C, - const int ldc) -{ - return gemm_impl(handle, transa, transb, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); -} - template ompBLAS_status gemv_impl(ompBLAS_handle& handle, const char trans, @@ -578,60 +443,6 @@ ompBLAS_status ger_batched(ompBLAS_handle& handle, } #endif -template -ompBLAS_status copy_impl(ompBLAS_handle& handle, - const int n, - const T* const x, - const int incx, - T* const y, - const int incy) -{ - PRAGMA_OFFLOAD("omp target teams distribute parallel for is_device_ptr(x, y)") - for (size_t i = 0; i < n; i++) - y[i * incy] = x[i * incx]; - return 0; -} - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const float* const x, - const int incx, - float* const y, - const int incy) -{ - return copy_impl(handle, n, x, incx, y, incy); -} - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const double* const x, - const int incx, - double* const y, - const int incy) -{ - return copy_impl(handle, n, x, incx, y, incy); -} - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const std::complex* const x, - const int incx, - std::complex* const y, - const int incy) -{ - return copy_impl(handle, n, x, incx, y, incy); -} - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const std::complex* const x, - const int incx, - std::complex* const y, - const int incy) -{ - return copy_impl(handle, n, x, incx, y, incy); -} - template ompBLAS_status copy_batched_impl(ompBLAS_handle& handle, const int n, diff --git a/src/Platforms/OMPTarget/ompBLAS.hpp b/src/Platforms/OMPTarget/ompBLAS.hpp index 278db707867..610ebf1062d 100644 --- a/src/Platforms/OMPTarget/ompBLAS.hpp +++ b/src/Platforms/OMPTarget/ompBLAS.hpp @@ -29,66 +29,6 @@ namespace ompBLAS using ompBLAS_status = int; using ompBLAS_handle = int; -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const float alpha, - const float* const A, - const int lda, - const float* const B, - const int ldb, - const float beta, - float* const C, - const int ldc); - -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const double alpha, - const double* const A, - const int lda, - const double* const B, - const int ldb, - const double beta, - double* const C, - const int ldc); - -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const std::complex alpha, - const std::complex* const A, - const int lda, - const std::complex* const B, - const int ldb, - const std::complex beta, - std::complex* const C, - const int ldc); - -ompBLAS_status gemm(ompBLAS_handle& handle, - const char transa, - const char transb, - const int M, - const int N, - const int K, - const std::complex alpha, - const std::complex* const A, - const int lda, - const std::complex* const B, - const int ldb, - const std::complex beta, - std::complex* const C, - const int ldc); - ompBLAS_status gemv(ompBLAS_handle& handle, const char trans, const int m, @@ -289,34 +229,6 @@ ompBLAS_status ger_batched(ompBLAS_handle& handle, const int lda, const int batch_count); -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const float* const x, - const int incx, - float* const y, - const int incy); - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const double* const x, - const int incx, - double* const y, - const int incy); - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const std::complex* const x, - const int incx, - std::complex* const y, - const int incy); - -ompBLAS_status copy(ompBLAS_handle& handle, - const int n, - const std::complex* const x, - const int incx, - std::complex* const y, - const int incy); - /** * @brief copy device data from x to y * From 157ec4c25bb65f49c66e5750e55ffe2dfe61fc84 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 14:34:40 -0500 Subject: [PATCH 040/168] extra line --- src/Platforms/OMPTarget/ompBLAS.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Platforms/OMPTarget/ompBLAS.cpp b/src/Platforms/OMPTarget/ompBLAS.cpp index 6c243b74ae2..0dce77844d3 100644 --- a/src/Platforms/OMPTarget/ompBLAS.cpp +++ b/src/Platforms/OMPTarget/ompBLAS.cpp @@ -443,6 +443,7 @@ ompBLAS_status ger_batched(ompBLAS_handle& handle, } #endif + template ompBLAS_status copy_batched_impl(ompBLAS_handle& handle, const int n, From 8feed230190b29faf683d15527f5ffacf797ef04 Mon Sep 17 00:00:00 2001 From: Hyeondeok-Shin Date: Wed, 18 Oct 2023 14:40:35 -0500 Subject: [PATCH 041/168] Update input style of testset for planewave orbitals --- .../solids/bccH_1x1x1_ae/qmc_short_pw.in.xml | 17 ++++++++--------- .../det_qmc_vmc_dmc_pw.in.xml | 19 +++++++++---------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/tests/solids/bccH_1x1x1_ae/qmc_short_pw.in.xml b/tests/solids/bccH_1x1x1_ae/qmc_short_pw.in.xml index 73ae33edfeb..8248e803d81 100644 --- a/tests/solids/bccH_1x1x1_ae/qmc_short_pw.in.xml +++ b/tests/solids/bccH_1x1x1_ae/qmc_short_pw.in.xml @@ -38,15 +38,14 @@ - - - - - - - - - + + + + + + + + diff --git a/tests/solids/diamondC_2x1x1_pp/det_qmc_vmc_dmc_pw.in.xml b/tests/solids/diamondC_2x1x1_pp/det_qmc_vmc_dmc_pw.in.xml index 560f0950945..a76f2964f14 100644 --- a/tests/solids/diamondC_2x1x1_pp/det_qmc_vmc_dmc_pw.in.xml +++ b/tests/solids/diamondC_2x1x1_pp/det_qmc_vmc_dmc_pw.in.xml @@ -41,16 +41,15 @@ - - - - - - - - - - + + + + + + + + + From 5821946cf2a6fff89c589c21fbbb66e1a84af9b7 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 15:03:22 -0500 Subject: [PATCH 042/168] remove dual space LCAO coeffs --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 6 +++--- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h | 6 +++--- src/QMCWaveFunctions/SPOSet.h | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index cd7a028c0b8..a9cd33893c0 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -79,7 +79,7 @@ void LCAOrbitalSet::setOrbitalSetSize(int norbs) Identity = false; OrbitalSetSize = norbs; - C = std::make_shared(OrbitalSetSize, BasisSetSize); + C = std::make_shared(OrbitalSetSize, BasisSetSize); Tempv.resize(OrbitalSetSize); Temphv.resize(OrbitalSetSize); Tempghv.resize(OrbitalSetSize); @@ -162,8 +162,8 @@ void LCAOrbitalSet::evaluateValue(const ParticleSet& P, int iat, ValueVector& ps } /** Find a better place for other user classes, Matrix should be padded as well */ -template -inline void Product_ABt(const VectorSoaContainer& A, const Matrix& B, VectorSoaContainer& C) +template +inline void Product_ABt(const VectorSoaContainer& A, const Matrix& B, VectorSoaContainer& C) { constexpr char transa = 't'; constexpr char transb = 'n'; diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h index 39e93e13cf1..cf6706df952 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.h @@ -38,7 +38,7 @@ struct LCAOrbitalSet : public SPOSet ///pointer to the basis set std::unique_ptr myBasisSet; /// pointer to matrix containing the coefficients - std::shared_ptr C; + std::shared_ptr C; /** constructor * @param bs pointer to the BasisSet @@ -55,7 +55,7 @@ struct LCAOrbitalSet : public SPOSet std::unique_ptr makeClone() const final; - void storeParamsBeforeRotation() final { C_copy = std::make_shared(*C); } + void storeParamsBeforeRotation() final { C_copy = std::make_shared(*C); } void applyRotation(const ValueMatrix& rot_mat, bool use_stored_copy) final; @@ -219,7 +219,7 @@ struct LCAOrbitalSet : public SPOSet ///number of Single-particle orbitals const IndexType BasisSetSize; /// a copy of the original C before orbital rotation is applied; - std::shared_ptr C_copy; + std::shared_ptr C_copy; ///true if C is an identity matrix bool Identity; diff --git a/src/QMCWaveFunctions/SPOSet.h b/src/QMCWaveFunctions/SPOSet.h index 4465dec0223..4f7c8e81750 100644 --- a/src/QMCWaveFunctions/SPOSet.h +++ b/src/QMCWaveFunctions/SPOSet.h @@ -59,8 +59,7 @@ class SPOSet : public QMCTraits using OffloadMWVGLArray = Array>; // [VGL, walker, Orbs] using OffloadMWVArray = Array>; // [walker, Orbs] template - using OffloadMatrix = Matrix>; - using OffloadValueMatrix = OffloadMatrix; + using OffloadMatrix = Matrix>; /** constructor */ SPOSet(const std::string& my_name); From a8072f345b34e6729bc29076f1f4e12c4c9b522a Mon Sep 17 00:00:00 2001 From: rcclay Date: Wed, 18 Oct 2023 14:31:30 -0600 Subject: [PATCH 043/168] Add hamiltonian to crowd estimator accumulate call --- src/Estimators/EstimatorManagerCrowd.cpp | 3 ++- src/Estimators/EstimatorManagerCrowd.h | 2 ++ src/Estimators/MagnetizationDensity.cpp | 1 + src/Estimators/MagnetizationDensity.h | 1 + src/Estimators/MomentumDistribution.cpp | 1 + src/Estimators/MomentumDistribution.h | 1 + src/Estimators/OneBodyDensityMatrices.cpp | 1 + src/Estimators/OneBodyDensityMatrices.h | 1 + src/Estimators/OperatorEstBase.h | 1 + src/Estimators/PerParticleHamiltonianLogger.cpp | 1 + src/Estimators/PerParticleHamiltonianLogger.h | 1 + src/Estimators/SpinDensityNew.cpp | 1 + src/Estimators/SpinDensityNew.h | 1 + src/Estimators/tests/FakeOperatorEstimator.h | 1 + src/Estimators/tests/test_EstimatorManagerCrowd.cpp | 2 +- src/Estimators/tests/test_MagnetizationDensity.cpp | 4 +++- src/Estimators/tests/test_MomentumDistribution.cpp | 5 ++++- .../tests/test_PerParticleHamiltonianLogger.cpp | 4 +++- src/Estimators/tests/test_SpinDensityNew.cpp | 13 +++++++++---- src/QMCDrivers/Crowd.h | 2 +- 20 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/Estimators/EstimatorManagerCrowd.cpp b/src/Estimators/EstimatorManagerCrowd.cpp index 6aec7bc1430..875bf911e21 100644 --- a/src/Estimators/EstimatorManagerCrowd.cpp +++ b/src/Estimators/EstimatorManagerCrowd.cpp @@ -25,6 +25,7 @@ EstimatorManagerCrowd::EstimatorManagerCrowd(EstimatorManagerNew& em) void EstimatorManagerCrowd::accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) { block_num_samples_ += walkers.size(); @@ -35,7 +36,7 @@ void EstimatorManagerCrowd::accumulate(const RefVector& walkers, for (int i = 0; i < num_scalar_estimators; ++i) scalar_estimators_[i]->accumulate(walkers); for (int i = 0; i < operator_ests_.size(); ++i) - operator_ests_[i]->accumulate(walkers, psets, wfns, rng); + operator_ests_[i]->accumulate(walkers, psets, wfns, hams, rng); } void EstimatorManagerCrowd::registerListeners(const RefVectorWithLeader& ham_list) diff --git a/src/Estimators/EstimatorManagerCrowd.h b/src/Estimators/EstimatorManagerCrowd.h index e97bc9e8f63..3ca7eaf9eb0 100644 --- a/src/Estimators/EstimatorManagerCrowd.h +++ b/src/Estimators/EstimatorManagerCrowd.h @@ -64,6 +64,7 @@ class EstimatorManagerCrowd * \param[in] walkers walkers in crowd * \param[in] psets walker particle sets * \param[in] wfns walker wavefunctions + * \param[in] hams walker Hamiltonians * \param[inout] rng crowd scope RandomGenerator * * walkers is especially questionable since its really just hiding the full sweep hamiltonian values from @@ -78,6 +79,7 @@ class EstimatorManagerCrowd void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng); ScalarEstimatorBase& get_main_estimator() { return *main_estimator_; } diff --git a/src/Estimators/MagnetizationDensity.cpp b/src/Estimators/MagnetizationDensity.cpp index fdaa29985a9..bfdc4bef4fc 100644 --- a/src/Estimators/MagnetizationDensity.cpp +++ b/src/Estimators/MagnetizationDensity.cpp @@ -49,6 +49,7 @@ size_t MagnetizationDensity::getFullDataSize() { return npoints_ * DIM; } void MagnetizationDensity::accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) { for (int iw = 0; iw < walkers.size(); ++iw) diff --git a/src/Estimators/MagnetizationDensity.h b/src/Estimators/MagnetizationDensity.h index caf20da3af9..a01137b8d76 100644 --- a/src/Estimators/MagnetizationDensity.h +++ b/src/Estimators/MagnetizationDensity.h @@ -60,6 +60,7 @@ class MagnetizationDensity : public OperatorEstBase void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) override; void collect(const RefVector& operator_estimators) override; diff --git a/src/Estimators/MomentumDistribution.cpp b/src/Estimators/MomentumDistribution.cpp index 53cdfa96e47..bc24a684c4d 100644 --- a/src/Estimators/MomentumDistribution.cpp +++ b/src/Estimators/MomentumDistribution.cpp @@ -234,6 +234,7 @@ void MomentumDistribution::startBlock(int steps) void MomentumDistribution::accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) { for (int iw = 0; iw < walkers.size(); ++iw) diff --git a/src/Estimators/MomentumDistribution.h b/src/Estimators/MomentumDistribution.h index 4e04b2ec91a..f11a485464a 100644 --- a/src/Estimators/MomentumDistribution.h +++ b/src/Estimators/MomentumDistribution.h @@ -98,6 +98,7 @@ class MomentumDistribution : public OperatorEstBase void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) override; /** this allows the EstimatorManagerNew to reduce without needing to know the details diff --git a/src/Estimators/OneBodyDensityMatrices.cpp b/src/Estimators/OneBodyDensityMatrices.cpp index bf2fcace480..82583750f10 100644 --- a/src/Estimators/OneBodyDensityMatrices.cpp +++ b/src/Estimators/OneBodyDensityMatrices.cpp @@ -397,6 +397,7 @@ void OneBodyDensityMatrices::calcDensityDrift(const Position& r, Real& dens, Pos void OneBodyDensityMatrices::accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) { implAccumulate(walkers, psets, wfns, rng); diff --git a/src/Estimators/OneBodyDensityMatrices.h b/src/Estimators/OneBodyDensityMatrices.h index 3419f90eaff..422226c72ba 100644 --- a/src/Estimators/OneBodyDensityMatrices.h +++ b/src/Estimators/OneBodyDensityMatrices.h @@ -166,6 +166,7 @@ class OneBodyDensityMatrices : public OperatorEstBase void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) override; void startBlock(int steps) override; diff --git a/src/Estimators/OperatorEstBase.h b/src/Estimators/OperatorEstBase.h index 49244e7b90c..fe107ea6ced 100644 --- a/src/Estimators/OperatorEstBase.h +++ b/src/Estimators/OperatorEstBase.h @@ -74,6 +74,7 @@ class OperatorEstBase virtual void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) = 0; /** Reduce estimator result data from crowds to rank diff --git a/src/Estimators/PerParticleHamiltonianLogger.cpp b/src/Estimators/PerParticleHamiltonianLogger.cpp index dfcaa0ab038..dbcf86c62d1 100644 --- a/src/Estimators/PerParticleHamiltonianLogger.cpp +++ b/src/Estimators/PerParticleHamiltonianLogger.cpp @@ -59,6 +59,7 @@ void PerParticleHamiltonianLogger::write(CrowdLogValues& cl_values, const std::v void PerParticleHamiltonianLogger::accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) { diff --git a/src/Estimators/PerParticleHamiltonianLogger.h b/src/Estimators/PerParticleHamiltonianLogger.h index cec53980827..54bf76132af 100644 --- a/src/Estimators/PerParticleHamiltonianLogger.h +++ b/src/Estimators/PerParticleHamiltonianLogger.h @@ -38,6 +38,7 @@ class PerParticleHamiltonianLogger : public OperatorEstBase void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) override; UPtr spawnCrowdClone() const override; diff --git a/src/Estimators/SpinDensityNew.cpp b/src/Estimators/SpinDensityNew.cpp index b72110995fe..206f4551b73 100644 --- a/src/Estimators/SpinDensityNew.cpp +++ b/src/Estimators/SpinDensityNew.cpp @@ -117,6 +117,7 @@ void SpinDensityNew::startBlock(int steps) void SpinDensityNew::accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) { auto& dp_ = derived_parameters_; diff --git a/src/Estimators/SpinDensityNew.h b/src/Estimators/SpinDensityNew.h index 5559f21e071..1b1aca657e3 100644 --- a/src/Estimators/SpinDensityNew.h +++ b/src/Estimators/SpinDensityNew.h @@ -85,6 +85,7 @@ class SpinDensityNew : public OperatorEstBase void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) override; /** this allows the EstimatorManagerNew to reduce without needing to know the details diff --git a/src/Estimators/tests/FakeOperatorEstimator.h b/src/Estimators/tests/FakeOperatorEstimator.h index ce75a66451f..8ce77806355 100644 --- a/src/Estimators/tests/FakeOperatorEstimator.h +++ b/src/Estimators/tests/FakeOperatorEstimator.h @@ -32,6 +32,7 @@ class FakeOperatorEstimator : public OperatorEstBase void accumulate(const RefVector& walkers, const RefVector& psets, const RefVector& wfns, + const RefVector& hams, RandomBase& rng) override {} diff --git a/src/Estimators/tests/test_EstimatorManagerCrowd.cpp b/src/Estimators/tests/test_EstimatorManagerCrowd.cpp index b4c0548572c..d3f8d42a73b 100644 --- a/src/Estimators/tests/test_EstimatorManagerCrowd.cpp +++ b/src/Estimators/tests/test_EstimatorManagerCrowd.cpp @@ -159,7 +159,7 @@ TEST_CASE("EstimatorManagerCrowd PerParticleHamiltonianLogger integration", "[es for (int iw = 0; iw < num_walkers; ++iw) savePropertiesIntoWalker(*(hams[iw]), walkers[iw]); - emc.accumulate(walker_refs, p_refs, twf_refs, rng); + emc.accumulate(walker_refs, p_refs, twf_refs, ham_refs, rng); } diff --git a/src/Estimators/tests/test_MagnetizationDensity.cpp b/src/Estimators/tests/test_MagnetizationDensity.cpp index 4028d3abb0e..68d97e85977 100644 --- a/src/Estimators/tests/test_MagnetizationDensity.cpp +++ b/src/Estimators/tests/test_MagnetizationDensity.cpp @@ -423,12 +423,14 @@ TEST_CASE("MagnetizationDensity::IntegrationTest", "[estimators]") for (int iw = 0; iw < nwalkers; iw++) updateWalker(walkers[iw], psets[iw], *(twfcs[iw])); + std::vector hams; auto ref_walkers(makeRefVector(walkers)); auto ref_psets(makeRefVector(psets)); auto ref_twfcs(convertUPtrToRefVector(twfcs)); + auto ref_hams(makeRefVector(hams)); FakeRandom rng; - magdensity.accumulate(ref_walkers, ref_psets, ref_twfcs, rng); + magdensity.accumulate(ref_walkers, ref_psets, ref_twfcs, ref_hams, rng); //Now the reference data // diff --git a/src/Estimators/tests/test_MomentumDistribution.cpp b/src/Estimators/tests/test_MomentumDistribution.cpp index 5382d25ec48..2703a3318f1 100644 --- a/src/Estimators/tests/test_MomentumDistribution.cpp +++ b/src/Estimators/tests/test_MomentumDistribution.cpp @@ -173,15 +173,18 @@ TEST_CASE("MomentumDistribution::accumulate", "[estimators]") } // Create ref vectors + std::vector hams; + auto ref_walkers = makeRefVector(walkers); auto ref_psets = makeRefVector(psets); auto ref_wfns = convertUPtrToRefVector(wfns); + auto ref_hams = makeRefVector(hams); // Setup RNG FakeRandom rng; // Perform accumulate - md.accumulate(ref_walkers, ref_psets, ref_wfns, rng); + md.accumulate(ref_walkers, ref_psets, ref_wfns, ref_hams, rng); // Check data std::vector& data = md.get_data(); diff --git a/src/Estimators/tests/test_PerParticleHamiltonianLogger.cpp b/src/Estimators/tests/test_PerParticleHamiltonianLogger.cpp index a16ae30876f..eee0c957757 100644 --- a/src/Estimators/tests/test_PerParticleHamiltonianLogger.cpp +++ b/src/Estimators/tests/test_PerParticleHamiltonianLogger.cpp @@ -95,10 +95,12 @@ TEST_CASE("PerParticleHamiltonianLogger_sum", "[estimators]") } std::vector wfns; + std::vector hams; auto ref_walkers = makeRefVector(walkers); auto ref_psets = makeRefVector(psets); auto ref_wfns = makeRefVector(wfns); + auto ref_hams = makeRefVector(hams); std::vector multi_walker_talkers{{"Talker1", nwalkers}, {"Talker2", nwalkers}, @@ -127,7 +129,7 @@ TEST_CASE("PerParticleHamiltonianLogger_sum", "[estimators]") using Walker = typename decltype(ref_walkers)::value_type::type; for(Walker& walker : ref_walkers) walker.ID = walker_id++; - crowd_oeb->accumulate(ref_walkers, ref_psets, ref_wfns, rng); + crowd_oeb->accumulate(ref_walkers, ref_psets, ref_wfns, ref_hams, rng); } RefVector crowd_loggers_refs = convertUPtrToRefVector(crowd_loggers); diff --git a/src/Estimators/tests/test_SpinDensityNew.cpp b/src/Estimators/tests/test_SpinDensityNew.cpp index 9e5857ce64e..3c71b971ef7 100644 --- a/src/Estimators/tests/test_SpinDensityNew.cpp +++ b/src/Estimators/tests/test_SpinDensityNew.cpp @@ -73,14 +73,16 @@ void accumulateFromPsets(int ncrowds, SpinDensityNew& sdn, UPtrVector wfns; + std::vector hams; auto ref_walkers = makeRefVector(walkers); auto ref_psets = makeRefVector(psets); auto ref_wfns = makeRefVector(wfns); + auto ref_hams = makeRefVector(hams); FakeRandom rng; - crowd_sdn.accumulate(ref_walkers, ref_psets, ref_wfns, rng); + crowd_sdn.accumulate(ref_walkers, ref_psets, ref_wfns, ref_hams, rng); } } @@ -111,14 +113,15 @@ void randomUpdateAccumulate(testing::RandomForTest& rft, UPtrVec } std::vector wfns; - + std::vector hams; auto ref_walkers = makeRefVector(walkers); auto ref_psets = makeRefVector(psets); auto ref_wfns = makeRefVector(wfns); + auto ref_hams = makeRefVector(hams); FakeRandom rng; - crowd_sdn.accumulate(ref_walkers, ref_psets, ref_wfns, rng); + crowd_sdn.accumulate(ref_walkers, ref_psets, ref_wfns, ref_hams, rng); } } @@ -215,14 +218,16 @@ TEST_CASE("SpinDensityNew::accumulate", "[estimators]") } std::vector wfns; + std::vector hams; auto ref_walkers = makeRefVector(walkers); auto ref_psets = makeRefVector(psets); auto ref_wfns = makeRefVector(wfns); + auto ref_hams = makeRefVector(hams); FakeRandom rng; - sdn.accumulate(ref_walkers, ref_psets, ref_wfns, rng); + sdn.accumulate(ref_walkers, ref_psets, ref_wfns, ref_hams, rng); std::vector& data_ref = sdn.get_data(); // There should be a check that the discretization of particle locations expressed in lattice coords diff --git a/src/QMCDrivers/Crowd.h b/src/QMCDrivers/Crowd.h index 73f0ae9e538..6de8b91195b 100644 --- a/src/QMCDrivers/Crowd.h +++ b/src/QMCDrivers/Crowd.h @@ -80,7 +80,7 @@ class Crowd { if (this->size() == 0) return; - estimator_manager_crowd_.accumulate(mcp_walkers_, walker_elecs_, walker_twfs_, rng); + estimator_manager_crowd_.accumulate(mcp_walkers_, walker_elecs_, walker_twfs_, walker_hamiltonians_, rng); } void setRNGForHamiltonian(RandomBase& rng); From 2a07baeba2f7962faebb75ccbbb49f49b16ed2f2 Mon Sep 17 00:00:00 2001 From: Peter Doak Date: Tue, 17 Oct 2023 18:27:23 -0400 Subject: [PATCH 044/168] add const to applyMinimumImage --- external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp | 1 + src/Particle/ParticleSet.BC.cpp | 2 +- src/Particle/ParticleSet.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp index 03440cfb1ea..036c3cae642 100644 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp +++ b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp @@ -15,6 +15,7 @@ struct buffer : mpi3::uvector{ int pos = 0; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) : make private buffer() = default; explicit buffer(std::size_t r) {reserve(r);} + explicit buffer(long r) {reserve(r);} buffer(buffer const&) = delete; buffer(buffer&&) = delete; buffer& operator=(buffer const&) = delete; diff --git a/src/Particle/ParticleSet.BC.cpp b/src/Particle/ParticleSet.BC.cpp index 2c1469f5567..d772a1bf61d 100644 --- a/src/Particle/ParticleSet.BC.cpp +++ b/src/Particle/ParticleSet.BC.cpp @@ -171,7 +171,7 @@ void ParticleSet::applyBC(ParticlePos& pos) } } -void ParticleSet::applyMinimumImage(ParticlePos& pinout) +void ParticleSet::applyMinimumImage(ParticlePos& pinout) const { if (getLattice().SuperCellEnum == SUPERCELL_OPEN) return; diff --git a/src/Particle/ParticleSet.h b/src/Particle/ParticleSet.h index 2c235021903..247f5ca66a3 100644 --- a/src/Particle/ParticleSet.h +++ b/src/Particle/ParticleSet.h @@ -426,7 +426,7 @@ class ParticleSet : public QMCTraits, public OhmmsElementBase, public PtclOnLatt void applyBC(const ParticlePos& pin, ParticlePos& pout); void applyBC(ParticlePos& pos); void applyBC(const ParticlePos& pin, ParticlePos& pout, int first, int last); - void applyMinimumImage(ParticlePos& pinout); + void applyMinimumImage(ParticlePos& pinout) const; /** load a Walker_t to the current ParticleSet * @param awalker the reference to the walker to be loaded From 2fc3823da2f2550b4e93a70d9abfe8839226ca11 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 16:19:46 -0500 Subject: [PATCH 045/168] clean up {} --- src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index e004335c644..49d055932a7 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -62,6 +62,7 @@ struct MultiFunctorAdapter inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const { r.updateFrom(); // TODO: remove after offload + const size_t nElec = r.size(0); const size_t Nxyz = r.size(1); // number of PBC images assert(nElec == u.size(0)); @@ -72,20 +73,13 @@ struct MultiFunctorAdapter auto* r_ptr = r.data(); auto* u_ptr = u.data(); - for (size_t ir = 0; ir < nR; ir++) - { if (r_ptr[ir] >= Rmax) - { for (size_t i = 0, n = Rnl.size(); i < n; ++i) u_ptr[ir * nRnl + i] = 0.0; - } else - { for (size_t i = 0, n = Rnl.size(); i < n; ++i) u_ptr[ir * nRnl + i] = Rnl[i]->f(r_ptr[ir]); - } - } u.updateTo(); // TODO: remove after offload } From c854e06f6312a233d568d917691e4c37f6b81b12 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 16:29:50 -0500 Subject: [PATCH 046/168] update vp function name and description --- src/Particle/VirtualParticleSet.cpp | 2 +- src/Particle/VirtualParticleSet.h | 5 +++-- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Particle/VirtualParticleSet.cpp b/src/Particle/VirtualParticleSet.cpp index 3746a5c17b9..96ae186576b 100644 --- a/src/Particle/VirtualParticleSet.cpp +++ b/src/Particle/VirtualParticleSet.cpp @@ -98,7 +98,7 @@ void VirtualParticleSet::releaseResource(ResourceCollection& collection, } -const RefVectorWithLeader VirtualParticleSet::extractDTRefList_vp( +const RefVectorWithLeader VirtualParticleSet::extractDTRefList( const RefVectorWithLeader& vp_list, int id) { diff --git a/src/Particle/VirtualParticleSet.h b/src/Particle/VirtualParticleSet.h index 7161f88a7ee..293b8e01209 100644 --- a/src/Particle/VirtualParticleSet.h +++ b/src/Particle/VirtualParticleSet.h @@ -98,10 +98,11 @@ class VirtualParticleSet : public ParticleSet inline size_t getTotalNum() const { return TotalNum; } /**Extract list of Distance Tables */ - static const RefVectorWithLeader extractDTRefList_vp( + static const RefVectorWithLeader extractDTRefList( const RefVectorWithLeader& vp_list, int id); - + /**Extract list of active VP coordinates, flattened over all walkers + */ static const std::vector extractCoordsRefList_vp( const RefVectorWithLeader& vp_list); /** move virtual particles to new postions and update distance tables diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index f92fc0513f8..e4fae6eb8b6 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -263,7 +263,7 @@ void SoaLocalizedBasisSet::mw_evaluateValueVPs(RefVectorWithLeader Date: Wed, 18 Oct 2023 17:40:24 -0500 Subject: [PATCH 047/168] Fix Lmax_ type and minimize code duplication in SoaSphericalTensor. --- src/Numerics/SoaSphericalTensor.h | 122 ++++-------------------------- 1 file changed, 13 insertions(+), 109 deletions(-) diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index e3b0ad67c67..515e4046b9c 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -59,13 +59,12 @@ struct SoaSphericalTensor SoaSphericalTensor(const SoaSphericalTensor& rhs) = default; ///compute Ylm - void evaluate_bare(T x, T y, T z, T* Ylm) const; - static void evaluate_bare_impl(T x, T y, T z, T* Ylm, const size_t Lmax_, const T* FacL, const T* FacLM); + static void evaluate_bare(T x, T y, T z, T* Ylm, const int Lmax_, const T* FacL, const T* FacLM); ///compute Ylm inline void evaluateV(T x, T y, T z, T* Ylm) const { - evaluate_bare(x, y, z, Ylm); + evaluate_bare(x, y, z, Ylm, Lmax, FactorL.data(), FactorLM.data()); for (int i = 0, nl = cYlm.size(); i < nl; i++) Ylm[i] *= NormFactor[i]; } @@ -100,8 +99,8 @@ struct SoaSphericalTensor is_device_ptr(xyz_devptr, Ylm_devptr)") for (size_t ir = 0; ir < nR; ir++) { - evaluate_bare_impl(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], - Ylm_devptr + (ir * Nlm), Lmax, fl_ptr, flm_ptr); + evaluate_bare(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], Ylm_devptr + (ir * Nlm), + Lmax, fl_ptr, flm_ptr); for (int i = 0; i < Nlm; i++) Ylm_devptr[ir * Nlm + i] *= NormFactor_ptr[i]; } @@ -111,7 +110,7 @@ struct SoaSphericalTensor inline void evaluateV(T x, T y, T z) { T* restrict Ylm = cYlm.data(0); - evaluate_bare(x, y, z, Ylm); + evaluate_bare(x, y, z, Ylm, Lmax, FactorL.data(), FactorLM.data()); for (int i = 0, nl = cYlm.size(); i < nl; i++) Ylm[i] *= NormFactor[i]; } @@ -213,13 +212,13 @@ inline SoaSphericalTensor::SoaSphericalTensor(const int l_max, bool addsign) PRAGMA_OFFLOAD("omp declare target") template -inline void SoaSphericalTensor::evaluate_bare_impl(const T x, - const T y, - const T z, - T* restrict Ylm, - const size_t Lmax_, - const T* FactorL_ptr, - const T* FactorLM_ptr) +inline void SoaSphericalTensor::evaluate_bare(const T x, + const T y, + const T z, + T* restrict Ylm, + const int Lmax_, + const T* FactorL_ptr, + const T* FactorLM_ptr) { constexpr T czero(0); constexpr T cone(1); @@ -314,106 +313,11 @@ inline void SoaSphericalTensor::evaluate_bare_impl(const T x, } PRAGMA_OFFLOAD("omp end declare target") -template -inline void SoaSphericalTensor::evaluate_bare(T x, T y, T z, T* restrict Ylm) const -{ - constexpr T czero(0); - constexpr T cone(1); - const T pi = 4.0 * std::atan(1.0); - const T omega = 1.0 / std::sqrt(4.0 * pi); - constexpr T eps2 = std::numeric_limits::epsilon() * std::numeric_limits::epsilon(); - - /* Calculate r, cos(theta), sin(theta), cos(phi), sin(phi) from input - coordinates. Check here the coordinate singularity at cos(theta) = +-1. - This also takes care of r=0 case. */ - T cphi, sphi, ctheta; - T r2xy = x * x + y * y; - T r = std::sqrt(r2xy + z * z); - if (r2xy < eps2) - { - cphi = czero; - sphi = cone; - ctheta = (z < czero) ? -cone : cone; - } - else - { - ctheta = z / r; - //protect ctheta, when ctheta is slightly >1 or <-1 - if (ctheta > cone) - ctheta = cone; - if (ctheta < -cone) - ctheta = -cone; - T rxyi = cone / std::sqrt(r2xy); - cphi = x * rxyi; - sphi = y * rxyi; - } - T stheta = std::sqrt(cone - ctheta * ctheta); - /* Now to calculate the associated legendre functions P_lm from the - recursion relation from l=0 to Lmax. Conventions of J.D. Jackson, - Classical Electrodynamics are used. */ - Ylm[0] = cone; - // calculate P_ll and P_l,l-1 - T fac = cone; - int j = -1; - for (int l = 1; l <= Lmax; l++) - { - j += 2; - fac *= -j * stheta; - int ll = index(l, l); - int l1 = index(l, l - 1); - int l2 = index(l - 1, l - 1); - Ylm[ll] = fac; - Ylm[l1] = j * ctheta * Ylm[l2]; - } - // Use recurence to get other plm's // - for (int m = 0; m < Lmax - 1; m++) - { - int j = 2 * m + 1; - for (int l = m + 2; l <= Lmax; l++) - { - j += 2; - int lm = index(l, m); - int l1 = index(l - 1, m); - int l2 = index(l - 2, m); - Ylm[lm] = (ctheta * j * Ylm[l1] - (l + m - 1) * Ylm[l2]) / (l - m); - } - } - // Now to calculate r^l Y_lm. // - T sphim, cphim, temp; - Ylm[0] = omega; //1.0/sqrt(pi4); - T rpow = 1.0; - for (int l = 1; l <= Lmax; l++) - { - rpow *= r; - //fac = rpow*sqrt(static_cast(2*l+1))*omega;//rpow*sqrt((2*l+1)/pi4); - //FactorL[l] = sqrt(2*l+1)/sqrt(4*pi) - fac = rpow * FactorL[l]; - int l0 = index(l, 0); - Ylm[l0] *= fac; - cphim = cone; - sphim = czero; - for (int m = 1; m <= l; m++) - { - temp = cphim * cphi - sphim * sphi; - sphim = sphim * cphi + cphim * sphi; - cphim = temp; - int lm = index(l, m); - fac *= FactorLM[lm]; - temp = fac * Ylm[lm]; - Ylm[lm] = temp * cphim; - lm = index(l, -m); - Ylm[lm] = temp * sphim; - } - } - //for (int i=0; i inline void SoaSphericalTensor::evaluateVGL(T x, T y, T z) { T* restrict Ylm = cYlm.data(0); - evaluate_bare(x, y, z, Ylm); + evaluate_bare(x, y, z, Ylm, Lmax, FactorL.data(), FactorLM.data()); constexpr T czero(0); constexpr T ahalf(0.5); From 6b0b89a0a896d834ad2c02e6bfcc5327090b3f56 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Wed, 18 Oct 2023 17:53:52 -0500 Subject: [PATCH 048/168] fix Lmax_ type and avoid code duplication in SoaCartesianTensor. --- src/Numerics/SoaCartesianTensor.h | 118 ++---------------------------- 1 file changed, 7 insertions(+), 111 deletions(-) diff --git a/src/Numerics/SoaCartesianTensor.h b/src/Numerics/SoaCartesianTensor.h index f86fe813977..419a796da61 100644 --- a/src/Numerics/SoaCartesianTensor.h +++ b/src/Numerics/SoaCartesianTensor.h @@ -45,7 +45,7 @@ struct SoaCartesianTensor using OffloadArray3D = Array>; ///maximum angular momentum - size_t Lmax; + int Lmax; ///normalization factor aligned_vector NormFactor; ///composite V,Gx,Gy,Gz,[L | H00, H01, H02, H11, H12, H12] @@ -60,13 +60,12 @@ struct SoaCartesianTensor explicit SoaCartesianTensor(const int l_max, bool addsign = false); ///compute Ylm - void evaluate_bare(T x, T y, T z, T* XYZ) const; - static void evaluate_bare_impl(T x, T y, T z, T* XYZ, const size_t Lmax_); + static void evaluate_bare(T x, T y, T z, T* XYZ, const int Lmax_); ///compute Ylm inline void evaluateV(T x, T y, T z, T* XYZ) const { - evaluate_bare(x, y, z, XYZ); + evaluate_bare(x, y, z, XYZ, Lmax); for (size_t i = 0, nl = cXYZ.size(); i < nl; i++) XYZ[i] *= NormFactor[i]; } @@ -74,7 +73,7 @@ struct SoaCartesianTensor ///compute Ylm inline void evaluateV(T x, T y, T z, T* XYZ) { - evaluate_bare(x, y, z, XYZ); + evaluate_bare(x, y, z, XYZ, Lmax); for (size_t i = 0, nl = cXYZ.size(); i < nl; i++) XYZ[i] *= NormFactor[i]; } @@ -108,8 +107,8 @@ struct SoaCartesianTensor is_device_ptr(xyz_devptr, XYZ_devptr)") for (size_t ir = 0; ir < nR; ir++) { - evaluate_bare_impl(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], - XYZ_devptr + (ir * Nlm), Lmax); + evaluate_bare(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], XYZ_devptr + (ir * Nlm), + Lmax); for (int i = 0; i < Nlm; i++) XYZ_devptr[ir * Nlm + i] *= NormFactor_ptr[i]; } @@ -181,7 +180,7 @@ SoaCartesianTensor::SoaCartesianTensor(const int l_max, bool addsign) : Lmax( PRAGMA_OFFLOAD("omp declare target") template -void SoaCartesianTensor::evaluate_bare_impl(T x, T y, T z, T* restrict XYZ, size_t Lmax_) +void SoaCartesianTensor::evaluate_bare(T x, T y, T z, T* restrict XYZ, int Lmax_) { const T x2 = x * x, y2 = y * y, z2 = z * z; const T x3 = x2 * x, y3 = y2 * y, z3 = z2 * z; @@ -284,109 +283,6 @@ void SoaCartesianTensor::evaluate_bare_impl(T x, T y, T z, T* restrict XYZ, s } PRAGMA_OFFLOAD("omp end declare target") -template -void SoaCartesianTensor::evaluate_bare(T x, T y, T z, T* restrict XYZ) const -{ - const T x2 = x * x, y2 = y * y, z2 = z * z; - const T x3 = x2 * x, y3 = y2 * y, z3 = z2 * z; - const T x4 = x3 * x, y4 = y3 * y, z4 = z3 * z; - const T x5 = x4 * x, y5 = y4 * y, z5 = z4 * z; - switch (Lmax) - { - case 6: - XYZ[83] = x2 * y2 * z2; // X2Y2Z2 - XYZ[82] = x * y2 * z3; // Z3Y2X - XYZ[81] = x2 * y * z3; // Z3X2Y - XYZ[80] = x * y3 * z2; // Y3Z2X - XYZ[79] = x2 * y3 * z; // Y3X2Z - XYZ[78] = x3 * y * z2; // X3Z2Y - XYZ[77] = x3 * y2 * z; // X3Y2Z - XYZ[76] = y3 * z3; // Y3Z3 - XYZ[75] = x3 * z3; // X3Z3 - XYZ[74] = x3 * y3; // X3Y3 - XYZ[73] = x * y * z4; // Z4XY - XYZ[72] = x * y4 * z; // Y4XZ - XYZ[71] = x4 * y * z; // X4YZ - XYZ[70] = y2 * z4; // Z4Y2 - XYZ[69] = x2 * z4; // Z4X2 - XYZ[68] = y4 * z2; // Y4Z2 - XYZ[67] = x2 * y4; // Y4X2 - XYZ[66] = x4 * z2; // X4Z2 - XYZ[65] = x4 * y2; // X4Y2 - XYZ[64] = y * z * z4; // Z5Y - XYZ[63] = x * z * z4; // Z5X - XYZ[62] = y * y4 * z; // Y5Z - XYZ[61] = x * y * y4; // Y5X - XYZ[60] = x * x4 * z; // X5Z - XYZ[59] = x * x4 * y; // X5Y - XYZ[58] = z * z5; // Z6 - XYZ[57] = y * y5; // Y6 - XYZ[56] = x * x5; // X6 - case 5: - XYZ[55] = x * y2 * z2; // YYZZX - XYZ[54] = x2 * y * z2; // XXZZY - XYZ[53] = x2 * y2 * z; // XXYYZ - XYZ[52] = x * y * z3; // ZZZXY - XYZ[51] = x * y3 * z; // YYYXZ - XYZ[50] = x3 * y * z; // XXXYZ - XYZ[49] = y2 * z3; // ZZZYY - XYZ[48] = x2 * z3; // ZZZXX - XYZ[47] = y3 * z2; // YYYZZ - XYZ[46] = x2 * y3; // YYYXX - XYZ[45] = x3 * z2; // XXXZZ - XYZ[44] = x3 * y2; // XXXYY - XYZ[43] = y * z4; // ZZZZY - XYZ[42] = x * z4; // ZZZZX - XYZ[41] = y4 * z; // YYYYZ - XYZ[40] = x * y4; // YYYYX - XYZ[39] = x4 * z; // XXXXZ - XYZ[38] = x4 * y; // XXXXY - XYZ[37] = z * z4; // ZZZZZ - XYZ[36] = y * y4; // YYYYY - XYZ[35] = x * x4; // XXXXX - case 4: - XYZ[34] = x * y * z2; // ZZXY - XYZ[33] = x * y2 * z; // YYXZ - XYZ[32] = x2 * y * z; // XXYZ - XYZ[31] = y2 * z2; // YYZZ - XYZ[30] = x2 * z2; // XXZZ - XYZ[29] = x2 * y2; // XXYY - XYZ[28] = y * z3; // ZZZY - XYZ[27] = x * z3; // ZZZX - XYZ[26] = y3 * z; // YYYZ - XYZ[25] = x * y3; // YYYX - XYZ[24] = x3 * z; // XXXZ - XYZ[23] = x3 * y; // XXXY - XYZ[22] = z4; // ZZZZ - XYZ[21] = y4; // YYYY - XYZ[20] = x4; // XXXX - case 3: - XYZ[19] = x * y * z; // XYZ - XYZ[18] = y * z2; // ZZY - XYZ[17] = x * z2; // ZZX - XYZ[16] = y2 * z; // YYZ - XYZ[15] = x * y2; // YYX - XYZ[14] = x2 * z; // XXZ - XYZ[13] = x2 * y; // XXY - XYZ[12] = z3; // ZZZ - XYZ[11] = y3; // YYY - XYZ[10] = x3; // XXX - case 2: - XYZ[9] = y * z; // YZ - XYZ[8] = x * z; // XZ - XYZ[7] = x * y; // XY - XYZ[6] = z2; // ZZ - XYZ[5] = y2; // YY - XYZ[4] = x2; // XX - case 1: - XYZ[3] = z; // Z - XYZ[2] = y; // Y - XYZ[1] = x; // X - case 0: - XYZ[0] = 1; // S - } -} - template void SoaCartesianTensor::evaluateVGL(T x, T y, T z) From 98ccd00a791a4a1bc1034b1a6745e0d5224a07fd Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:07:55 -0500 Subject: [PATCH 049/168] working on cleanup --- src/Numerics/SoaCartesianTensor.h | 24 +++--- src/Numerics/SoaSphericalTensor.h | 60 ++++++++------- src/Particle/SoaDistanceTableABOMPTarget.h | 9 +-- .../LCAO/MultiQuinticSpline1D.h | 67 +++++++++-------- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 73 +++++++++++-------- 5 files changed, 126 insertions(+), 107 deletions(-) diff --git a/src/Numerics/SoaCartesianTensor.h b/src/Numerics/SoaCartesianTensor.h index 419a796da61..48861a52e48 100644 --- a/src/Numerics/SoaCartesianTensor.h +++ b/src/Numerics/SoaCartesianTensor.h @@ -47,7 +47,7 @@ struct SoaCartesianTensor ///maximum angular momentum int Lmax; ///normalization factor - aligned_vector NormFactor; + Vector> NormFactor; ///composite V,Gx,Gy,Gz,[L | H00, H01, H02, H11, H12, H12] // {GH000, GH001, GH002, GH011, GH012, GH022, GH111, GH112, GH122, GH222 VectorSoaContainer cXYZ; @@ -60,7 +60,7 @@ struct SoaCartesianTensor explicit SoaCartesianTensor(const int l_max, bool addsign = false); ///compute Ylm - static void evaluate_bare(T x, T y, T z, T* XYZ, const int Lmax_); + static void evaluate_bare(T x, T y, T z, T* XYZ, int lmax); ///compute Ylm inline void evaluateV(T x, T y, T z, T* XYZ) const @@ -87,7 +87,7 @@ struct SoaCartesianTensor * @param [in] xyz electron positions [Nelec, Npbc, 3(x,y,z)] * @param [out] XYZ Cartesian tensor elements [Nelec, Npbc, Nlm] */ - inline void batched_evaluateV(OffloadArray3D& xyz, OffloadArray3D& XYZ) const + inline void batched_evaluateV(const OffloadArray3D& xyz, OffloadArray3D& XYZ) const { const size_t nElec = xyz.size(0); const size_t Npbc = xyz.size(1); // number of PBC images @@ -99,18 +99,17 @@ struct SoaCartesianTensor size_t nR = nElec * Npbc; // total number of positions to evaluate - auto* xyz_devptr = xyz.device_data(); - auto* XYZ_devptr = XYZ.device_data(); + auto* xyz_ptr = xyz.data(); + auto* XYZ_ptr = XYZ.data(); auto* NormFactor_ptr = NormFactor.data(); - PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:NormFactor_ptr[:Nlm]) \ - is_device_ptr(xyz_devptr, XYZ_devptr)") + PRAGMA_OFFLOAD( + "omp target teams distribute parallel for map(to:NormFactor_ptr[:Nlm], xyz_ptr[:3*nR], XYZ_ptr[:Nlm*nR])") for (size_t ir = 0; ir < nR; ir++) { - evaluate_bare(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], XYZ_devptr + (ir * Nlm), - Lmax); + evaluate_bare(xyz_ptr[0 + 3 * ir], xyz_ptr[1 + 3 * ir], xyz_ptr[2 + 3 * ir], XYZ_ptr + (ir * Nlm), Lmax); for (int i = 0; i < Nlm; i++) - XYZ_devptr[ir * Nlm + i] *= NormFactor_ptr[i]; + XYZ_ptr[ir * Nlm + i] *= NormFactor_ptr[i]; } } @@ -175,18 +174,19 @@ SoaCartesianTensor::SoaCartesianTensor(const int l_max, bool addsign) : Lmax( NormL); } } + NormFactor.updateTo(); } PRAGMA_OFFLOAD("omp declare target") template -void SoaCartesianTensor::evaluate_bare(T x, T y, T z, T* restrict XYZ, int Lmax_) +void SoaCartesianTensor::evaluate_bare(T x, T y, T z, T* restrict XYZ, int lmax) { const T x2 = x * x, y2 = y * y, z2 = z * z; const T x3 = x2 * x, y3 = y2 * y, z3 = z2 * z; const T x4 = x3 * x, y4 = y3 * y, z4 = z3 * z; const T x5 = x4 * x, y5 = y4 * y, z5 = z4 * z; - switch (Lmax_) + switch (lmax) { case 6: XYZ[83] = x2 * y2 * z2; // X2Y2Z2 diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index 515e4046b9c..9095dcd316d 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -44,13 +44,13 @@ struct SoaSphericalTensor ///maximum angular momentum for the center int Lmax; /// Normalization factors - aligned_vector NormFactor; + Vector> NormFactor; ///pre-evaluated factor \f$1/\sqrt{(l+m)\times(l+1-m)}\f$ - aligned_vector FactorLM; + Vector> FactorLM; ///pre-evaluated factor \f$\sqrt{(2l+1)/(4\pi)}\f$ - aligned_vector FactorL; + Vector> FactorL; ///pre-evaluated factor \f$(2l+1)/(2l-1)\f$ - aligned_vector Factor2L; + Vector> Factor2L; ///composite VectorSoaContainer cYlm; @@ -59,7 +59,7 @@ struct SoaSphericalTensor SoaSphericalTensor(const SoaSphericalTensor& rhs) = default; ///compute Ylm - static void evaluate_bare(T x, T y, T z, T* Ylm, const int Lmax_, const T* FacL, const T* FacLM); + static void evaluate_bare(T x, T y, T z, T* Ylm, const int lmax, const T* FacL, const T* FacLM); ///compute Ylm inline void evaluateV(T x, T y, T z, T* Ylm) const @@ -75,7 +75,7 @@ struct SoaSphericalTensor * @param [in] xyz electron positions [Nelec, Npbc, 3(x,y,z)] * @param [out] Ylm Spherical tensor elements [Nelec, Npbc, Nlm] */ - inline void batched_evaluateV(OffloadArray3D& xyz, OffloadArray3D& Ylm) const + inline void batched_evaluateV(const OffloadArray3D& xyz, OffloadArray3D& Ylm) const { const size_t nElec = xyz.size(0); const size_t Npbc = xyz.size(1); // number of PBC images @@ -87,22 +87,22 @@ struct SoaSphericalTensor size_t nR = nElec * Npbc; // total number of positions to evaluate - auto* xyz_devptr = xyz.device_data(); - auto* Ylm_devptr = Ylm.device_data(); - auto* flm_ptr = FactorLM.data(); - auto* fl_ptr = FactorL.data(); + auto* xyz_ptr = xyz.data(); + auto* Ylm_ptr = Ylm.data(); + auto* FactorLM_ptr = FactorLM.data(); + auto* FactorL_ptr = FactorL.data(); auto* NormFactor_ptr = NormFactor.data(); PRAGMA_OFFLOAD("omp target teams distribute parallel for \ - map(always, to:flm_ptr[:Nlm], fl_ptr[:Lmax+1], NormFactor_ptr[:Nlm]) \ - is_device_ptr(xyz_devptr, Ylm_devptr)") + map(to:FactorLM_ptr[:Nlm], FactorL_ptr[:Lmax+1], NormFactor_ptr[:Nlm], \ + xyz_ptr[:3*nR], Ylm_ptr[:Nlm*nR])") for (size_t ir = 0; ir < nR; ir++) { - evaluate_bare(xyz_devptr[0 + 3 * ir], xyz_devptr[1 + 3 * ir], xyz_devptr[2 + 3 * ir], Ylm_devptr + (ir * Nlm), - Lmax, fl_ptr, flm_ptr); + evaluate_bare(xyz_ptr[0 + 3 * ir], xyz_ptr[1 + 3 * ir], xyz_ptr[2 + 3 * ir], Ylm_ptr + (ir * Nlm), Lmax, + FactorL_ptr, FactorLM_ptr); for (int i = 0; i < Nlm; i++) - Ylm_devptr[ir * Nlm + i] *= NormFactor_ptr[i]; + Ylm_ptr[ir * Nlm + i] *= NormFactor_ptr[i]; } } @@ -208,17 +208,21 @@ inline SoaSphericalTensor::SoaSphericalTensor(const int l_max, bool addsign) FactorLM[index(l, m)] = fac2; FactorLM[index(l, -m)] = fac2; } + NormFactor.updateTo(); + FactorLM.updateTo(); + FactorL.updateTo(); + Factor2L.updateTo(); } PRAGMA_OFFLOAD("omp declare target") template -inline void SoaSphericalTensor::evaluate_bare(const T x, - const T y, - const T z, +inline void SoaSphericalTensor::evaluate_bare(T x, + T y, + T z, T* restrict Ylm, - const int Lmax_, - const T* FactorL_ptr, - const T* FactorLM_ptr) + int lmax, + const T* factorL, + const T* factorLM) { constexpr T czero(0); constexpr T cone(1); @@ -258,7 +262,7 @@ inline void SoaSphericalTensor::evaluate_bare(const T x, // calculate P_ll and P_l,l-1 T fac = cone; int j = -1; - for (int l = 1; l <= Lmax_; l++) + for (int l = 1; l <= lmax; l++) { j += 2; fac *= -j * stheta; @@ -269,10 +273,10 @@ inline void SoaSphericalTensor::evaluate_bare(const T x, Ylm[l1] = j * ctheta * Ylm[l2]; } // Use recurence to get other plm's // - for (int m = 0; m < Lmax_ - 1; m++) + for (int m = 0; m < lmax - 1; m++) { int j = 2 * m + 1; - for (int l = m + 2; l <= Lmax_; l++) + for (int l = m + 2; l <= lmax; l++) { j += 2; int lm = index(l, m); @@ -285,12 +289,12 @@ inline void SoaSphericalTensor::evaluate_bare(const T x, T sphim, cphim, temp; Ylm[0] = omega; //1.0/sqrt(pi4); T rpow = 1.0; - for (int l = 1; l <= Lmax_; l++) + for (int l = 1; l <= lmax; l++) { rpow *= r; //fac = rpow*sqrt(static_cast(2*l+1))*omega;//rpow*sqrt((2*l+1)/pi4); - //FactorL[l] = sqrt(2*l+1)/sqrt(4*pi) - fac = rpow * FactorL_ptr[l]; + //factorL[l] = sqrt(2*l+1)/sqrt(4*pi) + fac = rpow * factorL[l]; int l0 = index(l, 0); Ylm[l0] *= fac; cphim = cone; @@ -301,7 +305,7 @@ inline void SoaSphericalTensor::evaluate_bare(const T x, sphim = sphim * cphi + cphim * sphi; cphim = temp; int lm = index(l, m); - fac *= FactorLM_ptr[lm]; + fac *= factorLM[lm]; temp = fac * Ylm[lm]; Ylm[lm] = temp * cphim; lm = index(l, -m); diff --git a/src/Particle/SoaDistanceTableABOMPTarget.h b/src/Particle/SoaDistanceTableABOMPTarget.h index 8ef2ace4d1a..05dac1eaf13 100644 --- a/src/Particle/SoaDistanceTableABOMPTarget.h +++ b/src/Particle/SoaDistanceTableABOMPTarget.h @@ -298,14 +298,11 @@ class SoaDistanceTableABOMPTarget : public DTD_BConds, public Distance auto* input_ptr = offload_input.data(); const int num_sources_local = num_sources_; - const size_t num_r_dr = mw_r_dr.size(); - const size_t num_offload_input = offload_input.size(); - { ScopedTimer offload(dt_leader.offload_timer_); PRAGMA_OFFLOAD("omp target teams distribute collapse(2) num_teams(total_targets*num_teams) \ - map(always, to: input_ptr[:num_offload_input]) \ - depend(out:r_dr_ptr[:num_r_dr]) nowait") + map(always, to: input_ptr[:offload_input.size()]) \ + depend(out:r_dr_ptr[:mw_r_dr.size()]) nowait") for (int iat = 0; iat < total_targets; ++iat) for (int team_id = 0; team_id < num_teams; team_id++) { @@ -332,7 +329,7 @@ class SoaDistanceTableABOMPTarget : public DTD_BConds, public Distance if (!(modes_ & DTModes::MW_EVALUATE_RESULT_NO_TRANSFER_TO_HOST)) { PRAGMA_OFFLOAD( - "omp target update from(r_dr_ptr[:num_r_dr]) depend(inout:r_dr_ptr[:num_r_dr]) nowait") + "omp target update from(r_dr_ptr[:mw_r_dr.size()]) depend(inout:r_dr_ptr[:mw_r_dr.size()]) nowait") } // wait for computing and (optional) transferring back to host. // It can potentially be moved to ParticleSet to fuse multiple similar taskwait diff --git a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h index 126177fc849..14580edf385 100644 --- a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h +++ b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h @@ -58,10 +58,10 @@ struct LogGridLight } PRAGMA_OFFLOAD("omp declare target") - static inline T getCL(T r, int& loc, double OneOverLogDelta_, T lower_bound_, double dlog_ratio_) + static inline T getCL(T r, int& loc, double one_over_log_delta, T lower_bound, double log_delta) { - loc = static_cast(std::log(r / lower_bound_) * OneOverLogDelta_); - return r - lower_bound_ * std::exp(loc * dlog_ratio_); + loc = static_cast(std::log(r / lower_bound) * one_over_log_delta); + return r - lower_bound * std::exp(loc * log_delta); } PRAGMA_OFFLOAD("omp end declare target") @@ -152,7 +152,14 @@ class MultiQuinticSpline1D } } - inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, T Rmax) const + /** + * @brief evaluate MultiQuinticSpline1D for multiple electrons and multiple pbc images + * + * @param [in] r electron distances [Nelec, Npbc] + * @param [out] u value of all splines at all electron distances [Nelec, Npbc, Nsplines] + * @param Rmax spline will evaluate to zero for any distance greater than or equal to Rmax + */ + inline void batched_evaluate(const OffloadArray2D& r, OffloadArray3D& u, T Rmax) const { const size_t nElec = r.size(0); const size_t Nxyz = r.size(1); // number of PBC images @@ -161,49 +168,47 @@ class MultiQuinticSpline1D const size_t nRnl = u.size(2); // number of splines const size_t nR = nElec * Nxyz; // total number of positions to evaluate - auto* first_deriv_ptr = first_deriv.data(); - - double OneOverLogDelta = myGrid.OneOverLogDelta; - T lower_bound = myGrid.lower_bound; - T dlog_ratio = myGrid.LogDelta; + double one_over_log_delta = myGrid.OneOverLogDelta; + T lower_bound = myGrid.lower_bound; + T log_delta = myGrid.LogDelta; - auto* r_devptr = r.device_data(); - auto* u_devptr = u.device_data(); + auto* r_ptr = r.data(); + auto* u_ptr = u.data(); - auto* coeff_devptr = coeffs->device_data(); - auto* first_deriv_devptr = first_deriv.device_data(); - const size_t nCols = coeffs->cols(); - // const size_t coefsize = coeffs->size(); + auto* coeff_ptr = coeffs->data(); + auto* first_deriv_ptr = first_deriv.data(); + const size_t nCols = coeffs->cols(); + const size_t coefsize = coeffs->size(); PRAGMA_OFFLOAD("omp target teams distribute parallel for \ - is_device_ptr(r_devptr, u_devptr, coeff_devptr, first_deriv_devptr)") + map(to:r_ptr[:nR], u_ptr[:nRnl*nR], coeff_ptr[:coefsize], first_deriv_ptr[:num_splines_])") for (size_t ir = 0; ir < nR; ir++) { - if (r_devptr[ir] >= Rmax) + if (r_ptr[ir] >= Rmax) { for (size_t i = 0; i < num_splines_; ++i) - u_devptr[ir * nRnl + i] = 0.0; + u_ptr[ir * nRnl + i] = 0.0; } - else if (r_devptr[ir] < lower_bound) + else if (r_ptr[ir] < lower_bound) { - const T dr = r_devptr[ir] - lower_bound; + const T dr = r_ptr[ir] - lower_bound; for (size_t i = 0; i < num_splines_; ++i) - u_devptr[ir * nRnl + i] = coeff_devptr[i] + first_deriv_devptr[i] * dr; + u_ptr[ir * nRnl + i] = coeff_ptr[i] + first_deriv_ptr[i] * dr; } else { int loc; - const auto cL = LogGridLight::getCL(r_devptr[ir], loc, OneOverLogDelta, lower_bound, dlog_ratio); + const auto cL = LogGridLight::getCL(r_ptr[ir], loc, one_over_log_delta, lower_bound, log_delta); const size_t offset = loc * 6; - const T* restrict a = coeff_devptr + nCols * (offset + 0); - const T* restrict b = coeff_devptr + nCols * (offset + 1); - const T* restrict c = coeff_devptr + nCols * (offset + 2); - const T* restrict d = coeff_devptr + nCols * (offset + 3); - const T* restrict e = coeff_devptr + nCols * (offset + 4); - const T* restrict f = coeff_devptr + nCols * (offset + 5); + const T* restrict a = coeff_ptr + nCols * (offset + 0); + const T* restrict b = coeff_ptr + nCols * (offset + 1); + const T* restrict c = coeff_ptr + nCols * (offset + 2); + const T* restrict d = coeff_ptr + nCols * (offset + 3); + const T* restrict e = coeff_ptr + nCols * (offset + 4); + const T* restrict f = coeff_ptr + nCols * (offset + 5); for (size_t i = 0; i < num_splines_; ++i) - u_devptr[ir * nRnl + i] = a[i] + cL * (b[i] + cL * (c[i] + cL * (d[i] + cL * (e[i] + cL * f[i])))); + u_ptr[ir * nRnl + i] = a[i] + cL * (b[i] + cL * (c[i] + cL * (d[i] + cL * (e[i] + cL * f[i])))); } } } @@ -345,8 +350,8 @@ class MultiQuinticSpline1D out[(i * 6 + 5) * ncols + ispline] = static_cast(F[i]); } } - first_deriv.updateTo(); - coeffs->updateTo(); // FIXME: this is overkill, probably better to just do once after all splines added + first_deriv.updateTo(); // FIXME: this is overkill, probably better to just do once after all splines added + coeffs->updateTo(); // FIXME: this is overkill, probably better to just do once after all splines added } int getNumSplines() const { return num_splines_; } diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index f33a1721fcb..e2697bdc485 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -679,6 +679,25 @@ struct SoaAtomicBasisSet } } + /** + * @brief evaluate for multiple electrons + * + * This function should only assign to elements of psi in the range [[0:nElec],[BasisOffset:BasisOffset+BasisSetSize]]. + * These elements are assumed to be zero when passed to this function. + * This function only uses only one center (center_idx) from displ_list + * + * @param [in] atom_bs_list multi-walker list of SoaAtomicBasisSet [nWalkers] + * @param [in] lattice crystal lattice + * @param [in,out] psi wavefunction values for all electrons [nElec, nBasTot] + * @param [in] displ_list displacement from each electron to each center [NumCenters, nElec, 3] (flattened) + * @param [in] Tv_list translation vectors for computing overall phase factor [nElec, 3] (flattened) + * @param [in] nElec number of electrons + * @param [in] nBasTot total number of basis functions represented in psi + * @param [in] center_idx current center index (for indexing into displ_list) + * @param [in] BasisOffset index of first basis function of this center (for indexing into psi) + * @param [in] NumCenters total number of centers in system (for indexing into displ_list) + * + */ template inline void mw_evaluateV(const RefVectorWithLeader& atom_bs_list, const LAT& lattice, @@ -687,23 +706,18 @@ struct SoaAtomicBasisSet const Vector>& Tv_list, const size_t nElec, const size_t nBasTot, - const size_t c, + const size_t center_idx, const size_t BasisOffset, const size_t NumCenters) { assert(this == &atom_bs_list.getLeader()); auto& atom_bs_leader = atom_bs_list.template getCastedLeader>(); - /* - psi [nElec, nBasTot] (start at [0, BasisOffset]) - displ_list [3 * nElec * NumCenters] (start at [3*nElec*c]) - Tv_list [3 * nElec * NumCenters] - */ //TODO: use QMCTraits::DIM instead of 3? - int Nx = PBCImages[0] + 1; - int Ny = PBCImages[1] + 1; - int Nz = PBCImages[2] + 1; - int Nyz = Ny * Nz; - int Nxyz = Nx * Nyz; + // DIM==3 is baked into so many parts here that it's probably not worth it for now + const int Nx = PBCImages[0] + 1; + const int Ny = PBCImages[1] + 1; + const int Nz = PBCImages[2] + 1; + const int Nxyz = Nx * Ny * Nz; assert(psi.size(0) == nElec); assert(psi.size(1) == nBasTot); @@ -713,8 +727,8 @@ struct SoaAtomicBasisSet auto& dr = atom_bs_leader.mw_mem_handle_.getResource().dr; auto& r = atom_bs_leader.mw_mem_handle_.getResource().r; - size_t nRnl = RnlID.size(); - size_t nYlm = Ylm.size(); + const size_t nRnl = RnlID.size(); + const size_t nYlm = Ylm.size(); ylm_v.resize(nElec, Nxyz, nYlm); rnl_v.resize(nElec, Nxyz, nRnl); @@ -727,16 +741,16 @@ struct SoaAtomicBasisSet dr_pbc.resize(Nxyz, 3); correctphase.resize(nElec); - auto* dr_pbc_devptr = dr_pbc.device_data(); - auto* dr_new_devptr = dr.device_data(); - auto* r_new_devptr = r.device_data(); + auto* dr_pbc_ptr = dr_pbc.data(); + auto* dr_ptr = dr.data(); + auto* r_ptr = r.data(); - auto* correctphase_devptr = correctphase.device_data(); + auto* correctphase_ptr = correctphase.data(); - auto* Tv_list_devptr = Tv_list.device_data(); - auto* displ_list_devptr = displ_list.device_data(); + auto* Tv_list_ptr = Tv_list.data(); + auto* displ_list_ptr = displ_list.data(); - auto* psi_devptr = psi.device_data(); + auto* psi_ptr = psi.data(); // need to map Tensor vals to device auto* latR_ptr = lattice.R.data(); @@ -745,18 +759,17 @@ struct SoaAtomicBasisSet // should just do this once and store it (like with phase) { ScopedTimer local(pbc_timer_); - PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(always, to:latR_ptr[:9]) \ - is_device_ptr(dr_pbc_devptr) ") + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(to:latR_ptr[:9], dr_pbc_ptr[:3*Nxyz]) ") for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) { // i_xyz = k + Nz * (j + Ny * i) - int ij = i_xyz / Nz; - int k = i_xyz - ij * Nz; - int i = ij / Ny; - int j = ij - i * Ny; - int TransX = ((i % 2) * 2 - 1) * ((i + 1) / 2); - int TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); - int TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); + int ij = i_xyz / Nz; + int k = i_xyz - ij * Nz; + int i = ij / Ny; + int j = ij - i * Ny; + int TransX = ((i % 2) * 2 - 1) * ((i + 1) / 2); + int TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); + int TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); for (size_t i_dim = 0; i_dim < 3; i_dim++) dr_pbc_devptr[i_dim + 3 * i_xyz] = (TransX * latR_ptr[i_dim + 0 * 3] + TransY * latR_ptr[i_dim + 1 * 3] + TransZ * latR_ptr[i_dim + 2 * 3]); @@ -802,7 +815,7 @@ struct SoaAtomicBasisSet for (size_t i_dim = 0; i_dim < 3; i_dim++) { dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = - -(displ_list_devptr[i_dim + 3 * (i_e + c * nElec)] + dr_pbc_devptr[i_dim + 3 * i_xyz]); + -(displ_list_devptr[i_dim + 3 * (i_e + center_idx * nElec)] + dr_pbc_devptr[i_dim + 3 * i_xyz]); tmp_r2 += dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] * dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)]; } r_new_devptr[i_xyz + Nxyz * i_e] = std::sqrt(tmp_r2); From 73cd899a4b9b93dd4a574cb408767e62ce9e6a3c Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:26:44 -0500 Subject: [PATCH 050/168] cleanup in SoaAtomicBasisSet --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index e2697bdc485..9dbb732904b 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -750,8 +750,6 @@ struct SoaAtomicBasisSet auto* Tv_list_ptr = Tv_list.data(); auto* displ_list_ptr = displ_list.data(); - auto* psi_ptr = psi.data(); - // need to map Tensor vals to device auto* latR_ptr = lattice.R.data(); @@ -771,7 +769,7 @@ struct SoaAtomicBasisSet int TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); int TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); for (size_t i_dim = 0; i_dim < 3; i_dim++) - dr_pbc_devptr[i_dim + 3 * i_xyz] = + dr_pbc_ptr[i_dim + 3 * i_xyz] = (TransX * latR_ptr[i_dim + 0 * 3] + TransY * latR_ptr[i_dim + 1 * 3] + TransZ * latR_ptr[i_dim + 2 * 3]); } } @@ -781,21 +779,21 @@ struct SoaAtomicBasisSet ScopedTimer local_timer(phase_timer_); #if not defined(QMC_COMPLEX) - PRAGMA_OFFLOAD("omp target teams distribute parallel for is_device_ptr(correctphase_devptr) ") + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:correctphase_ptr[:nElec]) ") for (size_t i_e = 0; i_e < nElec; i_e++) - correctphase_devptr[i_e] = 1.0; + correctphase_ptr[i_e] = 1.0; #else auto* SuperTwist_ptr = SuperTwist.data(); - PRAGMA_OFFLOAD("omp target teams distribute parallel for map(always, to:SuperTwist_ptr[:9]) \ - is_device_ptr(Tv_list_devptr, correctphase_devptr) ") + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:SuperTwist_ptr[:9], \ + Tv_list_ptr[:nElec*3], correctphase_ptr[:nElec]) ") for (size_t i_e = 0; i_e < nElec; i_e++) { //RealType phasearg = dot(3, SuperTwist.data(), 1, Tv_list.data() + 3 * i_e, 1); RealType phasearg = 0; for (size_t i_dim = 0; i_dim < 3; i_dim++) - phasearg += SuperTwist[i_dim] * Tv_list_devptr[i_dim + 3 * i_e]; + phasearg += SuperTwist[i_dim] * Tv_list_ptr[i_dim + 3 * i_e]; RealType s, c; qmcplusplus::sincos(-phasearg, &s, &c); correctphase_devptr[i_e] = ValueType(c, s); @@ -806,7 +804,7 @@ struct SoaAtomicBasisSet { ScopedTimer local_timer(nelec_pbc_timer_); PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) \ - is_device_ptr(dr_new_devptr, dr_pbc_devptr, r_new_devptr, displ_list_devptr) ") + map(to: dr_ptr[:3*nElec*Nxyz], dr_pbc_ptr[:3*Nxyz], r_ptr[:nElec*Nxyz], displ_list_ptr[3*nElec*center_idx:3*nElec]) ") for (size_t i_e = 0; i_e < nElec; i_e++) { for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) @@ -814,11 +812,11 @@ struct SoaAtomicBasisSet RealType tmp_r2 = 0.0; for (size_t i_dim = 0; i_dim < 3; i_dim++) { - dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = - -(displ_list_devptr[i_dim + 3 * (i_e + center_idx * nElec)] + dr_pbc_devptr[i_dim + 3 * i_xyz]); - tmp_r2 += dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] * dr_new_devptr[i_dim + 3 * (i_xyz + Nxyz * i_e)]; + dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = + -(displ_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)] + dr_pbc_ptr[i_dim + 3 * i_xyz]); + tmp_r2 += dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] * dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)]; } - r_new_devptr[i_xyz + Nxyz * i_e] = std::sqrt(tmp_r2); + r_ptr[i_xyz + Nxyz * i_e] = std::sqrt(tmp_r2); } } } @@ -840,22 +838,22 @@ struct SoaAtomicBasisSet auto* NL_ptr = NL.data(); auto* psi_ptr = psi.data(); - auto* ylm_devptr = ylm_v.device_data(); - auto* rnl_devptr = rnl_v.device_data(); + auto* ylm_ptr = ylm_v.data(); + auto* rnl_ptr = rnl_v.data(); { ScopedTimer local_timer(psi_timer_); PRAGMA_OFFLOAD( - "omp target teams distribute parallel for collapse(2) map(always, to:phase_fac_ptr[:Nxyz], LM_ptr[:BasisSetSize], NL_ptr[:BasisSetSize]) \ - is_device_ptr(ylm_devptr, rnl_devptr, psi_devptr, correctphase_devptr) ") + "omp target teams distribute parallel for collapse(2) map(to:phase_fac_ptr[:Nxyz], LM_ptr[:BasisSetSize], NL_ptr[:BasisSetSize], \ + ylm_ptr[:nYlm*nElec*Nxyz], rnl_ptr[:nRnl*nElec*Nxyz], psi_ptr[:nBasTot*nElec], correctphase_ptr[:nElec]) ") for (size_t i_e = 0; i_e < nElec; i_e++) { for (size_t ib = 0; ib < BasisSetSize; ++ib) { for (size_t i_xyz = 0; i_xyz < Nxyz; i_xyz++) { - const ValueType Phase = phase_fac_ptr[i_xyz] * correctphase_devptr[i_e]; - psi_devptr[BasisOffset + ib + i_e * nBasTot] += ylm_devptr[(i_xyz + Nxyz * i_e) * nYlm + LM_ptr[ib]] * - rnl_devptr[(i_xyz + Nxyz * i_e) * nRnl + NL_ptr[ib]] * Phase; + const ValueType Phase = phase_fac_ptr[i_xyz] * correctphase_ptr[i_e]; + psi_ptr[BasisOffset + ib + i_e * nBasTot] += ylm_ptr[(i_xyz + Nxyz * i_e) * nYlm + LM_ptr[ib]] * + rnl_ptr[(i_xyz + Nxyz * i_e) * nRnl + NL_ptr[ib]] * Phase; } } } From 04d3f92acf14bf69cb81577b1b0fdeb7e1b16751 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:33:11 -0500 Subject: [PATCH 051/168] more cleanup in SoaLocBSet --- src/Numerics/SoaSphericalTensor.h | 2 +- src/QMCWaveFunctions/BasisSetBase.h | 2 +- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp | 4 +--- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index 9095dcd316d..268874e7ab8 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -59,7 +59,7 @@ struct SoaSphericalTensor SoaSphericalTensor(const SoaSphericalTensor& rhs) = default; ///compute Ylm - static void evaluate_bare(T x, T y, T z, T* Ylm, const int lmax, const T* FacL, const T* FacLM); + static void evaluate_bare(T x, T y, T z, T* Ylm, int lmax, const T* FacL, const T* FacLM); ///compute Ylm inline void evaluateV(T x, T y, T z, T* Ylm) const diff --git a/src/QMCWaveFunctions/BasisSetBase.h b/src/QMCWaveFunctions/BasisSetBase.h index b08f1988e19..d1c3b3ed5db 100644 --- a/src/QMCWaveFunctions/BasisSetBase.h +++ b/src/QMCWaveFunctions/BasisSetBase.h @@ -155,7 +155,7 @@ struct SoaBasisSetBase //Evaluates value for electron "iat". places it in a offload array for batched code. virtual void mw_evaluateValue(const RefVectorWithLeader& P_list, int iat, OffloadMWVArray& v) = 0; //Evaluates value for all the electrons of the virtual particles. places it in a offload array for batched code. - virtual void mw_evaluateValueVPs(RefVectorWithLeader>& basis_list, + virtual void mw_evaluateValueVPs(const RefVectorWithLeader>& basis_list, const RefVectorWithLeader& vp_list, OffloadMWVArray& v) = 0; //Evaluates value, gradient, and Hessian for electron "iat". Parks them into a temporary data structure "vgh". diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index e4fae6eb8b6..9420fefba37 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -249,7 +249,7 @@ void SoaLocalizedBasisSet::evaluateVGHGH(const ParticleSet& P, int ia template -void SoaLocalizedBasisSet::mw_evaluateValueVPs(RefVectorWithLeader>& basis_list, +void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLeader>& basis_list, const RefVectorWithLeader& vp_list, OffloadMWVArray& vp_basis_v) { @@ -280,13 +280,11 @@ void SoaLocalizedBasisSet::mw_evaluateValueVPs(RefVectorWithLeader * @param vp_list list of quantum virtual particleset (one for each walker) * @param v Array(n_walkers, BasisSetSize) */ - void mw_evaluateValueVPs(RefVectorWithLeader>& basis_list, + void mw_evaluateValueVPs(const RefVectorWithLeader>& basis_list, const RefVectorWithLeader& vp_list, OffloadMWVArray& v) override; From 5c6efb15c42dd2f2f714d0773902abb4e20ab71d Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:34:31 -0500 Subject: [PATCH 052/168] don't use explicit device ptr --- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index 9420fefba37..0b68bea82b9 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -293,11 +293,11 @@ void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLea displ_list_tr.updateTo(); // set AO data to zero on device - auto* vp_basis_v_devptr = vp_basis_v.device_data(); - PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) is_device_ptr(vp_basis_v_devptr) ") + auto* vp_basis_v_ptr = vp_basis_v.data(); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(to:vp_basis_v_ptr[:nVPs*BasisSetSize]) ") for (size_t i_vp = 0; i_vp < nVPs; i_vp++) for (size_t ib = 0; ib < BasisSetSize; ++ib) - vp_basis_v_devptr[ib + i_vp * BasisSetSize] = 0; + vp_basis_v_ptr[ib + i_vp * BasisSetSize] = 0; // TODO: group/sort centers by species? for (int c = 0; c < NumCenters; c++) From 59dce66f1c7822b913bd7b8594526f27f275f659 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:41:12 -0500 Subject: [PATCH 053/168] clearer extractCoordsRefList_vp --- src/Particle/VirtualParticleSet.cpp | 2 +- src/Particle/VirtualParticleSet.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Particle/VirtualParticleSet.cpp b/src/Particle/VirtualParticleSet.cpp index 96ae186576b..91ce702f31a 100644 --- a/src/Particle/VirtualParticleSet.cpp +++ b/src/Particle/VirtualParticleSet.cpp @@ -119,7 +119,7 @@ const std::vector VirtualParticleSet::extractCoordsRefList_v std::vector coords_list; for (const VirtualParticleSet& vp : vp_list) for (int iat = 0; iat < vp.getTotalNum(); iat++) - coords_list.push_back(vp.activeR(iat)); + coords_list.push_back(vp.R[iat]); return coords_list; } diff --git a/src/Particle/VirtualParticleSet.h b/src/Particle/VirtualParticleSet.h index 293b8e01209..1789a0dbcb7 100644 --- a/src/Particle/VirtualParticleSet.h +++ b/src/Particle/VirtualParticleSet.h @@ -101,7 +101,7 @@ class VirtualParticleSet : public ParticleSet static const RefVectorWithLeader extractDTRefList( const RefVectorWithLeader& vp_list, int id); - /**Extract list of active VP coordinates, flattened over all walkers + /**Extract list of VP coordinates, flattened over all walkers */ static const std::vector extractCoordsRefList_vp( const RefVectorWithLeader& vp_list); From e79bc240441069a82d1c028ea95f4bfd87705402 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:43:16 -0500 Subject: [PATCH 054/168] better name for extractVPCoords --- src/Particle/VirtualParticleSet.cpp | 2 +- src/Particle/VirtualParticleSet.h | 2 +- src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Particle/VirtualParticleSet.cpp b/src/Particle/VirtualParticleSet.cpp index 91ce702f31a..7f3b4b99408 100644 --- a/src/Particle/VirtualParticleSet.cpp +++ b/src/Particle/VirtualParticleSet.cpp @@ -113,7 +113,7 @@ const RefVectorWithLeader VirtualParticleSet::extractDTRe } -const std::vector VirtualParticleSet::extractCoordsRefList_vp( +const std::vector VirtualParticleSet::extractVPCoords( const RefVectorWithLeader& vp_list) { std::vector coords_list; diff --git a/src/Particle/VirtualParticleSet.h b/src/Particle/VirtualParticleSet.h index 1789a0dbcb7..61696e836ea 100644 --- a/src/Particle/VirtualParticleSet.h +++ b/src/Particle/VirtualParticleSet.h @@ -103,7 +103,7 @@ class VirtualParticleSet : public ParticleSet int id); /**Extract list of VP coordinates, flattened over all walkers */ - static const std::vector extractCoordsRefList_vp( + static const std::vector extractVPCoords( const RefVectorWithLeader& vp_list); /** move virtual particles to new postions and update distance tables * @param refp reference particle set diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index 0b68bea82b9..d650d56b199 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -264,7 +264,7 @@ void SoaLocalizedBasisSet::mw_evaluateValueVPs(const RefVectorWithLea const auto dt_list(vps_leader.extractDTRefList(vp_list, myTableIndex)); - const auto coordR_list(vps_leader.extractCoordsRefList_vp(vp_list)); + const auto coordR_list(vps_leader.extractVPCoords(vp_list)); // make these shared resource? PinnedDualAllocator? OffloadPinnedAllocator? Vector> Tv_list; From ce4e24987dc63badff9f86ac48a7e8a769aca6c2 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 13:56:06 -0500 Subject: [PATCH 055/168] add documentation for MultiFunctorAdapter::batched_evaluate --- src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index 49d055932a7..00a7b1746c7 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -59,7 +59,14 @@ struct MultiFunctorAdapter u[i] = Rnl[i]->f(r); } - inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const + /** + * @brief evaluate for multiple electrons and multiple pbc images + * + * @param [in] r electron distances [Nelec, Npbc] + * @param [out] u value of all splines at all electron distances [Nelec, Npbc, Nsplines] + * @param Rmax evaluate to zero for any distance greater than or equal to Rmax + */ + inline void batched_evaluate(const OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const { r.updateFrom(); // TODO: remove after offload From f7c9a002112e1df53907db5df823c6ef8fe8feaa Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 14:06:20 -0500 Subject: [PATCH 056/168] clean up multifunctor batched eval --- .../LCAO/MultiFunctorAdapter.h | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index 00a7b1746c7..573b9fdb51d 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -63,12 +63,12 @@ struct MultiFunctorAdapter * @brief evaluate for multiple electrons and multiple pbc images * * @param [in] r electron distances [Nelec, Npbc] - * @param [out] u value of all splines at all electron distances [Nelec, Npbc, Nsplines] + * @param [out] u value of all splines at all electron distances [Nelec, Npbc, nRnl] * @param Rmax evaluate to zero for any distance greater than or equal to Rmax */ - inline void batched_evaluate(const OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const + inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const { - r.updateFrom(); // TODO: remove after offload + r.updateFrom(); // TODO: remove after offload and make r const const size_t nElec = r.size(0); const size_t Nxyz = r.size(1); // number of PBC images @@ -77,16 +77,14 @@ struct MultiFunctorAdapter const size_t nRnl = u.size(2); // number of splines const size_t nR = nElec * Nxyz; // total number of positions to evaluate - auto* r_ptr = r.data(); - auto* u_ptr = u.data(); - - for (size_t ir = 0; ir < nR; ir++) - if (r_ptr[ir] >= Rmax) - for (size_t i = 0, n = Rnl.size(); i < n; ++i) - u_ptr[ir * nRnl + i] = 0.0; - else - for (size_t i = 0, n = Rnl.size(); i < n; ++i) - u_ptr[ir * nRnl + i] = Rnl[i]->f(r_ptr[ir]); + for (size_t i_e = 0; i_e < nElec; i_e++) + for (size_t i_xyz = 0; i_xyz < Nxyz; i_xyz++) + if (r(i_e, i_xyz) >= Rmax) + for (size_t i = 0, n = Rnl.size(); i < n; ++i) + u(i_e, i_xyz, i) = 0.0; + else + for (size_t i = 0, n = Rnl.size(); i < n; ++i) + u(i_e, i_xyz, i) = Rnl[i]->f(r(i_e, i_xyz)); u.updateTo(); // TODO: remove after offload } From 0771a5bc2abc3ea3df057b2951325103d6745170 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 14:13:35 -0500 Subject: [PATCH 057/168] add information about host/device data in multifunctor batched_evaluate --- src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index 573b9fdb51d..b6000c1e0e2 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -62,6 +62,12 @@ struct MultiFunctorAdapter /** * @brief evaluate for multiple electrons and multiple pbc images * + * r is assumed to be up-to-date on the device before entering this function, and + * u needs to be updated on the device before exiting this function + * + * Eventually, all computation should be done on the device to avoid transfers, but + * for now this is all done on the host + * * @param [in] r electron distances [Nelec, Npbc] * @param [out] u value of all splines at all electron distances [Nelec, Npbc, nRnl] * @param Rmax evaluate to zero for any distance greater than or equal to Rmax From 65a2d8ffd5b9cd3b055750f8770c9ab2e18e0295 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 14:22:56 -0500 Subject: [PATCH 058/168] updated description of multifunctor batched_evaluate --- src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index b6000c1e0e2..17316dac5f0 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -69,7 +69,7 @@ struct MultiFunctorAdapter * for now this is all done on the host * * @param [in] r electron distances [Nelec, Npbc] - * @param [out] u value of all splines at all electron distances [Nelec, Npbc, nRnl] + * @param [out] u value of all radial functions at all electron distances [Nelec, Npbc, nRnl] * @param Rmax evaluate to zero for any distance greater than or equal to Rmax */ inline void batched_evaluate(OffloadArray2D& r, OffloadArray3D& u, RealType Rmax) const From 9ded5e51a2e04f2185b50dc00d7dc5c1b65e7b57 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 15:45:40 -0500 Subject: [PATCH 059/168] compute pbc image displacements in builder (similar to phase computation) --- .../LCAO/LCAOrbitalBuilder.cpp | 72 ++++++++++++------- src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.h | 14 ++-- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 61 ++++++---------- .../LCAO/SoaLocalizedBasisSet.cpp | 10 +-- .../LCAO/SoaLocalizedBasisSet.h | 3 +- 5 files changed, 87 insertions(+), 73 deletions(-) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp index d524f60208e..ef5a8d3a675 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp @@ -128,7 +128,7 @@ LCAOrbitalBuilder::LCAOrbitalBuilder(ParticleSet& els, ParticleSet& ions, Commun if (cuspC == "yes") doCuspCorrection = true; //Evaluate the Phase factor. Equals 1 for OBC. - EvalPeriodicImagePhaseFactors(SuperTwist, PeriodicImagePhaseFactors); + EvalPeriodicImagePhaseFactors(SuperTwist, PeriodicImagePhaseFactors, PeriodicImageDisplacements); // no need to wait but load the basis set processChildren(cur, [&](const std::string& cname, const xmlNodePtr element) { @@ -364,7 +364,7 @@ LCAOrbitalBuilder::BasisSet_t* LCAOrbitalBuilder::createBasisSet(xmlNodePtr cur) cur = cur->next; } // done with basis set mBasisSet->setBasisSetSize(-1); - mBasisSet->setPBCParams(PBCImages, SuperTwist, PeriodicImagePhaseFactors); + mBasisSet->setPBCParams(PBCImages, SuperTwist, PeriodicImagePhaseFactors, PeriodicImageDisplacements); return mBasisSet; } @@ -448,7 +448,7 @@ LCAOrbitalBuilder::BasisSet_t* LCAOrbitalBuilder::createBasisSetH5() hin.close(); } mBasisSet->setBasisSetSize(-1); - mBasisSet->setPBCParams(PBCImages, SuperTwist, PeriodicImagePhaseFactors); + mBasisSet->setPBCParams(PBCImages, SuperTwist, PeriodicImagePhaseFactors, PeriodicImageDisplacements); return mBasisSet; } @@ -968,17 +968,36 @@ void LCAOrbitalBuilder::LoadFullCoefsFromH5(hdf_archive& hin, } /// Periodic Image Phase Factors computation to be determined -void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors(PosType SuperTwist, - std::vector& LocPeriodicImagePhaseFactors) +void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors( + PosType SuperTwist, + Vector>& LocPeriodicImagePhaseFactors, + Array>& LocPeriodicImageDisplacements) { - const int NbImages = (PBCImages[0] + 1) * (PBCImages[1] + 1) * (PBCImages[2] + 1); + const int Nx = PBCImages[0] + 1; + const int Ny = PBCImages[1] + 1; + const int Nz = PBCImages[2] + 1; + const int NbImages = Nx * Ny * Nz; LocPeriodicImagePhaseFactors.resize(NbImages); - for (size_t i = 0; i < NbImages; i++) - LocPeriodicImagePhaseFactors[i] = 1.0; + LocPeriodicImageDisplacements.resize(NbImages, 3); + for (size_t ix = 0; ix < Nx; ix++) + for (size_t iy = 0; iy < Ny; iy++) + for (size_t iz = 0; iz < Nz; iz++) + { + const size_t i = iz + Nz * (iy + Ny * ix); + int TransX = ((ix % 2) * 2 - 1) * ((ix + 1) / 2); + int TransY = ((iy % 2) * 2 - 1) * ((iy + 1) / 2); + int TransZ = ((iz % 2) * 2 - 1) * ((iz + 1) / 2); + LocPeriodicImagePhaseFactors[i] = 1.0; + for (size_t idim = 0; idim < 3; idim++) + LocPeriodicImageDisplacements(i, idim) = + TransX * Lattice(0, idim) + TransY * Lattice(1, idim) + TransZ * Lattice(2, idim); + } } -void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors(PosType SuperTwist, - std::vector>& LocPeriodicImagePhaseFactors) +void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors( + PosType SuperTwist, + Vector, OffloadPinnedAllocator>>& LocPeriodicImagePhaseFactors, + Array>& LocPeriodicImageDisplacements) { // Allow computation to continue with no HDF file if the system has open boundary conditions. // The complex build is usually only used with open BC for testing. @@ -1010,28 +1029,31 @@ void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors(PosType SuperTwist, int phase_idx = 0; int TransX, TransY, TransZ; RealType phase; - - for (int i = 0; i <= PBCImages[0]; i++) //loop Translation over X - { - TransX = ((i % 2) * 2 - 1) * ((i + 1) / 2); - for (int j = 0; j <= PBCImages[1]; j++) //loop Translation over Y - { - TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); - for (int k = 0; k <= PBCImages[2]; k++) //loop Translation over Z + const int Nx = PBCImages[0] + 1; + const int Ny = PBCImages[1] + 1; + const int Nz = PBCImages[2] + 1; + const int NbImages = Nx * Ny * Nz; + LocPeriodicImageDisplacements.resize(NbImages, 3); + for (size_t ix = 0; ix < Nx; ix++) + for (size_t iy = 0; iy < Ny; iy++) + for (size_t iz = 0; iz < Nz; iz++) { - TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); + const size_t i = iz + Nz * (iy + Ny * ix); + int TransX = ((ix % 2) * 2 - 1) * ((ix + 1) / 2); + int TransY = ((iy % 2) * 2 - 1) * ((iy + 1) / 2); + int TransZ = ((iz % 2) * 2 - 1) * ((iz + 1) / 2); RealType s, c; PosType Val; - Val[0] = TransX * Lattice(0, 0) + TransY * Lattice(1, 0) + TransZ * Lattice(2, 0); - Val[1] = TransX * Lattice(0, 1) + TransY * Lattice(1, 1) + TransZ * Lattice(2, 1); - Val[2] = TransX * Lattice(0, 2) + TransY * Lattice(1, 2) + TransZ * Lattice(2, 2); + for (size_t idim = 0; idim < 3; idim++) + { + Val[idim] = TransX * Lattice(0, idim) + TransY * Lattice(1, idim) + TransZ * Lattice(2, idim); + LocPeriodicImageDisplacements(i, idim) = Val[idim]; + } phase = dot(SuperTwist, Val); qmcplusplus::sincos(phase, &s, &c); - LocPeriodicImagePhaseFactors.emplace_back(c, s); + LocPeriodicImagePhaseFactors[i] = std::complex(c, s); } - } - } } } // namespace qmcplusplus diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.h b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.h index 9bc344b285b..a20a37e3021 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.h +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.h @@ -58,7 +58,8 @@ class LCAOrbitalBuilder : public SPOSetBuilder ///Coordinates Super Twist PosType SuperTwist; ///Periodic Image Phase Factors. Correspond to the phase from the PBCImages. Computed only once. - std::vector PeriodicImagePhaseFactors; + Vector> PeriodicImagePhaseFactors; + Array> PeriodicImageDisplacements; ///Store Lattice parameters from HDF5 to use in PeriodicImagePhaseFactors Tensor Lattice; @@ -90,9 +91,14 @@ class LCAOrbitalBuilder : public SPOSetBuilder bool MultiDet); // the dimensions of Creal are determined by the dataset on file void LoadFullCoefsFromH5(hdf_archive& hin, int setVal, PosType& SuperTwist, Matrix& Creal, bool Multidet); - void EvalPeriodicImagePhaseFactors(PosType SuperTwist, std::vector& LocPeriodicImagePhaseFactors); - void EvalPeriodicImagePhaseFactors(PosType SuperTwist, - std::vector>& LocPeriodicImagePhaseFactors); + void EvalPeriodicImagePhaseFactors( + PosType SuperTwist, + Vector>& LocPeriodicImagePhaseFactors, + Array>& LocPeriodicImageDisplacements); + void EvalPeriodicImagePhaseFactors( + PosType SuperTwist, + Vector, OffloadPinnedAllocator>>& LocPeriodicImagePhaseFactors, + Array>& LocPeriodicImageDisplacements); /** read matrix from h5 file * \param[in] hin: hdf5 arhive to be read from * \param setname: where to read from in hdf5 archive diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 9dbb732904b..7409c77d0f0 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -50,7 +50,8 @@ struct SoaAtomicBasisSet ///Coordinates of SuperTwist TinyVector SuperTwist; ///Phase Factor array - std::vector periodic_image_phase_factors; + OffloadVector periodic_image_phase_factors; + OffloadArray2D periodic_image_displacements; ///maximum radius of this center RealType Rmax; ///spherical harmonics @@ -113,11 +114,16 @@ struct SoaAtomicBasisSet */ void setPBCParams(const TinyVector& pbc_images, const TinyVector supertwist, - const std::vector& PeriodicImagePhaseFactors) + const OffloadVector& PeriodicImagePhaseFactors, + const OffloadArray2D& PeriodicImageDisplacements) { PBCImages = pbc_images; periodic_image_phase_factors = PeriodicImagePhaseFactors; + periodic_image_displacements = PeriodicImageDisplacements; SuperTwist = supertwist; + + periodic_image_phase_factors.updateTo(); + periodic_image_displacements.updateTo(); } @@ -690,7 +696,7 @@ struct SoaAtomicBasisSet * @param [in] lattice crystal lattice * @param [in,out] psi wavefunction values for all electrons [nElec, nBasTot] * @param [in] displ_list displacement from each electron to each center [NumCenters, nElec, 3] (flattened) - * @param [in] Tv_list translation vectors for computing overall phase factor [nElec, 3] (flattened) + * @param [in] Tv_list translation vectors for computing overall phase factor [NumCenters, nElec, 3] (flattened) * @param [in] nElec number of electrons * @param [in] nBasTot total number of basis functions represented in psi * @param [in] center_idx current center index (for indexing into displ_list) @@ -736,14 +742,11 @@ struct SoaAtomicBasisSet r.resize(nElec, Nxyz); // TODO: move these outside? - auto& dr_pbc = atom_bs_leader.mw_mem_handle_.getResource().dr_pbc; auto& correctphase = atom_bs_leader.mw_mem_handle_.getResource().correctphase; - dr_pbc.resize(Nxyz, 3); correctphase.resize(nElec); - auto* dr_pbc_ptr = dr_pbc.data(); - auto* dr_ptr = dr.data(); - auto* r_ptr = r.data(); + auto* dr_ptr = dr.data(); + auto* r_ptr = r.data(); auto* correctphase_ptr = correctphase.data(); @@ -753,27 +756,6 @@ struct SoaAtomicBasisSet // need to map Tensor vals to device auto* latR_ptr = lattice.R.data(); - // build dr_pbc: translation vectors to all images - // should just do this once and store it (like with phase) - { - ScopedTimer local(pbc_timer_); - PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(to:latR_ptr[:9], dr_pbc_ptr[:3*Nxyz]) ") - for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) - { - // i_xyz = k + Nz * (j + Ny * i) - int ij = i_xyz / Nz; - int k = i_xyz - ij * Nz; - int i = ij / Ny; - int j = ij - i * Ny; - int TransX = ((i % 2) * 2 - 1) * ((i + 1) / 2); - int TransY = ((j % 2) * 2 - 1) * ((j + 1) / 2); - int TransZ = ((k % 2) * 2 - 1) * ((k + 1) / 2); - for (size_t i_dim = 0; i_dim < 3; i_dim++) - dr_pbc_ptr[i_dim + 3 * i_xyz] = - (TransX * latR_ptr[i_dim + 0 * 3] + TransY * latR_ptr[i_dim + 1 * 3] + TransZ * latR_ptr[i_dim + 2 * 3]); - } - } - { ScopedTimer local_timer(phase_timer_); @@ -787,13 +769,13 @@ struct SoaAtomicBasisSet auto* SuperTwist_ptr = SuperTwist.data(); PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:SuperTwist_ptr[:9], \ - Tv_list_ptr[:nElec*3], correctphase_ptr[:nElec]) ") + Tv_list_ptr[3*nElec*center_idx:3*nElec], correctphase_ptr[:nElec]) ") for (size_t i_e = 0; i_e < nElec; i_e++) { //RealType phasearg = dot(3, SuperTwist.data(), 1, Tv_list.data() + 3 * i_e, 1); RealType phasearg = 0; for (size_t i_dim = 0; i_dim < 3; i_dim++) - phasearg += SuperTwist[i_dim] * Tv_list_ptr[i_dim + 3 * i_e]; + phasearg += SuperTwist[i_dim] * Tv_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)]; RealType s, c; qmcplusplus::sincos(-phasearg, &s, &c); correctphase_devptr[i_e] = ValueType(c, s); @@ -801,10 +783,11 @@ struct SoaAtomicBasisSet #endif } + auto* periodic_image_displacements_ptr = periodic_image_displacements.data(); { ScopedTimer local_timer(nelec_pbc_timer_); PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) \ - map(to: dr_ptr[:3*nElec*Nxyz], dr_pbc_ptr[:3*Nxyz], r_ptr[:nElec*Nxyz], displ_list_ptr[3*nElec*center_idx:3*nElec]) ") + map(to: dr_ptr[:3*nElec*Nxyz], periodic_image_displacements_ptr[:3*Nxyz], r_ptr[:nElec*Nxyz], displ_list_ptr[3*nElec*center_idx:3*nElec]) ") for (size_t i_e = 0; i_e < nElec; i_e++) { for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) @@ -812,8 +795,8 @@ struct SoaAtomicBasisSet RealType tmp_r2 = 0.0; for (size_t i_dim = 0; i_dim < 3; i_dim++) { - dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = - -(displ_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)] + dr_pbc_ptr[i_dim + 3 * i_xyz]); + dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = -(displ_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)] + + periodic_image_displacements_ptr[i_dim + 3 * i_xyz]); tmp_r2 += dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] * dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)]; } r_ptr[i_xyz + Nxyz * i_e] = std::sqrt(tmp_r2); @@ -891,11 +874,11 @@ struct SoaAtomicBasisSet return std::make_unique(*this); } - OffloadArray4D ylm_vgl; // [5][Nelec][PBC][NYlm] - OffloadArray4D rnl_vgl; // [5][Nelec][PBC][NRnl] - OffloadArray3D ylm_v; // [Nelec][PBC][NYlm] - OffloadArray3D rnl_v; // [Nelec][PBC][NRnl] - OffloadArray2D dr_pbc; // [PBC][xyz] translation vector for each image + OffloadArray4D ylm_vgl; // [5][Nelec][PBC][NYlm] + OffloadArray4D rnl_vgl; // [5][Nelec][PBC][NRnl] + OffloadArray3D ylm_v; // [Nelec][PBC][NYlm] + OffloadArray3D rnl_v; // [Nelec][PBC][NRnl] + // OffloadArray2D dr_pbc; // [PBC][xyz] translation vector for each image OffloadArray3D dr; // [Nelec][PBC][xyz] ion->elec displacement for each image OffloadArray2D r; // [Nelec][PBC] ion->elec distance for each image OffloadVector correctphase; // [Nelec] overall phase diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index d650d56b199..9b0a1329470 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -100,12 +100,14 @@ SoaLocalizedBasisSet::SoaLocalizedBasisSet(const SoaLocalizedBasisSet } template -void SoaLocalizedBasisSet::setPBCParams(const TinyVector& PBCImages, - const TinyVector Sup_Twist, - const std::vector& phase_factor) +void SoaLocalizedBasisSet::setPBCParams( + const TinyVector& PBCImages, + const TinyVector Sup_Twist, + const Vector>& phase_factor, + const Array>& pbc_displacements) { for (int i = 0; i < LOBasisSet.size(); ++i) - LOBasisSet[i]->setPBCParams(PBCImages, Sup_Twist, phase_factor); + LOBasisSet[i]->setPBCParams(PBCImages, Sup_Twist, phase_factor, pbc_displacements); SuperTwist = Sup_Twist; } diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 4de3714c73c..9b69e077396 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -91,7 +91,8 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase */ void setPBCParams(const TinyVector& PBCImages, const TinyVector Sup_Twist, - const std::vector& phase_factor); + const Vector>& phase_factor, + const Array>& pbc_displacements); /** set BasisSetSize and allocate mVGL container */ From 8c488fab692173395ab5b9599fdde18051db4804 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 15:46:43 -0500 Subject: [PATCH 060/168] fix devptr->ptr --- src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 7409c77d0f0..06287f6e9a7 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -778,7 +778,7 @@ struct SoaAtomicBasisSet phasearg += SuperTwist[i_dim] * Tv_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)]; RealType s, c; qmcplusplus::sincos(-phasearg, &s, &c); - correctphase_devptr[i_e] = ValueType(c, s); + correctphase_ptr[i_e] = ValueType(c, s); } #endif } From e1a8f030e5c6adea73a84867e1d772b25dc5e0a2 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 16:21:27 -0500 Subject: [PATCH 061/168] fix name --- src/Numerics/SoaSphericalTensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index 268874e7ab8..fe77b215f37 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -59,7 +59,7 @@ struct SoaSphericalTensor SoaSphericalTensor(const SoaSphericalTensor& rhs) = default; ///compute Ylm - static void evaluate_bare(T x, T y, T z, T* Ylm, int lmax, const T* FacL, const T* FacLM); + static void evaluate_bare(T x, T y, T z, T* Ylm, int lmax, const T* factorL, const T* factorLM); ///compute Ylm inline void evaluateV(T x, T y, T z, T* Ylm) const From f63db33e4148945e85c1a722205362600a8c8802 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 16:32:57 -0500 Subject: [PATCH 062/168] add batched VGL for SoaCartesianTensor, SoaSphericalTensor --- src/Numerics/SoaCartesianTensor.h | 97 ++++++++++++++---- src/Numerics/SoaSphericalTensor.h | 160 ++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+), 17 deletions(-) diff --git a/src/Numerics/SoaCartesianTensor.h b/src/Numerics/SoaCartesianTensor.h index 48861a52e48..209d405bd4a 100644 --- a/src/Numerics/SoaCartesianTensor.h +++ b/src/Numerics/SoaCartesianTensor.h @@ -43,6 +43,7 @@ struct SoaCartesianTensor using ggg_type = TinyVector, 3>; using OffloadArray2D = Array>; using OffloadArray3D = Array>; + using OffloadArray4D = Array>; ///maximum angular momentum int Lmax; @@ -62,6 +63,17 @@ struct SoaCartesianTensor ///compute Ylm static void evaluate_bare(T x, T y, T z, T* XYZ, int lmax); + static void evaluateVGL_impl(T x, + T y, + T z, + T* restrict XYZ, + T* restrict gr0, + T* restrict gr1, + T* restrict gr2, + T* restrict lap, + int lmax, + const T* normfactor, + int n_normfac); ///compute Ylm inline void evaluateV(T x, T y, T z, T* XYZ) const { @@ -112,7 +124,47 @@ struct SoaCartesianTensor XYZ_ptr[ir * Nlm + i] *= NormFactor_ptr[i]; } } + /** + * @brief evaluate VGL for multiple electrons and multiple pbc images + * + * when offload is enabled, xyz is assumed to be up to date on the device before entering the function + * XYZ_vgl will be up to date on the device (but not host) when this function exits + * + * @param [in] xyz electron positions [Nelec, Npbc, 3(x,y,z)] + * @param [out] XYZ_vgl Cartesian tensor elements [5(v, gx, gy, gz, lapl), Nelec, Npbc, Nlm] + */ + inline void batched_evaluateVGL(const OffloadArray3D& xyz, OffloadArray4D& XYZ_vgl) const + { + const size_t nElec = xyz.size(0); + const size_t Npbc = xyz.size(1); // number of PBC images + assert(xyz.size(2) == 3); + + assert(XYZ_vgl.size(0) == 5); + assert(XYZ_vgl.size(1) == nElec); + assert(XYZ_vgl.size(2) == Npbc); + const size_t Nlm = XYZ_vgl.size(3); + assert(NormFactor.size() == Nlm); + + size_t nR = nElec * Npbc; // total number of positions to evaluate + size_t offset = Nlm * nR; // stride for v/gx/gy/gz/l + auto* xyz_ptr = xyz.data(); + auto* XYZ_vgl_ptr = XYZ_vgl.data(); + auto* NormFactor_ptr = NormFactor.data(); + // TODO: make separate ptrs to start of v/gx/gy/gz/l? + // might be more readable? + // or just pass one ptr to evaluateVGL and apply stride/offset inside + + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:NormFactor_ptr[:Nlm], \ + xyz_ptr[:3*nR], XYZ_vgl_ptr[:5*nR*Nlm])") + for (size_t ir = 0; ir < nR; ir++) + { + evaluateVGL_impl(xyz_ptr[0 + 3 * ir], xyz_ptr[1 + 3 * ir], xyz_ptr[2 + 3 * ir], XYZ_vgl_ptr + (ir * Nlm), + XYZ_vgl_ptr + (ir * Nlm + offset * 1), XYZ_vgl_ptr + (ir * Nlm + offset * 2), + XYZ_vgl_ptr + (ir * Nlm + offset * 3), XYZ_vgl_ptr + (ir * Nlm + offset * 4), Lmax, + NormFactor_ptr, Nlm); + } + } ///makes a table of \f$ r^l S_l^m \f$ and their gradients up to Lmax. void evaluateVGL(T x, T y, T z); @@ -177,6 +229,14 @@ SoaCartesianTensor::SoaCartesianTensor(const int l_max, bool addsign) : Lmax( NormFactor.updateTo(); } +template +void SoaCartesianTensor::evaluateVGL(T x, T y, T z) +{ + constexpr T czero(0); + cXYZ = czero; + evaluateVGL_impl(x, y, z, cXYZ.data(0), cXYZ.data(1), cXYZ.data(2), cXYZ.data(3), cXYZ.data(4), Lmax, + NormFactor.data(), NormFactor.size()); +} PRAGMA_OFFLOAD("omp declare target") template @@ -284,23 +344,26 @@ void SoaCartesianTensor::evaluate_bare(T x, T y, T z, T* restrict XYZ, int lm PRAGMA_OFFLOAD("omp end declare target") +PRAGMA_OFFLOAD("omp declare target") template -void SoaCartesianTensor::evaluateVGL(T x, T y, T z) +void SoaCartesianTensor::evaluateVGL_impl(T x, + T y, + T z, + T* restrict XYZ, + T* restrict gr0, + T* restrict gr1, + T* restrict gr2, + T* restrict lap, + int lmax, + const T* normfactor, + int n_normfac) { - constexpr T czero(0); - cXYZ = czero; - const T x2 = x * x, y2 = y * y, z2 = z * z; const T x3 = x2 * x, y3 = y2 * y, z3 = z2 * z; const T x4 = x3 * x, y4 = y3 * y, z4 = z3 * z; const T x5 = x4 * x, y5 = y4 * y, z5 = z4 * z; - T* restrict XYZ = cXYZ.data(0); - T* restrict gr0 = cXYZ.data(1); - T* restrict gr1 = cXYZ.data(2); - T* restrict gr2 = cXYZ.data(3); - T* restrict lap = cXYZ.data(4); - switch (Lmax) + switch (lmax) { case 6: XYZ[83] = x2 * y2 * z2; // X2Y2Z2 @@ -639,16 +702,16 @@ void SoaCartesianTensor::evaluateVGL(T x, T y, T z) XYZ[0] = 1; // S } - const size_t ntot = NormFactor.size(); - for (size_t i = 0; i < ntot; i++) + for (size_t i = 0; i < n_normfac; i++) { - XYZ[i] *= NormFactor[i]; - gr0[i] *= NormFactor[i]; - gr1[i] *= NormFactor[i]; - gr2[i] *= NormFactor[i]; - lap[i] *= NormFactor[i]; + XYZ[i] *= normfactor[i]; + gr0[i] *= normfactor[i]; + gr1[i] *= normfactor[i]; + gr2[i] *= normfactor[i]; + lap[i] *= normfactor[i]; } } +PRAGMA_OFFLOAD("omp end declare target") template diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index fe77b215f37..8f1c1104a50 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -41,6 +41,7 @@ struct SoaSphericalTensor { using OffloadArray2D = Array>; using OffloadArray3D = Array>; + using OffloadArray4D = Array>; ///maximum angular momentum for the center int Lmax; /// Normalization factors @@ -60,6 +61,16 @@ struct SoaSphericalTensor ///compute Ylm static void evaluate_bare(T x, T y, T z, T* Ylm, int lmax, const T* factorL, const T* factorLM); + static void evaluateVGL_impl(const T x, + const T y, + const T z, + T* restrict Ylm_vgl, + int lmax, + const T* factorL, + const T* factorLM, + const T* factor2L, + const T* normfactor, + size_t offset); ///compute Ylm inline void evaluateV(T x, T y, T z, T* Ylm) const @@ -106,6 +117,47 @@ struct SoaSphericalTensor } } + /** + * @brief evaluate VGL for multiple electrons and multiple pbc images + * + * when offload is enabled, xyz is assumed to be up to date on the device before entering the function + * Ylm_vgl will be up to date on the device (but not host) when this function exits + * + * @param [in] xyz electron positions [Nelec, Npbc, 3(x,y,z)] + * @param [out] Ylm_vgl Spherical tensor elements [5(v, gx, gy, gz, lapl), Nelec, Npbc, Nlm] + */ + inline void batched_evaluateVGL(const OffloadArray3D& xyz, OffloadArray4D& Ylm_vgl) const + { + const size_t nElec = xyz.size(0); + const size_t Npbc = xyz.size(1); // number of PBC images + assert(xyz.size(2) == 3); + + assert(Ylm_vgl.size(0) == 5); + assert(Ylm_vgl.size(1) == nElec); + assert(Ylm_vgl.size(2) == Npbc); + const size_t Nlm = Ylm_vgl.size(3); + assert(NormFactor.size() == Nlm); + + const size_t nR = nElec * Npbc; // total number of positions to evaluate + const size_t offset = Nlm * nR; // stride for v/gx/gy/gz/l + + auto* xyz_ptr = xyz.data(); + auto* Ylm_vgl_ptr = Ylm_vgl.data(); + auto* FactorLM_ptr = FactorLM.data(); + auto* FactorL_ptr = FactorL.data(); + auto* Factor2L_ptr = Factor2L.data(); + auto* NormFactor_ptr = NormFactor.data(); + + + PRAGMA_OFFLOAD("omp target teams distribute parallel for \ + map(to:FactorLM_ptr[:Nlm], FactorL_ptr[:Lmax+1], NormFactor_ptr[:Nlm], Factor2L_ptr[:Lmax+1], \ + xyz_ptr[:nR*3], Ylm_vgl_ptr[:5*nR*Nlm])") + for (size_t ir = 0; ir < nR; ir++) + evaluateVGL_impl(xyz_ptr[0 + 3 * ir], xyz_ptr[1 + 3 * ir], xyz_ptr[2 + 3 * ir], Ylm_vgl_ptr + (ir * Nlm), Lmax, + FactorL_ptr, FactorLM_ptr, Factor2L_ptr, NormFactor_ptr, offset); + } + + ///compute Ylm inline void evaluateV(T x, T y, T z) { @@ -317,6 +369,114 @@ inline void SoaSphericalTensor::evaluate_bare(T x, } PRAGMA_OFFLOAD("omp end declare target") + +PRAGMA_OFFLOAD("omp declare target") +template +inline void SoaSphericalTensor::evaluateVGL_impl(const T x, + const T y, + const T z, + T* restrict Ylm_vgl, + int lmax, + const T* factorL, + const T* factorLM, + const T* factor2L, + const T* normfactor, + size_t offset) +{ + T* restrict Ylm = Ylm_vgl; + // T* restrict Ylm = cYlm.data(0); + evaluate_bare(x, y, z, Ylm, lmax, factorL, factorLM); + const size_t Nlm = (lmax + 1) * (lmax + 1); + + constexpr T czero(0); + constexpr T ahalf(0.5); + T* restrict gYlmX = Ylm_vgl + offset * 1; + T* restrict gYlmY = Ylm_vgl + offset * 2; + T* restrict gYlmZ = Ylm_vgl + offset * 3; + T* restrict lYlm = Ylm_vgl + offset * 4; // just need to set to zero + + // Calculating Gradient now// + for (int l = 1; l <= lmax; l++) + { + //T fac = ((T) (2*l+1))/(2*l-1); + T fac = factor2L[l]; + for (int m = -l; m <= l; m++) + { + int lm = index(l - 1, 0); + T gx, gy, gz, dpr, dpi, dmr, dmi; + const int ma = std::abs(m); + const T cp = std::sqrt(fac * (l - ma - 1) * (l - ma)); + const T cm = std::sqrt(fac * (l + ma - 1) * (l + ma)); + const T c0 = std::sqrt(fac * (l - ma) * (l + ma)); + gz = (l > ma) ? c0 * Ylm[lm + m] : czero; + if (l > ma + 1) + { + dpr = cp * Ylm[lm + ma + 1]; + dpi = cp * Ylm[lm - ma - 1]; + } + else + { + dpr = czero; + dpi = czero; + } + if (l > 1) + { + switch (ma) + { + case 0: + dmr = -cm * Ylm[lm + 1]; + dmi = cm * Ylm[lm - 1]; + break; + case 1: + dmr = cm * Ylm[lm]; + dmi = czero; + break; + default: + dmr = cm * Ylm[lm + ma - 1]; + dmi = cm * Ylm[lm - ma + 1]; + } + } + else + { + dmr = cm * Ylm[lm]; + dmi = czero; + //dmr = (l==1) ? cm*Ylm[lm]:0.0; + //dmi = 0.0; + } + if (m < 0) + { + gx = ahalf * (dpi - dmi); + gy = -ahalf * (dpr + dmr); + } + else + { + gx = ahalf * (dpr - dmr); + gy = ahalf * (dpi + dmi); + } + lm = index(l, m); + if (ma) + { + gYlmX[lm] = normfactor[lm] * gx; + gYlmY[lm] = normfactor[lm] * gy; + gYlmZ[lm] = normfactor[lm] * gz; + } + else + { + gYlmX[lm] = gx; + gYlmY[lm] = gy; + gYlmZ[lm] = gz; + } + } + } + for (int i = 0; i < Nlm; i++) + { + Ylm[i] *= normfactor[i]; + lYlm[i] = 0; + } + //for (int i=0; i inline void SoaSphericalTensor::evaluateVGL(T x, T y, T z) { From 0ba734736616854482389b5a19c634aa6798b489 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 16:49:00 -0500 Subject: [PATCH 063/168] resize phase vector --- src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp index ef5a8d3a675..72479801019 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp @@ -1033,6 +1033,7 @@ void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors( const int Ny = PBCImages[1] + 1; const int Nz = PBCImages[2] + 1; const int NbImages = Nx * Ny * Nz; + LocPeriodicImagePhaseFactors.resize(NbImages); LocPeriodicImageDisplacements.resize(NbImages, 3); for (size_t ix = 0; ix < Nx; ix++) for (size_t iy = 0; iy < Ny; iy++) From c04f50ec3c86cad99b67b5936c1a179adac8899a Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 17:16:13 -0500 Subject: [PATCH 064/168] reduce code duplication for SoaSphericalTensor VGL --- src/Numerics/SoaSphericalTensor.h | 90 ++----------------------------- 1 file changed, 4 insertions(+), 86 deletions(-) diff --git a/src/Numerics/SoaSphericalTensor.h b/src/Numerics/SoaSphericalTensor.h index 8f1c1104a50..cef63f4308e 100644 --- a/src/Numerics/SoaSphericalTensor.h +++ b/src/Numerics/SoaSphericalTensor.h @@ -59,8 +59,9 @@ struct SoaSphericalTensor SoaSphericalTensor(const SoaSphericalTensor& rhs) = default; - ///compute Ylm + ///compute Ylm for single position static void evaluate_bare(T x, T y, T z, T* Ylm, int lmax, const T* factorL, const T* factorLM); + ///compute Ylm_vgl for single position static void evaluateVGL_impl(const T x, const T y, const T z, @@ -480,91 +481,8 @@ PRAGMA_OFFLOAD("omp end declare target") template inline void SoaSphericalTensor::evaluateVGL(T x, T y, T z) { - T* restrict Ylm = cYlm.data(0); - evaluate_bare(x, y, z, Ylm, Lmax, FactorL.data(), FactorLM.data()); - - constexpr T czero(0); - constexpr T ahalf(0.5); - T* restrict gYlmX = cYlm.data(1); - T* restrict gYlmY = cYlm.data(2); - T* restrict gYlmZ = cYlm.data(3); - - // Calculating Gradient now// - for (int l = 1; l <= Lmax; l++) - { - //T fac = ((T) (2*l+1))/(2*l-1); - T fac = Factor2L[l]; - for (int m = -l; m <= l; m++) - { - int lm = index(l - 1, 0); - T gx, gy, gz, dpr, dpi, dmr, dmi; - const int ma = std::abs(m); - const T cp = std::sqrt(fac * (l - ma - 1) * (l - ma)); - const T cm = std::sqrt(fac * (l + ma - 1) * (l + ma)); - const T c0 = std::sqrt(fac * (l - ma) * (l + ma)); - gz = (l > ma) ? c0 * Ylm[lm + m] : czero; - if (l > ma + 1) - { - dpr = cp * Ylm[lm + ma + 1]; - dpi = cp * Ylm[lm - ma - 1]; - } - else - { - dpr = czero; - dpi = czero; - } - if (l > 1) - { - switch (ma) - { - case 0: - dmr = -cm * Ylm[lm + 1]; - dmi = cm * Ylm[lm - 1]; - break; - case 1: - dmr = cm * Ylm[lm]; - dmi = czero; - break; - default: - dmr = cm * Ylm[lm + ma - 1]; - dmi = cm * Ylm[lm - ma + 1]; - } - } - else - { - dmr = cm * Ylm[lm]; - dmi = czero; - //dmr = (l==1) ? cm*Ylm[lm]:0.0; - //dmi = 0.0; - } - if (m < 0) - { - gx = ahalf * (dpi - dmi); - gy = -ahalf * (dpr + dmr); - } - else - { - gx = ahalf * (dpr - dmr); - gy = ahalf * (dpi + dmi); - } - lm = index(l, m); - if (ma) - { - gYlmX[lm] = NormFactor[lm] * gx; - gYlmY[lm] = NormFactor[lm] * gy; - gYlmZ[lm] = NormFactor[lm] * gz; - } - else - { - gYlmX[lm] = gx; - gYlmY[lm] = gy; - gYlmZ[lm] = gz; - } - } - } - for (int i = 0; i < cYlm.size(); i++) - Ylm[i] *= NormFactor[i]; - //for (int i=0; i From 5ed0234259a960297e5bafbff154d03ec0195018 Mon Sep 17 00:00:00 2001 From: Steven Hahn Date: Thu, 19 Oct 2023 18:27:54 -0400 Subject: [PATCH 065/168] Remove unused template parameter Signed-off-by: Steven Hahn --- src/Containers/OhmmsSoA/PosTransformer.h | 10 +++++----- src/Containers/OhmmsSoA/VectorSoaContainer.h | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Containers/OhmmsSoA/PosTransformer.h b/src/Containers/OhmmsSoA/PosTransformer.h index 33ef4ec17f6..f7cec870d04 100644 --- a/src/Containers/OhmmsSoA/PosTransformer.h +++ b/src/Containers/OhmmsSoA/PosTransformer.h @@ -129,12 +129,12 @@ void PosAoS2SoA(int nrows, int ncols, const T1* restrict iptr, int lda, T2* rest * * Modeled after blas/lapack for lda/ldb */ -template -void PosSoA2AoS(int nrows, int ncols, const T1* restrict iptr, int lda, T2* restrict out, int ldb) +template +void PosSoA2AoS(int nrows, int ncols, const T* restrict iptr, int lda, T* restrict out, int ldb) { - const T1* restrict x = iptr; - const T1* restrict y = iptr + lda; - const T1* restrict z = iptr + 2 * lda; + const T* restrict x = iptr; + const T* restrict y = iptr + lda; + const T* restrict z = iptr + 2 * lda; #if !defined(__ibmxl__) #pragma omp simd aligned(x, y, z: QMC_SIMD_ALIGNMENT) #endif diff --git a/src/Containers/OhmmsSoA/VectorSoaContainer.h b/src/Containers/OhmmsSoA/VectorSoaContainer.h index 2a46be72deb..00ee232ae21 100644 --- a/src/Containers/OhmmsSoA/VectorSoaContainer.h +++ b/src/Containers/OhmmsSoA/VectorSoaContainer.h @@ -208,11 +208,9 @@ struct VectorSoaContainer * * The same sizes are assumed. */ - template - void copyIn(const Vector>& in) + void copyIn(const Vector>& in) { - //if(nLocal!=in.size()) resize(in.size()); - PosAoS2SoA(nLocal, D, reinterpret_cast(in.first_address()), D, myData, nGhosts); + PosAoS2SoA(nLocal, D, reinterpret_cast(in.first_address()), D, myData, nGhosts); } /** SoA to AoS : copy to Vector> From 8b2fe9318a7abad73944bc30260b847c05ce2ca4 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Thu, 19 Oct 2023 18:46:46 -0500 Subject: [PATCH 066/168] mw v/vgl for AOs --- src/QMCWaveFunctions/BasisSetBase.h | 10 +- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 7 +- .../LCAO/MultiFunctorAdapter.h | 59 ++++++ .../LCAO/MultiQuinticSpline1D.h | 91 ++++++++ src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h | 200 ++++++++++++++++++ .../LCAO/SoaLocalizedBasisSet.cpp | 109 ++++++++-- .../LCAO/SoaLocalizedBasisSet.h | 12 +- 7 files changed, 462 insertions(+), 26 deletions(-) diff --git a/src/QMCWaveFunctions/BasisSetBase.h b/src/QMCWaveFunctions/BasisSetBase.h index d1c3b3ed5db..57d6fc3cf87 100644 --- a/src/QMCWaveFunctions/BasisSetBase.h +++ b/src/QMCWaveFunctions/BasisSetBase.h @@ -151,9 +151,15 @@ struct SoaBasisSetBase //Evaluates value, gradient, and laplacian for electron "iat". Parks them into a temporary data structure "vgl". virtual void evaluateVGL(const ParticleSet& P, int iat, vgl_type& vgl) = 0; //Evaluates value, gradient, and laplacian for electron "iat". places them in a offload array for batched code. - virtual void mw_evaluateVGL(const RefVectorWithLeader& P_list, int iat, OffloadMWVGLArray& vgl) = 0; + virtual void mw_evaluateVGL(const RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& P_list, + int iat, + OffloadMWVGLArray& vgl) = 0; //Evaluates value for electron "iat". places it in a offload array for batched code. - virtual void mw_evaluateValue(const RefVectorWithLeader& P_list, int iat, OffloadMWVArray& v) = 0; + virtual void mw_evaluateValue(const RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& P_list, + int iat, + OffloadMWVArray& v) = 0; //Evaluates value for all the electrons of the virtual particles. places it in a offload array for batched code. virtual void mw_evaluateValueVPs(const RefVectorWithLeader>& basis_list, const RefVectorWithLeader& vp_list, diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index a9cd33893c0..65e2ccc9446 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -453,7 +453,8 @@ void LCAOrbitalSet::mw_evaluateVGLImplGEMM(const RefVectorWithLeader& sp { ScopedTimer local(basis_timer_); - myBasisSet->mw_evaluateVGL(P_list, iat, basis_vgl_mw); + auto basis_list = spo_leader.extractBasisRefList(spo_list); + myBasisSet->mw_evaluateVGL(basis_list, P_list, iat, basis_vgl_mw); } if (Identity) @@ -547,7 +548,9 @@ void LCAOrbitalSet::mw_evaluateValueImplGEMM(const RefVectorWithLeader& auto& basis_v_mw = spo_leader.mw_mem_handle_.getResource().basis_v_mw; basis_v_mw.resize(nw, BasisSetSize); - myBasisSet->mw_evaluateValue(P_list, iat, basis_v_mw); + auto basis_list = spo_leader.extractBasisRefList(spo_list); + myBasisSet->mw_evaluateValue(basis_list, P_list, iat, basis_v_mw); + basis_v_mw.updateFrom(); // TODO: remove this when gemm is implemented if (Identity) { diff --git a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h index 17316dac5f0..6a4d3465512 100644 --- a/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h +++ b/src/QMCWaveFunctions/LCAO/MultiFunctorAdapter.h @@ -36,6 +36,7 @@ struct MultiFunctorAdapter using single_type = FN; using OffloadArray2D = Array>; using OffloadArray3D = Array>; + using OffloadArray4D = Array>; aligned_vector> Rnl; MultiFunctorAdapter() = default; @@ -95,6 +96,64 @@ struct MultiFunctorAdapter u.updateTo(); // TODO: remove after offload } + /** + * @brief evaluate value, first deriv, second deriv for multiple electrons and multiple pbc images + * + * r is assumed to be up-to-date on the device before entering this function, and + * vgl needs to be updated on the device before exiting this function + * + * Eventually, all computation should be done on the device to avoid transfers, but + * for now this is all done on the host + * + * @param [in] r electron distances [Nelec, Npbc] + * @param [out] vgl val/d1/d2 of all radial functions at all electron distances [3(val,d1,d2), Nelec, Npbc, nRnl] + * @param Rmax radial function and derivs will evaluate to zero for any distance greater than or equal to Rmax + */ + inline void batched_evaluateVGL(OffloadArray2D& r, OffloadArray4D& vgl, RealType Rmax) const + { + r.updateFrom(); // TODO: remove when eval is offloaded + const size_t nElec = r.size(0); + const size_t Nxyz = r.size(1); // number of PBC images + assert(3 == vgl.size(0)); + assert(nElec == vgl.size(1)); + assert(Nxyz == vgl.size(2)); + const size_t nRnl = vgl.size(3); // number of primitives + const size_t nR = nElec * Nxyz; // total number of positions to evaluate + assert(nRnl == Rnl.size()); + + auto* r_ptr = r.data(); + auto* u_ptr = vgl.data_at(0, 0, 0, 0); + auto* du_ptr = vgl.data_at(1, 0, 0, 0); + auto* d2u_ptr = vgl.data_at(2, 0, 0, 0); + + for (size_t ir = 0; ir < nR; ir++) + { + // TODO: once this is offloaded, maybe better to just avoid branching and keep values past Rmax + if (r_ptr[ir] >= Rmax) + { + for (size_t i = 0; i < nRnl; ++i) + { + u_ptr[ir * nRnl + i] = 0.0; + du_ptr[ir * nRnl + i] = 0.0; + d2u_ptr[ir * nRnl + i] = 0.0; + } + } + else + { + const RealType r = r_ptr[ir]; + const RealType rinv = RealType(1) / r; + for (size_t i = 0; i < nRnl; ++i) + { + Rnl[i]->evaluateAll(r, rinv); + u_ptr[ir * nRnl + i] = Rnl[i]->Y; + du_ptr[ir * nRnl + i] = Rnl[i]->dY; + d2u_ptr[ir * nRnl + i] = Rnl[i]->d2Y; + } + } + } + vgl.updateTo(); // TODO: remove when eval is offloaded + } + inline void evaluate(RealType r, RealType* restrict u, RealType* restrict du, RealType* restrict d2u) { const RealType rinv = RealType(1) / r; diff --git a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h index 14580edf385..641c26df947 100644 --- a/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h +++ b/src/QMCWaveFunctions/LCAO/MultiQuinticSpline1D.h @@ -102,6 +102,7 @@ class MultiQuinticSpline1D using CoeffType = Matrix>; using OffloadArray2D = Array>; using OffloadArray3D = Array>; + using OffloadArray4D = Array>; private: ///number of splines @@ -213,6 +214,96 @@ class MultiQuinticSpline1D } } + /** + * @brief evaluate value, first deriv, second deriv of MultiQuinticSpline1D for multiple electrons and multiple pbc images + * + * r is assumed to be up-to-date on the device when entering this function, and + * vgl will be up to date on the device when exiting this function + * + * @param [in] r electron distances [Nelec, Npbc] + * @param [out] vgl val/d1/d2 of all splines at all electron distances [3(val,d1,d2), Nelec, Npbc, Nsplines] + * @param Rmax spline and derivatives will evaluate to zero for any distance greater than or equal to Rmax + */ + inline void batched_evaluateVGL(const OffloadArray2D& r, OffloadArray4D& vgl, T Rmax) const + { + const size_t nElec = r.size(0); + const size_t Nxyz = r.size(1); // number of PBC images + assert(3 == vgl.size(0)); + assert(nElec == vgl.size(1)); + assert(Nxyz == vgl.size(2)); + const size_t nRnl = vgl.size(3); // number of splines + const size_t nR = nElec * Nxyz; // total number of positions to evaluate + + double one_over_log_delta = myGrid.OneOverLogDelta; + T lower_bound = myGrid.lower_bound; + T dlog_ratio = myGrid.LogDelta; + + auto* r_ptr = r.data(); + auto* u_ptr = vgl.data_at(0, 0, 0, 0); + auto* du_ptr = vgl.data_at(1, 0, 0, 0); + auto* d2u_ptr = vgl.data_at(2, 0, 0, 0); + + auto* coeff_ptr = coeffs->data(); + auto* first_deriv_ptr = first_deriv.data(); + const size_t nCols = coeffs->cols(); + const size_t coefsize = coeffs->size(); + + constexpr T ctwo(2); + constexpr T cthree(3); + constexpr T cfour(4); + constexpr T cfive(5); + constexpr T csix(6); + constexpr T c12(12); + constexpr T c20(20); + + PRAGMA_OFFLOAD("omp target teams distribute parallel for \ + map(to: first_deriv_ptr[:num_splines_], coeff_ptr[:coefsize], \ + r_ptr[:nR], u_ptr[:nRnl*nR], du_ptr[:nRnl*nR], d2u_ptr[:nRnl*nR])") + for (size_t ir = 0; ir < nR; ir++) + { + if (r_ptr[ir] >= Rmax) + { + for (size_t i = 0; i < num_splines_; ++i) + { + u_ptr[ir * nRnl + i] = 0.0; + du_ptr[ir * nRnl + i] = 0.0; + d2u_ptr[ir * nRnl + i] = 0.0; + } + } + else if (r_ptr[ir] < lower_bound) + { + const T dr = r_ptr[ir] - lower_bound; + const T* restrict a = coeff_ptr; + // const T* restrict a = (*coeffs)[0]; + for (size_t i = 0; i < num_splines_; ++i) + { + u_ptr[ir * nRnl + i] = a[i] + first_deriv_ptr[i] * dr; + du_ptr[ir * nRnl + i] = first_deriv_ptr[i]; + d2u_ptr[ir * nRnl + i] = 0.0; + } + } + else + { + int loc; + const auto cL = LogGridLight::getCL(r_ptr[ir], loc, one_over_log_delta, lower_bound, dlog_ratio); + // const auto cL = myGrid.getCLForQuintic(r_list[ir], loc); + const size_t offset = loc * 6; + const T* restrict a = coeff_ptr + nCols * (offset + 0); + const T* restrict b = coeff_ptr + nCols * (offset + 1); + const T* restrict c = coeff_ptr + nCols * (offset + 2); + const T* restrict d = coeff_ptr + nCols * (offset + 3); + const T* restrict e = coeff_ptr + nCols * (offset + 4); + const T* restrict f = coeff_ptr + nCols * (offset + 5); + for (size_t i = 0; i < num_splines_; ++i) + { + u_ptr[ir * nRnl + i] = a[i] + cL * (b[i] + cL * (c[i] + cL * (d[i] + cL * (e[i] + cL * f[i])))); + du_ptr[ir * nRnl + i] = + b[i] + cL * (ctwo * c[i] + cL * (cthree * d[i] + cL * (cfour * e[i] + cL * f[i] * cfive))); + d2u_ptr[ir * nRnl + i] = ctwo * c[i] + cL * (csix * d[i] + cL * (c12 * e[i] + cL * f[i] * c20)); + } + } + } + } inline void evaluate(T r, T* restrict u, T* restrict du, T* restrict d2u) const { if (r < myGrid.lower_bound) diff --git a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h index 06287f6e9a7..c3a3f85f25c 100644 --- a/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaAtomicBasisSet.h @@ -685,6 +685,206 @@ struct SoaAtomicBasisSet } } + /** + * @brief evaluate VGL for multiple electrons + * + * This function should only assign to elements of psi in the range [[0:nElec],[BasisOffset:BasisOffset+BasisSetSize]]. + * These elements are assumed to be zero when passed to this function. + * This function only uses only one center (center_idx) from displ_list + * + * @param [in] atom_bs_list multi-walker list of SoaAtomicBasisSet [nWalkers] + * @param [in] lattice crystal lattice + * @param [in,out] psi_vgl wavefunction vgl for all electrons [5, nElec, nBasTot] + * @param [in] displ_list displacement from each electron to each center [NumCenters, nElec, 3] (flattened) + * @param [in] Tv_list translation vectors for computing overall phase factor [NumCenters, nElec, 3] (flattened) + * @param [in] nElec number of electrons + * @param [in] nBasTot total number of basis functions represented in psi_vgl + * @param [in] center_idx current center index (for indexing into displ_list) + * @param [in] BasisOffset index of first basis function of this center (for indexing into psi_vgl) + * @param [in] NumCenters total number of centers in system (for indexing into displ_list) + * + */ + + template + inline void mw_evaluateVGL(const RefVectorWithLeader& atom_bs_list, + const LAT& lattice, + Array>& psi_vgl, + const Vector>& displ_list, + const Vector>& Tv_list, + const size_t nElec, + const size_t nBasTot, + const size_t center_idx, + const size_t BasisOffset, + const size_t NumCenters) + { + assert(this == &atom_bs_list.getLeader()); + auto& atom_bs_leader = atom_bs_list.template getCastedLeader>(); + + int Nx = PBCImages[0] + 1; + int Ny = PBCImages[1] + 1; + int Nz = PBCImages[2] + 1; + int Nyz = Ny * Nz; + int Nxyz = Nx * Nyz; + + //assert(psi_vgl.size(0) == 5); + assert(psi_vgl.size(1) == nElec); + assert(psi_vgl.size(2) == nBasTot); + + + auto& ylm_vgl = atom_bs_leader.mw_mem_handle_.getResource().ylm_vgl; + auto& rnl_vgl = atom_bs_leader.mw_mem_handle_.getResource().rnl_vgl; + auto& dr = atom_bs_leader.mw_mem_handle_.getResource().dr; + auto& r = atom_bs_leader.mw_mem_handle_.getResource().r; + + size_t nRnl = RnlID.size(); + size_t nYlm = Ylm.size(); + + ylm_vgl.resize(5, nElec, Nxyz, nYlm); + rnl_vgl.resize(3, nElec, Nxyz, nRnl); + dr.resize(nElec, Nxyz, 3); + r.resize(nElec, Nxyz); + + + // TODO: move these outside? + // auto& dr_pbc = atom_bs_leader.mw_mem_handle_.getResource().dr_pbc; + auto& correctphase = atom_bs_leader.mw_mem_handle_.getResource().correctphase; + // dr_pbc.resize(Nxyz, 3); + correctphase.resize(nElec); + + auto* dr_ptr = dr.data(); + auto* r_ptr = r.data(); + + auto* correctphase_ptr = correctphase.data(); + + auto* Tv_list_ptr = Tv_list.data(); + auto* displ_list_ptr = displ_list.data(); + // TODO: should this be VT? + constexpr RealType cone(1); + constexpr RealType ctwo(2); + + //V,Gx,Gy,Gz,L + auto* restrict psi_ptr = psi_vgl.data_at(0, 0, 0); + auto* restrict dpsi_x_ptr = psi_vgl.data_at(1, 0, 0); + auto* restrict dpsi_y_ptr = psi_vgl.data_at(2, 0, 0); + auto* restrict dpsi_z_ptr = psi_vgl.data_at(3, 0, 0); + auto* restrict d2psi_ptr = psi_vgl.data_at(4, 0, 0); + + { + ScopedTimer local_timer(phase_timer_); +#if not defined(QMC_COMPLEX) + + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:correctphase_ptr[:nElec]) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + correctphase_ptr[i_e] = 1.0; + +#else + auto* SuperTwist_ptr = SuperTwist.data(); + + PRAGMA_OFFLOAD("omp target teams distribute parallel for map(to:SuperTwist_ptr[:9], \ + Tv_list_ptr[3*nElec*center_idx:3*nElec], correctphase_ptr[:nElec]) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + { + //RealType phasearg = dot(3, SuperTwist.data(), 1, Tv_list.data() + 3 * i_e, 1); + RealType phasearg = 0; + for (size_t i_dim = 0; i_dim < 3; i_dim++) + phasearg += SuperTwist[i_dim] * Tv_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)]; + RealType s, c; + qmcplusplus::sincos(-phasearg, &s, &c); + correctphase_ptr[i_e] = ValueType(c, s); + } +#endif + } + + auto* periodic_image_displacements_ptr = periodic_image_displacements.data(); + { + ScopedTimer local_timer(nelec_pbc_timer_); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) \ + map(to: dr_ptr[:3*nElec*Nxyz], periodic_image_displacements_ptr[:3*Nxyz], r_ptr[:nElec*Nxyz], displ_list_ptr[3*nElec*center_idx:3*nElec]) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + { + for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) + { + RealType tmp_r2 = 0.0; + for (size_t i_dim = 0; i_dim < 3; i_dim++) + { + dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] = -(displ_list_ptr[i_dim + 3 * (i_e + center_idx * nElec)] + + periodic_image_displacements_ptr[i_dim + 3 * i_xyz]); + tmp_r2 += dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)] * dr_ptr[i_dim + 3 * (i_xyz + Nxyz * i_e)]; + } + r_ptr[i_xyz + Nxyz * i_e] = std::sqrt(tmp_r2); + } + } + } + + { + ScopedTimer local(rnl_timer_); + MultiRnl.batched_evaluateVGL(r, rnl_vgl, Rmax); + } + { + ScopedTimer local(ylm_timer_); + Ylm.batched_evaluateVGL(dr, ylm_vgl); + } + + + auto* phase_fac_ptr = periodic_image_phase_factors.data(); + auto* LM_ptr = LM.data(); + auto* NL_ptr = NL.data(); + + RealType* restrict phi_ptr = rnl_vgl.data_at(0, 0, 0, 0); + RealType* restrict dphi_ptr = rnl_vgl.data_at(1, 0, 0, 0); + RealType* restrict d2phi_ptr = rnl_vgl.data_at(2, 0, 0, 0); + + + const RealType* restrict ylm_v_ptr = ylm_vgl.data_at(0, 0, 0, 0); //value + const RealType* restrict ylm_x_ptr = ylm_vgl.data_at(1, 0, 0, 0); //gradX + const RealType* restrict ylm_y_ptr = ylm_vgl.data_at(2, 0, 0, 0); //gradY + const RealType* restrict ylm_z_ptr = ylm_vgl.data_at(3, 0, 0, 0); //gradZ + const RealType* restrict ylm_l_ptr = ylm_vgl.data_at(4, 0, 0, 0); //lap + { + ScopedTimer local_timer(psi_timer_); + PRAGMA_OFFLOAD( + "omp target teams distribute parallel for collapse(2) map(to:phase_fac_ptr[:Nxyz], LM_ptr[:BasisSetSize], NL_ptr[:BasisSetSize], \ + ylm_v_ptr[:nYlm*nElec*Nxyz], ylm_x_ptr[:nYlm*nElec*Nxyz], ylm_y_ptr[:nYlm*nElec*Nxyz], ylm_z_ptr[:nYlm*nElec*Nxyz], ylm_l_ptr[:nYlm*nElec*Nxyz], \ + phi_ptr[:nRnl*nElec*Nxyz], dphi_ptr[:nRnl*nElec*Nxyz], d2phi_ptr[:nRnl*nElec*Nxyz], \ + psi_ptr[:nBasTot*nElec], dpsi_x_ptr[:nBasTot*nElec], dpsi_y_ptr[:nBasTot*nElec], dpsi_z_ptr[:nBasTot*nElec], d2psi_ptr[:nBasTot*nElec], \ + correctphase_ptr[:nElec], r_ptr[:nElec*Nxyz], dr_ptr[:3*nElec*Nxyz]) ") + for (size_t i_e = 0; i_e < nElec; i_e++) + { + for (size_t ib = 0; ib < BasisSetSize; ++ib) + { + const int nl(NL_ptr[ib]); + const int lm(LM_ptr[ib]); + for (int i_xyz = 0; i_xyz < Nxyz; i_xyz++) + { + const ValueType Phase = phase_fac_ptr[i_xyz] * correctphase_ptr[i_e]; + const RealType rinv = cone / r_ptr[i_xyz + Nxyz * i_e]; + const RealType x = dr_ptr[0 + 3 * (i_xyz + Nxyz * i_e)]; + const RealType y = dr_ptr[1 + 3 * (i_xyz + Nxyz * i_e)]; + const RealType z = dr_ptr[2 + 3 * (i_xyz + Nxyz * i_e)]; + const RealType drnloverr = rinv * dphi_ptr[nl + nRnl * (i_xyz + Nxyz * i_e)]; + const RealType ang = ylm_v_ptr[lm + nYlm * (i_xyz + Nxyz * i_e)]; + const RealType gr_x = drnloverr * x; + const RealType gr_y = drnloverr * y; + const RealType gr_z = drnloverr * z; + const RealType ang_x = ylm_x_ptr[lm + nYlm * (i_xyz + Nxyz * i_e)]; + const RealType ang_y = ylm_y_ptr[lm + nYlm * (i_xyz + Nxyz * i_e)]; + const RealType ang_z = ylm_z_ptr[lm + nYlm * (i_xyz + Nxyz * i_e)]; + const RealType vr = phi_ptr[nl + nRnl * (i_xyz + Nxyz * i_e)]; + + psi_ptr[BasisOffset + ib + i_e * nBasTot] += ang * vr * Phase; + dpsi_x_ptr[BasisOffset + ib + i_e * nBasTot] += (ang * gr_x + vr * ang_x) * Phase; + dpsi_y_ptr[BasisOffset + ib + i_e * nBasTot] += (ang * gr_y + vr * ang_y) * Phase; + dpsi_z_ptr[BasisOffset + ib + i_e * nBasTot] += (ang * gr_z + vr * ang_z) * Phase; + d2psi_ptr[BasisOffset + ib + i_e * nBasTot] += + (ang * (ctwo * drnloverr + d2phi_ptr[nl + nRnl * (i_xyz + Nxyz * i_e)]) + + ctwo * (gr_x * ang_x + gr_y * ang_y + gr_z * ang_z) + + vr * ylm_l_ptr[lm + nYlm * (i_xyz + Nxyz * i_e)]) * + Phase; + } + } + } + } + } /** * @brief evaluate for multiple electrons * diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp index 9b0a1329470..547ce58b341 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.cpp @@ -191,32 +191,56 @@ void SoaLocalizedBasisSet::evaluateVGL(const ParticleSet& P, int iat, } template -void SoaLocalizedBasisSet::mw_evaluateVGL(const RefVectorWithLeader& P_list, +void SoaLocalizedBasisSet::mw_evaluateVGL(const RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& P_list, int iat, OffloadMWVGLArray& vgl_v) { + assert(this == &basis_list.getLeader()); + auto& basis_leader = basis_list.template getCastedLeader>(); + const auto& IonID(ions_.GroupID); + auto& pset_leader = P_list.getLeader(); + + size_t Nw = P_list.size(); + assert(vgl_v.size(0) == 5); + assert(vgl_v.size(1) == Nw); + assert(vgl_v.size(2) == BasisSetSize); + + Vector> Tv_list; + Vector> displ_list_tr; + Tv_list.resize(3 * NumCenters * Nw); + displ_list_tr.resize(3 * NumCenters * Nw); + + for (size_t iw = 0; iw < P_list.size(); iw++) { - const auto& IonID(ions_.GroupID); const auto& coordR = P_list[iw].activeR(iat); const auto& d_table = P_list[iw].getDistTableAB(myTableIndex); - const auto& dist = (P_list[iw].getActivePtcl() == iat) ? d_table.getTempDists() : d_table.getDistRow(iat); const auto& displ = (P_list[iw].getActivePtcl() == iat) ? d_table.getTempDispls() : d_table.getDisplRow(iat); - - PosType Tv; - - // number of walkers * BasisSetSize - auto stride = vgl_v.size(1) * BasisSetSize; - assert(BasisSetSize == vgl_v.size(2)); - vgl_type vgl_iw(vgl_v.data_at(0, iw, 0), BasisSetSize, stride); - for (int c = 0; c < NumCenters; c++) - { - Tv[0] = (ions_.R[c][0] - coordR[0]) - displ[c][0]; - Tv[1] = (ions_.R[c][1] - coordR[1]) - displ[c][1]; - Tv[2] = (ions_.R[c][2] - coordR[2]) - displ[c][2]; - LOBasisSet[IonID[c]]->evaluateVGL(P_list[iw].getLattice(), dist[c], displ[c], BasisOffset[c], vgl_iw, Tv); - } + for (size_t idim = 0; idim < 3; idim++) + { + Tv_list[idim + 3 * (iw + c * Nw)] = (ions_.R[c][idim] - coordR[idim]) - displ[c][idim]; + displ_list_tr[idim + 3 * (iw + c * Nw)] = displ[c][idim]; + } + } +#if defined(QMC_COMPLEX) + Tv_list.updateTo(); +#endif + displ_list_tr.updateTo(); + // set AO data to zero on device + auto* vgl_v_ptr = vgl_v.data(); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(3) map(to:vgl_v_ptr[:5*Nw*BasisSetSize])") + for (size_t ivgl = 0; ivgl < 5; ivgl++) + for (size_t iw = 0; iw < Nw; iw++) + for (size_t ib = 0; ib < BasisSetSize; ib++) + vgl_v_ptr[ib + BasisSetSize * (iw + Nw * ivgl)] = 0; + + for (int c = 0; c < NumCenters; c++) + { + auto one_species_basis_list = extractOneSpeciesBasisRefList(basis_list, IonID[c]); + LOBasisSet[IonID[c]]->mw_evaluateVGL(one_species_basis_list, pset_leader.getLattice(), vgl_v, displ_list_tr, + Tv_list, Nw, BasisSetSize, c, BasisOffset[c], NumCenters); } } @@ -330,12 +354,57 @@ void SoaLocalizedBasisSet::evaluateV(const ParticleSet& P, int iat, O } template -void SoaLocalizedBasisSet::mw_evaluateValue(const RefVectorWithLeader& P_list, +void SoaLocalizedBasisSet::mw_evaluateValue(const RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& P_list, int iat, - OffloadMWVArray& v) + OffloadMWVArray& vals) { + assert(this == &basis_list.getLeader()); + auto& basis_leader = basis_list.template getCastedLeader>(); + const auto& IonID(ions_.GroupID); + auto& pset_leader = P_list.getLeader(); + + size_t Nw = P_list.size(); + assert(vals.size(0) == Nw); + assert(vals.size(1) == BasisSetSize); + + Vector> Tv_list; + Vector> displ_list_tr; + Tv_list.resize(3 * NumCenters * Nw); + displ_list_tr.resize(3 * NumCenters * Nw); + + for (size_t iw = 0; iw < P_list.size(); iw++) - evaluateV(P_list[iw], iat, v.data_at(iw, 0)); + { + const auto& coordR = P_list[iw].activeR(iat); + const auto& d_table = P_list[iw].getDistTableAB(myTableIndex); + const auto& displ = (P_list[iw].getActivePtcl() == iat) ? d_table.getTempDispls() : d_table.getDisplRow(iat); + + for (int c = 0; c < NumCenters; c++) + for (size_t idim = 0; idim < 3; idim++) + { + Tv_list[idim + 3 * (iw + c * Nw)] = (ions_.R[c][idim] - coordR[idim]) - displ[c][idim]; + displ_list_tr[idim + 3 * (iw + c * Nw)] = displ[c][idim]; + } + } +#if defined(QMC_COMPLEX) + Tv_list.updateTo(); +#endif + displ_list_tr.updateTo(); + // set AO data to zero on device + auto* vals_ptr = vals.data(); + PRAGMA_OFFLOAD("omp target teams distribute parallel for collapse(2) map(to:vals_ptr[:Nw*BasisSetSize])") + + for (size_t iw = 0; iw < Nw; iw++) + for (size_t ib = 0; ib < BasisSetSize; ib++) + vals_ptr[ib + BasisSetSize * iw] = 0; + + for (int c = 0; c < NumCenters; c++) + { + auto one_species_basis_list = extractOneSpeciesBasisRefList(basis_list, IonID[c]); + LOBasisSet[IonID[c]]->mw_evaluateV(one_species_basis_list, pset_leader.getLattice(), vals, displ_list_tr, Tv_list, + Nw, BasisSetSize, c, BasisOffset[c], NumCenters); + } } diff --git a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h index 9b69e077396..624c81915fc 100644 --- a/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h +++ b/src/QMCWaveFunctions/LCAO/SoaLocalizedBasisSet.h @@ -111,11 +111,15 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase void evaluateVGL(const ParticleSet& P, int iat, vgl_type& vgl) override; /** compute V using packed array with all walkers + * @param basis_list list of basis sets (one for each walker) * @param P_list list of quantum particleset (one for each walker) * @param iat active particle * @param v Array(n_walkers, BasisSetSize) */ - void mw_evaluateValue(const RefVectorWithLeader& P_list, int iat, OffloadMWVArray& v) override; + void mw_evaluateValue(const RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& P_list, + int iat, + OffloadMWVArray& v) override; /** compute V using packed array with all walkers * @param basis_list list of basis sets (one for each walker) @@ -128,11 +132,15 @@ class SoaLocalizedBasisSet : public SoaBasisSetBase /** compute VGL using packed array with all walkers + * @param basis_list list of basis sets (one for each walker) * @param P_list list of quantum particleset (one for each walker) * @param iat active particle * @param vgl Array(n_walkers, 5, BasisSetSize) */ - void mw_evaluateVGL(const RefVectorWithLeader& P_list, int iat, OffloadMWVGLArray& vgl) override; + void mw_evaluateVGL(const RefVectorWithLeader>& basis_list, + const RefVectorWithLeader& P_list, + int iat, + OffloadMWVGLArray& vgl) override; /** compute VGH * @param P quantum particleset From f8ad5f7a708e27bda397b01318815ea1ff6a8446 Mon Sep 17 00:00:00 2001 From: Doak P W Date: Fri, 20 Oct 2023 11:22:51 -0400 Subject: [PATCH 067/168] actually non required except for osx. --- external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp index 036c3cae642..03440cfb1ea 100644 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp +++ b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp @@ -15,7 +15,6 @@ struct buffer : mpi3::uvector{ int pos = 0; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) : make private buffer() = default; explicit buffer(std::size_t r) {reserve(r);} - explicit buffer(long r) {reserve(r);} buffer(buffer const&) = delete; buffer(buffer&&) = delete; buffer& operator=(buffer const&) = delete; From 0dc9b659e3e141eaed84d6e2e7701fa0e4fd4233 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Fri, 20 Oct 2023 14:31:18 -0500 Subject: [PATCH 068/168] working on mw LCAO test --- src/QMCWaveFunctions/tests/CMakeLists.txt | 7 +- .../tests/test_LCAO_diamondC_2x1x1.cpp | 106 ++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp diff --git a/src/QMCWaveFunctions/tests/CMakeLists.txt b/src/QMCWaveFunctions/tests/CMakeLists.txt index ec066f8735c..08b36ad9e9c 100644 --- a/src/QMCWaveFunctions/tests/CMakeLists.txt +++ b/src/QMCWaveFunctions/tests/CMakeLists.txt @@ -25,6 +25,7 @@ set(UTEST_HDF_INPUT8 ${qmcpack_SOURCE_DIR}/tests/molecules/LiH_ae_MSD/LiH.Multid set(UTEST_HDF_INPUT9 ${qmcpack_SOURCE_DIR}/tests/converter/test_Bi_dirac/gold.orbs.h5) set(UTEST_HDF_INPUT10 ${qmcpack_SOURCE_DIR}/src/QMCWaveFunctions/tests/lcao_spinor_molecule.h5) set(UTEST_HDF_INPUT11 ${qmcpack_SOURCE_DIR}/tests/solids/hcpBe_1x1x1_pp/pwscf.pwscf.h5) +set(UTEST_HDF_INPUT12 ${qmcpack_SOURCE_DIR}/tests/solids/diamondC_2x1x1-Gaussian_pp/C_Diamond.h5) maybe_symlink(${UTEST_HDF_INPUT0} ${UTEST_DIR}/diamondC_1x1x1.pwscf.h5) maybe_symlink(${UTEST_HDF_INPUT1} ${UTEST_DIR}/diamondC_2x1x1.pwscf.h5) @@ -38,6 +39,7 @@ maybe_symlink(${UTEST_HDF_INPUT8} ${UTEST_DIR}/LiH.Multidet.h5) maybe_symlink(${UTEST_HDF_INPUT9} ${UTEST_DIR}/Bi.orbs.h5) maybe_symlink(${UTEST_HDF_INPUT10} ${UTEST_DIR}/lcao_spinor_molecule.h5) maybe_symlink(${UTEST_HDF_INPUT11} ${UTEST_DIR}/hcpBe.pwscf.h5) +maybe_symlink(${UTEST_HDF_INPUT12} ${UTEST_DIR}/C_Diamond.h5) if(NOT QMC_DATA) message(VERBOSE "QMC_DATA not set. NiO_a16 based tests not added.") @@ -100,6 +102,9 @@ set(TRIALWF_SRC test_lattice_gaussian.cpp test_TWFGrads.cpp) +set(TMP_SRC + test_LCAO_diamondC_2x1x1.cpp) + set(SPOSET_SRC test_spo_collection_input_spline.cpp test_spo_collection_input_LCAO_xml.cpp @@ -166,7 +171,7 @@ if(ENABLE_OFFLOAD) set(DETERMINANT_SRC ${DETERMINANT_SRC} test_DiracMatrixComputeOMPTarget.cpp) endif(ENABLE_OFFLOAD) -foreach(CATEGORY common trialwf sposet jastrow determinant) +foreach(CATEGORY common trialwf sposet jastrow determinant tmp) set(UTEST_EXE test_${SRC_DIR}_${CATEGORY}) set(UTEST_NAME deterministic-unit_${UTEST_EXE}) string(TOUPPER "${CATEGORY}_SRC" SOURCE_FILE_VAR_NAME) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp new file mode 100644 index 00000000000..963f08434ed --- /dev/null +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -0,0 +1,106 @@ +////////////////////////////////////////////////////////////////////////////////////// +// This file is distributed under the University of Illinois/NCSA Open Source License. +// See LICENSE file in top directory for details. +// +// Copyright (c) 2016 Jeongnim Kim and QMCPACK developers. +// +// File developed by: Mark Dewing, markdewing@gmail.com, University of Illinois at Urbana-Champaign +// +// File created by: Mark Dewing, markdewing@gmail.com, University of Illinois at Urbana-Champaign +////////////////////////////////////////////////////////////////////////////////////// + + +#include "catch.hpp" + +#include "OhmmsData/Libxml2Doc.h" +#include "OhmmsPETE/OhmmsMatrix.h" +#include "Particle/ParticleSet.h" +#include "Particle/ParticleSetPool.h" +#include "QMCWaveFunctions/WaveFunctionComponent.h" +#include "LCAO/LCAOrbitalBuilder.h" +#include + +#include +#include +#include + +using std::string; + +namespace qmcplusplus +{ + +TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") +{ + Communicate* c = OHMMS::Controller; + + ParticleSet::ParticleLayout lattice; + lattice.R = {6.7463223, 6.7463223, 0.0, 0.0, 3.37316115, 3.37316115, 3.37316115, 0.0, 3.37316115}; + + + ParticleSetPool ptcl = ParticleSetPool(c); + ptcl.setSimulationCell(lattice); + auto ions_uptr = std::make_unique(ptcl.getSimulationCell()); + auto elec_uptr = std::make_unique(ptcl.getSimulationCell()); + ParticleSet& ions_(*ions_uptr); + ParticleSet& elec_(*elec_uptr); + + ions_.setName("ion0"); + ptcl.addParticleSet(std::move(ions_uptr)); + ions_.create({4}); + ions_.R[0] = {0.0, 0.0, 0.0}; + ions_.R[1] = {1.686580575, 1.686580575, 1.686580575}; + ions_.R[2] = {3.37316115, 3.37316115, 0.0}; + ions_.R[3] = {5.059741726, 5.059741726, 1.686580575}; + SpeciesSet& ispecies = ions_.getSpeciesSet(); + int Cidx = ispecies.addSpecies("C"); + + ions_.print(app_log()); + + elec_.setName("elec"); + ptcl.addParticleSet(std::move(elec_uptr)); + elec_.create({8, 8}); + elec_.R[0] = {0.0, 1.0, 0.0}; + elec_.R[1] = {0.0, 1.1, 0.0}; + elec_.R[2] = {0.0, 1.2, 0.0}; + elec_.R[3] = {0.0, 1.3, 0.0}; + + SpeciesSet& tspecies = elec_.getSpeciesSet(); + int upIdx = tspecies.addSpecies("u"); + int downIdx = tspecies.addSpecies("d"); + int chargeIdx = tspecies.addAttribute("charge"); + tspecies(chargeIdx, upIdx) = -1; + tspecies(chargeIdx, downIdx) = -1; + // const int ei_table_index = elec_.addTable(ions_); + + // diamondC_2x1x1 + // from tests/solids/diamondC_2x1x1-Gaussian_pp/C_Diamond-Twist0.wfj.xml + const std::string wf_xml_str = R"( + + + + + + + + + + + + + + + )"; + Libxml2Document doc; + bool okay = doc.parseFromString(wf_xml_str); + REQUIRE(okay); + + xmlNodePtr root = doc.getRoot(); + xmlNodePtr spo_xml = xmlFirstElementChild(root); + + LCAOrbitalBuilder lcaoSet(elec_, ions_, c, spo_xml); + auto spo = lcaoSet.createSPOSetFromXML(spo_xml); + REQUIRE(spo); + auto& lcao_spos = dynamic_cast(*spo); + REQUIRE(!lcao_spos.isIdentity()); +} +} // namespace qmcplusplus From a9375345443d38241a1ad6dc89977037b3e7e704 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Fri, 20 Oct 2023 14:45:54 -0500 Subject: [PATCH 069/168] change to sposet_collection input --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 963f08434ed..0e24f535208 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -76,18 +76,19 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // from tests/solids/diamondC_2x1x1-Gaussian_pp/C_Diamond-Twist0.wfj.xml const std::string wf_xml_str = R"( - - - - - - - - - - - - + + + + + + + + + + + + + )"; Libxml2Document doc; From ae1e19f0b5d45aecdb467800c371442e87869615 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Fri, 20 Oct 2023 17:40:23 -0500 Subject: [PATCH 070/168] test LCAO mw_evaluateDetRatios --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 84 ++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 0e24f535208..2b834a88dab 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -102,6 +102,88 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") auto spo = lcaoSet.createSPOSetFromXML(spo_xml); REQUIRE(spo); auto& lcao_spos = dynamic_cast(*spo); - REQUIRE(!lcao_spos.isIdentity()); + // CHECK(!lcao_spos.isIdentity()); + + const int norb = spo->getOrbitalSetSize(); + + + // test batched interfaces + + ParticleSet elec_2(elec_); + // interchange positions + elec_2.R[0] = elec_.R[1]; + elec_2.R[1] = elec_.R[0]; + RefVectorWithLeader p_list(elec_, {elec_, elec_2}); + + + std::unique_ptr spo_2(spo->makeClone()); + RefVectorWithLeader spo_list(*spo, {*spo, *spo_2}); + + ResourceCollection pset_res("test_pset_res"); + ResourceCollection spo_res("test_spo_res"); + + elec_.createResource(pset_res); + spo->createResource(spo_res); + + ResourceCollectionTeamLock mw_pset_lock(pset_res, p_list); + ResourceCollectionTeamLock mw_sposet_lock(spo_res, spo_list); + + // make VPs + const size_t nw = 2; + const size_t nvp_ = 4; + const size_t nvp_2 = 3; + const std::vector nvp_list = {nvp_, nvp_2}; + VirtualParticleSet VP_(elec_, nvp_); + VirtualParticleSet VP_2(elec_2, nvp_2); + + // move VPs + std::vector newpos_vp_(nvp_); + std::vector newpos_vp_2(nvp_2); + for (int i = 0; i < nvp_; i++) + newpos_vp_[i] = {std::cos(i), std::sin(i), i / 10}; + for (int i = 0; i < nvp_2; i++) + newpos_vp_2[i] = {-std::sin(i), i / 10, std::cos(i)}; + VP_.makeMoves(elec_, 0, newpos_vp_); + VP_2.makeMoves(elec_2, 0, newpos_vp_2); + + // make VP refvec + RefVectorWithLeader vp_list(VP_, {VP_, VP_2}); + ResourceCollection vp_res("test_vp_res"); + VP_.createResource(vp_res); + ResourceCollectionTeamLock mw_vpset_lock(vp_res, vp_list); + + // fill invrow with dummy data for each walker + std::vector psiMinv_data_(norb); + std::vector psiMinv_data_2(norb); + for (int i = 0; i < norb; i++) + { + psiMinv_data_[i] = std::cos(i); + psiMinv_data_2[i] = std::sin(i); + } + std::vector invRow_ptr_list{psiMinv_data_.data(), psiMinv_data_2.data()}; + + // ratios_list + std::vector> ratios_list(nw); + for (size_t iw = 0; iw < nw; iw++) + ratios_list[iw].resize(nvp_list[iw]); + + // just need dummy refvec with correct size + SPOSet::ValueVector tmp_psi_list(norb); + spo->mw_evaluateDetRatios(spo_list, RefVectorWithLeader(VP_, {VP_, VP_2}), + RefVector{tmp_psi_list}, invRow_ptr_list, ratios_list); + + + // app_log() << "calcRatioGrad \n" << std::setprecision(14); + // for (int iw = 0; iw < nw; iw++) + // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) + // app_log() << "CHECK(ratios_list[" << iw << "][" << ivp << "] == Approx(" << ratios_list[iw][ivp] << "));\n"; + + CHECK(ratios_list[0][0] == Approx(-206.77166092783)); + CHECK(ratios_list[0][1] == Approx(44.875898444974)); + CHECK(ratios_list[0][2] == Approx(24.96409353735)); + CHECK(ratios_list[0][3] == Approx(-53.413899456988)); + CHECK(ratios_list[1][0] == Approx(-306.54954579776)); + CHECK(ratios_list[1][1] == Approx(45.022133237688)); + CHECK(ratios_list[1][2] == Approx(183.86331674535)); } } // namespace qmcplusplus From 6e9e47329c5545cde915adec7c3f19719c50696d Mon Sep 17 00:00:00 2001 From: Hyeondeok-Shin Date: Sun, 22 Oct 2023 23:15:38 -0500 Subject: [PATCH 071/168] Add batched driver test for periodic LCAO. --- .../diamondC_1x1x1-Gaussian_pp/CMakeLists.txt | 68 ++++++++++++++++++ .../det_qmc_short_vmcbatch.in.xml | 24 +++++++ .../qmc_long_vmcbatch.in.xml | 23 ++++++ .../qmc_long_vmcbatch_dmcbatch.in.xml | 36 ++++++++++ .../qmc_short_vmcbatch.in.xml | 23 ++++++ .../qmc_short_vmcbatch_dmcbatch.in.xml | 36 ++++++++++ .../CMakeLists.txt | 24 +++++++ .../vmcbatch_long.in.xml | 22 ++++++ .../vmcbatch_short.in.xml | 22 ++++++ .../.qmc_long_vmc_dmc.in.xml.swp | Bin 0 -> 12288 bytes .../diamondC_2x1x1-Gaussian_pp/CMakeLists.txt | 48 +++++++++++++ .../det_qmc_short_vmcbatch.in.xml | 25 +++++++ .../qmc_long_vmcbatch.in.xml | 23 ++++++ .../qmc_long_vmcbatch_dmcbatch.in.xml | 35 +++++++++ .../qmc_short_vmcbatch.in.xml | 24 +++++++ .../qmc_short_vmcbatch_dmcbatch.in.xml | 36 ++++++++++ .../CMakeLists.txt | 24 +++++++ .../vmcbatch_long.in.xml | 22 ++++++ .../vmcbatch_short.in.xml | 22 ++++++ 19 files changed, 537 insertions(+) create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp/det_qmc_short_vmcbatch.in.xml create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_long.in.xml create mode 100644 tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_short.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/.qmc_long_vmc_dmc.in.xml.swp create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/det_qmc_short_vmcbatch.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_long.in.xml create mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_short.in.xml diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt b/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt index 3908c838f26..58595be702d 100644 --- a/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt @@ -20,6 +20,18 @@ qmc_run_and_check( DIAMOND_SCALARS # VMC ) +qmc_run_and_check( + short-diamondC_1x1x1_pp-vmcbatch_gaussian_sdj + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_1x1x1-Gaussian_pp" + qmc_short_vmcbatch + qmc_short_vmcbatch.in.xml + 1 + 16 + TRUE + 0 + DIAMOND_SCALARS # VMC +) + # # Long tests # @@ -38,6 +50,18 @@ qmc_run_and_check( LONG_DIAMOND_SCALARS # VMC ) +qmc_run_and_check( + long-diamondC_1x1x1_pp-vmcbatch_gaussian_sdj + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_1x1x1-Gaussian_pp" + qmc_long_vmcbatch + qmc_long_vmcbatch.in.xml + 1 + 16 + TRUE + 0 + LONG_DIAMOND_SCALARS # VMC +) + # Reference DMC run in qmc-ref "-10.532497 0.000470" list(APPEND LONG_DIAMOND_DMC_SCALARS "totenergy" "-10.532497 0.006127") @@ -53,6 +77,18 @@ qmc_run_and_check( LONG_DIAMOND_DMC_SCALARS # DMC ) +qmc_run_and_check( + long-diamondC_1x1x1_pp-dmcbatch_gaussian_sdj + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_1x1x1-Gaussian_pp" + qmc_long_vmcbatch_dmcbatch + qmc_long_vmcbatch_dmcbatch.in.xml + 1 + 16 + TRUE + 1 + LONG_DIAMOND_DMC_SCALARS # DMC +) + # Deterministic test if(QMC_MIXED_PRECISION) list(APPEND DET_DIAMOND_SCALARS "totenergy" "-10.26910069 0.000222") @@ -85,3 +121,35 @@ qmc_run_and_check( 0 DET_DIAMOND_SCALARS # VMC ) + +if(QMC_MIXED_PRECISION) + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "totenergy" "-10.82674512 0.000222") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "kinetic" "12.23702380 0.000221") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "potential" "-23.06376891 0.000005") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "eeenergy" "-3.00267948 0.000003") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "ionion" "-12.77567012 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "localecp" "-7.99462588 0.000007") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "nonlocalecp" "0.70920650 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "samples" "9 0.0") +else() + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "totenergy" "-10.82674404 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "kinetic" "12.23702514 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "potential" "-23.06376918 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "eeenergy" "-3.00267968 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "ionion" "-12.77566743 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "localecp" "-7.99462875 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "nonlocalecp" "0.70920645 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "samples" "9 0.0") +endif() + +qmc_run_and_check( + deterministic-diamondC_1x1x1_pp-vmcbatch_gaussian_sdj + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_1x1x1-Gaussian_pp" + det_qmc_short_vmcbatch + det_qmc_short_vmcbatch.in.xml + 1 + 1 + TRUE + 0 + DET_DIAMOND_VMCBATCH_SCALARS # VMC +) diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/det_qmc_short_vmcbatch.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp/det_qmc_short_vmcbatch.in.xml new file mode 100644 index 00000000000..f10f8365df1 --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/det_qmc_short_vmcbatch.in.xml @@ -0,0 +1,24 @@ + + + + batch + + + + + + + + + + + + + 1 + 3 + 3 + 2 + 0.3 + 3 + + diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml new file mode 100644 index 00000000000..41d6858bd09 --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml @@ -0,0 +1,23 @@ + + + + batch + + + + + + + + + + + + 16 + 100 + 100 + 2 + 0.3 + 100 + + diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml new file mode 100644 index 00000000000..5ae498fbca0 --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml @@ -0,0 +1,36 @@ + + + + batch + + + + + + + + + + + + + 256 + 1 + 20 + 1 + 1 + 1.0 + no + + + + 256 + no + 20 + 0.005 + 10 + 100 + no + + + diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml new file mode 100644 index 00000000000..6bc1a8f031a --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml @@ -0,0 +1,23 @@ + + + + batch + + + + + + + + + + + + 16 + 100 + 10 + 1 + 0.3 + 50 + + diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml new file mode 100644 index 00000000000..f77e84783e2 --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml @@ -0,0 +1,36 @@ + + + + batch + + + + + + + + + + + + + 2 + 1 + 100 + 1 + 2 + 1.0 + no + + + + 32 + no + 20 + 0.005 + 10 + 100 + no + + + diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/CMakeLists.txt b/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/CMakeLists.txt index 8637e2ef45a..9a792aa2502 100644 --- a/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/CMakeLists.txt +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/CMakeLists.txt @@ -23,6 +23,18 @@ if(QMC_COMPLEX) diamondC_1x1x1-Gaussian_pp_Tw_cplx_LONG_SCALARS #VMC ) + qmc_run_and_check( + long-diamondC_1x1x1-Gaussian_pp_Tw_cplx-vmcbatch + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx" + vmcbatch_long + vmcbatch_long.in.xml + 1 + 16 + TRUE + 0 + diamondC_1x1x1-Gaussian_pp_Tw_cplx_LONG_SCALARS #VMC + ) + # VMC short run with no Jastrows # Compare directly to Hartree Fock Energy of: E=-11.187187769284947 #Data generated using utils/make_ref_data.sh diamondC_1x1x1-Gaussian_pp_Tw_cplx_SHORT qmc_ref/qmc_short.s000.scalar.dat qmc_ref/qmc_ref_long.s000.scalar.dat @@ -46,6 +58,18 @@ if(QMC_COMPLEX) diamondC_1x1x1-Gaussian_pp_Tw_cplx_SHORT_SCALARS #VMC ) + qmc_run_and_check( + short-diamondC_1x1x1-Gaussian_pp_Tw_cplx-vmcbatch + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx" + vmcbatch_short + vmcbatch_short.in.xml + 1 + 16 + TRUE + 0 + diamondC_1x1x1-Gaussian_pp_Tw_cplx_SHORT_SCALARS #VMC + ) + else() message(VERBOSE "Skipping Complex Periodic LCAO if Complex code not build (QMC_COMPLEX=0)") endif() diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_long.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_long.in.xml new file mode 100644 index 00000000000..df5ee1c44f1 --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_long.in.xml @@ -0,0 +1,22 @@ + + + + batch + + + + + + + + + + + + 200 + 50 + 1 + 0.1 + 100 + + diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_short.in.xml b/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_short.in.xml new file mode 100644 index 00000000000..a656a27c49d --- /dev/null +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp_Tw_cplx/vmcbatch_short.in.xml @@ -0,0 +1,22 @@ + + + + btach + + + + + + + + + + + + 200 + 10 + 1 + 0.1 + 100 + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp/.qmc_long_vmc_dmc.in.xml.swp b/tests/solids/diamondC_2x1x1-Gaussian_pp/.qmc_long_vmc_dmc.in.xml.swp new file mode 100644 index 0000000000000000000000000000000000000000..62367df7808eacde2fe0ff85cbb8e4f9830bb39d GIT binary patch literal 12288 zcmeI2&yO256vw^XC@fTPW%W1Jrn!+z0dO-DT*h(`*%O) zH=+T;%Hk{)+5 zIhTd8T`!$?U1tiFxo)PC(&+5gcyQ7`>32R%YUfm<$JT~<>Y5g-CYfCvzQGXmc3uqU{E zW%X!p)$UwqSGUv<0U|&IhyW2F0z`la5CI}U1c(3;AOinG0!hZ$*RL`bU4irX|9}1a z|Jj?2{RVvxeF5Ev1oZP8jC~2c3;nsv*fZ!U^ds~YbOc?4{&=0SpP+A`&!G39J?Qt> z82bTw0v$qEp+OZdg$AvN=oTv#WmVkXrl|0CPiuH(uo7kwZVkbQI}1dVA63j{zh+`@y<)z4Cljb zhI>`wq^Y#Qp3+RAp^HSuZr$%CAbMMo-nOt{Prpy)6m&q?$>;NQz@wg~$fO z{lD`PVW=|kX3hJf0Q*qOYPJ+S&!)qW0?};~g*gGwQ3U4${PPfGts!vtc`1Ur`tdn(SxGZETwLe-~?1@|0J(ZlI z6Mb;!b~s$U+C-tmo9U|pzt>E@XT0FPnb}$s+?l#cVF|B^p5TjQE~m9l!&?J8&M7>b znhIMt7Qy(+x94r&H5#9_;ndWHDJK^#w}rA=r`_lmeLO5=8g!{8iEo;Eiv_wiUpj>{ ziiNLg*~UFO$8i;Tju6_AXA%Xcg85S;4Sc23qR!-+ + + + batch + + + + + + + + + + + + + 1 + 3 + 3 + 2 + 0.3 + 3 + + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml new file mode 100644 index 00000000000..ba8163c3d58 --- /dev/null +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch.in.xml @@ -0,0 +1,23 @@ + + + + batch + + + + + + + + + + + + 16 + 100 + 20 + 2 + 0.3 + 20 + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml new file mode 100644 index 00000000000..25dad52721d --- /dev/null +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_long_vmcbatch_dmcbatch.in.xml @@ -0,0 +1,35 @@ + + + + batch + + + + + + + + + + + + + 256 + 1 + 20 + 1 + 1 + 1.0 + no + + + + 256 + no + 20 + 0.01 + 5 + 80 + no + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml new file mode 100644 index 00000000000..1967833b7fd --- /dev/null +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch.in.xml @@ -0,0 +1,24 @@ + + + + batch + + + + + + + + + + + + 16 + 50 + 10 + 1 + 0.3 + 20 + + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml new file mode 100644 index 00000000000..01ad22f92b1 --- /dev/null +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp/qmc_short_vmcbatch_dmcbatch.in.xml @@ -0,0 +1,36 @@ + + + + batch + + + + + + + + + + + + + 2 + 1 + 100 + 1 + 2 + 1.0 + no + + + + 32 + no + 20 + 0.005 + 10 + 40 + no + + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/CMakeLists.txt b/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/CMakeLists.txt index ef28fd9d425..c66321de051 100644 --- a/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/CMakeLists.txt +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/CMakeLists.txt @@ -23,6 +23,18 @@ if(QMC_COMPLEX) diamondC_2x1x1-Gaussian_pp_cplx_LONG_SCALARS #VMC ) + qmc_run_and_check( + long-diamondC_2x1x1-Gaussian_pp_Tw_cplx-vmcbatch + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx" + vmcbatch_long + vmcbatch_long.in.xml + 1 + 16 + TRUE + 0 + diamondC_2x1x1-Gaussian_pp_cplx_LONG_SCALARS #VMC + ) + # VMC short run with no Jastrows # Compare directly to Hartree Fock Energy of: E=-21.30438154915548 #Data generated using utils/make_ref_data.sh diamondC_2x1x1-Gaussian_pp_cplx_SHORT qmc_ref/qmc_short.s000.scalar.dat qmc_ref/qmc_ref_long.s000.scalar.dat @@ -46,6 +58,18 @@ if(QMC_COMPLEX) diamondC_2x1x1-Gaussian_pp_cplx_SHORT_SCALARS #VMC ) + qmc_run_and_check( + short-diamondC_2x1x1-Gaussian_pp_Tw_cplx-vmcbatch + "${qmcpack_SOURCE_DIR}/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx" + vmcbatch_short + vmcbatch_short.in.xml + 1 + 16 + TRUE + 0 + diamondC_2x1x1-Gaussian_pp_cplx_SHORT_SCALARS #VMC + ) + else() message(VERBOSE "Skipping Complex Periodic LCAO if Complex code not build (QMC_COMPLEX=0)") endif() diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_long.in.xml b/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_long.in.xml new file mode 100644 index 00000000000..9db5a7253c9 --- /dev/null +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_long.in.xml @@ -0,0 +1,22 @@ + + + + batch + + + + + + + + + + + + 200 + 10 + 1 + 0.1 + 100 + + diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_short.in.xml b/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_short.in.xml new file mode 100644 index 00000000000..ed2e1f7ea18 --- /dev/null +++ b/tests/solids/diamondC_2x1x1-Gaussian_pp_cplx/vmcbatch_short.in.xml @@ -0,0 +1,22 @@ + + + + batch + + + + + + + + + + + + 200 + 3 + 1 + 0.1 + 100 + + From 8d11a7239a5b2db01a49621c412a9871f015dcb5 Mon Sep 17 00:00:00 2001 From: Hyeondeok-Shin Date: Mon, 23 Oct 2023 01:02:29 -0500 Subject: [PATCH 072/168] Update tolerance for mixed precision build --- tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt b/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt index 58595be702d..94ba99dda29 100644 --- a/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt +++ b/tests/solids/diamondC_1x1x1-Gaussian_pp/CMakeLists.txt @@ -129,7 +129,7 @@ if(QMC_MIXED_PRECISION) list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "eeenergy" "-3.00267948 0.000003") list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "ionion" "-12.77567012 0.000001") list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "localecp" "-7.99462588 0.000007") - list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "nonlocalecp" "0.70920650 0.000001") + list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "nonlocalecp" "0.70920650 0.000003") list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "samples" "9 0.0") else() list(APPEND DET_DIAMOND_VMCBATCH_SCALARS "totenergy" "-10.82674404 0.000001") From b2a04bc9e7c3ec35d1f17d68858b8d49d6b70d2e Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 10:43:17 -0500 Subject: [PATCH 073/168] simpler test data --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 2b834a88dab..a957e32b68d 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -31,6 +31,7 @@ namespace qmcplusplus TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") { + using VT = SPOSet::ValueType; Communicate* c = OHMMS::Controller; ParticleSet::ParticleLayout lattice; @@ -140,9 +141,9 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") std::vector newpos_vp_(nvp_); std::vector newpos_vp_2(nvp_2); for (int i = 0; i < nvp_; i++) - newpos_vp_[i] = {std::cos(i), std::sin(i), i / 10}; + newpos_vp_[i] = {double(1.0 * i / nvp_), double(2.0 * i / nvp_), double(i / (2.0 * nvp_))}; for (int i = 0; i < nvp_2; i++) - newpos_vp_2[i] = {-std::sin(i), i / 10, std::cos(i)}; + newpos_vp_2[i] = {double(2.0 * i / nvp_2), double(i / (2.0 * nvp_2)), double(1.0 * i / nvp_2)}; VP_.makeMoves(elec_, 0, newpos_vp_); VP_2.makeMoves(elec_2, 0, newpos_vp_2); @@ -157,8 +158,8 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") std::vector psiMinv_data_2(norb); for (int i = 0; i < norb; i++) { - psiMinv_data_[i] = std::cos(i); - psiMinv_data_2[i] = std::sin(i); + psiMinv_data_[i] = (i % 10) / 4.5 - 1.0; + psiMinv_data_2[i] = ((i + 5) % 10) / 4.5 - 1.0; } std::vector invRow_ptr_list{psiMinv_data_.data(), psiMinv_data_2.data()}; @@ -173,17 +174,17 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") RefVector{tmp_psi_list}, invRow_ptr_list, ratios_list); - // app_log() << "calcRatioGrad \n" << std::setprecision(14); + // app_log() << "ratios_list refvalues: \n" << std::setprecision(14); // for (int iw = 0; iw < nw; iw++) // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) // app_log() << "CHECK(ratios_list[" << iw << "][" << ivp << "] == Approx(" << ratios_list[iw][ivp] << "));\n"; - CHECK(ratios_list[0][0] == Approx(-206.77166092783)); - CHECK(ratios_list[0][1] == Approx(44.875898444974)); - CHECK(ratios_list[0][2] == Approx(24.96409353735)); - CHECK(ratios_list[0][3] == Approx(-53.413899456988)); - CHECK(ratios_list[1][0] == Approx(-306.54954579776)); - CHECK(ratios_list[1][1] == Approx(45.022133237688)); - CHECK(ratios_list[1][2] == Approx(183.86331674535)); + CHECK(ratios_list[0][0] == Approx(7.0447024716135)); + CHECK(ratios_list[0][1] == Approx(91.946906522354)); + CHECK(ratios_list[0][2] == Approx(34.424260462098)); + CHECK(ratios_list[0][3] == Approx(-1.6869723113315)); + CHECK(ratios_list[1][0] == Approx(-62.492756926476)); + CHECK(ratios_list[1][1] == Approx(88.860599314669)); + CHECK(ratios_list[1][2] == Approx(110.8285119408)); } } // namespace qmcplusplus From 0d8f865e1c5bab5a040a9fe0f72aa1ca7427c8d5 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 10:55:02 -0500 Subject: [PATCH 074/168] D2H ao VGL until better gemm interface is implemented --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index 65e2ccc9446..1831193984c 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -455,6 +455,7 @@ void LCAOrbitalSet::mw_evaluateVGLImplGEMM(const RefVectorWithLeader& sp ScopedTimer local(basis_timer_); auto basis_list = spo_leader.extractBasisRefList(spo_list); myBasisSet->mw_evaluateVGL(basis_list, P_list, iat, basis_vgl_mw); + basis_vgl_mw.updateFrom(); // TODO: remove this when gemm is implemented } if (Identity) From 6d69babb455b3b73d02b8bdec128d5bb6774c718 Mon Sep 17 00:00:00 2001 From: Hyeondeok-Shin Date: Mon, 23 Oct 2023 11:55:50 -0500 Subject: [PATCH 075/168] remove swap file --- .../.qmc_long_vmc_dmc.in.xml.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/solids/diamondC_2x1x1-Gaussian_pp/.qmc_long_vmc_dmc.in.xml.swp diff --git a/tests/solids/diamondC_2x1x1-Gaussian_pp/.qmc_long_vmc_dmc.in.xml.swp b/tests/solids/diamondC_2x1x1-Gaussian_pp/.qmc_long_vmc_dmc.in.xml.swp deleted file mode 100644 index 62367df7808eacde2fe0ff85cbb8e4f9830bb39d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&yO256vw^XC@fTPW%W1Jrn!+z0dO-DT*h(`*%O) zH=+T;%Hk{)+5 zIhTd8T`!$?U1tiFxo)PC(&+5gcyQ7`>32R%YUfm<$JT~<>Y5g-CYfCvzQGXmc3uqU{E zW%X!p)$UwqSGUv<0U|&IhyW2F0z`la5CI}U1c(3;AOinG0!hZ$*RL`bU4irX|9}1a z|Jj?2{RVvxeF5Ev1oZP8jC~2c3;nsv*fZ!U^ds~YbOc?4{&=0SpP+A`&!G39J?Qt> z82bTw0v$qEp+OZdg$AvN=oTv#WmVkXrl|0CPiuH(uo7kwZVkbQI}1dVA63j{zh+`@y<)z4Cljb zhI>`wq^Y#Qp3+RAp^HSuZr$%CAbMMo-nOt{Prpy)6m&q?$>;NQz@wg~$fO z{lD`PVW=|kX3hJf0Q*qOYPJ+S&!)qW0?};~g*gGwQ3U4${PPfGts!vtc`1Ur`tdn(SxGZETwLe-~?1@|0J(ZlI z6Mb;!b~s$U+C-tmo9U|pzt>E@XT0FPnb}$s+?l#cVF|B^p5TjQE~m9l!&?J8&M7>b znhIMt7Qy(+x94r&H5#9_;ndWHDJK^#w}rA=r`_lmeLO5=8g!{8iEo;Eiv_wiUpj>{ ziiNLg*~UFO$8i;Tju6_AXA%Xcg85S;4Sc23qR!-+ Date: Mon, 23 Oct 2023 13:03:43 -0500 Subject: [PATCH 076/168] get lattice in LCAO builder even if not complex --- .../LCAO/LCAOrbitalBuilder.cpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp index 72479801019..50b98e8546b 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp @@ -973,6 +973,33 @@ void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors( Vector>& LocPeriodicImagePhaseFactors, Array>& LocPeriodicImageDisplacements) { + // Allow computation to continue with no HDF file if the system has open boundary conditions. + // The complex build is usually only used with open BC for testing. + bool usesOpenBC = PBCImages[0] == 0 && PBCImages[1] == 0 && PBCImages[2] == 0; + + ///Exp(ik.g) where i is imaginary, k is the supertwist and g is the translation vector PBCImage. + if (h5_path != "" && !usesOpenBC) + { + hdf_archive hin(myComm); + if (myComm->rank() == 0) + { + if (!hin.open(h5_path, H5F_ACC_RDONLY)) + APP_ABORT("Could not open H5 file"); + + hin.push("Cell", false); + + hin.read(Lattice, "LatticeVectors"); + hin.close(); + } + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + myComm->bcast(Lattice(i, j)); + } + else if (!usesOpenBC) + { + APP_ABORT("Attempting to run PBC LCAO with no HDF5 support. Behaviour is unknown. Safer to exit"); + } + const int Nx = PBCImages[0] + 1; const int Ny = PBCImages[1] + 1; const int Nz = PBCImages[2] + 1; From 9f817fb21dfd213b685d2862be61e437e030302c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:15:20 +0000 Subject: [PATCH 077/168] Bump urllib3 from 2.0.6 to 2.0.7 in /nexus/sphinx_docs Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.6 to 2.0.7. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/2.0.6...2.0.7) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- nexus/sphinx_docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/sphinx_docs/requirements.txt b/nexus/sphinx_docs/requirements.txt index 1c9cc6c5860..68fd750be0e 100644 --- a/nexus/sphinx_docs/requirements.txt +++ b/nexus/sphinx_docs/requirements.txt @@ -69,7 +69,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -urllib3==2.0.6 +urllib3==2.0.7 # via requests # The following packages are considered to be unsafe in a requirements file: From aaf637b79e84d47bb6c0578f8c7deed262d95650 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 13:15:29 -0500 Subject: [PATCH 078/168] update vp test vals after lattice fix --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index a957e32b68d..ba783fc1591 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -179,12 +179,12 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) // app_log() << "CHECK(ratios_list[" << iw << "][" << ivp << "] == Approx(" << ratios_list[iw][ivp] << "));\n"; - CHECK(ratios_list[0][0] == Approx(7.0447024716135)); - CHECK(ratios_list[0][1] == Approx(91.946906522354)); - CHECK(ratios_list[0][2] == Approx(34.424260462098)); - CHECK(ratios_list[0][3] == Approx(-1.6869723113315)); - CHECK(ratios_list[1][0] == Approx(-62.492756926476)); - CHECK(ratios_list[1][1] == Approx(88.860599314669)); - CHECK(ratios_list[1][2] == Approx(110.8285119408)); + CHECK(ratios_list[0][0] == Approx(-0.11554491049855)); + CHECK(ratios_list[0][1] == Approx(0.19155774810121)); + CHECK(ratios_list[0][2] == Approx(-0.16063724839636)); + CHECK(ratios_list[0][3] == Approx(-0.37105113615831)); + CHECK(ratios_list[1][0] == Approx(-0.12705469585615)); + CHECK(ratios_list[1][1] == Approx(0.67930890998428)); + CHECK(ratios_list[1][2] == Approx(0.83583922544552)); } } // namespace qmcplusplus From 33767d58483334780b292220fb2d80fb4a5eaff9 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 13:51:21 -0500 Subject: [PATCH 079/168] add ref vals for batched VP test and fix LCAO detratios when C==Identity --- src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp | 3 +++ .../tests/test_LCAO_diamondC_2x1x1.cpp | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp index a9cd33893c0..8d8a2eda13d 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalSet.cpp @@ -597,6 +597,9 @@ void LCAOrbitalSet::evaluateDetRatios(const VirtualParticleSet& VP, Vector vTemp(Temp.data(0), BasisSetSize); Vector invTemp(Temp.data(1), BasisSetSize); + if (Identity) + BLAS::copy(psiinv.size(), psiinv.data(), invTemp.data()); + else { ScopedTimer local(mo_timer_); // when only a subset of orbitals is used, extract limited rows of C. diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index ba783fc1591..0497626f869 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -156,10 +156,14 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // fill invrow with dummy data for each walker std::vector psiMinv_data_(norb); std::vector psiMinv_data_2(norb); + SPOSet::ValueVector psiMinv_ref_0(norb); + SPOSet::ValueVector psiMinv_ref_1(norb); for (int i = 0; i < norb; i++) { psiMinv_data_[i] = (i % 10) / 4.5 - 1.0; psiMinv_data_2[i] = ((i + 5) % 10) / 4.5 - 1.0; + psiMinv_ref_0[i] = psiMinv_data_[i]; + psiMinv_ref_1[i] = psiMinv_data_2[i]; } std::vector invRow_ptr_list{psiMinv_data_.data(), psiMinv_data_2.data()}; @@ -173,6 +177,14 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") spo->mw_evaluateDetRatios(spo_list, RefVectorWithLeader(VP_, {VP_, VP_2}), RefVector{tmp_psi_list}, invRow_ptr_list, ratios_list); + std::vector ratios_ref_0(nvp_); + std::vector ratios_ref_1(nvp_2); + spo->evaluateDetRatios(VP_, tmp_psi_list, psiMinv_ref_0, ratios_ref_0); + spo_2->evaluateDetRatios(VP_2, tmp_psi_list, psiMinv_ref_1, ratios_ref_1); + for (int ivp = 0; ivp < nvp_; ivp++) + CHECK(ratios_list[0][ivp] == Approx(ratios_ref_0[ivp])); + for (int ivp = 0; ivp < nvp_2; ivp++) + CHECK(ratios_list[1][ivp] == Approx(ratios_ref_1[ivp])); // app_log() << "ratios_list refvalues: \n" << std::setprecision(14); // for (int iw = 0; iw < nw; iw++) From 7514577b847823bb97409cc448d1860164da96b5 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 13:54:34 -0500 Subject: [PATCH 080/168] fix test check with complex --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 0497626f869..9f9d9b6497b 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -189,14 +189,15 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // app_log() << "ratios_list refvalues: \n" << std::setprecision(14); // for (int iw = 0; iw < nw; iw++) // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) - // app_log() << "CHECK(ratios_list[" << iw << "][" << ivp << "] == Approx(" << ratios_list[iw][ivp] << "));\n"; - - CHECK(ratios_list[0][0] == Approx(-0.11554491049855)); - CHECK(ratios_list[0][1] == Approx(0.19155774810121)); - CHECK(ratios_list[0][2] == Approx(-0.16063724839636)); - CHECK(ratios_list[0][3] == Approx(-0.37105113615831)); - CHECK(ratios_list[1][0] == Approx(-0.12705469585615)); - CHECK(ratios_list[1][1] == Approx(0.67930890998428)); - CHECK(ratios_list[1][2] == Approx(0.83583922544552)); + // app_log() << "CHECK(std::real(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << ratios_list[iw][ivp] + // << "));\n"; + + CHECK(std::real(ratios_list[0][0]) == Approx(-0.11554491049855)); + CHECK(std::real(ratios_list[0][1]) == Approx(0.19155774810121)); + CHECK(std::real(ratios_list[0][2]) == Approx(-0.16063724839636)); + CHECK(std::real(ratios_list[0][3]) == Approx(-0.37105113615831)); + CHECK(std::real(ratios_list[1][0]) == Approx(-0.12705469585615)); + CHECK(std::real(ratios_list[1][1]) == Approx(0.67930890998428)); + CHECK(std::real(ratios_list[1][2]) == Approx(0.83583922544552)); } } // namespace qmcplusplus From 1c0d22a128620716ec707857b7979b9f041bf0c3 Mon Sep 17 00:00:00 2001 From: Peter Doak Date: Mon, 23 Oct 2023 14:56:23 -0400 Subject: [PATCH 081/168] only do OMP_TESTLOOP test if QMC_OMP is true --- CMakeLists.txt | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 271631d012e..6ea6544eafa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -371,29 +371,31 @@ message(STATUS "QMC_SIMD_ALIGNMENT is set to ${QMC_SIMD_ALIGNMENT}") #--------------------------------------------------------- # Determine if OpenMP taskloop works with the CXX compiler #--------------------------------------------------------- -include(TestOpenMPtaskloop) -option(ENABLE_OMP_TASKLOOP "Enable OpenMP taskloop" ${OMP_TASKLOOP_OKAY}) -message(STATUS "ENABLE_OMP_TASKLOOP is set to ${ENABLE_OMP_TASKLOOP}") - -#--------------------------------------------------------- -# Set up OpenMP offload compile options -#--------------------------------------------------------- -set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT OFF) -if(ENABLE_OFFLOAD AND DEFINED OPENMP_OFFLOAD_COMPILE_OPTIONS) - message(STATUS "OpenMP offload CXX flags: ${OPENMP_OFFLOAD_COMPILE_OPTIONS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPENMP_OFFLOAD_COMPILE_OPTIONS}") - if(${COMPILER} MATCHES "Clang" - AND OPENMP_OFFLOAD_COMPILE_OPTIONS MATCHES "gfx" - AND QMC_CUDA2HIP) - # As of 11/2021, QMC_OFFLOAD_MEM_ASSOCIATED=ON is needed for AMD and mainline LLVM compilers - # when using OpenMP offload to AMD GPU together with HIP. - set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT ON) +if(QMC_OMP) + include(TestOpenMPtaskloop) + option(ENABLE_OMP_TASKLOOP "Enable OpenMP taskloop" ${OMP_TASKLOOP_OKAY}) + message(STATUS "ENABLE_OMP_TASKLOOP is set to ${ENABLE_OMP_TASKLOOP}") + + #--------------------------------------------------------- + # Set up OpenMP offload compile options + #--------------------------------------------------------- + set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT OFF) + if(ENABLE_OFFLOAD AND DEFINED OPENMP_OFFLOAD_COMPILE_OPTIONS) + message(STATUS "OpenMP offload CXX flags: ${OPENMP_OFFLOAD_COMPILE_OPTIONS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPENMP_OFFLOAD_COMPILE_OPTIONS}") + if(${COMPILER} MATCHES "Clang" + AND OPENMP_OFFLOAD_COMPILE_OPTIONS MATCHES "gfx" + AND QMC_CUDA2HIP) + # As of 11/2021, QMC_OFFLOAD_MEM_ASSOCIATED=ON is needed for AMD and mainline LLVM compilers + # when using OpenMP offload to AMD GPU together with HIP. + set(QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT ON) + endif() endif() + # Some OpenMP offload runtime libraries have composibility issue with a vendor native runtime. + # A workaround is making the vendor native runtime responsible for memory allocations and OpenMP associate/disassocate them. + cmake_dependent_option(QMC_OFFLOAD_MEM_ASSOCIATED "Manage OpenMP memory allocations via the vendor runtime" + ${QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT} "ENABLE_OFFLOAD;ENABLE_CUDA" OFF) endif() -# Some OpenMP offload runtime libraries have composibility issue with a vendor native runtime. -# A workaround is making the vendor native runtime responsible for memory allocations and OpenMP associate/disassocate them. -cmake_dependent_option(QMC_OFFLOAD_MEM_ASSOCIATED "Manage OpenMP memory allocations via the vendor runtime" - ${QMC_OFFLOAD_MEM_ASSOCIATED_DEFAULT} "ENABLE_OFFLOAD;ENABLE_CUDA" OFF) #------------------------------------------------------------------------------------- # consider making this always on if OpenMP is no longer UB with Thread Support Library From c6feeeb19a97e94e930f58376adb447605acf9f6 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 13:03:43 -0500 Subject: [PATCH 082/168] get lattice in LCAO builder even if not complex --- .../LCAO/LCAOrbitalBuilder.cpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp index 72479801019..50b98e8546b 100644 --- a/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp +++ b/src/QMCWaveFunctions/LCAO/LCAOrbitalBuilder.cpp @@ -973,6 +973,33 @@ void LCAOrbitalBuilder::EvalPeriodicImagePhaseFactors( Vector>& LocPeriodicImagePhaseFactors, Array>& LocPeriodicImageDisplacements) { + // Allow computation to continue with no HDF file if the system has open boundary conditions. + // The complex build is usually only used with open BC for testing. + bool usesOpenBC = PBCImages[0] == 0 && PBCImages[1] == 0 && PBCImages[2] == 0; + + ///Exp(ik.g) where i is imaginary, k is the supertwist and g is the translation vector PBCImage. + if (h5_path != "" && !usesOpenBC) + { + hdf_archive hin(myComm); + if (myComm->rank() == 0) + { + if (!hin.open(h5_path, H5F_ACC_RDONLY)) + APP_ABORT("Could not open H5 file"); + + hin.push("Cell", false); + + hin.read(Lattice, "LatticeVectors"); + hin.close(); + } + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + myComm->bcast(Lattice(i, j)); + } + else if (!usesOpenBC) + { + APP_ABORT("Attempting to run PBC LCAO with no HDF5 support. Behaviour is unknown. Safer to exit"); + } + const int Nx = PBCImages[0] + 1; const int Ny = PBCImages[1] + 1; const int Nz = PBCImages[2] + 1; From 32db80f37f079a14e03727500935fc57f152ba7e Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 14:15:44 -0500 Subject: [PATCH 083/168] vgl tests --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 9f9d9b6497b..c5d25153798 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -109,11 +109,14 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // test batched interfaces + const size_t nw = 2; ParticleSet elec_2(elec_); // interchange positions elec_2.R[0] = elec_.R[1]; elec_2.R[1] = elec_.R[0]; + elec_.update(); + elec_2.update(); RefVectorWithLeader p_list(elec_, {elec_, elec_2}); @@ -129,8 +132,86 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") ResourceCollectionTeamLock mw_pset_lock(pset_res, p_list); ResourceCollectionTeamLock mw_sposet_lock(spo_res, spo_list); + SPOSet::ValueVector psiref_0(norb); + SPOSet::GradVector dpsiref_0(norb); + SPOSet::ValueVector d2psiref_0(norb); + SPOSet::ValueVector psiref_1(norb); + SPOSet::GradVector dpsiref_1(norb); + SPOSet::ValueVector d2psiref_1(norb); + spo->evaluateVGL(elec_, 0, psiref_0, dpsiref_0, d2psiref_0); + spo_2->evaluateVGL(elec_2, 0, psiref_1, dpsiref_1, d2psiref_1); + + // app_log() << "vgl_refvalues: \n" << std::setprecision(14); + // for (int iorb = 0; iorb < 2; iorb++) + // { + // app_log() << "CHECK(std::real(psiref_0[" << iorb << "]) == Approx(" << psiref_0[iorb] << "));\n"; + // app_log() << "CHECK(std::real(d2psiref_0[" << iorb << "]) == Approx(" << d2psiref_0[iorb] << "));\n"; + // for (int idim = 0; idim < 3; idim++) + // app_log() << "CHECK(std::real(dpsiref_0[" << iorb << "][" << idim << "]) == Approx(" << dpsiref_0[iorb][idim] << "));\n"; + // } + // for (int iorb = 0; iorb < 2; iorb++) + // { + // app_log() << "CHECK(std::real(psiref_1[" << iorb << "]) == Approx(" << psiref_1[iorb] << "));\n"; + // app_log() << "CHECK(std::real(d2psiref_1[" << iorb << "]) == Approx(" << d2psiref_1[iorb] << "));\n"; + // for (int idim = 0; idim < 3; idim++) + // app_log() << "CHECK(std::real(dpsiref_1[" << iorb << "][" << idim << "]) == Approx(" << dpsiref_1[iorb][idim] << "));\n"; + // } + CHECK(std::real(psiref_0[0]) == Approx(0.23966351080363)); + CHECK(std::real(d2psiref_0[0]) == Approx(-0.52701907904562)); + CHECK(std::real(dpsiref_0[0][0]) == Approx(-0.0057821587526225)); + CHECK(std::real(dpsiref_0[0][1]) == Approx(-0.10996780053441)); + CHECK(std::real(dpsiref_0[0][2]) == Approx(0.0057818514970319)); + CHECK(std::real(psiref_0[1]) == Approx(0.26673125102484)); + CHECK(std::real(d2psiref_0[1]) == Approx(-0.56873091511411)); + CHECK(std::real(dpsiref_0[1][0]) == Approx(-1.5495607896545e-06)); + CHECK(std::real(dpsiref_0[1][1]) == Approx(-0.49161303927196)); + CHECK(std::real(dpsiref_0[1][2]) == Approx(-2.5223735120643e-07)); + + CHECK(std::real(psiref_1[0]) == Approx(0.22735347471476)); + CHECK(std::real(d2psiref_1[0]) == Approx(-0.41247534585435)); + CHECK(std::real(dpsiref_1[0][0]) == Approx(-0.0064508940042537)); + CHECK(std::real(dpsiref_1[0][1]) == Approx(-0.13393939202647)); + CHECK(std::real(dpsiref_1[0][2]) == Approx(0.0064505548318947)); + CHECK(std::real(psiref_1[1]) == Approx(0.21979955506832)); + CHECK(std::real(d2psiref_1[1]) == Approx(-0.31185449762657)); + CHECK(std::real(dpsiref_1[1][0]) == Approx(-1.7377316644163e-06)); + CHECK(std::real(dpsiref_1[1][1]) == Approx(-0.44562353474568)); + CHECK(std::real(dpsiref_1[1][2]) == Approx(2.5297265181413e-07)); + + SPOSet::ValueVector psi_v_1(norb); + SPOSet::ValueVector psi_v_2(norb); + RefVector psi_v_list{psi_v_1, psi_v_2}; + spo->mw_evaluateValue(spo_list, p_list, 0, psi_v_list); + + SPOSet::ValueVector psi_1(norb); + SPOSet::GradVector dpsi_1(norb); + SPOSet::ValueVector d2psi_1(norb); + SPOSet::ValueVector psi_2(norb); + SPOSet::GradVector dpsi_2(norb); + SPOSet::ValueVector d2psi_2(norb); + RefVector psi_list = {psi_1, psi_2}; + RefVector dpsi_list = {dpsi_1, dpsi_2}; + RefVector d2psi_list = {d2psi_1, d2psi_2}; + spo->mw_evaluateVGL(spo_list, p_list, 0, psi_list, dpsi_list, d2psi_list); + + for (size_t iorb = 0; iorb < norb; iorb++) + { + CHECK(psi_list[0].get()[iorb] == Approx(psiref_0[iorb])); + CHECK(psi_list[1].get()[iorb] == Approx(psiref_1[iorb])); + CHECK(d2psi_list[0].get()[iorb] == Approx(d2psiref_0[iorb])); + CHECK(d2psi_list[1].get()[iorb] == Approx(d2psiref_1[iorb])); + for (size_t idim = 0; idim < SPOSet::DIM; idim++) + { + CHECK(dpsi_list[0].get()[iorb][idim] == Approx(dpsiref_0[iorb][idim])); + CHECK(dpsi_list[1].get()[iorb][idim] == Approx(dpsiref_1[iorb][idim])); + } + } + + for (size_t iorb = 0; iorb < norb; iorb++) + for (size_t iw = 0; iw < nw; iw++) + CHECK(psi_v_list[iw].get()[iorb] == Approx(psi_list[iw].get()[iorb])); + // make VPs - const size_t nw = 2; const size_t nvp_ = 4; const size_t nvp_2 = 3; const std::vector nvp_list = {nvp_, nvp_2}; From 2b5cfe6279a282e613d837f8e23c34fb0de9dbde Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 14:30:07 -0500 Subject: [PATCH 084/168] fix tests for complex build --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index c5d25153798..93c2ea81058 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -194,23 +194,39 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") RefVector d2psi_list = {d2psi_1, d2psi_2}; spo->mw_evaluateVGL(spo_list, p_list, 0, psi_list, dpsi_list, d2psi_list); + for (size_t iorb = 0; iorb < norb; iorb++) { - CHECK(psi_list[0].get()[iorb] == Approx(psiref_0[iorb])); - CHECK(psi_list[1].get()[iorb] == Approx(psiref_1[iorb])); - CHECK(d2psi_list[0].get()[iorb] == Approx(d2psiref_0[iorb])); - CHECK(d2psi_list[1].get()[iorb] == Approx(d2psiref_1[iorb])); + CHECK(std::real(psi_list[0].get()[iorb]) == Approx(std::real(psiref_0[iorb]))); + CHECK(std::real(psi_list[1].get()[iorb]) == Approx(std::real(psiref_1[iorb]))); + CHECK(std::real(d2psi_list[0].get()[iorb]) == Approx(std::real(d2psiref_0[iorb]))); + CHECK(std::real(d2psi_list[1].get()[iorb]) == Approx(std::real(d2psiref_1[iorb]))); for (size_t idim = 0; idim < SPOSet::DIM; idim++) { - CHECK(dpsi_list[0].get()[iorb][idim] == Approx(dpsiref_0[iorb][idim])); - CHECK(dpsi_list[1].get()[iorb][idim] == Approx(dpsiref_1[iorb][idim])); + CHECK(std::real(dpsi_list[0].get()[iorb][idim]) == Approx(std::real(dpsiref_0[iorb][idim]))); + CHECK(std::real(dpsi_list[1].get()[iorb][idim]) == Approx(std::real(dpsiref_1[iorb][idim]))); } } - for (size_t iorb = 0; iorb < norb; iorb++) for (size_t iw = 0; iw < nw; iw++) - CHECK(psi_v_list[iw].get()[iorb] == Approx(psi_list[iw].get()[iorb])); - + CHECK(std::real(psi_v_list[iw].get()[iorb]) == Approx(std::real(psi_list[iw].get()[iorb]))); +#ifdef QMC_COMPLEX + for (size_t iorb = 0; iorb < norb; iorb++) + { + CHECK(std::imag(psi_list[0].get()[iorb]) == Approx(std::imag(psiref_0[iorb]))); + CHECK(std::imag(psi_list[1].get()[iorb]) == Approx(std::imag(psiref_1[iorb]))); + CHECK(std::imag(d2psi_list[0].get()[iorb]) == Approx(std::imag(d2psiref_0[iorb]))); + CHECK(std::imag(d2psi_list[1].get()[iorb]) == Approx(std::imag(d2psiref_1[iorb]))); + for (size_t idim = 0; idim < SPOSet::DIM; idim++) + { + CHECK(std::imag(dpsi_list[0].get()[iorb][idim]) == Approx(std::imag(dpsiref_0[iorb][idim]))); + CHECK(std::imag(dpsi_list[1].get()[iorb][idim]) == Approx(std::imag(dpsiref_1[iorb][idim]))); + } + } + for (size_t iorb = 0; iorb < norb; iorb++) + for (size_t iw = 0; iw < nw; iw++) + CHECK(std::imag(psi_v_list[iw].get()[iorb]) == Approx(std::imag(psi_list[iw].get()[iorb]))); +#endif // make VPs const size_t nvp_ = 4; const size_t nvp_2 = 3; @@ -263,15 +279,26 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") spo->evaluateDetRatios(VP_, tmp_psi_list, psiMinv_ref_0, ratios_ref_0); spo_2->evaluateDetRatios(VP_2, tmp_psi_list, psiMinv_ref_1, ratios_ref_1); for (int ivp = 0; ivp < nvp_; ivp++) - CHECK(ratios_list[0][ivp] == Approx(ratios_ref_0[ivp])); + CHECK(std::real(ratios_list[0][ivp]) == Approx(std::real(ratios_ref_0[ivp]))); + for (int ivp = 0; ivp < nvp_2; ivp++) + CHECK(std::real(ratios_list[1][ivp]) == Approx(std::real(ratios_ref_1[ivp]))); +#ifdef QMC_COMPLEX + for (int ivp = 0; ivp < nvp_; ivp++) + CHECK(std::imag(ratios_list[0][ivp]) == Approx(std::imag(ratios_ref_0[ivp]))); for (int ivp = 0; ivp < nvp_2; ivp++) - CHECK(ratios_list[1][ivp] == Approx(ratios_ref_1[ivp])); + CHECK(std::imag(ratios_list[1][ivp]) == Approx(std::imag(ratios_ref_1[ivp]))); +#endif // app_log() << "ratios_list refvalues: \n" << std::setprecision(14); // for (int iw = 0; iw < nw; iw++) // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) // app_log() << "CHECK(std::real(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << ratios_list[iw][ivp] // << "));\n"; + app_log() << "ratios_list refvalues: \n" << std::setprecision(14); + for (int iw = 0; iw < nw; iw++) + for (int ivp = 0; ivp < nvp_list[iw]; ivp++) + app_log() << "CHECK(std::imag(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << ratios_list[iw][ivp] + << "));\n"; CHECK(std::real(ratios_list[0][0]) == Approx(-0.11554491049855)); CHECK(std::real(ratios_list[0][1]) == Approx(0.19155774810121)); From fbfacb1736b2e2447e3a5ab847f90fc4d4c29caf Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 14:32:53 -0500 Subject: [PATCH 085/168] fix detratio tests for complex --- .../tests/test_LCAO_diamondC_2x1x1.cpp | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 9f9d9b6497b..4f55173145b 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -109,11 +109,14 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // test batched interfaces + const size_t nw = 2; ParticleSet elec_2(elec_); // interchange positions elec_2.R[0] = elec_.R[1]; elec_2.R[1] = elec_.R[0]; + elec_.update(); + elec_2.update(); RefVectorWithLeader p_list(elec_, {elec_, elec_2}); @@ -130,7 +133,6 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") ResourceCollectionTeamLock mw_sposet_lock(spo_res, spo_list); // make VPs - const size_t nw = 2; const size_t nvp_ = 4; const size_t nvp_2 = 3; const std::vector nvp_list = {nvp_, nvp_2}; @@ -182,14 +184,25 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") spo->evaluateDetRatios(VP_, tmp_psi_list, psiMinv_ref_0, ratios_ref_0); spo_2->evaluateDetRatios(VP_2, tmp_psi_list, psiMinv_ref_1, ratios_ref_1); for (int ivp = 0; ivp < nvp_; ivp++) - CHECK(ratios_list[0][ivp] == Approx(ratios_ref_0[ivp])); + CHECK(std::real(ratios_list[0][ivp]) == Approx(std::real(ratios_ref_0[ivp]))); for (int ivp = 0; ivp < nvp_2; ivp++) - CHECK(ratios_list[1][ivp] == Approx(ratios_ref_1[ivp])); + CHECK(std::real(ratios_list[1][ivp]) == Approx(std::real(ratios_ref_1[ivp]))); +#ifdef QMC_COMPLEX + for (int ivp = 0; ivp < nvp_; ivp++) + CHECK(std::imag(ratios_list[0][ivp]) == Approx(std::imag(ratios_ref_0[ivp]))); + for (int ivp = 0; ivp < nvp_2; ivp++) + CHECK(std::imag(ratios_list[1][ivp]) == Approx(std::imag(ratios_ref_1[ivp]))); +#endif // app_log() << "ratios_list refvalues: \n" << std::setprecision(14); // for (int iw = 0; iw < nw; iw++) // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) - // app_log() << "CHECK(std::real(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << ratios_list[iw][ivp] + // app_log() << "CHECK(std::real(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << std::real(ratios_list[iw][ivp]) + // << "));\n"; + // app_log() << "ratios_list refvalues: \n" << std::setprecision(14); + // for (int iw = 0; iw < nw; iw++) + // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) + // app_log() << "CHECK(std::imag(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << std::imag(ratios_list[iw][ivp]) // << "));\n"; CHECK(std::real(ratios_list[0][0]) == Approx(-0.11554491049855)); From c0896706e5adf4ea079453bf12caebb8fed7b273 Mon Sep 17 00:00:00 2001 From: Kevin Gasperich Date: Mon, 23 Oct 2023 14:34:21 -0500 Subject: [PATCH 086/168] fix merge duplicate --- src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp index 5fadb9fa44c..8b319508702 100644 --- a/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp +++ b/src/QMCWaveFunctions/tests/test_LCAO_diamondC_2x1x1.cpp @@ -299,11 +299,6 @@ TEST_CASE("LCAO DiamondC_2x1x1", "[wavefunction]") // for (int ivp = 0; ivp < nvp_list[iw]; ivp++) // app_log() << "CHECK(std::imag(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << std::imag(ratios_list[iw][ivp]) // << "));\n"; - app_log() << "ratios_list refvalues: \n" << std::setprecision(14); - for (int iw = 0; iw < nw; iw++) - for (int ivp = 0; ivp < nvp_list[iw]; ivp++) - app_log() << "CHECK(std::imag(ratios_list[" << iw << "][" << ivp << "]) == Approx(" << ratios_list[iw][ivp] - << "));\n"; CHECK(std::real(ratios_list[0][0]) == Approx(-0.11554491049855)); CHECK(std::real(ratios_list[0][1]) == Approx(0.19155774810121)); From 61aa9646172746214fcf1e65e5977d03d68d28fd Mon Sep 17 00:00:00 2001 From: Alfredo Correa Date: Mon, 23 Oct 2023 12:37:09 -0700 Subject: [PATCH 087/168] remove mpi3 subtree --- external_codes/mpi_wrapper/mpi3/.clang-format | 233 - external_codes/mpi_wrapper/mpi3/.clang-tidy | 32 - .../mpi_wrapper/mpi3/.cppcheck-suppressions | 5 - .../mpi_wrapper/mpi3/.deepsource.toml | 9 - external_codes/mpi_wrapper/mpi3/.gitignore | 5 - .../mpi_wrapper/mpi3/.gitlab-ci.yml | 239 - external_codes/mpi_wrapper/mpi3/.gitmodules | 0 .../mpi_wrapper/mpi3/CMakeLists.txt | 30 - external_codes/mpi_wrapper/mpi3/LICENSE | 23 - external_codes/mpi_wrapper/mpi3/README.md | 968 --- .../mpi_wrapper/mpi3/examples/01-hello.cpp | 21 - .../mpi_wrapper/mpi3/examples/02-send.cpp | 47 - .../mpi_wrapper/mpi3/examples/03-isend.cpp | 34 - .../mpi_wrapper/mpi3/examples/04-array.cpp | 68 - .../mpi3/examples/05-array_mult.cpp | 109 - .../mpi_wrapper/mpi3/examples/average.cpp | 52 - .../mpi_wrapper/mpi3/examples/broadcast.cpp | 131 - .../mpi3/examples/calculate_pi.cpp | 49 - .../mpi_wrapper/mpi3/examples/groups.cpp | 23 - .../mpi_wrapper/mpi3/examples/hello.cpp | 21 - .../mpi_wrapper/mpi3/examples/hello_world.cpp | 23 - .../mpi3/examples/managed_shared_memory.cpp | 30 - .../mpi3/examples/managed_shared_memory_0.cpp | 57 - .../mpi_wrapper/mpi3/examples/pi_reduce.cpp | 56 - .../mpi_wrapper/mpi3/examples/ping_pong.cpp | 30 - .../mpi_wrapper/mpi3/examples/random_walk.cpp | 82 - .../mpi_wrapper/mpi3/examples/rank.cpp | 35 - .../mpi3/examples/reduce_average.cpp | 54 - .../mpi_wrapper/mpi3/examples/ring.cpp | 28 - .../mpi3/examples/send_receive.cpp | 26 - .../mpi3/examples/shared_window.cpp | 50 - .../mpi3/examples/shared_window_2.cpp | 47 - .../mpi3/examples/shared_window_3.cpp | 50 - .../mpi_wrapper/mpi3/examples/shm_vector.cpp | 86 - .../mpi_wrapper/mpi3/examples/split.cpp | 28 - .../mpi_wrapper/mpi3/examples/test3.cpp | 137 - .../mpi_wrapper/mpi3/examples/test4.cpp | 155 - .../mpi_wrapper/mpi3/examples/test5.cpp | 125 - .../mpi3/fake/examples/MPI_Barrier.c | 15 - .../fake/examples/MPI_Comm_call_errhandler.c | 40 - .../mpi3/fake/examples/MPI_Comm_compare.c | 36 - .../mpi3/fake/examples/MPI_Comm_dup.c | 25 - .../mpi3/fake/examples/MPI_Comm_free.c | 90 - .../mpi3/fake/examples/MPI_Comm_rank.c | 14 - .../fake/examples/MPI_Comm_set_errhandler.c | 45 - .../mpi3/fake/examples/MPI_Gather.c | 58 - .../mpi3/fake/examples/MPI_Gatherv.c | 39 - .../mpi_wrapper/mpi3/fake/examples/MPI_Recv.c | 39 - .../mpi3/fake/examples/MPI_Reduce.c | 43 - .../mpi_wrapper/mpi3/fake/examples/MPI_Send.c | 39 - .../mpi3/fake/examples/MPI_Sendrecv_replace.c | 24 - .../mpi3/fake/examples/MPI_Type_contiguos.c | 29 - .../mpi3/fake/examples/MPI_Type_dup.c | 31 - .../mpi3/fake/examples/MPI_Type_get_extent.c | 62 - .../mpi3/fake/examples/MPI_Type_size.c | 62 - .../mpi3/fake/examples/MPI_Type_vector.c | 45 - .../mpi3/fake/examples/MPI_Wtick.c | 14 - .../mpi3/fake/examples/MPI_Wtime.c | 17 - .../mpi_wrapper/mpi3/fake/examples/Makefile | 25 - .../mpi_wrapper/mpi3/fake/examples/abort.cpp | 10 - .../mpi_wrapper/mpi3/fake/examples/testme | 3 - external_codes/mpi_wrapper/mpi3/fake/load | 2 - external_codes/mpi_wrapper/mpi3/fake/mpi.h | 2763 ------- .../mpi3/fake/mpi_qmcpack_compile.h | 741 -- external_codes/mpi_wrapper/mpi3/fake/mpic++ | 2 - external_codes/mpi_wrapper/mpi3/fake/mpicc | 2 - external_codes/mpi_wrapper/mpi3/fake/mpirun | 2 - .../mpi_wrapper/mpi3/fake/test/CMakeLists.txt | 33 - .../mpi3/fake/test/test_c_mpi_init.c | 11 - .../mpi3/fake/test/test_mpi_init.cpp | 11 - .../mpi3/fake/test/test_wrapper_mpi_init.cpp | 11 - .../mpi_wrapper/mpi3/include/mpi3/FILE.hpp | 145 - .../mpi_wrapper/mpi3/include/mpi3/address.hpp | 53 - .../mpi3/include/mpi3/allocator.hpp | 129 - .../mpi_wrapper/mpi3/include/mpi3/buffer.hpp | 111 - .../include/mpi3/cartesian_communicator.hpp | 348 - .../mpi3/include/mpi3/communication_mode.hpp | 69 - .../mpi3/include/mpi3/communicator.hpp | 3233 -------- .../mpi3/communicator/operatorators.hpp | 35 - .../include/mpi3/communicator/operators.hpp | 36 - .../mpi3/include/mpi3/communicator_fwd.hpp | 24 - .../include/mpi3/communicator_iterator.hpp | 55 - .../mpi3/include/mpi3/config/NODISCARD.hpp | 27 - .../mpi_wrapper/mpi3/include/mpi3/core.hpp | 36 - .../mpi3/detail/basic_communicator.hpp | 336 - .../mpi3/include/mpi3/detail/buffer.hpp | 42 - .../mpi3/include/mpi3/detail/call.hpp | 75 - .../mpi3/include/mpi3/detail/datatype.hpp | 205 - .../mpi3/include/mpi3/detail/equality.hpp | 46 - .../mpi3/include/mpi3/detail/iterator.hpp | 170 - .../include/mpi3/detail/iterator_traits.hpp | 115 - .../mpi3/include/mpi3/detail/just.hpp | 159 - .../mpi3/include/mpi3/detail/package.hpp | 167 - .../mpi3/include/mpi3/detail/tuple_offset.hpp | 51 - .../mpi3/include/mpi3/detail/value_traits.hpp | 83 - .../mpi3/include/mpi3/dummy/.dummy | 0 .../mpi3/include/mpi3/dynamic_window.hpp | 63 - .../mpi3/include/mpi3/environment.hpp | 315 - .../mpi_wrapper/mpi3/include/mpi3/error.hpp | 99 - .../mpi3/include/mpi3/error_handler.hpp | 103 - .../mpi3/include/mpi3/exception.hpp | 21 - .../mpi_wrapper/mpi3/include/mpi3/future.hpp | 49 - .../mpi3/include/mpi3/generalized_request.hpp | 158 - .../mpi3/include/mpi3/graph_communicator.hpp | 140 - .../mpi_wrapper/mpi3/include/mpi3/group.hpp | 142 - .../mpi_wrapper/mpi3/include/mpi3/handle.hpp | 139 - .../mpi_wrapper/mpi3/include/mpi3/info.hpp | 96 - .../mpi_wrapper/mpi3/include/mpi3/main.hpp | 45 - .../mpi3/include/mpi3/main_environment.hpp | 31 - .../mpi_wrapper/mpi3/include/mpi3/match.hpp | 26 - .../mpi_wrapper/mpi3/include/mpi3/message.hpp | 71 - .../mpi_wrapper/mpi3/include/mpi3/mutex.hpp | 379 - .../mpi3/include/mpi3/nccl/communicator.hpp | 367 - .../mpi3/nccl/detail/basic_datatype.hpp | 48 - .../mpi3/nccl/detail/basic_reduction.hpp | 37 - .../include/mpi3/nccl/test/CMakeLists.txt | 145 - .../mpi3/nccl/test/nccl_constructor.cu | 63 - .../mpi3/nccl/universal_communicator.hpp | 57 - .../mpi3/include/mpi3/operation.hpp | 222 - .../mpi_wrapper/mpi3/include/mpi3/ostream.hpp | 198 - .../mpi3/include/mpi3/package_archive.hpp | 451 -- .../mpi_wrapper/mpi3/include/mpi3/pointer.hpp | 205 - .../mpi_wrapper/mpi3/include/mpi3/port.hpp | 94 - .../mpi_wrapper/mpi3/include/mpi3/process.hpp | 120 - .../mpi_wrapper/mpi3/include/mpi3/request.hpp | 180 - .../mpi3/include/mpi3/rma/counter.hpp | 38 - .../mpi3/include/mpi3/rma/memory.hpp | 266 - .../mpi3/include/mpi3/rma/mutex.hpp | 40 - .../serialization_hack/archive_exception.cpp | 160 - .../mpi3/serialization_hack/basic_archive.cpp | 87 - .../serialization_hack/basic_iarchive.cpp | 602 -- .../serialization_hack/basic_iserializer.cpp | 35 - .../serialization_hack/basic_oarchive.cpp | 474 -- .../serialization_hack/basic_oserializer.cpp | 34 - .../serialization_hack/extended_type_info.cpp | 194 - .../extended_type_info_typeid.cpp | 168 - .../mpi3/serialization_hack/singleton.cpp | 38 - .../mpi3/include/mpi3/shared_communicator.hpp | 139 - .../mpi3/include/mpi3/shared_main.hpp | 39 - .../mpi3/include/mpi3/shared_mutex.hpp | 124 - .../mpi3/include/mpi3/shared_window.hpp | 110 - .../mpi3/include/mpi3/shm/allocator.hpp | 135 - .../mpi3/shm/managed_shared_memory.hpp | 187 - .../mpi3/include/mpi3/shm/memory.hpp | 162 - .../mpi3/include/mpi3/shm/mutex.hpp | 69 - .../mpi3/include/mpi3/shm/pool.hpp | 47 - .../mpi_wrapper/mpi3/include/mpi3/shm/ptr.hpp | 261 - .../mpi3/include/mpi3/shm/vector.hpp | 195 - .../mpi_wrapper/mpi3/include/mpi3/status.hpp | 74 - .../mpi3/include/mpi3/timed_terminate.hpp | 32 - .../mpi_wrapper/mpi3/include/mpi3/type.hpp | 522 -- .../mpi_wrapper/mpi3/include/mpi3/types.hpp | 13 - .../mpi_wrapper/mpi3/include/mpi3/vector.hpp | 123 - .../mpi_wrapper/mpi3/include/mpi3/version.hpp | 108 - .../mpi3/include/mpi3/wall_clock.hpp | 68 - .../mpi_wrapper/mpi3/include/mpi3/window.hpp | 301 - external_codes/mpi_wrapper/mpi3/pre-push | 17 - external_codes/mpi_wrapper/mpi3/pres/.gitkeep | 0 .../QMCPack_Alfredo_MPI3Wrapper_slides.pdf | Bin 138556 -> 0 bytes .../mpi_wrapper/mpi3/test/CMakeLists.txt | 272 - .../mpi_wrapper/mpi3/test/all_reduce.cpp | 114 - .../mpi3/test/async_interaction.cpp | 33 - .../mpi_wrapper/mpi3/test/broadcast.cpp | 32 - .../mpi_wrapper/mpi3/test/cartesian.cpp | 280 - .../mpi3/test/communicator_abort.cpp | 10 - .../mpi3/test/communicator_all_gather.cpp | 99 - .../mpi3/test/communicator_all_gatherv.cpp | 76 - ...mmunicator_all_gatherv_output_iterator.cpp | 29 - .../mpi3/test/communicator_all_to_all.cpp | 55 - .../mpi3/test/communicator_attributes.cpp | 37 - .../mpi3/test/communicator_barrier.cpp | 13 - .../mpi3/test/communicator_cctor.cpp | 76 - .../mpi3/test/communicator_create.hpp | 20 - .../test/communicator_custom_attributes.cpp | 57 - .../mpi3/test/communicator_divide.cpp | 20 - .../mpi3/test/communicator_error.cpp | 32 - .../mpi3/test/communicator_exception.cpp | 21 - .../mpi3/test/communicator_gather.cpp | 112 - .../mpi3/test/communicator_gather2.cpp | 29 - .../mpi3/test/communicator_grip_handle.cpp | 41 - .../mpi3/test/communicator_ibroadcast.cpp | 32 - .../mpi3/test/communicator_igather.cpp | 42 - .../mpi3/test/communicator_iprobe.cpp | 28 - .../mpi3/test/communicator_ireceive.cpp | 36 - .../mpi3/test/communicator_issend.cpp | 30 - .../mpi3/test/communicator_list.cpp | 37 - .../mpi3/test/communicator_main.cpp | 18 - .../test/communicator_main.cpp.openmpi.supp | 6558 ----------------- .../mpi3/test/communicator_mutable.cpp | 161 - .../mpi3/test/communicator_operator.cpp | 45 - .../mpi3/test/communicator_ostream.cpp | 54 - .../mpi3/test/communicator_pack.cpp | 65 - .../mpi3/test/communicator_passby.cpp | 35 - .../mpi3/test/communicator_reduce.cpp | 66 - .../test/communicator_reduce_in_place.cpp | 50 - .../mpi3/test/communicator_scatter.cpp | 100 - .../mpi3/test/communicator_scatterv.cpp | 47 - .../mpi3/test/communicator_send.cpp | 57 - .../mpi3/test/communicator_send_async.cpp | 55 - .../mpi3/test/communicator_send_class.cpp | 198 - .../communicator_send_class_nonintrusive.cpp | 92 - .../mpi3/test/communicator_send_receive.cpp | 102 - .../test/communicator_set_error_handler.cpp | 23 - .../mpi3/test/communicator_split.cpp | 30 - .../mpi_wrapper/mpi3/test/compare.cpp | 25 - .../mpi_wrapper/mpi3/test/datatype.cpp | 28 - .../mpi3/test/datatype_struct_vector3.cpp | 107 - .../mpi3/test/deino_all_to_all.cpp | 117 - .../mpi_wrapper/mpi3/test/deino_broadcast.cpp | 59 - .../mpi_wrapper/mpi3/test/deino_op_create.cpp | 60 - .../mpi_wrapper/mpi3/test/empty_main.cpp | 10 - .../mpi3/test/environment_self.cpp | 43 - .../mpi3/test/environment_thread.cpp | 15 - .../mpi_wrapper/mpi3/test/gather2.cpp | 23 - .../mpi_wrapper/mpi3/test/group.cpp | 37 - .../mpi3/test/group_operations.cpp | 63 - .../mpi_wrapper/mpi3/test/ibarrier.cpp | 21 - .../mpi_wrapper/mpi3/test/keyval.cpp | 133 - .../mpi_wrapper/mpi3/test/legacy_fftw3.cpp | 65 - .../mpi_wrapper/mpi3/test/library_check.cpp | 12 - .../mpi_wrapper/mpi3/test/library_main.cpp | 35 - .../mpi_wrapper/mpi3/test/minimal.cpp | 26 - .../mpi_wrapper/mpi3/test/multi_array.cpp | 94 - .../mpi_wrapper/mpi3/test/package.cpp | 45 - .../mpi3/test/parallel_dot_product.dat | 12 - .../mpi_wrapper/mpi3/test/process.cpp | 159 - .../mpi_wrapper/mpi3/test/process_vector.cpp | 47 - .../mpi_wrapper/mpi3/test/reduce_maxloc.cpp | 44 - .../mpi3/test/request_test_some.cpp | 38 - .../mpi_wrapper/mpi3/test/request_wait.cpp | 27 - .../mpi3/test/request_wait_all.cpp | 25 - .../mpi3/test/request_wait_any.cpp | 27 - .../mpi3/test/request_wait_some.cpp | 32 - external_codes/mpi_wrapper/mpi3/test/ring.cpp | 27 - .../mpi_wrapper/mpi3/test/send_complex.cpp | 45 - .../mpi3/test/serialization_intrusive.cpp | 80 - .../mpi3/test/shared_communicator.cpp | 88 - .../mpi_wrapper/mpi3/test/shared_mutex.cpp | 11 - .../mpi_wrapper/mpi3/test/shared_window.cpp | 43 - .../mpi_wrapper/mpi3/test/shared_window2.cpp | 74 - .../mpi_wrapper/mpi3/test/shared_window3.c | 62 - .../mpi_wrapper/mpi3/test/shared_window3.cpp | 73 - .../mpi_wrapper/mpi3/test/shm_allocator.cpp | 37 - .../mpi3/test/shm_mapped_region.cpp | 32 - .../mpi_wrapper/mpi3/test/shm_multi_array.cpp | 97 - .../mpi_wrapper/mpi3/test/shm_vector.cpp | 77 - .../mpi3/test/simple_broadcast.cpp | 59 - .../mpi3/test/simple_send_receive.cpp | 51 - .../mpi_wrapper/mpi3/test/spinor.cpp | 50 - .../mpi_wrapper/mpi3/test/status.cpp | 41 - external_codes/mpi_wrapper/mpi3/test/test.cpp | 38 - .../mpi_wrapper/mpi3/test/test_some.cpp | 38 - .../mpi_wrapper/mpi3/test/type_commit.cpp | 55 - .../mpi_wrapper/mpi3/test/type_size.cpp | 80 - .../mpi_wrapper/mpi3/test/uniform_abort.cpp | 231 - .../mpi_wrapper/mpi3/test/wall_time.cpp | 26 - .../mpi3/test/window_accumulate.cpp | 47 - .../mpi3/test/window_blocking_put.cpp | 42 - .../mpi_wrapper/mpi3/test/window_group.cpp | 27 - .../mpi_wrapper/mpi3/test/window_put_get.cpp | 33 - 260 files changed, 34669 deletions(-) delete mode 100644 external_codes/mpi_wrapper/mpi3/.clang-format delete mode 100644 external_codes/mpi_wrapper/mpi3/.clang-tidy delete mode 100644 external_codes/mpi_wrapper/mpi3/.cppcheck-suppressions delete mode 100644 external_codes/mpi_wrapper/mpi3/.deepsource.toml delete mode 100644 external_codes/mpi_wrapper/mpi3/.gitignore delete mode 100644 external_codes/mpi_wrapper/mpi3/.gitlab-ci.yml delete mode 100644 external_codes/mpi_wrapper/mpi3/.gitmodules delete mode 100644 external_codes/mpi_wrapper/mpi3/CMakeLists.txt delete mode 100644 external_codes/mpi_wrapper/mpi3/LICENSE delete mode 100644 external_codes/mpi_wrapper/mpi3/README.md delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/01-hello.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/02-send.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/03-isend.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/04-array.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/05-array_mult.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/average.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/broadcast.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/calculate_pi.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/groups.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/hello.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/hello_world.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory_0.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/pi_reduce.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/ping_pong.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/random_walk.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/rank.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/reduce_average.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/ring.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/send_receive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/shared_window.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/shared_window_2.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/shared_window_3.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/shm_vector.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/split.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/test3.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/test4.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/examples/test5.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Barrier.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_call_errhandler.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_compare.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_dup.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_free.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_rank.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_set_errhandler.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Gather.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Gatherv.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Recv.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Reduce.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Send.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Sendrecv_replace.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_contiguos.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_dup.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_get_extent.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_size.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_vector.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtick.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtime.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/Makefile delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/examples/abort.cpp delete mode 100755 external_codes/mpi_wrapper/mpi3/fake/examples/testme delete mode 100755 external_codes/mpi_wrapper/mpi3/fake/load delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/mpi.h delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/mpi_qmcpack_compile.h delete mode 100755 external_codes/mpi_wrapper/mpi3/fake/mpic++ delete mode 100755 external_codes/mpi_wrapper/mpi3/fake/mpicc delete mode 100755 external_codes/mpi_wrapper/mpi3/fake/mpirun delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/test/CMakeLists.txt delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/test/test_c_mpi_init.c delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/test/test_mpi_init.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/fake/test/test_wrapper_mpi_init.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/FILE.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/address.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/allocator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/buffer.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/cartesian_communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/communication_mode.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operatorators.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operators.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_fwd.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_iterator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/config/NODISCARD.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/core.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/basic_communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/call.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/datatype.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/equality.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator_traits.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/just.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/package.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/tuple_offset.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/detail/value_traits.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/dummy/.dummy delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/dynamic_window.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/environment.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/error.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/error_handler.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/exception.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/future.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/generalized_request.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/graph_communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/group.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/handle.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/info.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/main.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/main_environment.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/match.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/message.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/mutex.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_datatype.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_reduction.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/CMakeLists.txt delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/nccl_constructor.cu delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/universal_communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/operation.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/ostream.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/package_archive.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/pointer.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/port.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/process.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/request.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/rma/counter.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/rma/memory.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/rma/mutex.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/archive_exception.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_archive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iarchive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iserializer.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oarchive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oserializer.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info_typeid.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/singleton.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shared_communicator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shared_main.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shared_mutex.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shared_window.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/allocator.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/managed_shared_memory.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/memory.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/mutex.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/pool.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/ptr.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/shm/vector.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/status.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/timed_terminate.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/type.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/types.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/vector.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/version.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/wall_clock.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/include/mpi3/window.hpp delete mode 100755 external_codes/mpi_wrapper/mpi3/pre-push delete mode 100644 external_codes/mpi_wrapper/mpi3/pres/.gitkeep delete mode 100644 external_codes/mpi_wrapper/mpi3/pres/QMCPack_Alfredo_MPI3Wrapper_slides.pdf delete mode 100644 external_codes/mpi_wrapper/mpi3/test/CMakeLists.txt delete mode 100644 external_codes/mpi_wrapper/mpi3/test/all_reduce.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/async_interaction.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/broadcast.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/cartesian.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_abort.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_all_gather.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv_output_iterator.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_all_to_all.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_attributes.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_barrier.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_cctor.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_create.hpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_custom_attributes.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_divide.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_error.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_exception.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_gather.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_gather2.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_grip_handle.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_ibroadcast.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_igather.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_iprobe.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_ireceive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_issend.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_list.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp.openmpi.supp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_mutable.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_operator.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_ostream.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_pack.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_passby.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_reduce.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_reduce_in_place.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_scatter.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_scatterv.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_send.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_send_async.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_send_class.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_send_class_nonintrusive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_send_receive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_set_error_handler.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/communicator_split.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/compare.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/datatype.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/datatype_struct_vector3.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/deino_all_to_all.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/deino_broadcast.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/deino_op_create.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/empty_main.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/environment_self.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/environment_thread.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/gather2.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/group.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/group_operations.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/ibarrier.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/keyval.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/legacy_fftw3.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/library_check.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/library_main.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/minimal.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/multi_array.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/package.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/parallel_dot_product.dat delete mode 100644 external_codes/mpi_wrapper/mpi3/test/process.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/process_vector.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/reduce_maxloc.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/request_test_some.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/request_wait.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/request_wait_all.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/request_wait_any.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/request_wait_some.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/ring.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/send_complex.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/serialization_intrusive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shared_communicator.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shared_mutex.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shared_window.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shared_window2.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shared_window3.c delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shared_window3.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shm_allocator.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shm_mapped_region.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shm_multi_array.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/shm_vector.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/simple_broadcast.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/simple_send_receive.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/spinor.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/status.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/test.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/test_some.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/type_commit.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/type_size.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/uniform_abort.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/wall_time.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/window_accumulate.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/window_blocking_put.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/window_group.cpp delete mode 100644 external_codes/mpi_wrapper/mpi3/test/window_put_get.cpp diff --git a/external_codes/mpi_wrapper/mpi3/.clang-format b/external_codes/mpi_wrapper/mpi3/.clang-format deleted file mode 100644 index 02fed447340..00000000000 --- a/external_codes/mpi_wrapper/mpi3/.clang-format +++ /dev/null @@ -1,233 +0,0 @@ ---- -Language: Cpp -# BasedOnStyle: Google -#AccessModifierOffset: -1 -AlignAfterOpenBracket: BlockIndent # Align -AlignArrayOfStructures: Right -#AlignConsecutiveMacros: None -AlignConsecutiveAssignments: Consecutive # None -#AlignConsecutiveBitFields: None -AlignConsecutiveDeclarations: Consecutive -#AlignEscapedNewlines: Left -AlignOperands: AlignAfterOperator -AlignTrailingComments: false -#AllowAllArgumentsOnNextLine: true -#AllowAllParametersOfDeclarationOnNextLine: true -#AllowShortEnumsOnASingleLine: true -#AllowShortBlocksOnASingleLine: Never -AllowShortCaseLabelsOnASingleLine: true # false -#AllowShortFunctionsOnASingleLine: All -#AllowShortLambdasOnASingleLine: All -AllowShortIfStatementsOnASingleLine: AllIfsAndElse -#AllowShortLoopsOnASingleLine: true -#AlwaysBreakAfterDefinitionReturnType: None -#AlwaysBreakAfterReturnType: None -#AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: No # Yes -#AttributeMacros: -# - __capability -#BinPackArguments: true -#BinPackParameters: true -BraceWrapping: -# AfterCaseLabel: false -# AfterClass: false -# AfterControlStatement: Never -# AfterEnum: false -# AfterFunction: false -# AfterNamespace: false -# AfterObjCDeclaration: false -# AfterStruct: false -# AfterUnion: false -# AfterExternBlock: false -# BeforeCatch: false -# BeforeElse: false -# BeforeLambdaBody: false -# BeforeWhile: false -# IndentBraces: false - SplitEmptyFunction: false # true -# SplitEmptyRecord: true -# SplitEmptyNamespace: true -#BreakBeforeBinaryOperators: None -#BreakBeforeConceptDeclarations: true -#BreakBeforeBraces: Attach -#BreakBeforeInheritanceComma: false -BreakInheritanceList: BeforeComma -#BreakBeforeTernaryOperators: true -#BreakConstructorInitializersBeforeComma: false -#BreakConstructorInitializers: BeforeColon -#BreakAfterJavaFieldAnnotations: false -#BreakStringLiterals: true -ColumnLimit: 0 -#CommentPragmas: '^ IWYU pragma:' -QualifierAlignment: Right # Leave -#CompactNamespaces: false -ConstructorInitializerIndentWidth: 0 -ContinuationIndentWidth: 99 -#Cpp11BracedListStyle: true -#DeriveLineEnding: true -#DerivePointerAlignment: true -#DisableFormat: false -#EmptyLineAfterAccessModifier: Never -#EmptyLineBeforeAccessModifier: LogicalBlock -#ExperimentalAutoDetectBinPacking: false -#PackConstructorInitializers: NextLine -#BasedOnStyle: '' -#ConstructorInitializerAllOnOneLineOrOnePerLine: false -#AllowAllConstructorInitializersOnNextLine: true -FixNamespaceComments: true -#ForEachMacros: -# - foreach -# - Q_FOREACH -# - BOOST_FOREACH -#IfMacros: -# - KJ_IF_MAYBE -#IncludeBlocks: Regroup -#IncludeCategories: -# - Regex: '^' -# Priority: 2 -# SortPriority: 0 -# CaseSensitive: false -# - Regex: '^<.*\.h>' -# Priority: 1 -# SortPriority: 0 -# CaseSensitive: false -# - Regex: '^<.*' -# Priority: 2 -# SortPriority: 0 -# CaseSensitive: false -# - Regex: '.*' -# Priority: 3 -# SortPriority: 0 -# CaseSensitive: false -#IncludeIsMainRegex: '([-_](test|unittest))?$' -#IncludeIsMainSourceRegex: '' -#IndentAccessModifiers: true # false -AccessModifierOffset: -98 # 2 -#IndentCaseLabels: true -#IndentCaseBlocks: false -#IndentGotoLabels: true -#IndentPPDirectives: None -#IndentExternBlock: AfterExternBlock -#IndentRequires: false -IndentWidth: 99 -#IndentWrappedFunctionNames: false -#InsertTrailingCommas: None -#InsertBraces: true # clang format 15 -#JavaScriptQuotes: Leave -#JavaScriptWrapImports: true -#KeepEmptyLinesAtTheStartOfBlocks: false -#LambdaBodyIndentation: Signature -#MacroBlockBegin: '' -#MacroBlockEnd: '' -#MaxEmptyLinesToKeep: 1 -#NamespaceIndentation: None -#ObjCBinPackProtocolList: Never -#ObjCBlockIndentWidth: 2 -#ObjCBreakBeforeNestedBlockParam: true -#ObjCSpaceAfterProperty: false -#ObjCSpaceBeforeProtocolList: true -#PenaltyBreakAssignment: 2 -#PenaltyBreakBeforeFirstCallParameter: 1 -#PenaltyBreakComment: 300 -#PenaltyBreakFirstLessLess: 120 -#PenaltyBreakOpenParenthesis: 0 -#PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -#PenaltyExcessCharacter: 1000000 -#PenaltyReturnTypeOnItsOwnLine: 200 -#PenaltyIndentedWhitespace: 0 -PointerAlignment: Left -#PPIndentWidth: -1 -#RawStringFormats: -# - Language: Cpp -# Delimiters: -# - cc -# - CC -# - cpp -# - Cpp -# - CPP -# - 'c++' -# - 'C++' -# CanonicalDelimiter: '' -# BasedOnStyle: google -# - Language: TextProto -# Delimiters: -# - pb -# - PB -# - proto -# - PROTO -# EnclosingFunctions: -# - EqualsProto -# - EquivToProto -# - PARSE_PARTIAL_TEXT_PROTO -# - PARSE_TEST_PROTO -# - PARSE_TEXT_PROTO -# - ParseTextOrDie -# - ParseTextProtoOrDie -# - ParseTestProto -# - ParsePartialTestProto -# CanonicalDelimiter: pb -# BasedOnStyle: google -#ReferenceAlignment: Pointer -#ReflowComments: true -#RemoveBracesLLVM: false -#SeparateDefinitionBlocks: Leave -#ShortNamespaceLines: 1 -#SortIncludes: CaseSensitive -#SortJavaStaticImport: Before -#SortUsingDeclarations: true -#SpaceAfterCStyleCast: false -#SpaceAfterLogicalNot: false -SpaceAfterTemplateKeyword: false # true -#SpaceBeforeAssignmentOperators: true -#SpaceBeforeCaseColon: false -#SpaceBeforeCpp11BracedList: false -#SpaceBeforeCtorInitializerColon: true -#SpaceBeforeInheritanceColon: true -SpaceBeforeParens: Custom # ControlStatements -SpaceBeforeParensOptions: - AfterControlStatements: false # true -# AfterForeachMacros: true -# AfterFunctionDefinitionName: false -# AfterFunctionDeclarationName: false -# AfterIfMacros: true -# AfterOverloadedOperator: false -# BeforeNonEmptyParentheses: false -#SpaceAroundPointerQualifiers: Default -#SpaceBeforeRangeBasedForLoopColon: true -#SpaceInEmptyBlock: false -#SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 -#SpacesInAngles: Never -#SpacesInConditionalStatement: false -#SpacesInContainerLiterals: true -#SpacesInCStyleCastParentheses: false -SpacesInLineCommentPrefix: - Minimum: 1 -# Maximum: -1 -#SpacesInParentheses: false -#SpacesInSquareBrackets: false -#SpaceBeforeSquareBrackets: false -#BitFieldColonSpacing: Both -Standard: c++17 -#StatementAttributeLikeMacros: -# - Q_EMIT -#StatementMacros: -# - Q_UNUSED -# - QT_REQUIRE_VERSION -TabWidth: 99 -#UseCRLF: false -UseTab: ForContinuationAndIndentation # Never -WhitespaceSensitiveMacros: - - BOOST_REQUIRE - - BOOST_TEST - - BOOST_TEST_REQUIRE - - assert - - MPI_ -#WhitespaceSensitiveMacros: -# - STRINGIZE -# - PP_STRINGIZE -# - BOOST_PP_STRINGIZE -# - NS_SWIFT_NAME -# - CF_SWIFT_NAME -... diff --git a/external_codes/mpi_wrapper/mpi3/.clang-tidy b/external_codes/mpi_wrapper/mpi3/.clang-tidy deleted file mode 100644 index bb16502d09f..00000000000 --- a/external_codes/mpi_wrapper/mpi3/.clang-tidy +++ /dev/null @@ -1,32 +0,0 @@ -# -*-indent-tabs-mode:nil;c-basic-offset:2;tab-width:2;autowrap:nil;-*- ---- -Checks: '*, - -altera-struct-pack-align, - -cppcoreguidelines-avoid-magic-numbers, - -cppcoreguidelines-pro-bounds-array-to-pointer-decay, - -fuchsia-default-arguments-declarations, - -fuchsia-default-arguments-calls, - -fuchsia-overloaded-operator, - -fuchsia-trailing-return, - -google-runtime-references, - -hicpp-no-array-decay, - -llvmlibc-callee-namespace, - -llvm-header-guard, - -llvmlibc-implementation-in-namespace, - -llvmlibc-restrict-system-libc-headers, - -modernize-use-trailing-return-type, - -modernize-concat-nested-namespaces, - -modernize-use-nodiscard, - -readability-identifier-length, - -readability-magic-numbers -' -CheckOptions: - - { key: readability-identifier-naming.NamespaceCase, value: lower_case } - - { key: readability-identifier-naming.ClassCase, value: lower_case } - - { key: readability-identifier-naming.PrivateMemberSufix, value: _ } - - { key: readability-identifier-naming.StructCase, value: lower_case } - - { key: readability-identifier-naming.FunctionCase, value: lower_case } - - { key: readability-identifier-naming.TemplateParameterCase, value: CamelCase } -WarningsAsErrors: '*' -HeaderFilterRegex: '.' -FormatStyle: file diff --git a/external_codes/mpi_wrapper/mpi3/.cppcheck-suppressions b/external_codes/mpi_wrapper/mpi3/.cppcheck-suppressions deleted file mode 100644 index 0f9cf7386cd..00000000000 --- a/external_codes/mpi_wrapper/mpi3/.cppcheck-suppressions +++ /dev/null @@ -1,5 +0,0 @@ -#syntaxError -#missingInclude -missingIncludeSystem -#unmatchedSuppression -#preprocessorErrorDirective diff --git a/external_codes/mpi_wrapper/mpi3/.deepsource.toml b/external_codes/mpi_wrapper/mpi3/.deepsource.toml deleted file mode 100644 index 0ea2ab62c10..00000000000 --- a/external_codes/mpi_wrapper/mpi3/.deepsource.toml +++ /dev/null @@ -1,9 +0,0 @@ -version = 1 - -[[analyzers]] -name = "shell" -enabled = true - -[[analyzers]] -name = "test-coverage" -enabled = true \ No newline at end of file diff --git a/external_codes/mpi_wrapper/mpi3/.gitignore b/external_codes/mpi_wrapper/mpi3/.gitignore deleted file mode 100644 index 95c242d2c8f..00000000000 --- a/external_codes/mpi_wrapper/mpi3/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -build* -.build* -.vscode - - diff --git a/external_codes/mpi_wrapper/mpi3/.gitlab-ci.yml b/external_codes/mpi_wrapper/mpi3/.gitlab-ci.yml deleted file mode 100644 index 4f77c2d2707..00000000000 --- a/external_codes/mpi_wrapper/mpi3/.gitlab-ci.yml +++ /dev/null @@ -1,239 +0,0 @@ -# -*-indent-tabs-mode:nil;c-basic-offset:2;tab-width:4;-*- -# Copyright 2020-2023 Alfredo A. Correa - -image: correaadock/gnudev:v2 - -variables: - GIT_SUBMODULE_STRATEGY: recursive - -openmpi: - stage: build - script: - - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make g++ git libboost-serialization-dev - - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 - - cd test - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="--oversubscribe" - - cmake .. -DCMAKE_BUILD_TYPE=Debug - - cmake --build . --parallel 2 || make VERBOSE=1 - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - -icpc-intelmpi: - stage: build - image: intel/oneapi-hpckit:latest - allow_failure: true - script: - - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl g++ git make libboost-test-dev libboost-serialization-dev - - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 - - cd test - - mkdir build && cd build - - icpc --version - - CXX=icpc CXXFLAGS="-diag-disable=593,2196,1786,1478" cmake .. -DCMAKE_BUILD_TYPE=Debug #https://community.intel.com/t5/Intel-C-Compiler/Inline-and-no-inline-warning/td-p/1216764 - - cmake --build . --parallel 2 || cmake --build . --verbose - - ctest --output-on-failure - needs: ["openmpi"] - -icpx-intelmpi: - stage: build - image: intel/oneapi-hpckit:latest - allow_failure: true - script: - - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl g++ git make libboost-test-dev libboost-serialization-dev - - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 - - cd test - - mkdir build && cd build - - icpx --version - - CXX=icpx CXXFLAGS="-O1" cmake .. -DCMAKE_BUILD_TYPE=Debug - - cmake --build . --parallel 2 || make VERBOSE=1 - - ctest --output-on-failure - needs: ["openmpi"] - -openmpi-clang: - stage: build - script: - - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang g++ git libstdc++-12-dev libboost-serialization-dev - - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 - - cd test - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="--oversubscribe" - - clang++ --version - - mpirun --version - - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug - - cmake --build . --parallel 2 || make VERBOSE=1 - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - needs: ["openmpi"] - -openmpi-clang20: - stage: build - script: - - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang git libstdc++-12-dev libboost-serialization-dev - - cd test - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="--oversubscribe" - - clang++ --version - - mpirun --version - - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 - - cmake --build . --parallel 2 || make VERBOSE=1 - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - needs: ["openmpi-clang"] - -openmpi-clang-tidy: - stage: build - script: - - apt update -qq && apt install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin clang libstdc++-12-dev clang-tidy cmake git make libboost-serialization-dev - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="--oversubscribe" - - clang++ --version - - clang-tidy --version - - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_CLANG_TIDY="clang-tidy" - - make --jobs=2 || make VERBOSE=1 - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - needs: ["openmpi-clang"] - -openmpi-cppcheck: - stage: build - script: - - apt update -qq && apt install -qq -y --no-install-recommends libopenmpi-dev openmpi-bin g++ libstdc++-12-dev ca-certificates cmake cppcheck git make libboost-serialization-dev - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="--oversubscribe" - - g++ --version - - cppcheck --version - - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_CPPCHECK="cppcheck;--force;--enable=all;--inline-suppr;--language=c++;--suppress=missingIncludeSystem;--suppress=syntaxError;--suppress=unmatchedSuppression;--std=c++17;--error-exitcode=666;-UEXCLUDE_CPPCHECK" - - make --jobs=2 || make VERBOSE=1 - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - needs: ["openmpi"] - -mpich-debug: - stage: build - script: - - apt update -qq && apt install -qq -y --no-install-recommends libmpich-dev mpich - - cd test - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="" - - cmake .. -DCMAKE_BUILD_TYPE=Debug - - cmake --build . --parallel 2 || cmake --build . --verbose - - ctest --output-on-failure - -mpich-valgrind: - stage: build - allow_failure: true - script: - - apt update -qq && apt-get install -qq -y --no-install-recommends libmpich-dev mpich - - mpirun --version - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="" - - export VALGRIND_EXE="valgrind --leak-check=full --track-origins=yes --show-leak-kinds=all --suppressions=.valgrind_suppressions --gen-suppressions=all --error-exitcode=1 " - - cmake .. -DCMAKE_BUILD_TYPE=Debug - - make --jobs=2 || make VERBOSE=1 - - ulimit -n # reports current value - - ulimit -n 1024 # workaround neededed by valgrind in docker running in Fedora 37 - - ctest --output-on-failure - needs: ["mpich-debug"] - -qmcpack-openmpi: - stage: test - script: - - apt-get -qq update && apt-get -qq install --no-install-recommends -y libblas-dev liblapack-dev libfftw3-dev libboost-serialization-dev libopenmpi-dev gfortran g++ cmake make git ca-certificates numdiff python3 python3-numpy python3-h5py python3-mpi4py python3-scipy libxml2-dev libhdf5-dev - - git clone https://github.com/QMCPACK/qmcpack.git - - cd qmcpack - - git config --global user.email "alfredo.correa@gmail.com" && git config --global user.name "Alfredo Correa" - - git rm -r external_codes/mpi_wrapper/mpi3 && git commit -m "remove mpi3 subtree" - - git subtree add --squash -P external_codes/mpi_wrapper/mpi3 https://gitlab.com/correaa/boost-mpi3.git $CI_COMMIT_BRANCH - - cd build - - cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DBUILD_PPCONVERT=1 -DQMC_MIXED_PRECISION=1 -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS="--allow-run-as-root;--bind-to;none" .. #-DCMAKE_CXX_FLAGS="-Werror" - - make --jobs=2 || make VERBOSE=1 # afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R afqmc --output-on-failure - needs: ["openmpi"] - -qmcpack-cuda-runner: - allow_failure: true - image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 - tags: - - nvidia-docker - stage: test - script: - - apt-get -qq update && apt-get -qq install --no-install-recommends -y libblas-dev liblapack-dev libfftw3-dev libboost-serialization-dev libopenmpi-dev gfortran g++ cmake make git ca-certificates numdiff python3 python3-numpy python3-h5py python3-mpi4py python3-scipy libxml2-dev libhdf5-dev - - cmake --version - - git clone --depth=1 https://github.com/QMCPACK/qmcpack.git - - cd qmcpack - - git config --global user.email "alfredo.correa@gmail.com" && git config --global user.name "Alfredo Correa" - - git rm -r external_codes/mpi3 && git commit -m "remove mpi3 subtree" - - git subtree add --squash -P external_codes/mpi3 $CI_REPOSITORY_URL $CI_COMMIT_BRANCH # e.g. https://gitlab.com/correaa/boost-multi.git - - cd ../qmcpack - - cd build - - CUDACXX=/usr/local/cuda/bin/nvcc cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DQMC_CXX_STANDARD=17 -DENABLE_CUDA=1 -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_HOST_COMPILER=g++ -DCMAKE_CXX_FLAGS="-Wno-deprecated -Wno-deprecated-declarations" .. - - make -j4 afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R afqmc --output-on-failure - needs: ["openmpi-cuda-11", "qmcpack-openmpi"] - -inq-openmpi: - stage: test - tags: - - cpu - script: - - apt-get update && apt-get install --no-install-recommends -y --quiet libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-serialization-dev libopenmpi-dev libhdf5-dev gfortran g++ cmake pkg-config make git ca-certificates wget - - cmake --version - - git clone https://gitlab.com/npneq/inq.git --recurse-submodules - - cd inq - - cd external_libs/mpi3 - - git checkout $CI_COMMIT_BRANCH - - cd ../.. - - mkdir build && cd build - - ../configure --prefix=$HOME --disable-debug - - make --jobs=2 || make VERBOSE=1 - - make install - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - needs: ["openmpi"] - -# - export DEBIAN_FRONTEND=noninteractive -# - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates git gnupg software-properties-common -# - apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/3bf863cc.pub -# - add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/ /" -# - add-apt-repository contrib -# - apt-get update -# - apt-get -y install cuda - -# TODO(correaa) -> openmpi-cuda-11 -openmpi-cuda-11: - stage: build - allow_failure: false - image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 - script: - - apt-get update && apt-get install --no-install-recommends -y cmake libboost-test-dev libboost-serialization-dev libopenmpi-dev make git - - /usr/local/cuda-11/bin/nvcc --version - - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 - - cd test - - mkdir build && cd build - - export MPI_OVERSUBSCRIBE="--oversubscribe" - - cmake .. -DCMAKE_CUDA_COMPILER=/usr/local/cuda-11/bin/nvcc -DCMAKE_BUILD_TYPE=Debug -DENABLE_CUDA=1 - - cmake --build . --parallel 2 || cmake --build . --verbose - - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure - needs: ["openmpi"] - -# - wget https://cmake.org/files/v3.21/cmake-3.21.3-linux-x86_64.sh --no-verbose # following https://askubuntu.com/a/865294/15943 -# - mkdir /opt/cmake -# - sh cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/opt/cmake -# - ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake - -inq-cuda-11-openmpi-compileonly: - stage: build - allow_failure: true - image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 - tags: - - nvidia-docker - script: - - apt-get update && apt-get install --no-install-recommends -y --quiet cmake libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-serialization-dev libopenmpi-dev libhdf5-dev gfortran g++ pkg-config make git ca-certificates wget - - cmake --version - - git clone https://gitlab.com/npneq/inq.git --recurse-submodules - - cd inq - - cd external_libs/mpi3 - - git checkout $CI_COMMIT_BRANCH - - cd ../.. - - mkdir build && cd build - - /usr/local/cuda-11/bin/nvcc -V - - CUDA_ARCH_OVERRIDE=1 ../configure --prefix=$HOME --enable-cuda --with-cuda-prefix=/usr/local/cuda --pass-thru -DCMAKE_CUDA_COMPILER=/usr/local/cuda-11/bin/nvcc -DCMAKE_CUDA_ARCHITECTURES=61 - - make silicon --jobs=2 - - make install - - ctest -R silicon - needs: - - openmpi-cuda-11 diff --git a/external_codes/mpi_wrapper/mpi3/.gitmodules b/external_codes/mpi_wrapper/mpi3/.gitmodules deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/external_codes/mpi_wrapper/mpi3/CMakeLists.txt b/external_codes/mpi_wrapper/mpi3/CMakeLists.txt deleted file mode 100644 index 74f0a7cbd57..00000000000 --- a/external_codes/mpi_wrapper/mpi3/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -# -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -cmake_minimum_required(VERSION 3.16) - -project( - mpi3 - VERSION 0.79.0 - DESCRIPTION "B-MPI3 is a C++ library wrapper for version 3.1 of the MPI standard interface that simplifies the utilization and maintenance of MPI code." - HOMEPAGE_URL "https://gitlab.com/correaa/boost-mpi3" - LANGUAGES CXX -) - -include(GNUInstallDirs) - -add_library(${PROJECT_NAME} INTERFACE) - -target_include_directories(${PROJECT_NAME} INTERFACE $ $ $) - -target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_17) -#set_target_properties(${PROJECT_NAME} PROPERTIES CXX_EXTENSIONS OFF) - -# this makes CM FetchContent friendly https://www.foonathan.net/2022/06/cmake-fetchcontent/ -if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) - return() -endif() - -include(CTest) - -enable_testing() - -add_subdirectory(test) diff --git a/external_codes/mpi_wrapper/mpi3/LICENSE b/external_codes/mpi_wrapper/mpi3/LICENSE deleted file mode 100644 index 36b7cd93cdf..00000000000 --- a/external_codes/mpi_wrapper/mpi3/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/external_codes/mpi_wrapper/mpi3/README.md b/external_codes/mpi_wrapper/mpi3/README.md deleted file mode 100644 index e949010eaaa..00000000000 --- a/external_codes/mpi_wrapper/mpi3/README.md +++ /dev/null @@ -1,968 +0,0 @@ - -[comment]: # (Comment) - -# B.MPI3 -*Alfredo A. Correa* - - -[//]: <> () - -B-MPI3 is a C++ library wrapper for version 3.1 of the MPI standard interface that simplifies the utilization and maintenance of MPI code. -B-MPI3 C++ aims to provide a more convenient, powerful and an interface less prone to errors than the standard C-based MPI interface. - -B-MPI3 simplifies the utilization of MPI without completely changing the communication model, allowing for a seamless transition from C-MPI. -B-MPI3 also provides allocators and facilities to manipulate MPI-mediated Remote Access and shared memory. - -For example, pointers are not utilized directly and it is replaced by an iterator-based interface and most data, in particular custom type objects are serialized automatically into messages by the library. -B-MPI3 interacts well with the C++ standard library, containers and custom data types (classes). - -B.MPI3 is written from [scratch](https://octo-repo-visualization.vercel.app/?repo=llnl%2Fb-mpi3) in C++17 and it has been tested with many MPI library implementations and compilers, OpenMPI +1.9, MPICH +3.2.1, MVAPICH or Spectrum MPI, using the following compilers gcc +5.4.1, clang +6.0, PGI 18.04. -(Any standard compliant MPI library can be used.) - -B.MPI3 is not an official Boost library, but is designed following the principles of Boost and the STL. -B.MPI3 is not a derivative of Boost.MPI and it is unrelated to the, [now deprecated](https://web.archive.org/web/20170421220544/http://blogs.cisco.com/performance/the-mpi-c-bindings-what-happened-and-why/), official MPI-C++ interface. -It adds features which were missing in Boost.MPI (which only covers MPI-1), with an iterator-based interface and MPI-3 features (RMA and Shared memory). - -B.MPI3 optionally depends on Boost +1.53 for automatic serialization. - -## Contents -[[_TOC_]] - -## Introduction - -MPI is a large library for run-time parallelism where several paradigms coexist. -It was is originally designed as standardized and portable message-passing system to work on a wide variety of parallel computing architectures. - -The last standard, MPI-3, uses a combination of techniques to achieve parallelism, Message Passing (MP), (Remote Memory Access (RMA) and Shared Memory (SM). -We try here to give a uniform interface and abstractions for these features by means of wrapper function calls and concepts brought familiar to C++ and the STL. - -## Motivation: The problem with the standard interface - -A typical C-call for MP looks like this, - -```cpp -int status_send = MPI_Send(&numbers, 10, MPI_INT, 1, 0, MPI_COMM_WORLD); -assert(status_send == MPI_SUCCESS); -... // concurrently with -int status_recv = MPI_Recv(&numbers, 10, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); -assert(status_recv == MPI_SUCCESS); -``` - -In principle this call can be made from a C++ program. -However there are obvious drawbacks from using this standard interface. - -Here we enumerate some of problems, - -* Function calls have many arguments (e.g. 6 or 7 arguments in average) -* Many mandatory arguments are redundant or could easily have a default natural value (e.g. message tags are not always necessary). -* Use of raw pointers and sizes, (e.g. `&number` and `1`) -* Data argument are type-erased into `void*`. -* Only primitive types (e.g. `MPI_INT`) can be passed. -* Consistency between pointer types and data-types is responsibility of the user. -* Only contiguous memory blocks can be used with this interface. -* Error codes are stored and had to be checked after each function call. -* Use of handles (such as `MPI_COMM_WORLD`), handles do not have a well defined semantics. - -A call of this type would be an improvement: - -```cpp -world.send(numbers.begin(), numbers.end(), 1); -... // concurrently with -world.receive(numbers.begin(), numbers.end(), 0); -``` - -For other examples, see here: [http://mpitutorial.com/tutorials/mpi-send-and-receive/](http://mpitutorial.com/tutorials/mpi-send-and-receive/) - -MPI used to ship with a C++-style interfaces. -It turns out that this interface was a very minimal change over the C version, and for good reasons it was dropped. - -The B.MPI3 library was designed to use simultaneously (interleaved) with the standard C interface of MPI. -In this way, changes to existing code can be made incrementally. - -## Installation - -The library is "header-only"; no separate compilation is necessary. -In order to compile it requires an MPI distribution (e.g. OpenMPI or MPICH2) and the corresponding compiler-wrapper (`mpic++` or `mpicxx`). -This library requires C++14 and the Boost library installed. -A typical compilation/run command looks like this: - -```bash -$ mpic++ -std=c++14 -O3 mpi3/test/communicator_send.cpp -o communicator_send.x -lboost_serialization -$ mpirun -n 8 ./communicator_send.x -``` - -In a system such as Red Hat, the dependencies can by installed by - -```bash -dnf install gcc-c++ boost-devel openmpi-devel mpich-devel -``` - -Some systems require loading the MPI module before compiling and using MPI programs, `module load mpi/mpich`. - -The library is tested frequently against `openmpi` and `mpich`, and less frequently with `mvapich2`. - -## Testing - -The library has a basic `ctest` based testing system. - -```bash -# module load mpi/mpich # or mpi/openmpi , needed in systems like Fedora -cd mpi3/test -mkdir build && cd build -cmake .. -cmake --build .. -ctest -``` - -## Initialization - -Like MPI, B.MPI3 requires some global library initialization. -The library includes a convenience `mpi3/main.hpp` which wraps around this initialization steps and *simulates* a main function. -In this way, a parallel program looks very much like normal programs, except that the main function has a third argument with the default global communicator passed in. - -```cpp -#include "mpi3/version.hpp" -#include "mpi3/main.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - if(world.rank() == 0) cout << mpi3::version() << '\n'; - return 0; -} -``` - -Here `world` is a communicator object that is a wrapper over MPI communicator handle. - -Changing the `main` program to this syntax in existing code can be too intrusive. -For this reason a more traditional initialization is also possible. -The alternative initialization is done by instantiating the `mpi3::environment` object (from with the global communicator `.world()` is extracted). - -```cpp -#include "mpi3/environment.hpp" -int main(int argc, char** argv){ - mpi3::environment env(argc, argv); - auto world = env.world(); // communicator is extracted from the environment - // ... code here - return 0; -} -``` - -## Communicators - -In the last example, `world` is a global communicator (not necessarily the same as `MPI_COMM_WORLD`, but a copy of it). -There is no global communicator variable `world` that can be accessed directly in a nested function. -The idea behind this is to avoid using the global communicators in nested functions of the program unless they are explicitly passed in the function call. -Communicators are usually passed by reference to nested functions. -Even in traditional MPI it is a mistake to assume that the `MPI_COMM_WORLD` is the only available communicator. - -`mpi3::communicator` represent communicators with value-semantics. -This means that `mpi3::communicator` can be copied or passed by reference. -A communicator and their copies are different entities that compare equal. -Communicators can be empty, in a state that is analogous to `MPI_COMM_NULL` but with proper value semantics. - -Like in MPI communicators can be duplicated (copied into a new instance) or split. -They can be also compared. - -```cpp -mpi3::communicator world2 = world; -assert( world2 == world ); -mpi3::communicator hemisphere = world/2; -mpi3::communicator interleaved = world%2; -``` - -This program for example splits the global communicator in two sub-communicators one of size 2 (including process 0 and 1) and one with size 6 (including 2, 3, ... 7); - -```cpp -#include "mpi3/main.hpp" -#include "mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - assert(world.size() == 8); // this program can only be run in 8 processes - mpi3::communicator comm = (world <= 1); - assert(!comm || (comm && comm.size() == 2)); - return 0; -} -``` - -Communicators give also index access to individual `mpi3::processes` ranging from `0` to `comm.size()`. -For example, `world[0]` referrers to process 0 or the global communicator. -An `mpi3::process` is simply a rank inside a communicator. -This concept doesn't exist explicit in the standard C interface, but it simplifies the syntax for message passing. - -Splitting communicators can be done more traditionally via the `communicator::split` member function. - -Communicators are used to pass messages and to create memory windows. -A special type of communicator is a shared-communicator `mpi3::shared_communicator`. - -## Message Passing - -This section describes the features related to the message passing (MP) functions in the MPI library. -In C-MPI information is passed via pointers to memory. -This is expected in a C-based interface and it is also very efficient. -In Boost.MPI, information is passed exclusively by value semantics. -Although there are optimizations that amortize the cost, we decided to generalize the pointer interface and leave the value-based message passing for a higher-level syntax. - -Here we replicate the design of STL to process information, that is, aggregated data is passed mainly via iterators. (Pointer is a type of iterator). - -For example in STL data is copied between ranges in this way. -```cpp -std::copy(origin.begin(), origin.end(), destination.begin()); -``` - -The caller of function copy doesn't need to worry about he type of the `origin` and `destination` containers, it can mix pointers and iterators and the function doesn't need more redundant information than the information passed. -The programmer is responsible for managing the memory and making sure that design is such that the algorithm can access the data referred by the passed iterators. - -Contiguous iterators (to built-in types) are particularity efficient because they can be mapped to pointers at compile time. This in turn is translated into a MPI primitive function call. -The interface for other type of iterators or contiguous iterators to non-build-in type are simulated, mainly via buffers and serialization. -The idea behind this is that generic message passing function calls can be made to work with arbitrary data types. - -The main interface for message passing in B.MPI3 are member functions of the communicator. -For example `communicator::send`, `::receive` and `::barrier`. -The functions `::rank` and `::size` allows each process to determine their unique identity inside the communicator. - -```cpp -int mpi3::main(int argc, char* argv[], mpi3::communicator world) { - assert(world.size() == 2); - if(world.rank() == 0) { - std::vector v = {1.,2.,3.}; - world.send(v.begin(), v.end(), 1); // send to rank 1 - } else if(world.rank() == 1) { - std::vector v(3); - world.receive(v.begin(), v.end(), 0); // receive from rank 1 - assert( v == std::vector{1.,2.,3.} ); - } - world.barrier(); // synchronize execution here - return 0; -} -``` - -Other important functions are `::gather`, `::broadcast` and `::accumulate`. -This syntax has a more or less obvious (but simplified) mapping to the standard C-MPI interface. -In Boost.MPI3 however all, these functions have reasonable defaults that make the function call shorted and less prone to errors and with the C-MPI interface. - -For more examples, look into `./mpi3/tests/`, `./mpi3/examples/` and `./mpi3/exercises/`. - -The interface described above is iterator based and is a direct generalization of the C-interface which works with pointers. -If the iterators are contiguous and the associated value types are primitive MPI types, the function is directly mapped to the C-MPI call. - -Alternatively, value-based interface can be used. -We will show the terse syntax, using the process objects. - -```cpp -int mpi3::main(int, char**, mpi3::communicator world) { - assert(world.size() == 2); - if(world.rank() == 0) { - double v = 5.; - world[1] << v; - } else if(world.rank() == 1) { - double v = -1.; - world[0] >> v; - assert(v == 5.); - } - return 0; -} -``` - -## Remote Memory Access - -Remote Memory (RM) is handled by `mpi3::window` objects. -`mpi3::window`s are created by `mpi3::communicator` via a collective (member) functions. -Since `mpi3::window`s represent memory, it cannot be copied (but can be moved). - -```cpp -mpi3::window w = world.make_window(begin, end); -``` - -Just like in the MPI interface, local access and remote access is synchronized by a `window::fence` call. -Read and write remote access is performed via put and get functions. - -```cpp -w.fence(); -w.put(begin, end, rank); -w.fence(); -``` - -This is minimal example using `put` and `get` functions. - -```cpp -#include "mpi3/main.hpp" -#include - -namespace mpi3 = boost::mpi3; using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world) { - - std::vector darr(world.rank()?0:100); - mpi3::window w = world.make_window(darr.data(), darr.size()); - w.fence(); - if(world.rank() == 0) { - std::vector a = {5., 6.}; - w.put(a.begin(), a.end(), 0); - } - world.barrier(); - w.fence(); - std::vector b(2); - w.get(b.begin(), b.end(), 0); - w.fence(); - assert( b[0] == 5.); - world.barrier(); - - return 0; -} -``` - -In this example, memory from process 0 is shared across the communicator, and accessible through a common window. -Process 0 writes (`window::put`s) values in the memory (this can be done locally or remotely). -Later all processes read from this memory. -`put` and `get` functions take at least 3 arguments (and at most 4). -The first two is a range of iterators, while the third is the destination/source process rank (called "target_rank"). - -Relevant examples and test are located in For more examples, look into `./mpi3/tests/`, `./mpi3/examples/` and `./mpi3/exercises/`. - -`mpi3::window`s may carry type information (as `mpi3::window`) or not (`mpi3::window<>`) - -## Shared Memory - -Shared memory (SM) uses the underlying capability of the operating system to share memory from process within the same node. -Historically shared memory has an interface similar to that of remove access. -Only communicators that comprise a single node can be used to create a share memory window. -A special type of communicator can be created by splitting a given communicator. - -`mpi3::shared_communicator node = world.split_shared();` - -If the job is launched in single node, `node` will be equal (congruent) to `world`. -Otherwise the global communicator will be split into a number of (shared) communicators equal to the number of nodes. - -`mpi3::shared_communicator`s can create `mpi3::shared_window`s. -These are special type of memory windows. - -```cpp -#include "mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world) { - - mpi3::shared_communicator node = world.split_shared(); - mpi3::shared_window win = node.make_shared_window(node.rank()==0?1:0); - - assert(win.base() != nullptr and win.size() == 1); - - win.lock_all(); - if(node.rank()==0) *win.base(0) = 42; - for (int j=1; j != node.size(); ++j){ - if(node.rank()==0) node.send_n((int*)nullptr, 0, j);//, 666); - else if(node.rank()==j) node.receive_n((int*)nullptr, 0, 0);//, 666); - } - win.sync(); - - int l = *win.base(0); - win.unlock_all(); - - int minmax[2] = {-l,l}; - node.all_reduce_n(&minmax[0], 2, mpi3::max<>{}); - assert( -minmax[0] == minmax[1] ); - cout << "proc " << node.rank() << " " << l << std::endl; - - return 0; -} -``` - -For more examples, look into `./mpi3/tests/`, `./mpi3/examples/` and `./mpi3/exercises/`. - -# Beyond MP: RMA and SHM - -MPI provides a very low level abstraction to inter-process communication. -Higher level of abstractions can be constructed on top of MPI and by using the wrapper the works is simplified considerably. - -## Mutex - -Mutexes can be implemented fairly simply on top of RMA. -Mutexes are used similarly than in threaded code, -it prevents certain blocks of code to be executed by more than one process (rank) at a time. - -```cpp -#include "mpi3/main.hpp" -#include "mpi3/mutex.hpp" - -#include - -namespace mpi3 = boost::mpi3; using std::cout; - -int mpi3::main(int, char**, mpi3::communicator world) { - - mpi3::mutex m(world); - { - m.lock(); - cout << "locked from " << world.rank() << '\n'; - cout << "never interleaved " << world.rank() << '\n'; - cout << "forever blocked " << world.rank() << '\n'; - cout << std::endl; - m.unlock(); - } - return 0; -} -``` - -(Recursive mutexes are not implemented yet) - -Mutexes themselves can be used to implement atomic operations on data. - -# Ongoing work - -We are implementing memory allocators for remote memory, atomic classes and asynchronous remote function calls. -Higher abstractions and use patterns will be implemented, specially those that fit into the patterns of the STL algorithms and containers. - -# Advanced Topics - -## Thread safety - -If you are not using threads at all, you can skip this section; -however here you can find some rationale behind design decisions taken by the library and learn how to use `mpi3::communicator` as a member of a class. - -Thread-safety with MPI is extremely complicated, as there are various aspects to it, from the data communicated, to the communicator itself, to operations order, to asynchronous messaging, to the runtime system. -This library doesn't try to hide this fact; in place, it leverages the tools available to C++ to deal with this complication. -As we will see, there are certain steps to make the code _compatible_ with threads to difference degrees. - -Absolute thread-safety is a very strong guarantee and it would come at a very steep performance cost. -Almost no general purpose library guarantees complete thread safety. -In opposition to thread-safety, we will discuss thread-compatibility, which is a more reasonable goal. -Thread-compatibility refers to the property of a system to be able to be thread-safe if extra steps are taken and that you have the option to take these steps only when needed. - -The first condition for thread compatibility is to have an MPI environment that supports threads. -If you have an MPI system provides only a `thread_support` at the level of `mpi3::thread::single` it means that there is probably no way to make MPI calls from different threads an expect correct results. -If your program expects to call MPI in concurrent sections, your only option would be to change to a system that supports MPI threading. - -In this small example, we assume that the program expects threading and MPI by completely rejecting the run if the any level different from `single` is not provided. -This is not at all terrible choice, _optionally_ supporting threading in a big program can be prohibitive from a design point of view. - -```cpp -int main() { - mpi3::environment env{mpi3::thread::multiple}; - switch( env.thread_support() ) { - case mpi3::thread::single : throw std::logic_error{"threads not supported"}; - case mpi3::thread::funneled : std::cout<<"funneled" < mpi3::single`, since the levels `multiple > serialized > funneled > single` are ordered. - -### From C to C++ - -The MPI-C standard interface is expressed in the C language (and Fortran). -The C-language doesn't have many ways to deal with threads except by thorough documentation. -This indicates that any level of thread assurance that we can express in a C++ interface cannot be derived by the C-interface syntax alone; -it has to be derived, at best, from the documentation and when documentation is lacking from common sense and common practice in existing MPI implementations. - -The modern C++ language has several tools to deal with thread safety: the C++11 memory model, the `const`, `mutable` and `thread_local` attributes and a few other standard types and functions, such as `std::mutex`, `std::call_once`, etc. - -### Data and threads - -Even if MPI operations are called outside concurrent sections it is still your responsibility to make sure that the *data* involved in communication is synchronized; this is always the case. -Clear ownership and scoping of *data* helps a lot towards thread safety. -Avoiding mutable shared data between threads also helps. -Perhaps as a last resort, data can be locked with mutex objects to be written or accessed one thread at time. - -### Communicator and threads - -The library doesn't control or owns the communicated data for the most part, therefore the main concern of the library regarding threading is within the communicator class itself. - -The C-MPI interface briefly mentions thread-safety, for example most MPI operations are accompanied by the following note (e.g. https://www.mpich.org/static/docs/latest/www3/MPI_Send.html): - -> **Thread and Interrupt Safety** -> -> This routine is thread-safe. This means that this routine may be safely used by multiple threads without the need for any user-provided thread locks. However, the routine is not interrupt safe. Typically, this is due to the use of memory allocation routines such as malloc or other non-MPICH runtime routines that are themselves not interrupt-safe. - -This doesn't mean that that _all_ calls can be safely done from different threads concurrently, only some of them, those that refer to completely different argument can be safe. - -In practice it is observable that for most MPI operations the "state" of the communicator can change in time. -Even if after the operation the communicator seems to be in the same state as before the call the operation itself changes, at least briefly, the state of the communicator object. -This internal state can be observed from another thread even through undefined behavior, even if transiently. -A plausible model to explain this behavior is that internal buffers are used by individual communicators during communication. - -In modern C++, this is enough to mark communicator operations non-`const` (i.e. an operation than can be applied only on a mutable instance of the communicator). - -(`MPI_Send` has "tags" to differentiate separate communications and may help with concurrent calls, but this is still not a enough since the tags are runtime variables, of which the library doesn't know the origin. -Besides, the use of tags are not a general solution since collective operation do not use tags at all. -It has been known for a while that the identity of the communicator in some sense serves as a tag for collective communications. -This is why it is so useful to be able to duplicate communicators to distinguish between collective communication messages.) - -This explains why most member functions of `mpi3::communicator` are non-`const`, and also why most of the time `mpi3::communicators` must either be passed either by non-`const` reference or by value (depending on the intended use, see below.) -Be aware that passing by `const`-reference `mpi3::communicator const&` is not very productive because no communication operation can be performed with this instance (not even duplication to obtain a new instance). -(This behavior is not unheard of in C++: standard "printing" streams generally need be _mutable_ to be useful (e.g. `std::cout` or `std::ofstream`), even though they don't seem to have a changing state.) - -This brings us to the important topic of communicator construction and assignment. - -More material: ["C++ and Beyond 2012: Herb Sutter - You don't know const and mutable"](https://web.archive.org/web/20170119232617/https://channel9.msdn.com/posts/C-and-Beyond-2012-Herb-Sutter-You-dont-know-blank-and-blank) and ["related"](https://web.archive.org/web/20160924183715/https://channel9.msdn.com/Shows/Going+Deep/C-and-Beyond-2012-Herb-Sutter-Concurrency-and-Parallelism). - -### Duplication of communicator - -In C, custom structures do not have special member functions that indicate copying. -In general this is provided by free functions operating in pointer or _handle_ types, and in general in their signature ignores `const`ness. - -In C-MPI, the main function to duplicate a communicator is `int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)`. -When translating from C to C++ we have to understand that `MPI_Comm` is a handle to a communicator, that is, it behaves like a pointer. -In a general library the source (first argument) is conceptually constant (unmodified) during a copy, so we could be tempted to mark it as `const` when translating to C++. - -```cpp -struct communicator { - ... - communicator bad_duplicate() const; // returns a new communicator, congruent to the current communicator, -}; -``` -Furthermore, we could be tempted to call it `copy` or even to make it part of the copy-constructor. - -But, alas, this is not the case according to the rules we delineated earlier. -We know that duplication is an operation that requires communication and it is observable (through concurrent threads) that the internal state of the _original_ communicator is changed *while* it is duplicated. -Therefore to be honest with the semantics of communicator duplication we are forced to implement this function as non-`const`. - -```cpp -struct communicator { - ... - communicator duplicate(); // returns a new communicator, congruent to the current communicator -}; -``` - -The consequence of this line of though is that a `const` communicator (i.e. non-mutable) cannot be duplicated. -That is, not only such communicator cannot do any communication operation but it cannot be duplicated itself. - -```cpp -mpi3::communicator new_comm{comm.duplicate()}; -``` - -This syntax also makes very explicit what the operation really does. - -### Pass-by-value or pass-by-reference - -As indicated earlier, a useful communicator is one that is mutable. -Therefore when passing a communicator to a function we have two main options, either pass by reference (non-const reference) or by value. - -```cpp -void f(mpi3::communicator& comm); -void g(mpi3::communicator comm); -``` - -These two cases have different meanings and different things can be done with the corresponding communicators. - -Case `f` implies that, first, we are reusing a communicator, even if all communication operations are internal to the function or second, that `f` can communicate messages with a communicator that is external to the function. - -Although reusing a communicator sound reasonable (since duplicating communicators can be an expensive operation), even if all communication is contained in `f` there is a risk that some communicator is mixed inadvertedly with communication external to `f`. -The logic of collective operation would be distributed in different functions in the code, which is possible but difficult or impossible to reason about. -If `f` is running in a multi-threaded environment, it could be dealing with a communicator that is being shared with other threads. - -Case `g` is different in the sense that it knows that it has exclusive access to the communicator, send and receive operations cannot be captured by other functions and collective operations only need to be fully contained inside `g`. - -For completeness we can also imagine a function declared as pass-by-const-reference. - -```cpp -void h(mpi3::communicator const& comm); -``` - -In the current system, this is not very useful since only a handful of operations, which do not include communication or duplication, can be done. -(An example is probing the `.size()` of the communicator.) -Take into account that, inside the `h` function it is also "too late" to produce a duplicate of the communicator. - -## Communicator as an implementation detail - -Note that so far we didn't indicate how to use `mpi3::communicator` member with threads, we are simply following the logic being transparent of what each MPI operation is likely to perform behind the scenes regarding the (transient) state of the communicator. - -In C++ it is very useful to include a communicator to each object that requires to perform communication to maintain its internal consistency. -Suppose we have a data that is distributed across many processes, and that we store a instance of the communicator containing these processes. -Such class could have operations that modify its state and others that do not. -The correct design is to mark the function in the latter category as `const`. - -```cpp -struct distributed_data { - void set() { ... } - void print() const { ... } - - private: - mpi3::communicator comm_; -}; -``` - -However such design doesn't work, because for `print` to do any actual communication (e.g. to communicate some data to the root process) would need to have access to a mutable communicator, the `const` mark prevents that. - -One option is to make `print` non-`const`, this is bad because we will lose any concept of mutability just because an implementation detail. -The other option to remove const by force, -```cpp - void print() const { const_cast(comm_).do_something()... } -``` -which would work but it is not very idiomatic. -Besides, this class would become now **hostile** to threads, because two simultaneous `print` calls (which are marked as `const`) on the same class could overlap, the messages could be mixed and weird behavior can appear under threads and we would need to look inside the implementation of `print`. -Ending up with hostile class is an basically a show stopped for threading and must be avoided. - -Note that making the communicator member a pointer `mpi3::communicator* comm_;` doesn't solve any problem, it just kick the can down the road. - -This leads to a more modern design which would use the keyword `mutable`. - -```cpp -struct distributed_data { - void set() { ... } - void print() const { ... } - - public: - mutable mpi3::communicator comm_; -}; -``` - -This will allow the use of the communicator from internals of `print() const` without the use of `const_cast`. -This doesn't save us from the problem of using the communicator concurrently but at least it is clear in the declaration of the class. -As a matter of fact this `mutable` attribute is exactly what marks the class a thread unsafe. -(A mutable member without a synchronization mechanism is a red flag in itself.) -If a single instance of the class is never used across threads *or* the program is single threaded there is nothing else that one needs to do. - -Note also that different instances of the class can also be used from different threads, since they don't share anything, nor internal data or their internal communicator. - -What if you want to make your class, that contains a communicator thread-safe, at least safe for calling concurrently non mutating (`const`) members? -For that you need to implement your own synchronization or locking mechanism. -There is no single recipe for that, you can use a single mutex to lock access for the communicator alone or both the communicator and data. - -```cpp -struct distributed_data { - void set() { ... } - void print() const { std::lock_guard guard{mtx_}; ... use comm_ ... } - - private: - mutable std::mutex mtx_; - mutable mpi3::communicator comm_; -}; -``` - -I don't recommend doing this specifically; the code above is just to illustrate the point. -I can not give a general recipe beyond this point, because there are many possible choices on how to make class thread safe (e.g. data-safe) or thread safe to some specific level (operation-safe). -Ideally concurrent data structure should be able to do some of the work without the synchronization bottleneck. -The whole point is that the library gives you this option, to trade-off safety and efficiency to the desired degree but no more. - -In fact a (bad) blanket way to make the library thread safe could be to wrap every communicator in class with a mutex and make all most communication operations `const`. -This would force, from a design perspective, an unacceptable operation cost. - -### Not a copy-constructor, but a duplicate-constructor - -So far we have shown the `duplicate` interface function as a mechanism for duplicating communicators (used as `auto new_comm{comm.duplicate()}`), which is nice because it makes the operation very explicit, but it also makes it difficult to integrate generically with other parts of C++. - -A reasonable copy constructor of the class containing a communicator would be: - -```cpp -struct distributed_data { - distributed_data(distributed_data const& other) : comm_{other.comm_.duplicate()} {} - - private: - ... - mutable mpi3::communicator comm_; -}; -``` -Note that this code is valid because `comm_` is a mutable member of `other`. -The worst part of forcing us to use the "non-standard" `duplicate` function is that we can no longer "default" the copy constructor. - -Copying in C++ is usually delegated to special member functions such as the copy-constructor or copy-assignment. -However these function take their source argument as `const` reference and as such it cannot call the `duplicate` member. -(And even if we could we would be lying to the compiler in the sense that we could make the system crash by copying concurrently a single (supposedly) `const` communicator that is shared in two threads.) - -However the language is general enough to allow a constructor by non-const reference. -The signature of this constructor is this one: - -```cpp -communicator::communicator(communicator & other) {...} // "duplicate" constructor? -communicator::communicator(communicator const& ) = delete; // no copy constructor -``` - -There is no standard name for this type of constructor, I choose to call it here "duplicate"-constructor, or mutable-copy-constructor. -This function does internally call `MPI_Comm_dup`, and like `duplicate()` it can only be called with a source that is mutable. -This makes the copy constructor of the containing class more standard, or even can be implemented as `= default;`. - -```cpp -struct distributed_data { - distributed_data(distributed_data const& other) : comm_{other.comm_} {} // or = default; - ... - private: - mutable mpi3::communicator comm_; -}; -``` - -**In summary**, -1) all important communication operations are non-`const` because according to the rules and practice of modern C++ the internal state of the communicator is affected by these operations, -2) ... including the `duplicate` operation; -3) `mutable` is a good marker to indicate the _possible_ need for custom (thread) synchronization mechanism; it also makes possible the use of communicator as member of a class. -4) the need may be critical or not (the user of the library decides), -5) mutable instances of communicators (i.e. non-`const` variables or mutable members) can be duplicated using standard C++ syntax, via "duplicate"-constructor or via `duplicate` member functions. -6) In general, it is likely to be a good idea to duplicate communicator for specific threads *before* creating them; otherwise duplication will happen "too late" with a shared (non-const) communicator. - -(Thanks Arthur O'Dwyer for the critical reading of this section.) - -## NCCL (GPU communication) - -If the underlying MPI distribution is GPU-aware, in principle you can pass GPU pointers to the communication routines. -This is generally faster than copying back and forth to CPU. - -Nvidia's NCCL conceptually implements a subset of MPI operations and it might be faster than GPU-aware MPI. -To obtain an NCCL communicator you pass an MPI communicator. - -```cpp - mpi3::nccl::communicator gpu_comm{mpi_comm}; -``` - -The original MPI communicator is assumed be working with non-overlapping devices (e.g. one process per GPU). -This can be achieved by `cudaSetDevice(world.rank() % num_devices);` generally at the start of the program or autommically by using certain ways to run the MPI program (e.g. `lrun` tries to attach each MPI process to a different GPU device). - -With some limitations, the NCCL communicator can be used to perform operations on GPU memory without the need to obtaining raw pointers. -By default it works with `thrust[::cuda]::device_ptr` or `thrust[::cuda]::universal_ptr`. -For example this produces a reduction in GPU across processes (even processes in different nodes): - -```cpp -// thust::device_vector> A(1000, gpu_comm.rank()); - thrust::device_vector> A(1000, gpu_comm.rank()); - - gpu_comm.all_reduce_n(A.data(), A.size(), A.data()); -``` - -Like B-MPI3 communicator the NCCL communicator is destroyed automatically when leaving the scope. - -The implementation is preliminary, the NCCL communicator is moveable but not copyable (or duplicable). -Congruent NCCL communicators can be constructed from the same (or congruent) B-MPI3 communicator (at the cost of a regular MPI broadcast). -There is not mechanism to create NCCL subcommunicators from other NCCL communicators, except using MPI subcommunicators as constructor arguments. - -# Conclusion - -The goal is to provide a type-safe, efficient, generic interface for MPI. -We achieve this by leveraging template code and classes that C++ provides. -Typical low-level use patterns become extremely simple, and that exposes higher-level patterns. - -# Mini tutorial - -This section describes the process of bringing a C++ program that uses the original MPI interface to one that uses B.MPI3. -Below it is a valid C++ MPI program using send and receive function. -Due to the legacy nature of MPI, C and C++ idioms are mixed. - -```cpp -#include - -#include -#include -#include - -int main(int argc, char **argv) { - MPI_Init(&argc, &argv); - MPI_Comm comm = MPI_COMM_WORLD; - - int count = 10; - - std::vector xsend(count); iota(begin(xsend), end(xsend), 0); - std::vector xrecv(count, -1); - - int rank = -1; - int nprocs = -1; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &nprocs); - if(nprocs%2 == 1) { - if(rank == 0) {std::cerr<<"Must be called with an even number of processes"< -#include -#include - -namespace bmpi3 = boost::mpi3; - -int main(int argc, char **argv) try { - bmpi3::environment::initialize(argc, argv); - MPI_Comm comm = &bmpi3::environment::get_world_instance(); assert(comm == MPI_COMM_WORLD) -... - bmpi3::environment::finalize(); - return 0; -} -``` - -Notice that we are getting a reference to the global communicator using the `get_world_instance`, then, with the ampersand (`&`) operator, we obtain a `MPI_Comm` handle than can be used with the rest of the code untouched. - -Since `finalize` will need to be executed in any path, it is preferable to use an RAII object to represent the environment. -Just like in classic MPI, it is wrong to create more than one environment. - -Both, accessing the global communicator directly is in general considered problematic. -For this reason it makes more sense to ask for a duplicate of the global communicator. - -```cpp -int main(int argc, char **argv) { - bmpi3::environment env(argc, argv); - bmpi3::communicator world = env.world(); - MPI_Comm comm = &world; assert(comm != MPI_COMM_WORLD); -... - return 0; -} -``` - -This ensures that `finalize` is always called (by the destructor) and that we are not using the original global communicator, but a duplicate. - -Since this pattern is very common, a convenient "main" function is declared by the library as a replacement declared in the `mpi3/main.hpp` header. - -```cpp -#include "../../mpi3/main.hpp" - -#include -#include -#include - -namespace bmpi3 = boost::mpi3; - -int bmpi3::main(int, char **, bmpi3::communicator world) { - MPI_Comm comm = &world; assert(comm != MPI_COMM_WORLD); -... - return 0; -} -``` - -The next step is to replace the use of the MPI communicator handle by a proper `mpi3::communicator` object. -Since `world` is already a duplicate of the communicator we can directly use it. -The `size` and `rank` are methods of this object which naturally return their values. - -```cpp -... - int rank = world.rank(); - int nprocs = world.size(); -... -``` - -Similarly the calls to send and receive data can be transformed. -Notice that the all the irrelevant or redundant arguments (including the receive source) can be omitted. - -```cpp -... - world.send_n (xsend.data(), count, partner_rank); - world.receive_n(xrecv.data(), count); -... -``` - -(We use the `_n` suffix interface to emphasize that we are using element count (container size) as argument.) - -The condition `(rank == 0)` is so common that can be replaced by the `communicator`'s method `is_root()`: - -```cpp - if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< -#include -#include - -namespace bmpi3 = boost::mpi3; - -int bmpi3::main(int /*argc*/, char ** /*argv*/, bmpi3::communicator world) try { - int count = 10; - - std::vector xsend(count); iota(begin(xsend), end(xsend), 0); - std::vector xrecv(count, -1); - - if(world.size()%2 == 1) { - if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< -#include -#include - -namespace bmpi3 = boost::mpi3; - -int bmpi3::main(int /*argc*/, char ** /*argv*/, bmpi3::communicator world) { - if(world.size()%2 == 1) { - if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< xsend(10); iota(begin(xsend), end(xsend), 0); - std::vector xrecv(xsend.size(), -1); - - world.send_receive(cbegin(xsend), cend(xsend), (world.rank()/2)*2 + (world.rank()+1)%2, begin(xrecv)); - - assert(xrecv[5] == 5); - if(world.is_root()) {std::cerr<<"successfully completed"< - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - if(world.rank() == 0) cout << mpi3::version() << '\n'; - - cout << "hello from task " << world.rank() << " in host " << mpi3::processor_name() << std::endl; - - if(world.rank() == 0) cout << "numer of tasks " << world.size() << std::endl; - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/02-send.cpp b/external_codes/mpi_wrapper/mpi3/examples/02-send.cpp deleted file mode 100644 index 0673223604d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/02-send.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/version.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], boost::mpi3::communicator& world){ - - if(world.size() % 2 != 0){ - if(world.root()) cout << "Quitting. Need an even number of tasks: numtasks = " << world.size() << "\n"; - }else{ - int message = -1; - if(world.root()) cout << "MASTER: number of mpi tasks is " << world.size() << "\n"; - if(world.rank() < world.size()/2){ - int partner = world.rank() + world.size()/2; - int rank = world.rank(); - world.send_value(rank, partner); - world.receive_value(message, partner); - }else if(world.rank() >= world.size()/2){ - int partner = world.rank() - world.size()/2; - world.receive_value(message, partner); - int rank = world.rank(); - world.send_value(rank, partner); - } - cout << "Task " << world.rank() << " is partner with " << message << "\n"; - } - - if(world.rank()==0){ - std::vector code = {1.,2.,3.,4.}; - world.send_n(code.begin(), 4, 1); - } - - if(world.rank()==1){ - std::vector code(4); - world.receive_n(code.begin(), 4, 0); - assert(( code == std::vector{1.,2.,3.,4.} )); - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/03-isend.cpp b/external_codes/mpi_wrapper/mpi3/examples/03-isend.cpp deleted file mode 100644 index 10c861988c0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/03-isend.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++17 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/version.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - if(world.size() % 2 != 0){ - if(world.root()) cout << "Quitting. Need an even number of tasks: numtasks = " << world.size() << "\n"; - return 1; - } - - int message = -1; - cout << "Hello from task " << world.rank() << " on host " << mpi3::processor_name() << "\n"; - if(world.root()) cout << "MASTER: number of mpi tasks is " << world.size() << "\n"; - - int partner = world.rank() - -using std::cout; -using std::endl; - -double update(std::vector& data, int offset, int chunk_size){ - double sum = 0; - for(int i = offset; i != offset + chunk_size; ++i){ - data[i] += i*1.0; - sum += data[i]; - } - return sum; -} - -int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator const& world){ - if(world.size() % 4 != 0){ - if(world.master()) cout << "Quitting. Need an even number of tasks: numtasks = " << world.size() << std::endl; - return 1; - } - - int const array_size = 16000000; - int chunk_size = array_size/world.size(); - - if(world.master()){ - std::vector data(array_size); - { - double sum = 0; - for(int i = 0; i != data.size(); ++i){ - data[i] = i*1.0; - sum += data[i]; - } - cout << "serial sum " << sum << endl; - } - - int offset = chunk_size; - for(int dest = 1; dest != world.size(); ++dest){ - world.send_n(data.begin() + offset, chunk_size, dest); - offset += chunk_size; - } - - double partial_sum = 0; - for(int i = 0; i != chunk_size; ++i) - partial_sum += data[i]; - - double parallel_sum = world.reduce(partial_sum); - cout << "parallel sum is " << parallel_sum << endl; - }else{ - std::vector partial_data(chunk_size); - world.receive_n(partial_data.begin(), chunk_size, 0); - - double partial_sum = 0; - for(int i = 0; i != partial_data.size(); ++i) - partial_sum += partial_data[i]; - - world.reduce(partial_sum); - } - - -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/05-array_mult.cpp b/external_codes/mpi_wrapper/mpi3/examples/05-array_mult.cpp deleted file mode 100644 index 806ee98f2c5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/05-array_mult.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++17 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -// based on https://computing.llnl.gov/tutorials/mpi/samples/C/array_mm.c -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/version.hpp" -#include "alf/boost/mpi3/processor_name.hpp" -#include "alf/boost/multi/array.hpp" - -#include - -using std::cout; -using std::endl; - -int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator const& world){ - assert( world.size() > 1 ); - - if(world.master()){ - - std::string name = "master"; - std::list l = {1.,2.,3.,4.,5.,6.}; - - boost::multi::array a({{62, 15}}); - boost::multi::array b({{15,7}}); - - for(int i : a.extensions()[0]) - for(int j : a.extensions()[1]) - a[i][j] = i + j; - - for(int i : b.extensions()[0]) - for(int j : b.extensions()[1]) - b[i][j] = i*j; - - boost::multi::array c_serial({{62,7}}); - - for(int i : c_serial.extensions()[0]){ - for(int j : c_serial.extensions()[1]){ - c_serial[i][j] = 0; - for(int k : a.extensions()[1]){ - c_serial[i][j] += a[i][k] * b[k][j]; - } - } - } - - auto averow = a.extensions()[0].size()/(world.size()-1); - auto extra = a.extensions()[0].size()%(world.size()-1); - - auto offset = 0; - for(int dest = 1; dest != world.size(); ++dest){ - auto p = world[dest]; - p << name; - world.send_n(l.begin(), l.size(), dest); - int rows = (dest < extra + 1)?(averow + 1): averow; - p - << offset - << rows - ; - world.send_n(&a[offset][0], rows*a.extensions()[1].size(), dest); - world.send_n(&b[0][0], b.num_elements(), dest); - offset += rows; - } - - boost::multi::array c({{62,7}}); - for(int source = 1; source != world.size(); ++source){ - auto proc = world[source]; - int rows; - proc - >> offset - >> rows - ; - world.receive_n(&c[offset][0], rows*c.extensions()[1].size(), source); - } - for(int i : c.extensions()[0]) - for(int j : c.extensions()[1]) - assert( c[i][j] - c_serial[i][j] == 0 ); - }else{ - auto proc = world[0]; - std::string name; - proc >> name; - std::list l(6); - world.receive_n(l.begin(), l.size(), 0); - assert(( l == std::list{1.,2.,3.,4.,5.,6.} )); - int offset; - int rows; - proc >> offset >> rows; - boost::multi::array a_partial({{rows, 15}}); - world.receive_n(a_partial.data(), a_partial.num_elements(), 0); - boost::multi::array b({{15,7}}); - world.receive_n(b.data(), b.num_elements(), 0); - - boost::multi::array c_partial({{rows,7}}); - for(int i = 0; i != c_partial.extensions()[0].size(); ++i){ - for(int j = 0; j != c_partial.extensions()[1].size(); ++j){ - c_partial[i][j] = 0; - for(int k = 0; k != a_partial.extensions()[1].size(); ++k){ - c_partial[i][j] += a_partial[i][k] * b[k][j]; - } - } - } - proc - >> offset - >> rows - ; - world.send_n(c_partial.data(), c_partial.num_elements(), 0); - } - cout << "ok" << endl; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/average.cpp b/external_codes/mpi_wrapper/mpi3/examples/average.cpp deleted file mode 100644 index e13ada314f5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/average.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -//#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/environment.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/process.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -std::random_device rd; -std::mt19937 gen(rd()); - -int main(int, char*[]){//, mpi3::communicator& world){ - - mpi3::environment env; - auto& world = env.world(); - - int elements_per_proc = 5000; - - std::vector v; - if(world.rank() == 0){ - std::uniform_real_distribution<> dis; - v.resize(elements_per_proc*world.size(), -1.); - std::generate(v.begin(), v.end(), [&](){return dis(gen);}); - cout << "serial average " << std::accumulate(v.begin(), v.end(), 0.)/v.size() << '\n'; - } - - std::vector partial_v(elements_per_proc, -1.); - world.scatter_from(partial_v.begin(), partial_v.end(), v.begin(), 0); - double partial_ave = std::accumulate(partial_v.begin(), partial_v.end(), 0.)/partial_v.size(); - { - std::vector partial_aves = world.gather_value(partial_ave, 0); // std::vector subaves = (world[0] += subave); - if(world.rank() == 0){ - assert(partial_aves.size() == (unsigned)world.size()); - cout << "parallel average: " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << '\n'; - }else{ - assert(partial_aves.empty()); - } - } - { - std::vector partial_aves = (world |= partial_ave); - assert(partial_aves.size() == (unsigned)world.size()); - cout << "parallel average on " << world.rank() << ": " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << std::endl; - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/broadcast.cpp b/external_codes/mpi_wrapper/mpi3/examples/broadcast.cpp deleted file mode 100644 index 8e5a39feb4e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/broadcast.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/process.hpp" -#include "alf/boost/mpi3/detail/package_archive.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -void my_broadcast(mpi3::communicator& comm, std::vector& data, int root){ - if(comm.rank() == root){ - for(int i = 0; i != comm.size(); ++i){ - if(i != comm.rank()) comm[i] << data; - } - }else{ - comm[root] >> data; - } -} - -void my_bcast(void* data, int count, MPI_Datatype datatype, int root, - MPI_Comm communicator) { - int world_rank; - MPI_Comm_rank(communicator, &world_rank); - int world_size; - MPI_Comm_size(communicator, &world_size); - - if (world_rank == root) { - // If we are the root process, send our data to everyone - int i; - for (i = 0; i < world_size; i++) { - if (i != world_rank) { - MPI_Send(data, count, datatype, i, 0, communicator); - } - } - } else { - // If we are a receiver process, receive the data from the root - MPI_Recv(data, count, datatype, root, 0, communicator, MPI_STATUS_IGNORE); - } -} - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - { - std::vector data; - if(world.rank() == 0){data.resize(10); std::iota(data.begin(), data.end(), 0);} - my_broadcast(world, data, 0); - assert( data[5] == 5 ); - } - // ... less efficient but same effect as ... - { - std::vector data; - if(world.rank() == 0){data.resize(10); std::iota(data.begin(), data.end(), 0);} - world[0] & data; - assert( data[5] == 5 ); - } - int num_elements = 100000; - int num_trials = 1000; - { - int world_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); - - double total_my_bcast_time = 0.0; - double total_mpi_bcast_time = 0.0; - int i; - double* data = (double*)malloc(sizeof(double) * num_elements); - std::iota(data, data + num_elements, 0); - assert(data != NULL); - - for (i = 0; i < num_trials; i++) { - // Time my_bcast - // Synchronize before starting timing - MPI_Barrier(MPI_COMM_WORLD); - total_my_bcast_time -= MPI_Wtime(); - my_bcast(data, num_elements, MPI_DOUBLE, 0, MPI_COMM_WORLD); - // Synchronize again before obtaining final time - MPI_Barrier(MPI_COMM_WORLD); - total_my_bcast_time += MPI_Wtime(); - - // Time MPI_Bcast - MPI_Barrier(MPI_COMM_WORLD); - total_mpi_bcast_time -= MPI_Wtime(); - MPI_Bcast(data, num_elements, MPI_DOUBLE, 0, MPI_COMM_WORLD); - MPI_Barrier(MPI_COMM_WORLD); - total_mpi_bcast_time += MPI_Wtime(); - } - - // Print off timing information - if (world_rank == 0) { - printf("Data size = %d, Trials = %d\n", num_elements * (int)sizeof(double), - num_trials); - printf("Avg my_bcast time = %lf\n", total_my_bcast_time / num_trials); - printf("Avg MPI_Bcast time = %lf\n", total_mpi_bcast_time / num_trials); - } - free(data); - } - { - - std::vector data(num_elements); std::iota(data.begin(), data.end(), 0); - - double total_my_broadcast_time = 0.0; - double total_broadcast_time = 0.0; - for(int i = 0; i != num_trials; ++i){ - // if(world.rank() == 0) cout << i << std::endl; - world.barrier(); - total_my_broadcast_time -= mpi3::wall_time(); - my_broadcast(world, data, 0); - world.barrier(); - total_my_broadcast_time += mpi3::wall_time(); - world.barrier(); - total_broadcast_time -= mpi3::wall_time(); - // world.broadcast(data.begin(), data.end(), 0); - // MPI_Bcast(data.data(), num_elements, MPI_DOUBLE, 0, MPI_COMM_WORLD); - // world[0] & data; - int n = data.size(); world[0] & n; data.resize(n); - world.broadcast(data.begin(), data.end(), 0); - world.barrier(); - total_broadcast_time += mpi3::wall_time(); - } - if(world.rank() == 0){ - cout << "data size = " << num_elements * (int)sizeof(double) << ", trials = " << num_trials << '\n'; - cout << "avg my_broadcast time = " << total_my_broadcast_time / num_trials << '\n'; - cout << "avg broadcast time = " << total_broadcast_time / num_trials << '\n'; - } - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/calculate_pi.cpp b/external_codes/mpi_wrapper/mpi3/examples/calculate_pi.cpp deleted file mode 100644 index e72aab92537..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/calculate_pi.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/process.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -std::random_device rd; -std::mt19937 gen(rd()); -std::uniform_real_distribution<> dis; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - - int elements_per_proc = 5000; - - std::vector v; - if(world.rank() == 0){ - v.resize(elements_per_proc*world.size(), -1.); - std::generate(v.begin(), v.end(), [&](){return dis(gen);}); - cout << "serial average " << std::accumulate(v.begin(), v.end(), 0.)/v.size() << '\n'; - } - - std::vector partial_v(elements_per_proc, -1.); - world.scatter_from(partial_v.begin(), partial_v.end(), v.begin(), 0); - double partial_ave = std::accumulate(partial_v.begin(), partial_v.end(), 0.)/partial_v.size(); - { - std::vector partial_aves = world.gather_value(partial_ave, 0); // std::vector subaves = (world[0] += subave); - if(world.rank() == 0){ - assert(partial_aves.size() == (unsigned)world.size()); - cout << "parallel average: " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << '\n'; - }else{ - assert(partial_aves.empty()); - } - } - { - // std::vector partial_aves = world.all_gather_value(partial_ave); - std::vector partial_aves = (world |= partial_ave); - assert(partial_aves.size() == (unsigned)world.size()); - cout << "parallel average on " << world.rank() << ": " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << std::endl; - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/groups.cpp b/external_codes/mpi_wrapper/mpi3/examples/groups.cpp deleted file mode 100644 index 7b5957a7a97..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/groups.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 15 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - - mpi3::communicator prime = world.create(world.group().include( {2, 3, 5, 7, 11, 13} )); - - // If this rank isn't in the new communicator, it will be an invalid (null) communicator - // Using rank() or size() in a null communicator is erroneous - cout << "world " << world.rank() << "/" << world.size() << " ---> "; - if(prime) cout << "prime " << prime.rank() << "/" << prime.size() << std::endl; - else cout << "not in prime comm" << std::endl; - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/hello.cpp b/external_codes/mpi_wrapper/mpi3/examples/hello.cpp deleted file mode 100644 index 44d02a5c420..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/hello.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++17 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/version.hpp" -#include "alf/boost/mpi3/processor_name.hpp" - -#include - -using std::cout; -using std::endl; - -int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator const& world){ - if(world.rank() == 0) cout << boost::mpi3::version() << '\n'; - - cout << "hello from task " << world.rank() << " in host " << boost::mpi3::processor_name() << endl; - - if(world.rank() == 0) cout << "numer of tasks " << world.size() << endl; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/hello_world.cpp b/external_codes/mpi_wrapper/mpi3/examples/hello_world.cpp deleted file mode 100644 index 58e4fd9ae62..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/hello_world.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -#export PATH=/home/correaa/prj/alf/boost/mpi3/fake:$PATH -mpic++ $0 -o $0x&&mpirun -n 4 $0x $@&&rm $0x;exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - - cout - << "Hello world from processor '" << mpi3::processor_name() - << "' rank " << world.rank() - << " out of " << world.size() << " processors\n" - ; - - return 0; - -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory.cpp b/external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory.cpp deleted file mode 100644 index 2146a1abd6b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -time mpicxx -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x -lboost_system && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/shm/managed_shared_memory.hpp" // there is a bug in boost 1.64, this needs to be included first -#include "alf/boost/mpi3/main.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - mpi3::communicator node = world.split_shared(0); - int N = 1000; - - mpi3::shm::managed_shared_memory msm(node, 100000); - std::atomic& i = *msm.construct>(0); - node.barrier(); - for(int j = 0; j != N; ++j) ++i; - node.barrier(); - int snapshot = i; - if(node.rank() == 0) assert(snapshot == node.size()*N); - msm.destroy>(i); - node.barrier(); - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory_0.cpp b/external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory_0.cpp deleted file mode 100644 index 46e73743d0e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/managed_shared_memory_0.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -time mpicxx -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x -lboost_system && time mpirun -np 1 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/shm/managed_shared_memory.hpp" // there is a bug in boost 1.64, this needs to be included first -#include "alf/boost/mpi3/main.hpp" - -#include -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -struct list_node{ -// boost::interprocess::offset_ptr next; - list_node* next; - int value; -}; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - mpi3::communicator node = world.split_shared(0); - - mpi3::shm::managed_shared_memory segment(node, 65536); - list_node* prev = 0; - list_node* current; - list_node* first; - - list_node* ln = static_cast(segment.allocate(sizeof(list_node))); - assert(ln != nullptr); - ln -> value = 0; - ln -> next = 0; - list_node* ln2 = static_cast(segment.allocate(sizeof(list_node))); - assert(ln2 != nullptr); - ln2 -> value = 1; -// ln2 -> next = ln; - - for(int i = 0; i < 10; ++i, prev = current){ - current = new list_node; - // current = static_cast(segment.allocate(sizeof(list_node))); - current->value = i; - current->next = 0; - if(!prev) first = current; - else prev->next = current; - } - -#if 0 - for(current = first; current; ){ - prev = current; - current = current->next; - segment.deallocate(prev);//.get()); - } -#endif - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/pi_reduce.cpp b/external_codes/mpi_wrapper/mpi3/examples/pi_reduce.cpp deleted file mode 100644 index 99431503cbc..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/pi_reduce.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -OMPI_CXX=$CXXX mpicxx $CXXFLAGS $0 -o $0x&&mpirun --oversubscribe -n 6 $0x&&rm $0x;exit -#endif - -#include "../examples/../process.hpp" -#include "../examples/../main_environment.hpp" - -#include - -namespace mpi3 = boost::mpi3; - -// this program is based on https://computing.llnl.gov/tutorials/mpi/samples/C/mpi_pi_reduce.c - - - -double pi_approx(int n_samples){ - - std::random_device rd; - static std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis(-1.0, +1.0); - - int score = 0; - for(int i = 0; i != n_samples; ++i){ - double x = dis(gen); - double y = dis(gen); - if(x*x + y*y <= 1.0) ++score; - } - return 4.*score/static_cast(n_samples); // pi -} - -int mpi3::main(int, char*[], mpi3::environment& menv){ - auto world = menv.world(); - - constexpr int n_samples = 50000000; - - for(int s = 0; s != world.size(); ++s){ - auto t0 = mpi3::wall_time(); - if(auto continent = (world <= s)){ - - double pi = (continent += pi_approx(n_samples/continent.size())/continent.size()); - - if(world.root()){ - std::cout - <<"After "<< n_samples <<" throws " - <<"(in "<< continent.size() <<" procs), " - <<"average value of pi = " << pi - <<" (vs. real value of pi = "<< M_PI <<")" - <<" (time "<< (mpi3::wall_time() - t0).count() <<" sec)" - <> count; - cout << "proc " << world.rank() << " received count " << count << " from proc " << partner << '\n'; - } - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/random_walk.cpp b/external_codes/mpi_wrapper/mpi3/examples/random_walk.cpp deleted file mode 100644 index 2b771306419..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/random_walk.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/process.hpp" -#include "alf/boost/mpi3/detail/package_archive.hpp" - -#include - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -struct walker{ - int location; - int steps_left; - template - void serialize(Archive& ar, const unsigned int){ - ar & location & steps_left; - } -}; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - int domain_size = 20; - int max_walk_size = 100; - int num_walkers_per_proc = 10; - - // decompose domain - assert(world.size() <= domain_size); - int subdomain_start = domain_size/world.size()*world.rank(); - int subdomain_size = domain_size/world.size(); - if(world.rank() == world.size() - 1) subdomain_size += domain_size%world.size(); - - // initialize walkers - std::vector incomming_walkers(num_walkers_per_proc); - std::vector outgoing_walkers; - - std::random_device rd; - std::mt19937 gen(rd()*world.rank()); - std::uniform_int_distribution<> dis(0, max_walk_size); - std::generate( - incomming_walkers.begin(), incomming_walkers.end(), - [&]{return walker{subdomain_start, dis(gen)};} - ); - - cout << "process " << world.rank() << " initialized " << num_walkers_per_proc << " walkers in subdomain " << subdomain_start << " - " << subdomain_start + subdomain_size - 1 << std::endl; - - // determine the maximum amount of sends and receives needed to complete all walkers - int maximum_sends_recvs = max_walk_size/(domain_size/world.size()) + 1; - for(int m = 0; m != maximum_sends_recvs; ++m){ - // process all incomming_walkers - for(auto& w : incomming_walkers){ - while(w.steps_left){ - if(w.location == subdomain_start + subdomain_size){ - if(w.location == domain_size) w.location = 0; - outgoing_walkers.push_back(w); - break; - }else{ - --w.steps_left; - ++w.location; - } - } - } - cout << "process " << world.rank() << " sending " << outgoing_walkers.size() << " outgoing walkers to process " << (world.rank() + 1)%world.size() << std::endl; - if(world.rank()%2 == 0){ - world[(world.rank() + 1)%world.size()] << outgoing_walkers; - outgoing_walkers.clear(); - world[world.rank()?world.rank()-1:world.size()-1] >> incomming_walkers; - }else{ - world[world.rank()?world.rank()-1:world.size()-1] >> incomming_walkers; - world[(world.rank() + 1)%world.size()] << outgoing_walkers; - outgoing_walkers.clear(); - } - cout << "process " << world.rank() << " received " << incomming_walkers.size() << " incomming_walkers" << std::endl; - } - cout << "process " << world.rank() << " done" << std::endl; - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/rank.cpp b/external_codes/mpi_wrapper/mpi3/examples/rank.cpp deleted file mode 100644 index 199014ea6d0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/rank.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/process.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -std::random_device rd; -std::mt19937 gen(rd()); -std::uniform_real_distribution<> dis; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - - double number = dis(gen); - std::vector numbers = world.gather_value(number, 0); - - std::vector ranks; - if(world.rank() == 0){ - assert( not numbers.empty() ); - ranks.resize(numbers.size()); - std::iota(ranks.begin(), ranks.end(), 0); - std::sort(ranks.begin(), ranks.end(), [&numbers](auto a, auto b){return numbers[a] < numbers[b];}); - } else assert( numbers.empty() ); - - int rank = world.scatter_value(ranks); - cout << "processor " << world.rank() << " has value " << number << " and ranks " << rank << std::endl; - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/reduce_average.cpp b/external_codes/mpi_wrapper/mpi3/examples/reduce_average.cpp deleted file mode 100644 index eeaf8d70707..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/reduce_average.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/process.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -std::random_device rd; -std::mt19937 gen(rd()); -std::uniform_real_distribution<> dis; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - - int elements_per_proc = 5000; - - std::vector v(elements_per_proc); - std::generate(v.begin(), v.end(), [&](){return dis(gen);}); - - double local_sum = std::accumulate(v.begin(), v.end(), 0.); - cout << "local sum for proc " << world.rank() << " is " << local_sum << " average is " << local_sum/v.size() << std::endl; - - {// reduce -// double global_sum = world.reduce_value(local_sum, mpi3::sum, 0); - mpi3::optional global_sum = (world[0] += local_sum); - if(global_sum){ - assert(world.rank() == 0); - cout << "total sum is " << *global_sum << ", average is " << *global_sum/(world.size()*elements_per_proc) << std::endl; - } - } - - {// all reduce, all reduce is necessary to compute standard deviations, so everyone knows the mean - double global_sum = world.all_reduce_value(local_sum, mpi3::sum); - double mean = global_sum /(elements_per_proc*world.size()); - double local_squares = 0.; - for(auto& e : v){ - local_squares += (e - mean)*(e - mean); - } - // double global_squares = world.reduce_value(local_squares, mpi3::sum, 0); - mpi3::optional global_squares = (world[0] += local_squares); - if(global_squares){ - double stddev = sqrt( *global_squares / elements_per_proc / world.size() ); - cout << "mean = " << mean << ", standard deviation = " << stddev << std::endl; - } - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/ring.cpp b/external_codes/mpi_wrapper/mpi3/examples/ring.cpp deleted file mode 100644 index c0f5f9eb542..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/ring.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x && time mpirun -np 8 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - int token = -1; - - if(world.rank() != 0){ - world[world.rank() - 1] << token; - cout << "proc " << world.rank() << " received token " << token << " from proc " << world.rank() -1 << '\n'; - }; - - world[(world.rank() + 1)%world.size()] << token; - - if(world.rank() == 0){ - world[world.size() - 1] >> token; - cout << "proc " << world.rank() << " received token " << token << " from proc " << world.size() - 1 << '\n'; - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/send_receive.cpp b/external_codes/mpi_wrapper/mpi3/examples/send_receive.cpp deleted file mode 100644 index 1cb6e012cd2..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/send_receive.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -export PATH=/home/correaa/prj/alf/boost/mpi3/fake:$PATH -mpic++ -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x && time mpirun -np 1 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - if( world.size() < 2 ) throw std::runtime_error("needs 2 processes"); - - int number = 0; - if(world.rank() == 0){ - number = -1; - world[1] << number; - }else if(world.rank() == 1){ - world[0] >> number; - cout << "process 1 received number " << number << " from process 0\n"; - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/shared_window.cpp b/external_codes/mpi_wrapper/mpi3/examples/shared_window.cpp deleted file mode 100644 index bda652af5cf..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/shared_window.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/shared_window.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char *argv[], mpi3::communicator world){ - int rank = world.rank(); - int size = world.size(); - - mpi3::shared_window sw = world.make_shared_window(world.rank()?0:size); - int* shared = (int*)sw.base(0); - - if(rank==0){ - // sw.lock_exclusive(0); - for(int i = 0; i < size; i++) shared[i] = -1; - // sw.unlock(0); - } - - world.barrier(); - - std::vector local(size); - - sw.lock_shared(0); - for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); - cout << "processor " << world.rank() << " (before) : "; - for(int i = 0; i != size; ++i) cout << local[i] << " "; - cout << std::endl; - sw.unlock(0); - - sw.lock_exclusive(0); - sw.put_n(&rank, 1, 0, rank); - sw.unlock(0); - - sw.lock_shared(0); - for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); - cout << "processor " << world.rank() << " (after) : "; - for(int i = 0; i != size; ++i) cout << local[i] << ' '; - cout << std::endl; - sw.unlock(0); - - world.barrier(); - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/shared_window_2.cpp b/external_codes/mpi_wrapper/mpi3/examples/shared_window_2.cpp deleted file mode 100644 index d2b76aa3281..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/shared_window_2.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/mutex.hpp" -#include "alf/boost/mpi3/shared_window.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char *argv[], mpi3::communicator& world){ - - mpi3::communicator node = world.split_shared(0); - - mpi3::shared_window win = node.make_shared_window(1); - win.lock_all(); - win.sync(); - - node.barrier(); - - std::vector b_size(node.size()); - std::vector b_disp(node.size()); - std::vector buf(node.size()); - -// mpi3::mutex m(node); - if(node.rank() != 0){ - for(int i = 0; i != node.size(); ++i){ - b_size[i] = win.size(i); - b_disp[i] = win.disp_unit(i); - buf[i] = static_cast(win.base(i)); - if(i == node.rank()){ - *buf[i] = node.rank() + 100; - cout << "node rank " << node.rank() << ": *buf[" << i << "] = " << *buf[i] << std::endl; - } - } - }else cout << "node rank " << node.rank() << " master just watches." << std::endl; - - win.unlock_all(); - - if(node.rank() != 0) - for(int i=0; i != node.size(); ++i) - if(buf[i]) cout << "node rank " << node.rank() << ", target = " << i << ", buf = " << *buf[i] << ", b_size = " << b_size[i] << ", b_disp = " << b_disp[i] << std::endl; - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/shared_window_3.cpp b/external_codes/mpi_wrapper/mpi3/examples/shared_window_3.cpp deleted file mode 100644 index f038ce42cb6..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/shared_window_3.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x && time mpirun -np 10 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/shared_window.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char *argv[], mpi3::communicator& world){ - int rank = world.rank(); - int size = world.size(); - - mpi3::shared_window sw = world.make_shared_window(world.rank()?0:size); - int* shared = (int*)sw.base(0); - - if(rank==0){ - // sw.lock_exclusive(0); - for(int i = 0; i < size; i++) shared[i] = -1; - // sw.unlock(0); - } - - world.barrier(); - - std::vector local(size); - - sw.lock_shared(0); - for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); - cout << "processor " << world.rank() << " (before) : "; - for(int i = 0; i != size; ++i) cout << local[i] << " "; - cout << std::endl; - sw.unlock(0); - - sw.lock_exclusive(0); - sw.put_n(&rank, 1, 0, rank); - sw.unlock(0); - - sw.lock_shared(0); - for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); - cout << "processor " << world.rank() << " (after) : "; - for(int i = 0; i != size; ++i) cout << local[i] << ' '; - cout << std::endl; - sw.unlock(0); - - world.barrier(); - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/shm_vector.cpp b/external_codes/mpi_wrapper/mpi3/examples/shm_vector.cpp deleted file mode 100644 index 4f25ffa3def..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/shm_vector.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -time mpicxx -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/shm/vector.hpp" - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/mutex.hpp" - -#include -#include // lock_guard -#include -#include // sleep_for - -int rand(int lower, int upper){ - static std::random_device rd; - static std::mt19937 rng(rd()); - static std::uniform_int_distribution uni(lower, upper); - return uni(rng); -} -int rand(int upper = RAND_MAX){return rand(0, upper);} - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - - mpi3::shm::managed_shared_memory mpi3mshm(world); - - using elem = double; - - cout << "about to construct" << std::endl; - mpi3::shm::vector v(10, mpi3mshm.get_allocator()); - assert(v.data()); - v.resize(55); - cout << v.data() << std::endl; - assert(v.data()); - assert(not v.empty() and v.front() == 0 and std::equal(std::next(v.begin()), v.end(), v.begin()) ); - return 0; - - mpi3::mutex m(world); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - { - std::lock_guard lock(m); // m.lock(); - for(int i = 0; i != v.size(); ++i){ - v[i] = world.rank(); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - } // m.unlock(); - } - world.barrier(); - - if(world.rank() == 0){ - for(int i = 0; i != v.size(); ++i) cout << v[i] << " "; - cout << std::endl; - } - assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); - cout << "about to resize" << std::endl; - assert(v.data()); - v.resize(20); - assert(v.data()); - cout << "after resize" << std::endl; - world.barrier(); - assert(v.size()==20); - cout << "before assign" << std::endl; - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - { - std::lock_guard lock(m); - for(int i = 0; i != v.size(); ++i){ - v[i] = world.rank(); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - } - } - cout << "after assign" << std::endl; - world.barrier(); - cout << "after barrier" << std::endl; - - if(world.rank() == 0){ - // for(int i = 0; i != v.size(); ++i) cout << v[i] << " "; - cout << std::endl; - } -// assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); - cout << "end" << std::endl; - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/split.cpp b/external_codes/mpi_wrapper/mpi3/examples/split.cpp deleted file mode 100644 index fb4e03aeaee..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/split.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 16 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - - { - // int color = world.rank() / 4; - // mpi3::communicator row = world.split(color, world.rank()); - mpi3::communicator row = world / 4; // shortcut - cout << "world " << world.rank() << '/' << world.size() << " ---> row " << row.rank() << '/' << row.size() << '\n'; - } - { - // int color = world.rank() % 4; - // mpi3::communicator col = world.split(color, world.rank()); - mpi3::communicator col = world % 4; // shortcut - cout << "world " << world.rank() << '/' << world.size() << " ---> col " << col.rank() << '/' << col.size() << '\n'; - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/examples/test3.cpp b/external_codes/mpi_wrapper/mpi3/examples/test3.cpp deleted file mode 100644 index c1ae4cd6fea..00000000000 --- a/external_codes/mpi_wrapper/mpi3/examples/test3.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/shared_main.hpp" - -#include - -#include -#include -#include - -void mySum(void *in_, void *out_, int *len, MPI_Datatype *dtype) { - double* in = reinterpret_cast(in_); - double* out = reinterpret_cast(out_); - MPI_Aint sz,lb; - MPI_Type_get_extent(*dtype,&lb,&sz); - for(int i=0, iend=sz/sizeof(double); i,2> A(extents[ni][nw*node.size()]); // 2x2 matrix - boost::multi_array,2> B(extents[ni][nw]); // 2x2 matrix - boost::multi_array,2> C(extents[ni][nw]); // 2x2 matrix - - std::vector nr{10000,100000,250000,500000,ni}; - - for(std::size_t n=0; n != nr.size(); n++) { - ni = nr[n]; - mpi3::type tA = mpi3::double_.vector(ni, 2*nw, 2*A.strides()[0]); -// MPI_Datatype typeA; -// MPI_Type_vector(ni, 2*nw, 2*A.strides()[0], MPI_DOUBLE, &typeA); - tA.commit(); -// MPI_Type_commit(&typeA); -// MPI_Op myOpSum; -// MPI_Op_create(mySum,true,&myOpSum); -// mpi3::operation my_sum = - mpi3::commutative_operation my_sum(&mySum); -// MPI_Allreduce(MPI_IN_PLACE,A.data(),2*ni*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - node.all_reduce_in_place_n(A.data(), A.num_elements(), std::plus<>{}); - - int nt = 4; - node.barrier(); -// MPI_Barrier(MPI_COMM_WORLD); - { - boost::timer::auto_cpu_timer t("All reduce"); -// t1=getTime(); - for(int i = 0; i != nt; i++) - node.all_reduce_in_place_n(A.data(), A.num_elements(), std::plus<>{}); -/// MPI_Allreduce(MPI_IN_PLACE,A.data(),2*ni*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - } -// t2=getTime(); -// double tv1=(t2-t1)/nt; - { - for(int r=0, p=0; r != node.size(); r++, p+=nw) -// if(rank==r) node.reduce_in_place_n(A.data() + p, 1, my_sum, - MPI_Reduce(MPI_IN_PLACE, A.data()+p, 1, tA.impl_, myOpSum, r, node.impl_); - else - MPI_Reduce(A.data()+p, NULL, 1, tA.impl_, myOpSum, r, MPI_COMM_WORLD); - } - node.barrier(); - { - boost::timer::auto_cpu_timer t; - for(int i=0; i != nt; i++) - for(int r=0, p=0; r != size; r++, p+=nw) - if(rank==r) - MPI_Reduce(MPI_IN_PLACE, A.data()+p, 1, A.impl_, myOpSum, r, node.impl_); - else - MPI_Reduce(A.data()+p, NULL, 1, typeA, myOpSum, r, node.impl_); - } - std::vector req(node.size()); -// std::vector st(node.size()); - for(int r=0; r != node.size(); r++) - if(node.rank()==r) - MPI_Ireduce(C.data(), B.data(),2*ni*nw,MPI_DOUBLE,MPI_SUM,r, node.impl_, &req[r].impl_); - else - MPI_Ireduce(C.data(), B.data(),2*ni*nw,MPI_DOUBLE,MPI_SUM,r, node.impl_, &req[r].impl_); - mpi3::wait_all(req); - node.barrier(); - - { - boost::timer::auto_cpu_timer t("multiple Reduce"); - for(int i=0; i!=nt; i++){ - for(int r=0; r != node.size(); r++) - if(node.rank() == r) - MPI_Ireduce(C.data(), B.data(), 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_, &req[r].impl_); - else - MPI_Ireduce(C.data(), B.data(), 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_, &req[r].impl_); - mpi3::wait_all(req); - } - } - - for(int r=0; r != node.size(); r++) - if(node.rank()==r) - MPI_Reduce(MPI_IN_PLACE, B.data(), 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_); - else - MPI_Reduce(B.data(), NULL, 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_); - - node.barrier(); - - { - boost::timer::auto_cpu_timer t("multiple Ireduce"); - for(int i=0; i != nt; i++) - for(int r=0; r -#include -#include -#include -#include -#include -#include - -double getTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return double(tv.tv_sec)+double(tv.tv_usec)/1000000.0; -} - -void mySum(void *in_, void *out_, int *len, MPI_Datatype *dtype) { - double* in = reinterpret_cast(in_); - double* out = reinterpret_cast(out_); - MPI_Aint sz,lb; - MPI_Type_get_extent(*dtype,&lb,&sz); - for(int i=0, iend=sz/sizeof(double); i1) blk=atoi(argv[1]); - - boost::multi_array,2> A(extents[ni][nw*size]); // 2x2 matrix - boost::multi_array,2> B(extents[ni][nw]); // 2x2 matrix - boost::multi_array,2> C(extents[ni][nw]); // 2x2 matrix - - std::vector nr{10000,100000,250000,500000,ni}; - - for(int n=0; n ni) continue; - int nblk = ni/blk; - ni = nblk*blk; - - MPI_Allreduce(MPI_IN_PLACE,A.data(),2*ni*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - - int nt=4; - MPI_Barrier(MPI_COMM_WORLD); - t1=getTime(); - for(int i=0; i req(size*nblk); - std::vector st(size*nblk); - for(int r=0; r0) { - MPI_Waitall(cnt,req.data(),st.data()); - cnt=0; - } - } - t2=getTime(); - double tv5=(t2-t1)/nt; - - if(rank==0) cout<<" n, times (Allreduce, multiple IReduce 1, multiple IReduce 2): " < -#include -#include -#include -#include -#include -#include - -double getTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return double(tv.tv_sec)+double(tv.tv_usec)/1000000.0; -} - -void mySum(void *in_, void *out_, int *len, MPI_Datatype *dtype) { - double* in = reinterpret_cast(in_); - double* out = reinterpret_cast(out_); - MPI_Aint sz,lb; - MPI_Type_get_extent(*dtype,&lb,&sz); - for(int i=0, iend=sz/sizeof(double); i1) blk=atoi(argv[1]); - std::vector nr{1,10,25,50,75}; - - boost::multi_array,2> A(extents[ni*75][nw*size]); // 2x2 matrix - boost::multi_array,2> A_(extents[ni*75][nw*size]); // 2x2 matrix - boost::multi_array,2> B(extents[ni*75][nw]); // 2x2 matrix - boost::multi_array,2> C(extents[ni*75][nw]); // 2x2 matrix - - - for(int n=0; n ni_) continue; - int nblk = ni_/blk; - ni_ = nblk*blk; - - MPI_Allreduce(A_.data(),A.data(),2*ni_*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_GROUP); - - int nt=4; - MPI_Barrier(MPI_COMM_WORLD); - t1=getTime(); - for(int i=0; i req(nmmax); - std::vector st(nmmax); - MPI_Barrier(MPI_COMM_WORLD); - t1=getTime(); - for(int i=0; i0) { - MPI_Waitall(cnt,req.data(),st.data()); - cnt=0; - } - } - MPI_Barrier(MPI_COMM_WORLD); - t2=getTime(); - double tv5=(t2-t1)/nt; - - if(global_rank==0) cout<<" n, times (Allreduce, multiple IReduce): " < -int main(int argc, char *argv[]) -{ - int rank, nprocs; - - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD,&nprocs); - MPI_Comm_rank(MPI_COMM_WORLD,&rank); - MPI_Barrier(MPI_COMM_WORLD); - printf("Hello, world. I am %d of %d\n", rank, nprocs);fflush(stdout); - MPI_Finalize(); - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_call_errhandler.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_call_errhandler.c deleted file mode 100644 index 03a23511b6e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_call_errhandler.c +++ /dev/null @@ -1,40 +0,0 @@ -#include "mpi.h" -#include - -static int calls = 0; -static int errs = 0; -static MPI_Comm mycomm; - -void eh( MPI_Comm *comm, int *err, ... ) -{ - if (*err != MPI_ERR_OTHER) { - errs++; - printf( "Unexpected error code\n" );fflush(stdout); - } - if (*comm != mycomm) { - errs++; - printf( "Unexpected communicator\n" );fflush(stdout); - } - calls++; - return; -} - -int main( int argc, char *argv[] ) -{ - MPI_Comm comm; - MPI_Errhandler newerr; - - MPI_Init( &argc, &argv ); - comm = MPI_COMM_WORLD; - mycomm = comm; - MPI_Comm_create_errhandler( eh, &newerr ); - MPI_Comm_set_errhandler( comm, newerr ); - MPI_Comm_call_errhandler( comm, MPI_ERR_OTHER ); - MPI_Errhandler_free( &newerr ); - if (calls != 1) { - errs++; - printf( "Error handler not called\n" );fflush(stdout); - } - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_compare.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_compare.c deleted file mode 100644 index d22fd4a9807..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_compare.c +++ /dev/null @@ -1,36 +0,0 @@ -#include "mpi.h" -#include - -int main( int argc, char *argv[] ) -{ - int errs = 0; - int size, dims[2], periods[2], remain[2]; - int result; - MPI_Comm comm, newcomm; - - MPI_Init( &argc, &argv ); - - /* First, create a 1-dim cartesian communicator */ - periods[0] = 0; - MPI_Comm_size( MPI_COMM_WORLD, &size ); - dims[0] = size; - MPI_Cart_create( MPI_COMM_WORLD, 1, dims, periods, 0, &comm ); - - /* Now, extract a communicator with no dimensions */ - remain[0] = 0; - MPI_Cart_sub( comm, remain, &newcomm ); - - /* This should be congruent to MPI_COMM_SELF */ - MPI_Comm_compare( MPI_COMM_SELF, newcomm, &result ); - if (result != MPI_CONGRUENT) { - errs++; - printf( "cart sub to size 0 did not give self\n" );fflush(stdout); - } - - /* Free the new communicator */ - MPI_Comm_free( &newcomm ); - MPI_Comm_free( &comm ); - - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_dup.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_dup.c deleted file mode 100644 index ea7292436ca..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_dup.c +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include - -int main(int argc, char* argv[] ) -{ - MPI_Comm dup_comm_world, world_comm; - MPI_Group world_group; - int world_rank, world_size, rank, size; - - MPI_Init(&argc, &argv); - MPI_Comm_rank( MPI_COMM_WORLD, &world_rank ); - MPI_Comm_size( MPI_COMM_WORLD, &world_size ); - MPI_Comm_dup( MPI_COMM_WORLD, &dup_comm_world ); - /* Exercise Comm_create by creating an equivalent to dup_comm_world (sans attributes) */ - MPI_Comm_group( dup_comm_world, &world_group ); - MPI_Comm_create( dup_comm_world, world_group, &world_comm ); - MPI_Comm_rank( world_comm, &rank ); - if (rank != world_rank) { - printf( "incorrect rank in world comm: %d\n", rank );fflush(stdout); - MPI_Abort(MPI_COMM_WORLD, 3001 ); - } - MPI_Finalize(); - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_free.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_free.c deleted file mode 100644 index 8179a075f10..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_free.c +++ /dev/null @@ -1,90 +0,0 @@ -#include "mpi.h" -#include -#include - -/* Test that communicators have reference count semantics */ -#define NELM 128 -#define NCOMM 1020 - -int main( int argc, char *argv[] ) -{ - int errs = 0; - int rank, size, source, dest, i; - MPI_Comm comm; - MPI_Comm tmpComm[NCOMM]; - MPI_Status status; - MPI_Request req; - int *buf=0; - - MPI_Init( &argc, &argv ); - MPI_Comm_dup( MPI_COMM_WORLD, &comm ); - /* This is similar to the datatype test, except that we post - an irecv on a simple data buffer but use a rank-reordered communicator. - In this case, an error in handling the reference count will most - likely cause the program to hang, so this should be run only - if (a) you are confident that the code is correct or (b) - a timeout is set for mpiexec - */ - MPI_Comm_rank( comm, &rank ); - MPI_Comm_size( comm, &size ); - if (size < 2) { - fprintf( stderr, "This test requires at least two processes." );fflush(stderr); - MPI_Abort( MPI_COMM_WORLD, 1 ); - } - source = 0; - dest = size - 1; - if (rank == dest) { - buf = (int *)malloc( NELM * sizeof(int) ); - for (i=0; i -int main(int argc, char *argv[]) -{ - int rank, nprocs; - - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD,&nprocs); - MPI_Comm_rank(MPI_COMM_WORLD,&rank); - printf("Hello, world. I am %d of %d\n", rank, nprocs);fflush(stdout); - MPI_Finalize(); - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_set_errhandler.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_set_errhandler.c deleted file mode 100644 index 5f7ca5c0273..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Comm_set_errhandler.c +++ /dev/null @@ -1,45 +0,0 @@ -#include "mpi.h" -#include -#include -static int calls = 0; -static int errs = 0; -static MPI_Comm mycomm; - -void eh( MPI_Comm *comm, int *err, ... ) -{ - printf( "eh called\n" );fflush(stdout); - if (*err != MPI_ERR_OTHER) { - errs++; - printf( "Unexpected error code\n" );fflush(stdout); - } - if (*comm != mycomm) { - errs++; - printf( "Unexpected communicator\n" );fflush(stdout); - } - calls++; - return; -} - -int main( int argc, char *argv[] ) -{ - MPI_Comm comm; - MPI_Errhandler newerr; - - MPI_Init( &argc, &argv ); - comm = MPI_COMM_WORLD; - mycomm = comm; - MPI_Comm_create_errhandler( eh, &newerr ); - int s = -1; - s = MPI_Comm_set_errhandler( comm, newerr ); // fatal error in OpenMPI - assert(s == MPI_SUCCESS); - int rank = -1; - MPI_Comm_rank( comm, &rank ); // error but handler not called - MPI_Comm_call_errhandler( comm, MPI_ERR_OTHER ); // ok, it seems - MPI_Errhandler_free( &newerr ); - if (calls != 1) { - errs++; - printf( "Error handler not called\n" );fflush(stdout); - } - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Gather.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Gather.c deleted file mode 100644 index d37a374bdf0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Gather.c +++ /dev/null @@ -1,58 +0,0 @@ -#include "mpi.h" -#include -#include - -/* Gather data from a vector to contiguous */ - -int main( int argc, char **argv ) -{ - MPI_Datatype vec; - MPI_Comm comm; - double *vecin, *vecout; - int minsize = 2, count; - int root, i, n, stride, errs = 0; - int rank, size; - - MPI_Init( &argc, &argv ); - comm = MPI_COMM_WORLD; - /* Determine the sender and receiver */ - MPI_Comm_rank( comm, &rank ); - MPI_Comm_size( comm, &size ); - - for (root=0; root - -int main(int argc, char *argv[]) -{ - int buffer[6]; - int rank, size, i; - int receive_counts[4] = { 0, 1, 2, 3 }; - int receive_displacements[4] = { 0, 0, 1, 3 }; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &size); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (size != 4) - { - if (rank == 0) - { - printf("Please run with 4 processes\n");fflush(stdout); - } - MPI_Finalize(); - return 0; - } - for (i=0; i - -int main(int argc, char *argv[]) -{ - int rank, size, i; - int buffer[10]; - MPI_Status status; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &size); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (size < 2) - { - printf("Please run with two processes.\n");fflush(stdout); - MPI_Finalize(); - return 0; - } - if (rank == 0) - { - for (i=0; i<10; i++) - buffer[i] = i; - MPI_Send(buffer, 10, MPI_INT, 1, 123, MPI_COMM_WORLD); - } - if (rank == 1) - { - for (i=0; i<10; i++) - buffer[i] = -1; - MPI_Recv(buffer, 10, MPI_INT, 0, 123, MPI_COMM_WORLD, &status); - for (i=0; i<10; i++) - { - if (buffer[i] != i) - printf("Error: buffer[%d] = %d but is expected to be %d\n", i, buffer[i], i); - } - fflush(stdout); - } - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Reduce.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Reduce.c deleted file mode 100644 index e2097bfbfe9..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Reduce.c +++ /dev/null @@ -1,43 +0,0 @@ -#include "mpi.h" -#include -#include - -/* A simple test of Reduce with all choices of root process */ -int main( int argc, char *argv[] ) -{ - int errs = 0; - int rank, size, root; - int *sendbuf, *recvbuf, i; - int minsize = 2, count; - MPI_Comm comm; - - MPI_Init( &argc, &argv ); - - comm = MPI_COMM_WORLD; - /* Determine the sender and receiver */ - MPI_Comm_rank( comm, &rank ); - MPI_Comm_size( comm, &size ); - - for (count = 1; count < 130000; count = count * 2) { - sendbuf = (int *)malloc( count * sizeof(int) ); - recvbuf = (int *)malloc( count * sizeof(int) ); - for (root = 0; root < size; root ++) { - for (i=0; i - -int main(int argc, char *argv[]) -{ - int rank, size, i; - int buffer[10]; - MPI_Status status; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &size); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (size < 2) - { - printf("Please run with two processes.\n");fflush(stdout); - MPI_Finalize(); - return 0; - } - if (rank == 0) - { - for (i=0; i<10; i++) - buffer[i] = i; - MPI_Send(buffer, 10, MPI_INT, 1, 123, MPI_COMM_WORLD); - } - if (rank == 1) - { - for (i=0; i<10; i++) - buffer[i] = -1; - MPI_Recv(buffer, 10, MPI_INT, 0, 123, MPI_COMM_WORLD, &status); - for (i=0; i<10; i++) - { - if (buffer[i] != i) - printf("Error: buffer[%d] = %d but is expected to be %d\n", i, buffer[i], i); - } - fflush(stdout); - } - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Sendrecv_replace.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Sendrecv_replace.c deleted file mode 100644 index 17f9574414b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Sendrecv_replace.c +++ /dev/null @@ -1,24 +0,0 @@ -#include "mpi.h" -#include - -int main(int argc, char *argv[]) -{ - int myid, numprocs, left, right; - int buffer[10]; -/* MPI_Request request;*/ - MPI_Status status; - - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &numprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &myid); - - right = (myid + 1) % numprocs; - left = myid - 1; - if (left < 0) - left = numprocs - 1; - - MPI_Sendrecv_replace(buffer, 10, MPI_INT, left, 123, right, 123, MPI_COMM_WORLD, &status); - - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_contiguos.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_contiguos.c deleted file mode 100644 index d39081543c2..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_contiguos.c +++ /dev/null @@ -1,29 +0,0 @@ -#include "mpi.h" -#include - -int main(int argc, char *argv[]) -{ - int myrank; - MPI_Status status; - MPI_Datatype type; - int buffer[100]; - - MPI_Init(&argc, &argv); - - MPI_Type_contiguous( 100, MPI_CHAR, &type ); - MPI_Type_commit(&type); - - MPI_Comm_rank(MPI_COMM_WORLD, &myrank); - - if (myrank == 0) - { - MPI_Send(buffer, 1, type, 1, 123, MPI_COMM_WORLD); - } - else if (myrank == 1) - { - MPI_Recv(buffer, 1, type, 0, 123, MPI_COMM_WORLD, &status); - } - - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_dup.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_dup.c deleted file mode 100644 index af1cbcf9076..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_dup.c +++ /dev/null @@ -1,31 +0,0 @@ -#include "mpi.h" -#include - -int main(int argc, char *argv[]) -{ -// int myrank; -// MPI_Status status; - MPI_Datatype type, type2; -// int buffer[100]; - - MPI_Init(&argc, &argv); - - MPI_Type_contiguous( 100, MPI_CHAR, &type ); - MPI_Type_commit(&type); - MPI_Type_dup(type, &type2); - MPI_Type_free(&type); - -/* MPI_Comm_rank(MPI_COMM_WORLD, &myrank); - - if (myrank == 0) - { - MPI_Send(buffer, 1, type2, 1, 123, MPI_COMM_WORLD); - } - else if (myrank == 1) - { - MPI_Recv(buffer, 1, type2, 0, 123, MPI_COMM_WORLD, &status); - } -*/ - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_get_extent.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_get_extent.c deleted file mode 100644 index e4fcfd74521..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_get_extent.c +++ /dev/null @@ -1,62 +0,0 @@ -#include "mpi.h" -#include -#include - -int main(int argc, char **argv) -{ - int /*mpi_err,*/ errs = 0, size; - MPI_Aint lb, ub, extent; - MPI_Datatype type; - struct { float a; int b; } foo; - - MPI_Init(&argc, &argv); - - type = MPI_INT; - MPI_Type_size(type, &size); - if (size != sizeof(int)) { - fprintf(stderr, "MPI_Type_size of MPI_INT incorrect size (%d); should be %d.\n", size, (int) sizeof(int));fflush(stderr); - errs++; - } - - MPI_Type_get_extent(type, &lb, &extent); - if (extent != sizeof(int)) { - fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(int));fflush(stderr); - errs++; - } - if (lb != 0) { - fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); - errs++; - } - - MPI_Type_ub(type, &ub); - if (ub != extent - lb) { - fprintf(stderr, "MPI_Type_ub of MPI_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); - errs++; - } - - type = MPI_FLOAT_INT; - MPI_Type_size(type, &size); - if (size != sizeof(float) + sizeof(int)) { - fprintf(stderr, "MPI_Type_size of MPI_FLOAT_INT returned incorrect size (%d); should be %d.\n", size, (int) (sizeof(float) + sizeof(int)));fflush(stderr); - errs++; - } - - MPI_Type_get_extent(type, &lb, &extent); - if (extent != sizeof(foo)) { - fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(foo));fflush(stderr); - errs++; - } - if (lb != 0) { - fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); - errs++; - } - - MPI_Type_ub(type, &ub); - if (ub != extent - lb) { - fprintf(stderr, "MPI_Type_ub of MPI_FLOAT_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); - errs++; - } - - MPI_Finalize(); - return errs; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_size.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_size.c deleted file mode 100644 index e4fcfd74521..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_size.c +++ /dev/null @@ -1,62 +0,0 @@ -#include "mpi.h" -#include -#include - -int main(int argc, char **argv) -{ - int /*mpi_err,*/ errs = 0, size; - MPI_Aint lb, ub, extent; - MPI_Datatype type; - struct { float a; int b; } foo; - - MPI_Init(&argc, &argv); - - type = MPI_INT; - MPI_Type_size(type, &size); - if (size != sizeof(int)) { - fprintf(stderr, "MPI_Type_size of MPI_INT incorrect size (%d); should be %d.\n", size, (int) sizeof(int));fflush(stderr); - errs++; - } - - MPI_Type_get_extent(type, &lb, &extent); - if (extent != sizeof(int)) { - fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(int));fflush(stderr); - errs++; - } - if (lb != 0) { - fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); - errs++; - } - - MPI_Type_ub(type, &ub); - if (ub != extent - lb) { - fprintf(stderr, "MPI_Type_ub of MPI_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); - errs++; - } - - type = MPI_FLOAT_INT; - MPI_Type_size(type, &size); - if (size != sizeof(float) + sizeof(int)) { - fprintf(stderr, "MPI_Type_size of MPI_FLOAT_INT returned incorrect size (%d); should be %d.\n", size, (int) (sizeof(float) + sizeof(int)));fflush(stderr); - errs++; - } - - MPI_Type_get_extent(type, &lb, &extent); - if (extent != sizeof(foo)) { - fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(foo));fflush(stderr); - errs++; - } - if (lb != 0) { - fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); - errs++; - } - - MPI_Type_ub(type, &ub); - if (ub != extent - lb) { - fprintf(stderr, "MPI_Type_ub of MPI_FLOAT_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); - errs++; - } - - MPI_Finalize(); - return errs; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_vector.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_vector.c deleted file mode 100644 index 17e28164220..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Type_vector.c +++ /dev/null @@ -1,45 +0,0 @@ -#include "mpi.h" -#include - -int main(int argc, char *argv[]) -{ - int rank, size, i; - MPI_Datatype type, type2; - int buffer[24]; - MPI_Status status; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &size); - if (size < 2) - { - printf("Please run with 2 processes.\n"); - MPI_Finalize(); - return 1; - } - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - MPI_Type_contiguous(3, MPI_INT, &type2); - MPI_Type_commit(&type2); - MPI_Type_vector(3, 2, 3, type2, &type); - MPI_Type_commit(&type); - - if (rank == 0) - { - for (i=0; i<24; i++) - buffer[i] = i; - MPI_Send(buffer, 1, type, 1, 123, MPI_COMM_WORLD); - } - - if (rank == 1) - { - for (i=0; i<24; i++) - buffer[i] = -1; - MPI_Recv(buffer, 1, type, 0, 123, MPI_COMM_WORLD, &status); - for (i=0; i<24; i++) - printf("buffer[%d] = %d\n", i, buffer[i]); - fflush(stdout); - } - - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtick.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtick.c deleted file mode 100644 index 25356b25c91..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtick.c +++ /dev/null @@ -1,14 +0,0 @@ -#include "mpi.h" -#include - -int main( int argc, char *argv[] ) -{ - double tick; - - MPI_Init( 0, 0 ); - tick = MPI_Wtick(); - printf("A single MPI tick is %0.9f seconds\n", tick); - fflush(stdout); - MPI_Finalize( ); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtime.c b/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtime.c deleted file mode 100644 index c3a920e9264..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/MPI_Wtime.c +++ /dev/null @@ -1,17 +0,0 @@ -#include "mpi.h" -//#include -#include -#include - -int main( int argc, char *argv[] ) -{ - double t1, t2; - - MPI_Init( 0, 0 ); - t1 = MPI_Wtime(); - sleep(5); - t2 = MPI_Wtime(); - printf("MPI_Wtime measured a 1 second sleep to be: %1.2f\n", t2-t1);fflush(stdout); - MPI_Finalize( ); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/Makefile b/external_codes/mpi_wrapper/mpi3/fake/examples/Makefile deleted file mode 100644 index b3acdb995ce..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -all : abort.x MPI_Send.x MPI_Recv.x MPI_Comm_dup.x - mpirun -np 1 ./abort.x - mpirun -np 1 ./MPI_Send.x - mpirun -np 1 ./MPI_Recv.x - mpirun -np 1 ./MPI_Comm_dup.x - rm -f $^ - -abort.x : abort.cpp - mpic++ abort.cpp -o abort.x - -MPI_Send.x : MPI_Send.cpp - mpic++ MPI_Send.cpp -o MPI_Send.x - -MPI_Recv.x : MPI_Recv.cpp - mpic++ MPI_Recv.cpp -o MPI_Recv.x - -MPI_Comm_dup.x : MPI_Comm_dup.cpp - mpic++ MPI_Comm_dup.cpp -o MPI_Comm_dup.x - -clean: - rm -f *.x - -load: - module unload mpi - diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/abort.cpp b/external_codes/mpi_wrapper/mpi3/fake/examples/abort.cpp deleted file mode 100644 index 3ef5a87a642..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/abort.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include -int main(int argc, char *argv[]) -{ - MPI_Init(NULL, NULL); - MPI_Abort(MPI_COMM_WORLD, 911); - /* No further code will execute */ - assert(0); - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/examples/testme b/external_codes/mpi_wrapper/mpi3/fake/examples/testme deleted file mode 100755 index 64b6838036a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/examples/testme +++ /dev/null @@ -1,3 +0,0 @@ -$2mpicc $1 -g -o $1.x -Wall -Wno-unused-parameter `#-Wfatal-errors`; $2mpirun -np 1 $1.x; -#rm $1.x - diff --git a/external_codes/mpi_wrapper/mpi3/fake/load b/external_codes/mpi_wrapper/mpi3/fake/load deleted file mode 100755 index 139597f9cb0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/load +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/external_codes/mpi_wrapper/mpi3/fake/mpi.h b/external_codes/mpi_wrapper/mpi3/fake/mpi.h deleted file mode 100644 index 9a621567b6e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/mpi.h +++ /dev/null @@ -1,2763 +0,0 @@ -#ifndef FAKE_MPI_H//-*-indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4;-*- -#define FAKE_MPI_H -// Fake MPI for serial execution - -#include // HOST_NAME_MAX -#include // gethostname -#include // strlen -#include // assert -#include // exit -#include // clock_gettime -#include // puts for debug - -#include -#ifdef __cplusplus -#include -#endif -#ifndef __cplusplus -#include -#endif - - -// Use weak linking on functions and variables to avoid multiple-definition -// errors at link time. - -#ifdef __cplusplus -#define WEAK inline -#else -#define WEAK __attribute__((weak)) -#endif - -#define WEAKVAR __attribute__((weak)) - - -typedef int MPI_Count; -typedef long long int MPI_Offset; - - -const int MPI_MAX_PROCESSOR_NAME = HOST_NAME_MAX; -const int MPI_MAX_INFO_KEY = 128; -const int MPI_MAX_INFO_VAL = 128; -const int MPI_MAX_ERROR_STRING = 128; - -enum {//: int { // error classes - MPI_SUCCESS, // No error - MPI_ERR_BUFFER, // Invalid buffer pointer - MPI_ERR_COUNT, // Invalid count argument - MPI_ERR_TYPE, // Invalid datatype argument - MPI_ERR_TAG, // Invalid tag argument - MPI_ERR_COMM, // Invalid communicator - MPI_ERR_RANK, // Invalid rank - MPI_ERR_REQUEST, // Invalid request (handle) - MPI_ERR_ROOT, // Invalid root - MPI_ERR_GROUP, // Invalid group - MPI_ERR_OP, // Invalid operation - MPI_ERR_TOPOLOGY, // Invalid topology - MPI_ERR_DIMS, // Invalid dimension argument - MPI_ERR_ARG, // Invalid argument of some other kind - MPI_ERR_UNKNOWN, // Unknown error - MPI_ERR_TRUNCATE, // Message truncated on receive - MPI_ERR_OTHER, // Known error not in this list - MPI_ERR_INTERN, // Internal MPI (implementation) error - MPI_ERR_IN_STATUS, // Error code is in status - MPI_ERR_PENDING, // Pending request - MPI_ERR_KEYVAL, // Invalid keyval has been passed - MPI_ERR_NO_MEM, // MPI_ALLOC_MEM failed because memory is exhausted - MPI_ERR_BASE, // Invalid base passed to MPI_FREE_MEM - MPI_ERR_INFO_KEY, // Key longer than MPI_MAX_INFO_KEY - MPI_ERR_INFO_VALUE, // Value longer than MPI_MAX_INFO_VAL - MPI_ERR_INFO_NOKEY, // Invalid key passed to MPI_INFO_DELETE - MPI_ERR_SPAWN, // Error in spawning processes - MPI_ERR_PORT, // Invalid port name passed to MPI_COMM_CONNECT - MPI_ERR_SERVICE, // Invalid service name passed to MPI_UNPUBLISH_NAME - MPI_ERR_NAME, // Invalid service name passed to MPI_LOOKUP_NAME - MPI_ERR_WIN, // Invalid win argument - MPI_ERR_SIZE, // Invalid size argument - MPI_ERR_DISP, // Invalid disp argument - MPI_ERR_INFO, // Invalid info argument - MPI_ERR_LOCKTYPE, // Invalid locktype argument - MPI_ERR_ASSERT, // Invalid assert argument - MPI_ERR_RMA_CONFLICT, // Conflicting accesses to window - MPI_ERR_RMA_SYNC, // Wrong synchronization of RMA calls - MPI_ERR_RMA_RANGE, // Target memory is not part of the window (in the case of a window created with MPI_WIN_CREATE_DYNAMIC, target memory is not attached) - MPI_ERR_RMA_ATTACH, // Memory cannot be attached (e.g., because of resource exhaustion) - MPI_ERR_RMA_SHARED, // Memory cannot be shared (e.g., some process in the group of the specified communicator cannot expose shared memory) - MPI_ERR_RMA_FLAVOR, // Passed window has the wrong flavor for the called function - MPI_ERR_FILE, // Invalid file handle - MPI_ERR_NOT_SAME, // Collective argument not identical on all processes, or collective routines called in a different order by different processes - MPI_ERR_AMODE, // Error related to the amode passed to MPI_FILE_OPEN - MPI_ERR_UNSUPPORTED_DATAREP, // Unsupported datarep passed to MPI_FILE_SET_VIEW - MPI_ERR_UNSUPPORTED_OPERATION, // Unsupported operation, such as seeking on a file which supports sequential access only - MPI_ERR_NO_SUCH_FILE, // File does not exist - MPI_ERR_FILE_EXISTS, // File exists - MPI_ERR_BAD_FILE, // Invalid file name (e.g., path name too long) - MPI_ERR_ACCESS, // Permission denied - MPI_ERR_NO_SPACE, // Not enough space - MPI_ERR_QUOTA, // Quota exceeded - MPI_ERR_READ_ONLY, // Read-only file or file system - MPI_ERR_FILE_IN_USE, // File operation could not be completed, as the file is currently open by some process - MPI_ERR_DUP_DATAREP, // Conversion functions could not be registered because a data representation identifier that was already defined was passed to MPI_REGISTER_DATAREP - MPI_ERR_CONVERSION, // An error occurred in a user supplied data conversion function. - MPI_ERR_IO, // Other I/O error - MPI_ERR_LASTCODE // Last error code -}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm - -enum { - MPI_PROC_NULL, - MPI_ANY_SOURCE, - MPI_ANY_TAG, - MPI_UNDEFINED, - MPI_BSEND_OVERHEAD, - MPI_KEYVAL_INVALID, - MPI_LOCK_EXCLUSIVE, - MPI_LOCK_SHARED, - MPI_ROOT -}; - -enum { - MPI_MODE_APPEND, - MPI_MODE_CREATE, - MPI_MODE_DELETE_ON_CLOSE, - MPI_MODE_EXCL, - MPI_MODE_NOCHECK, - MPI_MODE_NOPRECEDE, - MPI_MODE_NOPUT, - MPI_MODE_NOSTORE, - MPI_MODE_NOSUCCEED, - MPI_MODE_RDONLY, - MPI_MODE_RDWR, - MPI_MODE_SEQUENTIAL, - MPI_MODE_UNIQUE_OPEN, - MPI_MODE_WRONLY -}; - -// Forward declarations - -struct MPI_Comm_impl_; -typedef struct MPI_Comm_impl_* MPI_Comm; - -MPI_Comm MPI_COMM_NULL WEAKVAR = NULL;//{{&fatal_error_}}; -MPI_Comm MPI_COMM_WORLD WEAKVAR = NULL; -MPI_Comm MPI_COMM_SELF WEAKVAR = NULL; - -struct MPI_Datatype_impl_; -typedef struct MPI_Datatype_impl_* MPI_Datatype; - -int MPI_Comm_rank(MPI_Comm comm, int* rank); -int MPI_Comm_size(MPI_Comm comm, int *size); - -// Communicator split type constants -const int MPI_COMM_TYPE_SHARED = 1; - -// MPI Requests -typedef enum { - MPI_REQUEST_NULL -} MPI_Request; - -typedef struct { - int MPI_ERROR; - int MPI_SOURCE; - int MPI_TAG; -} MPI_Status; - -// Constants specifying empty or ignored input - -static MPI_Status *MPI_STATUS_IGNORE = NULL; -static MPI_Status *MPI_STATUSES_IGNORE = NULL; - -static char **MPI_ARGV_NULL = NULL; -static char ***MPI_ARGVS_NULL = NULL; -static int *MPI_ERRCODES_IGNORE = NULL; - -typedef struct {} MPI_Message; - -struct MPI_Group_impl_ {}; -typedef struct MPI_Group_impl_* MPI_Group; - - -static MPI_Group MPI_GROUP_NULL = NULL; -struct MPI_Group_impl_ MPI_GROUP_EMPTY_impl WEAKVAR; -MPI_Group MPI_GROUP_EMPTY WEAKVAR = &MPI_GROUP_EMPTY_impl; - -//struct MPI_Info{}; -//const struct MPI_Info MPI_INFO_NULL; - -typedef enum { //MPI_Info { - MPI_INFO_NULL -} MPI_Info ; - -//typedef struct {} MPI_Info; - -const void* MPI_IN_PLACE WEAKVAR = (const void*)(-1); - -typedef enum { // redefined operations are supplied for MPI_REDUCE - MPI_OP_NULL, // TODO: is this an operator? - MPI_MAX, // maximum - MPI_MIN, // minimum - MPI_SUM, // sum - MPI_PROD, // product - MPI_LAND, // logical and - MPI_BAND, // bitwise and - MPI_LOR, // logical or - MPI_BOR, // bitwise or - MPI_LXOR, // logical exclusive or (xor) - MPI_BXOR, // bitwise excluse or (xor) - MPI_MAXLOC, // max value and location - MPI_MINLOC, // min value and location -// MPI_REPLACE, // TODO: is this an operator? - MPI_NO_OP, // TODO: is this an operator? - MPI_OP_LASTCODE // not standard? -} MPI_Op; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node112.htm - - -// Forward declartions for Chapter 8 - MPI Environment Management - -typedef void MPI_Comm_errhandler_function(MPI_Comm *, int *, ...); - -#ifdef USE_MPI_REMOVED_FUNCTIONS -// Deprecated in MPI-2.0. Removed in MPI_3.0 -typedef MPI_Comm_errhandler_function MPI_Handler_function; -#endif - -struct MPI_Errhandler_impl_{ - MPI_Comm_errhandler_function* func_; -}; - -typedef struct MPI_Errhandler_impl_* MPI_Errhandler; - -struct MPI_Comm_impl_{ - MPI_Errhandler errhandler_; -}; - -MPI_Errhandler MPI_ERRORS_ARE_FATAL WEAKVAR; -MPI_Errhandler MPI_ERRORS_RETURN WEAKVAR; - -MPI_Errhandler MPI_ERRHANDLER_NULL WEAKVAR = NULL; - -int MPI_Comm_call_errhandler(MPI_Comm comm, int errorcode); - - -// ----------------------------------------------------------------------------- -// Chapter 9 - The Info Object -// ----------------------------------------------------------------------------- - -// MPI Info handling. -// Would not be too hard to create an std::map implementation - -int MPI_Info_create(MPI_Info *info); - -int MPI_Info_free(MPI_Info *info); - -int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo); - -int MPI_Info_delete(MPI_Info info, const char *key); - -int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, int *flag); - -int MPI_Info_get_nkeys(MPI_Info info, int *nkeys); - -int MPI_Info_get_nthkey(MPI_Info info, int n, char *key); - -int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen, int *flag); - -int MPI_Info_set(MPI_Info info, const char *key, const char *value); - -// ----------------------------------------------------------------------------- - - -enum { // level of thread support - MPI_THREAD_SINGLE, - MPI_THREAD_FUNNELED, - MPI_THREAD_SERIALIZED, - MPI_THREAD_MULTIPLE -}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm - -// ----------------------------------------------------------------------------- -// Chapter 3 - Point-to-Point Communication -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 3.2 Blocking Send and Receive Operations -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Send( // Performs a blocking send - const void *buf, // [in] initial address of send buffer (choice) - int count, // [in] number of elements in send buffer (nonnegat...) - MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) - int dest, // [in] rank of destination (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm // [in] communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node47.htm#Node47 - int rank = -1; MPI_Comm_rank(comm, &rank); - assert(rank != dest); - int size = -1; MPI_Comm_size(comm, &size); - assert(dest < size); // Invalid rank has value 1 but must be nonnegative and less than 1 - return MPI_SUCCESS; -} - -WEAK -int MPI_Recv( // Blocking receive for a message - void *buf, // [out] initial address of receive buffer (choice) - int count, // [in] maximum number of elements in receive buffer... - MPI_Datatype datatype, // [in] datatype of each receive buffer element... - int source, // [in] rank of source (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Status *status // [out] status object (Status) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node50.htm#Node50 - int rank = -1; MPI_Comm_rank(comm, &rank); - assert(rank != source); - assert(0); - return MPI_SUCCESS; -} - -WEAK -int MPI_Get_count( // Gets the number of "top level" elements - const MPI_Status *status, // [in] return status of receive operation (Status) - MPI_Datatype datatype, // [in] datatype of each receive buffer element (handle) - int *count // [out] number of received elements (integer) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node51.htm - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.2 Communication Modes -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Bsend( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Ssend( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Rsend( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.6 Buffer Allocation and Usage -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Buffer_attach( - void *buffer, - int size -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Buffer_detach( - void *buffer_addr, - int *size -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.10 Send-Receive -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Sendrecv( // Sends and receives a message - const void *sendbuf, // [in] initial address of send buffer (choice) - int sendcount, // [in] number of elements in send buffer (integer) - MPI_Datatype sendtype, // [in] type of elements in send buffer (handle) - int dest, // [in] rank of destination (integer) - int sendtag, // [in] send tag (integer) - void *recvbuf, // [out] initial address of receive buffer (choice) - int recvcount, // [in] number of elements in receive buffer (integer) - MPI_Datatype recvtype, // [in] type of elements in receive buffer (handle) - int source, // [in] rank of source (integer) - int recvtag, // [in] receive tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Status *status // [out] status object (Status). This refers to the receive operation. -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Sendrecv_replace( // Sends and receives a message - const void *buf, // [inout] address of buffer (choice) - int sendcount, // [in] number of elements in send buffer (integer) - MPI_Datatype datatype, // [in] type of elements in send buffer (handle) - int dest, // [in] rank of destination (integer) - int sendtag, // [in] send tag (integer) - int source, // [in] rank of source (integer) - int recvtag, // [in] receive tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Status *status // [out] status object (Status). This refers to the receive operation. -) -{ - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.7 Nonblocking Communication -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Isend( // Start a nonblocking send - const void *buf, // [in] initial address of send buffer (choice) - int count, // [in] number of elements in send buffer (nonnegat...) - MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) - int dest, // [in] rank of destination (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Request *request // [out] communication request (handle) -){ - return MPI_SUCCESS; -} - -WEAK -int MPI_Ibsend( // Start a nonblocking send in buffered mode - const void *buf, // [in] initial address of send buffer (choice) - int count, // [in] number of elements in send buffer (nonnegat...) - MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) - int dest, // [in] rank of destination (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Request *request // [out] communication request (handle) -){ - return MPI_SUCCESS; -} - -WEAK -int MPI_Issend( // Start a nonblocking send in synchronous mode - const void *buf, // [in] initial address of send buffer (choice) - int count, // [in] number of elements in send buffer (nonnegat...) - MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) - int dest, // [in] rank of destination (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Request *request // [out] communication request (handle) -){ - return MPI_SUCCESS; -} - -WEAK -int MPI_Irsend( // Start a nonblocking send in ready mode - const void *buf, // [in] initial address of send buffer (choice) - int count, // [in] number of elements in send buffer (nonnegat...) - MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) - int dest, // [in] rank of destination (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Request *request // [out] communication request (handle) -){ - return MPI_SUCCESS; -} - -WEAK -int MPI_Irecv( // Nonblocking receive for a message - void *buf, // [out] initial address of receive buffer (choice) - int count, // [in] maximum number of elements in receive buffer... - MPI_Datatype datatype, // [in] datatype of each receive buffer element... - int source, // [in] rank of source (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Request *request // [out] communication request (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node50.htm#Node50 - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.7.3 Communication Completion -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Wait( // Waits for an MPI request to complete - MPI_Request *request, // [in] request (handle) - MPI_Status *status // [out] status object (Status). May be MPI_STATUS_IGNORE. -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node64.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Test( - MPI_Request *request, - int *flag, - MPI_Status *status -){ - return MPI_SUCCESS; -} - -WEAK -int MPI_Request_free( - MPI_Request *request -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.7.5 Multiple Completions -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Waitany( - int count, - MPI_Request array_of_requests[], - int *index, - MPI_Status *status -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Testany( - int count, - MPI_Request array_of_requests[], - int *index, - int *flag, - MPI_Status *status -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Waitall( - int count, - MPI_Request array_of_requests[], - MPI_Status array_of_statuses[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Testall( - int count, - MPI_Request array_of_requests[], - int *flag, - MPI_Status array_of_statuses[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Waitsome( - int incount, - MPI_Request array_of_requests[], - int *outcount, - int array_of_indices[], - MPI_Status array_of_statuses[]) -{ - if (outcount) *outcount = 0; - return MPI_SUCCESS; -} - -WEAK -int MPI_Testsome( - int incount, - MPI_Request array_of_requests[], - int *outcount, - int array_of_indices[], - MPI_Status array_of_statuses[]) -{ - if (outcount) *outcount = 0; - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.7.6 Non-destructive Test of status -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Request_get_status( - MPI_Request request, - int *flag, - MPI_Status *status -) { - if (flag) *flag = -1; // true - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.8 Probe and Cancel -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 3.8.1 Probe -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Iprobe( // Nonblocking test for a message - int source, // [in] source rank, or MPI_ANY_SOURCE (integer) - int tag, // [in] tag value or MPI_ANY_TAG (integer) - MPI_Comm comm, // [in] communicator (handle) - int *flag, // [out] True if a message with the specified source, tag... - MPI_Status *status // [out] status object (Status) -) { - return MPI_SUCCESS; -} - -// Blocking test for a message -WEAK -int MPI_Probe( // like MPI_IMPROBE except that it is a blocking call that returns only after a matching message has been found. - int source, // [in] source rank, or MPI_ANY_SOURCE (integer) - int tag, // [in] tag value or MPI_ANY_TAG (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Status *status // [out] status object (Status) -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.8.2 Matching Probe -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Mprobe( - int source, // rank of source or MPI_ANY_SOURCE (integer) - int tag, // message tag or MPI_ANY_TAG (integer) - MPI_Comm comm, // communicator (handle) - MPI_Message *message, // returned message (handle) - MPI_Status *status // status object (Status) -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.8.3 Matched Receives -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Mrecv( - void* buf, // initial address of receive buffer (choice) - int count, // number of elements in receive buffer (non-negati) - MPI_Datatype datatype, // datatype of each receive buffer element (handle) - MPI_Message *message, // message (handle) - MPI_Status *status // status object (Status) -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.8.4 Cancel -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Cancel( - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Test_cancelled( - const MPI_Status *status, - int *flag -) { - if (flag) *flag = -1; // -1 true - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 3.9 Persistent Communication Requests -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Send_init( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Bsend_init( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Ssend_init( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Rsend_init( - const void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Recv_init( - const void *buf, - int count, - MPI_Datatype datatype, - int source, - int tag, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Start( - MPI_Request *request -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Startall( - int count, - MPI_Request array_of_requests[] -) { - return MPI_SUCCESS; -} - - - - -// ----------------------------------------------------------------------------- -// Chapter 4 - Datatypes -// ----------------------------------------------------------------------------- - - -//typedef unsigned long long MPI_Aint; -typedef ssize_t MPI_Aint; - -struct MPI_Datatype_impl_{ -// MPI_Aint lb; -// MPI_Aint extent; - int bytes; // for basic types - needs to be first for gcc compilation of the DEFINE_FAKE_MPI_DATATYPE macros - bool is_basic; - int count; // [in] number of blocks (integer) -- also number of entries in arrays array_of_types , array_of_displacements and array_of_blocklengths - int* blocklens; // [in] number of elements in each block (array) - MPI_Aint* indices; // [in] byte displacement of each block (array) - MPI_Datatype* old_types; // [in] type of elements in each block (array of handles to datatype objects) -}; - -#define DEFINE_FAKE_MPI_DATATYPE(s) \ - struct MPI_Datatype_impl_ DEF##s WEAKVAR = {.bytes =0, .is_basic=true, .count=0, .blocklens=NULL, .indices=NULL, .old_types=NULL}; \ - MPI_Datatype s WEAKVAR =&DEF##s; \ - - -// MPI Datatype with associated C/C++ type -#define DEFINE_FAKE_MPI_DATATYPE2(s, t) \ - struct MPI_Datatype_impl_ DEF##s WEAKVAR = {.bytes=sizeof(t), .is_basic=true, .count=0, .blocklens=NULL, .indices=NULL, .old_types=NULL}; \ - MPI_Datatype s WEAKVAR =&DEF##s; - - -DEFINE_FAKE_MPI_DATATYPE(MPI_DATATYPE_NULL) -DEFINE_FAKE_MPI_DATATYPE2(MPI_CHAR, char) -DEFINE_FAKE_MPI_DATATYPE2(MPI_SHORT, short int) -DEFINE_FAKE_MPI_DATATYPE2(MPI_INT, int) -DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG, long) -DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG_LONG_INT, long long) -DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG_LONG, long long) -DEFINE_FAKE_MPI_DATATYPE2(MPI_SIGNED_CHAR, char) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_CHAR, unsigned char) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_SHORT, unsigned short) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED, unsigned) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_LONG, unsigned long) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_LONG_LONG, unsigned long long) -DEFINE_FAKE_MPI_DATATYPE2(MPI_FLOAT, float) -DEFINE_FAKE_MPI_DATATYPE2(MPI_DOUBLE, double) -DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG_DOUBLE, long double) -DEFINE_FAKE_MPI_DATATYPE2(MPI_WCHAR, wchar_t) -#ifdef __cplusplus -DEFINE_FAKE_MPI_DATATYPE(MPI_C_BOOL) -#else -DEFINE_FAKE_MPI_DATATYPE2(MPI_C_BOOL, _Bool) -#endif -DEFINE_FAKE_MPI_DATATYPE2(MPI_INT8_T, int8_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_INT16_T, int16_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_INT32_T, int32_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_INT64_T, int64_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT8_T, uint8_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT16_T, uint16_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT32_T, uint32_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT64_T, uint64_t) -DEFINE_FAKE_MPI_DATATYPE2(MPI_AINT, MPI_Aint) -DEFINE_FAKE_MPI_DATATYPE2(MPI_COUNT, MPI_Count) -DEFINE_FAKE_MPI_DATATYPE2(MPI_OFFSET, MPI_Offset) -DEFINE_FAKE_MPI_DATATYPE2(MPI_C_COMPLEX, float _Complex) -DEFINE_FAKE_MPI_DATATYPE2(MPI_C_FLOAT_COMPLEX, float _Complex) -DEFINE_FAKE_MPI_DATATYPE2(MPI_C_DOUBLE_COMPLEX, double _Complex) -DEFINE_FAKE_MPI_DATATYPE(MPI_BYTE) -DEFINE_FAKE_MPI_DATATYPE(MPI_PACKED) -#ifdef __cplusplus -DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_BOOL, bool) -DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_FLOAT_COMPLEX, std::complex) -DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_DOUBLE_COMPLEX, std::complex) -DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_LONG_DOUBLE_COMPLEX, std::complex) -#else -DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_BOOL) -DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_FLOAT_COMPLEX) -DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_DOUBLE_COMPLEX) -DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_LONG_DOUBLE_COMPLEX) -#endif -DEFINE_FAKE_MPI_DATATYPE(MPI_FLOAT_INT) -DEFINE_FAKE_MPI_DATATYPE(MPI_DOUBLE_INT) -DEFINE_FAKE_MPI_DATATYPE(MPI_LONG_INT) -DEFINE_FAKE_MPI_DATATYPE(MPI_2INT) -DEFINE_FAKE_MPI_DATATYPE(MPI_SHORT_INT) -DEFINE_FAKE_MPI_DATATYPE(MPI_LONG_DOUBLE_INT) -#ifdef USE_MPI_REMOVED_FUNCTIONS -DEFINE_FAKE_MPI_DATATYPE(MPI_LB) -DEFINE_FAKE_MPI_DATATYPE(MPI_UB) -#endif - -// ----------------------------------------------------------------------------- -// Chapter 4.1.2 Datatype Constructors -// ----------------------------------------------------------------------------- - -int MPI_Type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent); - -WEAK -int MPI_Type_contiguous( // Creates a contiguous datatype - int count, // [in] replication count (nonnegative integer) - MPI_Datatype oldtype, // [in] old datatype (handle) - MPI_Datatype *newtype // [out] new datatype (handle) -){ -// MPI_Aint oldlb; -// MPI_Aint oldextent; -// int s = MPI_Type_get_extent(oldtype, &oldlb, &oldextent); - *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); - if(oldtype->count == 1){ - (*newtype)->count = 1; - (*newtype)->blocklens = (int*)malloc(sizeof(int)); - (*newtype)->blocklens[0] = count; - (*newtype)->indices = (MPI_Aint*)malloc(sizeof(MPI_Aint)); - (*newtype)->indices[0] = 0; - (*newtype)->old_types = (MPI_Datatype*)malloc(count*sizeof(MPI_Datatype)); - (*newtype)->old_types[0] = MPI_BYTE; - }else assert(0); -// if(count < 0) return MPI_ERR_INTERN; -// if(!newtype) return MPI_ERR_INTERN; -// (*newtype)->lb = 0; -// if(s != MPI_SUCCESS) return MPI_ERR_TYPE; -// (*newtype)->extent = count*oldextent; - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_vector( // Creates a vector (strided) datatype - int count, // [in] number of blocks (nonnegative integer) - int blocklength, // [in] number of elements in each block (nonnegative integer) - int stride, // [in] number of elements between start of each block (integer) - MPI_Datatype old_type, // [in] old datatype (handle) - MPI_Datatype *newtype_p // [out] new datatype (handle) -){ - assert(0); - if(count < 0) return MPI_ERR_INTERN; - *newtype_p = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); - if(!newtype_p) return MPI_ERR_INTERN; -// (*newtype)->lb = 0; - MPI_Aint oldlb; - MPI_Aint oldextent; - int s = MPI_Type_get_extent(old_type, &oldlb, &oldextent); -// (*newtype) - if(s != MPI_SUCCESS) return MPI_ERR_TYPE; -// (*newtype) -// (*newtype)->extent = count*oldextent; - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_create_hvector( // Creates a vector (strided) datatype - int count, // [in] number of blocks (nonnegative integer) - int blocklength, // [in] number of elements in each block (nonnegative integer) - MPI_Aint stride, // [in] number of bytes between start of each block (integer) - MPI_Datatype old_type, // [in] old datatype (handle) - MPI_Datatype *newtype_p // [out] new datatype (handle) -) { - return MPI_SUCCESS; -} - -#ifdef USE_MPI_REMOVED_FUNCTIONS -#define MPI_Type_hvector MPI_Type_create_hvector -#endif - -WEAK -int MPI_Type_indexed( - int count, - const int array_of_blocklengths[], - const int array_of_displacements[], - MPI_Datatype oldtype, - MPI_Datatype *newtype -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_hindexed( - int count, - const int array_of_blocklengths[], - const MPI_Aint array_of_displacements[], - MPI_Datatype oldtype, - MPI_Datatype *newtype -) { - return MPI_SUCCESS; -} - - -WEAK -int MPI_Type_create_struct( - int count, - const int array_of_blocklengths[], - const MPI_Aint array_of_displacements[], - const MPI_Datatype array_of_types[], - MPI_Datatype *newtype) -{ - return MPI_SUCCESS; -} - - - -// Removed in 3.0 -#ifdef USE_MPI_REMOVED_FUNCTIONS -WEAK -int MPI_Type_struct( // Creates a struct datatype - int count, // [in] number of blocks (integer) -- also number of entries in arrays array_of_types , array_of_displacements and array_of_blocklengths - int blocklens[], // [in] number of elements in each block (array) - MPI_Aint indices[], // [in] byte displacement of each block (array) - MPI_Datatype old_types[], // [in] type of elements in each block (array of handles to datatype objects) - MPI_Datatype *newtype // [out] new datatype (handle) -){ - *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); - (*newtype)->count = count; - (*newtype)->blocklens = (int*)malloc(count*sizeof(int)); - (*newtype)->indices = (MPI_Aint*)malloc(count*sizeof(MPI_Aint)); - (*newtype)->old_types = (MPI_Datatype*)malloc(count*sizeof(MPI_Datatype)); - memcpy((*newtype)->blocklens, blocklens, count*sizeof(int)); - memcpy((*newtype)->indices, indices, count*sizeof(MPI_Aint)); - - return MPI_SUCCESS; -} -#endif - -// ----------------------------------------------------------------------------- -// Chapter 4.1.3 Subarray Datatype Constructor -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 4.1.4 Distributed Array Datatype Constructor -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 4.1.5 Address and Size Functions -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Get_address( - const void *location, - MPI_Aint *address) -{ - return MPI_SUCCESS; -} - -#ifdef USE_MPI_REMOVED_FUNCTIONS -#define MPI_Address MPI_Get_address -#endif - -WEAK -int MPI_Type_size( // Return the number of bytes occupied by entries in the datatype - MPI_Datatype datatype, // [in] datatype (handle) - int *size // [out] datatype size (integer) -){ -// assert(size); - if(datatype->is_basic){ - *size = datatype->bytes; - return MPI_SUCCESS; - } - - if(datatype->count == 1){ - int oldsize = -1; - MPI_Type_size(datatype->old_types[0], &oldsize); - *size = datatype->blocklens[0]*oldsize; - }else assert(0); -// *size = datatype->extent; -/* switch(datatype){ - MPI_INT : *size = sizeof(int); break; - MPI_FLOAT : *size = sizeof(float); break; - MPI_DOUBLE : *size = sizeof(double); break; - MPI_FLOAT_INT: *size = sizeof(float) + sizeof(int); break; - default: assert(0); - }*/ - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 4.1.7 Extent and Bounds of Datatypes -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Type_extent( // Returns the extent of a datatype, deprecated->MPI_Type_get_extent - MPI_Datatype datatype, // [in] datatype (handle) - MPI_Aint *extent // [out] datatype extent (integer) -){ - assert(0); - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_get_extent( // Get the lower bound and extent for a Datatype - MPI_Datatype datatype, // [in] datatype to get information on (handle) - MPI_Aint *lb, // [out] lower bound of datatype (integer) - MPI_Aint *extent // [out] extent of datatype (integer) -){ - if(!lb || !extent) return MPI_ERR_ARG; - if(datatype == MPI_BYTE){ - *lb = 0; - *extent = 1; - return MPI_SUCCESS; - } - if(datatype->count == 1){ - MPI_Aint oldlb = 0; - MPI_Aint oldextent = 0; - int s = MPI_Type_get_extent(datatype->old_types[0], &oldlb, &oldextent); - assert(s == MPI_SUCCESS); - *lb = 0; - *extent = datatype->blocklens[0]*oldextent; - }else assert(0); - return MPI_SUCCESS; -} - -// Removed from the 3.0 standard -#ifdef USE_MPI_REMOVED_FUNCTIONS -WEAK -int MPI_Type_lb( // Returns the lower bound of a datatype - MPI_Datatype datatype, // [in] datatype (handle) - MPI_Aint *displacement -){ - if(!displacement) return MPI_ERR_ARG; - if(!datatype) return MPI_ERR_TYPE; - MPI_Aint lb = -1; - MPI_Aint extent = -1; - MPI_Type_get_extent(datatype, &lb, &extent); - *displacement = lb + extent; - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_ub( // Returns the upper bound of a datatype - MPI_Datatype datatype, // [in] datatype (handle) - MPI_Aint *displacement -){ - if(!displacement) return MPI_ERR_ARG; - if(!datatype) return MPI_ERR_TYPE; - MPI_Aint lb = -1; - MPI_Aint extent = -1; - MPI_Type_get_extent(datatype, &lb, &extent); - *displacement = lb + extent; - return MPI_SUCCESS; -} -#endif - -// ----------------------------------------------------------------------------- -// Chapter 4.1.7 True Extent of Datatypes -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 4.1.9 Commit and Free -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Type_commit( // Commits the datatype - MPI_Datatype *datatype // [in] datatype (handle) -){ - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_free( // Frees the datatype - MPI_Datatype *datatype // [in] datatype that is freed (handle) -){ - int c; -// *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); -// (*newtype)->count = oldtype->count; -// ; = (MPI_Datatype*)malloc(oldtype->count*sizeof(MPI_Datatype)); - for(c = 0 ; c != (*datatype)->count; ++c){ - MPI_Type_free((*datatype)->old_types + c); - } - free((*datatype)->old_types); - free((*datatype)->indices); - free((*datatype)->blocklens); - free((*datatype)); - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 4.1.10 Duplicating a Datatype -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Type_dup( // MPI_Type_dup - MPI_Datatype oldtype, // [in] datatype (handle) - MPI_Datatype *newtype // [out] copy of type (handle) -){ - int c; - *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); - (*newtype)->count = oldtype->count; - (*newtype)->blocklens = (int*)malloc(oldtype->count*sizeof(int)); - (*newtype)->indices = (MPI_Aint*)malloc(oldtype->count*sizeof(MPI_Aint)); - (*newtype)->old_types = (MPI_Datatype*)malloc(oldtype->count*sizeof(MPI_Datatype)); - for(c = 0 ; c != oldtype->count; ++c){ - (*newtype)->blocklens[c] = oldtype->blocklens[c]; - (*newtype)->indices[c] = oldtype->indices[c]; - MPI_Type_dup(oldtype->old_types[c], (*newtype)->old_types + c); - } - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 4.1.11 Use of General Datatypes in Communication -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Get_elements( - const MPI_Status *status, - MPI_Datatype datatype, - int *count -) { - return MPI_SUCCESS; - -} -WEAK -int MPI_Get_elements_x( - const MPI_Status *status, - MPI_Datatype datatype, - MPI_Count *count -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 4.1.13 Decoding a Datatype -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Type_get_envelope( - MPI_Datatype datatype, - int *num_integers, - int *num_addresses, - int *num_datatypes, - int *combiner -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Type_get_contents( - MPI_Datatype datatype, - int max_integers, - int max_addresses, - int max_datatypes, - int array_of_integers[], - MPI_Aint array_of_addresses[], - MPI_Datatype array_of_datatypes[] -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 4.2 Pack and Unpack -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Pack( - const void *inbuf, - int incount, - MPI_Datatype datatype, - void *outbuf, - int outsize, - int *position, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Unpack( - const void* inbuf, - int insize, - int *position, - void *outbuf, - int outcount, - MPI_Datatype datatype, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// Returns the upper bound on the amount of space needed to pack a message -WEAK -int MPI_Pack_size( - int incount, - MPI_Datatype datatype, - MPI_Comm comm, - int *size -) { - return MPI_SUCCESS; -} - - - - -// ----------------------------------------------------------------------------- -// Chapter 5 - Collective Communication -// ----------------------------------------------------------------------------- - - -// ----------------------------------------------------------------------------- -// Chapter 5.3 Barrier Synchronization -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Barrier( // Blocks until all processes in the communicator have reached this routine. - MPI_Comm comm // [in] communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node100.htm - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.4 Broadcast -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Bcast( // Broadcasts a message from the process with rank "root" to all other processes of the communicator - void* buffer, // starting address of buffer (choice) - int count, // number of entries in buffer (non-negative integer) - MPI_Datatype datatype, // data type of buffer (handle) - int root, // rank of broadcast root (integer) - MPI_Comm comm // communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node101.htm - if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.5 Gather -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Gather( // Gathers together values from a group of processes - const void *sendbuf, // [in] starting address of send buffer (choice) - int sendcnt, // [in] number of elements in send buffer (integer) - MPI_Datatype sendtype, // [in] data type of send buffer elements (handle) - void *recvbuf, // [out] address of receive buffer (choice, significant only at root) - int recvcnt, // [in] number of elements for any single receive (integer, significant only at root) - MPI_Datatype recvtype, // [in] data type of recv buffer elements (significant only at root) (handle) - int root, // [in] rank of receiving process (integer) - MPI_Comm comm // [in] communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node103.htm - if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; - assert(root == 0); - int sendsize = -1; - int recvsize = -1; -#define fake_mpi_max(a,b) ((a) > (b) ? (a) : (b)) - MPI_Type_size(sendtype, &sendsize); - MPI_Type_size(recvtype, &recvsize); - memcpy((char*)recvbuf, (const char*)sendbuf, fake_mpi_max(sendcnt*sendsize, recvcnt*recvsize)); - return MPI_SUCCESS; -#undef fake_mpi_max -} - -WEAK -int MPI_Gatherv( - const void *sendbuf, - int sendcount, - MPI_Datatype sendtype, - void * recvbuf, - const int recvcounts[], - const int displs[], - MPI_Datatype recvtype, - int root, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.6 Scatter -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Scatter( - const void* sendbuf, - int sendcount, - MPI_Datatype sendtype, - void* recvbuf, - int recvcount, - MPI_Datatype recvtype, - int root, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Scatterv( - const void* sendbuf, - const int sendcounts[], - const int displs[], - MPI_Datatype sendtype, - void* recvbuf, - int recvcount, - MPI_Datatype recvtype, - int root, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.7 Gather-to-all -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Allgather( - const void* sendbuf, - int sendcount, - MPI_Datatype sendtype, - void* recvbuf, - int recvcount, - MPI_Datatype recvtype, - MPI_Comm comm -) { - int sendsize = -1; - int recvsize = -1; -#define fake_mpi_max(a,b) ((a) > (b) ? (a) : (b)) - MPI_Type_size(sendtype, &sendsize); - MPI_Type_size(recvtype, &recvsize); - memcpy((char*)recvbuf, (const char*)sendbuf, fake_mpi_max(sendcount*sendsize, recvcount*recvsize)); - return MPI_SUCCESS; -#undef fake_mpi_max -} - -WEAK -int MPI_Allgatherv( - const void* sendbuf, - int sendcount, - MPI_Datatype sendtype, - void* recvbuf, - const int recvcounts[], - const int displs[], - MPI_Datatype recvtype, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.8 All-to-All Scatter/Gather -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Alltoall( - const void* sendbuf, - int sendcount, - MPI_Datatype sendtype, - void *recvbuf, - int recvcount, - MPI_Datatype recvtype, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Alltoallv( - const void* sendbuf, - const int sendcounts[], - const int sdispls[], - MPI_Datatype sendtype, - void *recvbuf, - const int recvcounts[], - const int rdispls[], - MPI_Datatype recvtype, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Alltoallw( - const void* sendbuf, - const int sendcounts[], - const int sdispls[], - MPI_Datatype sendtypes[], - void *recvbuf, - const int recvcounts[], - const int rdispls[], - const MPI_Datatype recvtypes[], - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.9.1 Reduce -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Reduce( - const void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - int root, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.9.5 User-Defined Reduction Operators -// ----------------------------------------------------------------------------- - -typedef void (MPI_User_function)(void *a, void *b, int *len, MPI_Datatype *); - -WEAK -int MPI_Op_create( - MPI_User_function *user_fn, - int commute, - MPI_Op *op -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Op_free( - MPI_Op *op -) { - if (op) *op = MPI_NO_OP; - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.9.6 All-reduce -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Allreduce( // Combines values from all processes and distributes the result back to all processes - const void* sendbuf, // starting address of send buffer (choice) - void* recvbuf, // starting address of receive buffer (choice) - int count, // number of elements in send buffer (non-negative) - MPI_Datatype datatype, // data type of elements of send buffer (handle) - MPI_Op op, // operation (handle) - MPI_Comm comm // communicator (handle) -) // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node117.htm#Node117 -{ - assert(&comm != &MPI_COMM_NULL); - //assert(sendbuf == MPI_IN_PLACE); - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.10 Reduce-Scatter -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 5.10.1 MPI_REDUCE_SCATTER_BLOCK -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Reduce_scatter_block( - const void* sendbuf, - void *recvbuf, - int recvcount, - MPI_Datatype datatype, - MPI_Op op, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.10.2 MPI_REDUCE_SCATTER -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Reduce_scatter( - const void* sendbuf, - void *recvbuf, - const int recvcounts[], - MPI_Datatype datatype, - MPI_Op op, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.11 Scan -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 5.11.1 Inclusive Scan -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Scan( - const void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.11.2 Exclusive Scan -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Exscan( - const void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - MPI_Comm comm -) { - return MPI_SUCCESS; -} - - - -// ----------------------------------------------------------------------------- -// Chapter 5.12 Nonblocking Collective Operations -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 5.12.2 Nonblocking Broadcast -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Ibcast( - void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - - -// ----------------------------------------------------------------------------- -// Chapter 5.12.3 Nonblocking Gather -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Igather( // Gathers together values from a group of processes - const void *sendbuf, // [in] starting address of send buffer (choice) - int sendcnt, // [in] number of elements in send buffer (integer) - MPI_Datatype sendtype, // [in] data type of send buffer elements (handle) - void *recvbuf, // [out] address of receive buffer (choice, significant only at root) - int recvcnt, // [in] number of elements for any single receive (integer, significant only at root) - MPI_Datatype recvtype, // [in] data type of recv buffer elements (significant only at root) (handle) - int root, // [in] rank of receiving process (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Request *request // [out] communicatio request (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node130.htm - if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; - assert(0); // TODO implementation - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 5.12.5 Nonblocking Gather-to-all -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Iallgather( - const void* sendbuf, - int sendcount, - MPI_Datatype sendtype, - void *recvbuf, - int recvcount, - MPI_Datatype recvtype, - MPI_Comm comm, - MPI_Request *request -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6 - Groups, Context, Communicators, and Caching -// ----------------------------------------------------------------------------- - -enum { - MPI_IDENT, - MPI_CONGRUENT, - MPI_SIMILAR, - MPI_UNEQUAL -}; - -// ----------------------------------------------------------------------------- -// Chapter 6.3 Group Management -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 6.3.1 Group Accessors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Group_size( // Returns the size of a group - MPI_Group group, // [in] group (handle) - int *size // [out] number of processes in the group (integer) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_rank( // Determines the rank of the calling process in the communicator - MPI_Group group, // group (handle) - int *rank // rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_translate_ranks( // Translates the ranks of processes in one group to those in another group - MPI_Group group1, // [in] group1 (handle) - int n, // [in] number of ranks in ranks1 and ranks2 arrays (integer) - const int ranks1[], // [in] array of zero or more valid ranks in group1 - MPI_Group group2, // [in] group2 (handle) - int ranks2[] // [out] array of corresponding ranks in group2, MPI_UNDEFINED when no correspondence exists. -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_compare( // Compares two groups - MPI_Group group1, // [in] group1 (handle) - MPI_Group group2, // [in] group2 (handle) - int *result // [out] integer which is MPI_IDENT if the order and members of the two groups are the same, MPI_SIMILAR if only the members are the same, and MPI_UNEQUAL otherwise -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm#Node151 - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6.3.2 Group Constructors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_group( - MPI_Comm comm, - MPI_Group *group -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_union( // Produces a group by combining two groups - MPI_Group group1, // first group (handle) - MPI_Group group2, // second group (handle) - MPI_Group *newgroup // union group (handle) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_intersection( // Produces a group as the intersection of two existing groups - MPI_Group group1, // [in] first group (handle) - MPI_Group group2, // [in] second group (handle) - MPI_Group *newgroup // [out] intersection group (handle) -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_difference( // Produces a group as the difference of two existing groups - MPI_Group group1, // [in] first group (handle) - MPI_Group group2, // [in] second group (handle) - MPI_Group *newgroup // [out] difference group (handle) -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_incl( // Produces a group by reordering an existing group and taking only listed members - MPI_Group group, // [in] group (handle) - int n, // [in] number of elements in array ranks (and size of newgroup ) (integer) - const int ranks[], - MPI_Group *newgroup // [in] ranks of processes in group to appear in newgroup (array of integers) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_excl( // Produces a group by reordering an existing group and taking only unlisted members - MPI_Group group, //[in] group (handle) - int n, // [in] number of elements in array ranks (integer) - const int ranks[], // [in] array of integer ranks in group not to appear in newgroup - MPI_Group *newgroup // [out] new group derived from above, preserving the order defined by group (handle) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_range_incl( // Creates a new group from ranges of ranks in an existing group - MPI_Group group, // [in] group (handle) - int n, // [in] number of triplets in array ranges (integer) - int ranges[][3], // [in] a one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in group or processes to be included in newgroup. - MPI_Group *newgroup // [out] new group derived from above, in the order defined by ranges (handle) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_range_excl( // Produces a group by excluding ranges of processes from an existing group - MPI_Group group, // [in] group (handle) - int n, // [in] number of elements in array ranks (integer) - int ranges[][3], // [in] a one-dimensional array of integer triplets of the form (first rank, last rank, stride), indicating the ranks in group of processes to be excluded from the output group newgroup . - MPI_Group *newgroup // [out] new group derived from above, preserving the order in group (handle) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Group_free( // Frees a group - MPI_Group *group // group (handle) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node153.htm - return MPI_SUCCESS; -} - - -// ----------------------------------------------------------------------------- -// Chapter 6.4 Communicator Management -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 6.4.1 Communicator Accessors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_size( // Determines the size of the group associated with a communicator - MPI_Comm comm, // communicator (handle) - int *size // number of processes in the group of comm (integer) -){ - //if(comm == MPI_COMM_NULL){ - // int error = MPI_ERR_COMM; - // comm->errhandler_->func_(&comm, &error); - // return error; - //} - *size = 1; - return MPI_SUCCESS; -} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node155.htm - -WEAK -int MPI_Comm_rank( // MPI_Group_rank Returns the rank of this process in the given group - MPI_Comm comm, // [in] group (handle) - int* rank // [out] rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer) -){ - //if(comm == MPI_COMM_NULL){ - // MPI_Comm_call_errhandler(comm, MPI_ERR_COMM); - // return MPI_ERR_COMM; - //} - *rank = 0; - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_compare( // Compares two communicators - MPI_Comm comm1, // [in] comm1 (handle) - MPI_Comm comm2, // [in] comm2 (handle) - int *result // [out] integer which is MPI_IDENT if the contexts and groups are the same, MPI_CONGRUENT if different contexts but identical groups, MPI_SIMILAR if different contexts but similar groups, and MPI_UNEQUAL otherwise -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node155.htm - if(&comm1 == &MPI_COMM_NULL || &comm2 == &MPI_COMM_NULL) return MPI_ERR_COMM; - if(result == NULL) return MPI_ERR_ARG; - *result = MPI_CONGRUENT; //MPI_IDENT; - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6.4.2 Communicator Constructors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_dup( // Duplicates an existing communicator with all its cached information - MPI_Comm comm, // communicator (handle) - MPI_Comm *newcomm // copy of comm (handle) -) -{ - if(&comm == &MPI_COMM_NULL) return MPI_ERR_COMM; - *newcomm = (struct MPI_Comm_impl_*)malloc(sizeof(struct MPI_Comm_impl_)); - assert(*newcomm != MPI_COMM_NULL); - (**newcomm).errhandler_ = comm->errhandler_; - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_create( // Creates a new communicator - MPI_Comm comm, // [in] communicator (handle) - MPI_Group group, // [in] group, which is a subset of the group of comm (handle) - MPI_Comm *newcomm // [out] new communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm - if(&comm == &MPI_COMM_NULL) return MPI_ERR_COMM; - if(group == MPI_GROUP_NULL) return MPI_ERR_GROUP; - newcomm = (MPI_Comm*)malloc(sizeof(MPI_Comm)); - return MPI_SUCCESS; -} - -int MPI_Comm_create_group( // must be called by all processes in group, which is a subgroup of the group of comm - MPI_Comm comm, // intracommunicator (handle) - MPI_Group group, // group, which is a subset of the group of comm (handle) - int tag, // tag (integer) - MPI_Comm *newcomm // new communicator (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm - -WEAK -int MPI_Comm_split( // Creates new communicators based on colors and keys - MPI_Comm comm, // [in] communicator (handle) - int color, // [in] control of subset assignment (integer) - int key, // [in] control of rank assigment (integer) - MPI_Comm *newcomm // [out] new communicator (handle) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6.4.3 Communicator Destructors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_free( // Marks the communicator object for deallocation - MPI_Comm *comm // [in] Communicator to be destroyed (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node157.htm - *comm = MPI_COMM_NULL; - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6.6 Inter-Communication -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 6.6.1 Inter-communicator Accessors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_test_inter( - MPI_Comm comm, - int *flag -) { - return MPI_SUCCESS; -} - -// Determines the size of the remote group associated with an inter-communictor -WEAK -int MPI_Comm_remote_size( - MPI_Comm comm, - int *size -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_remote_group( - MPI_Comm comm, - MPI_Group *group -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6.6.2 Inter-communicator Operations -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Intercomm_create( // Creates an intercommuncator from two intracommunicators - MPI_Comm local_comm, // [in] Local (intra)communicator - int local_leader, // [in] Rank in local_comm of leader (often 0) - MPI_Comm peer_comm, // [in] Communicator used to communicate between a designated process in the other communicator. Significant only at the process in local_comm with rank local_leader. - int remote_leader, // [in] Rank in peer_comm of remote leader (often 0) - int tag, // [in] Message tag to use in constructing intercommunicator; if multiple MPI_Intercomm_creates are being made, they should use different tags (more precisely, ensure that the local and remote leaders are using different tags for each MPI_intercomm_create). - MPI_Comm *newintercomm // [out] Created intercommunicator -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node168.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Intercomm_merge( - MPI_Comm intercomm, - int high, - MPI_Comm *newintracomm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 6.7 Caching -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 6.7.2 Communicators -// ----------------------------------------------------------------------------- - -typedef int MPI_Comm_copy_attr_function(MPI_Comm oldcomm, int comm_keyval, - void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); - -typedef int MPI_Comm_delete_attr_function(MPI_Comm comm, int comm_keyval, - void *attribute_val, void *extra_state); - -WEAK -int MPI_Comm_create_keyval( - MPI_Comm_copy_attr_function *comm_copy_attr_fn, - MPI_Comm_delete_attr_function *comm_delete_attr_fn, - int *comm_keyval, - void *extra_state -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_free_keyval( - int *comm_keyval -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_set_attr( - MPI_Comm comm, - int comm_keyval, - void *attribute_val -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_get_attr( - MPI_Comm comm, - int comm_keyval, - void *attribute_val, - int *flag -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_delete_attr( - MPI_Comm comm, - int comm_keyval -) { - return MPI_SUCCESS; -} - -#ifdef USE_MPI_REMOVED_FUNCTIONS -#define MPI_Attr_get MPI_Comm_get_attr -#define MPI_Attr_put MPI_Comm_set_attr -#endif - -enum { - MPI_HOST, - MPI_IO, - MPI_WTIME_IS_GLOBAL, - MPI_APPNUM, - MPI_UNIVERSE_SIZE, - MPI_LASTUSEDCODE, - MPI_TAG_UB -}; - -// ----------------------------------------------------------------------------- -// Chapter 6.8 Naming Objects -// ----------------------------------------------------------------------------- - -const int MPI_MAX_OBJECT_NAME = 128; - -WEAK -int MPI_Comm_set_name( // Sets the print name for a communicator - MPI_Comm comm, // [in] communicator to name (handle) - const char *comm_name // [in] Name for communicator (string) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_get_name( // Return the print name from the communicator - MPI_Comm comm, // communicator whose name is to be returned (handle) - char *comm_name, // the name previously stored on the communicator, or an... - int *resultlen // length of returned name (integer) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm - return MPI_SUCCESS; -} - -inline int MPI_Type_get_name(MPI_Datatype datatype, char *type_name, int *resultlen) -{ - if (resultlen) *resultlen = 0; - return MPI_SUCCESS; -} - -inline int MPI_Type_set_name(MPI_Datatype datatype, const char *type_name) -{ - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 7 - Process Topologies -// ----------------------------------------------------------------------------- - - -enum { - MPI_GRAPH, - MPI_CART, - MPI_DIST_GRAPH -}; - -// ----------------------------------------------------------------------------- -// Chapter 7.5 Topology Constructors -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Cart_create( // Makes a new communicator to which topology information has been attached - MPI_Comm comm_old, // [in] input communicator (handle) - int ndims, // [in] number of dimensions of cartesian grid (integer) - int *dims, // [in] integer array of size ndims specifying the number of processes in each dimension - int *periods, // [in] logical array of size ndims specifying whether the grid is periodic (true) or not (false) in each dimension - int reorder, // [in] ranking may be reordered (true) or not (false) (logical) - MPI_Comm *comm_cart // [out] communicator with new cartesian topology (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node192.htm - if(dims == NULL) return MPI_ERR_DIMS; - for(int i = 0; i != ndims; ++i) if(dims[i] <= 0) return MPI_ERR_DIMS; - *comm_cart = comm_old; // TODO: allocate unique handle - return MPI_SUCCESS; -} - -WEAK -int MPI_Dims_create( - int nnodes, - int ndims, - int dims[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Graph_create( - MPI_Comm comm_old, - int nnodes, - const int index[], - const int edges[], - int reorder, - MPI_Comm *comm_graph -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Dist_graph_create_adjacent( - MPI_Comm comm_old, - int indegree, - const int sources[], - const int sourceweights[], - int outdegree, - const int destinations[], - const int destweights[], - MPI_Info info, - int reorder, - MPI_Comm *comm_dist_graph -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Dist_graph_create( - MPI_Comm comm_old, - int n, - const int sources[], - const int degrees[], - const int destinations[], - const int weights[], - MPI_Info info, - int reorder, - MPI_Comm *comm_dist_graph -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 7.5.5 Topology Inquiry Functions -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Topo_test( - MPI_Comm comm, - int *status -) { - if (status) *status = MPI_UNDEFINED; - return MPI_SUCCESS; -} - -WEAK -int MPI_Graphdims_get( - MPI_Comm comm, - int *nnodes, - int *nedges -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Graph_get( - MPI_Comm comm, - int maxindex, - int maxedges, - int index[], - int edges[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Cartdim_get( - MPI_Comm comm, - int *ndims -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Cart_get( - MPI_Comm comm, - int maxdims, - int dims[], - int periods[], - int coords[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Cart_rank( - MPI_Comm comm, - const int coords[], - int *rank -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Cart_coords( - MPI_Comm comm, - int rank, - int maxdims, - int coords[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Graph_neighbors_count( - MPI_Comm comm, - int rank, - int *nneighbors -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Graph_neighbors( - MPI_Comm comm, - int rank, - int maxneighbors, - int neighbors[] -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Dist_graph_neighbors_count( - MPI_Comm comm, - int *indegree, - int *outdegree, - int *weighted -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Dist_graph_neighbors( - MPI_Comm comm, - int maxindegree, - int sources[], - int sourceweights[], - int maxoutdegree, - int destinations[], - int destweights[] -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 7.5.6 Cartesian Shift Coordinates -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Cart_shift( - MPI_Comm comm, - int direction, - int disp, - int *rank_source, - int *rank_dest -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 7.5.7 Partitioning of Cartesian Structures -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Cart_sub( // Partitions a communicator into subgroups which form lower-dimensional cartesian subgrids - MPI_Comm comm, // [in] communicator with cartesian structure (handle) - int *remain_dims, // [in] the ith entry of remain_dims specifies whether the ith dimension is kept in the subgrid (true) or is dropped (false) (logical vector) - MPI_Comm *newcomm // [out] communicator containing the subgrid that includes the calling process (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node198.htm - if(&comm == &MPI_COMM_NULL) return MPI_ERR_COMM; - *newcomm = comm; // TODO: allocate unique handle - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 7.5.8 Low-Level Topology Functions -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Cart_map( - MPI_Comm comm, - int ndims, - const int dims[], - const int periods[], - int *newrank -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Graph_map( - MPI_Comm comm, - int nnodes, - const int index[], - const int edges[], - int *newrank -) { - return MPI_SUCCESS; -} - - -// ----------------------------------------------------------------------------- -// Chapter 8 - MPI Environmental Management -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 8.1 Implementation Information -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 8.1.1 Version Inquiries -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Get_version( // Return the version number of MPI - int* version, // [out] Version of MPI - int* subversion // [out] Suversion of MPI -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node209.htm - *version = 3; - *subversion = 1; - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 8.1.2 Environmental Inquiries -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Get_processor_name( // the name of the processor on which it was called at the moment of the call. - char *name, // A unique specifier for the actual (as opposed to virtual) node. - int *resultlen // Length (in printable characters) of the result returned in name -){ - if(gethostname(name, MPI_MAX_PROCESSOR_NAME) > 0) - return MPI_ERR_UNKNOWN; - *resultlen = strlen(name); - return MPI_SUCCESS; -} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node210.htm#Node215 - -// ----------------------------------------------------------------------------- -// Chapter 8.2 Memory Allocation -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Alloc_mem( - MPI_Aint size, - MPI_Info info, - void *baseptr -) { - //if (baseptr) *(void **)baseptr = malloc(size); - if (baseptr) *(void **)baseptr = 0; - return MPI_SUCCESS; -} - -WEAK -int MPI_Free_mem( - void *base -) { - //free(base); - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 8.3 Error Handling -// ----------------------------------------------------------------------------- - -// http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node218.htm - -WEAK -int MPI_Comm_create_errhandler( // Create a communicator error handler - MPI_Comm_errhandler_function *comm_errhandler_fn, // [in] user defined error handling procedure (function) - MPI_Errhandler *errhandler // [out] MPI error handler (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node218.htm - *errhandler = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); - (*errhandler)->func_ = comm_errhandler_fn; - return MPI_SUCCESS; -} - -WEAK -int MPI_Comm_set_errhandler( // Set the error handler for a communicator - MPI_Comm comm, // [in] communicator (handle) - MPI_Errhandler errhandler // [in] new error handler for communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node218.htm -// assert(comm != MPI_COMM_NULL); - if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; - comm->errhandler_ = errhandler; //->func_ = errhandler->func_; - return MPI_SUCCESS; -} - - -WEAK -int MPI_Comm_get_errhandler( // Get the error handler attached to a communicator - MPI_Comm comm, // [in] communicator (handle) - MPI_Errhandler *errhandler // [out] handler currently associated with communicator (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node220.htm - return MPI_SUCCESS; -} - - -void fatal_error_(MPI_Comm * comm, int * errcode, ...); -WEAK -void no_op_error_(MPI_Comm * comm, int * errcode, ...){} - -#ifdef USE_MPI_REMOVED_FUNCTIONS -WEAK -int MPI_Errhandler_create( - MPI_Handler_function *errhandler_fn, - MPI_Errhandler *errhandler -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Errhandler_set( - MPI_Comm comm, - MPI_Errhandler errhandler -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Errhandler_get( - MPI_Comm comm, - MPI_Errhandler *errhandler -) { - return MPI_SUCCESS; -} -#endif - - - -// ----------------------------------------------------------------------------- -// Chapter 8.3.4 Freeing Errohandlers and Retrieving Error Strings -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Errhandler_free( // Frees an MPI-style errorhandler - MPI_Errhandler *errhandler // [in-out] MPI error handler (handle) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node221.htm - *errhandler = MPI_ERRHANDLER_NULL; - return MPI_SUCCESS; -} - -WEAK -int MPI_Error_string( // Return a string for a given error code - int errorcode, // [in] Error code returned by an MPI routine or an MPI error class - char *string, // [out] Text that corresponds to the errorcode - int *resultlen // [out] Length of string -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node221.htm#Node221 - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 8.4 Error Codes and Classes -// ----------------------------------------------------------------------------- - -int MPI_Error_class( // Converts an error code into an error class - int errorcode, // Error code returned by an MPI routine - int *errorclass // Error class associated with errorcode -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm#Node222 - -// ----------------------------------------------------------------------------- -// Chapter 8.5 Error Classes, Error Codes, and Error Handlers -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_call_errhandler( // Call the error handler installed on a communicator - MPI_Comm comm, // [in] communicator with error handler (handle) - int errorcode // [in] error code (integer) -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node223.htm -// if(comm == MPI_COMM_NULL){ -// return MPI_ERR_COMM; -// } - comm->errhandler_->func_(&comm, &errorcode); - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 8.6 Timers and Synchronization -// ----------------------------------------------------------------------------- - -WEAK -double MPI_Wtime( // Returns an elapsed time on the calling processor -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm - struct timespec tw; - clock_gettime(CLOCK_MONOTONIC, &tw); - return 1.0*tw.tv_sec + 1e-9*tw.tv_nsec;; -} - -WEAK -double MPI_Wtick( // Returns the resolution of MPI_Wtime -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm - return 1e-9; -} - -// ----------------------------------------------------------------------------- -// Chapter 8.7 Startup -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Init( // Initialize the MPI execution environment - int *argc, // [in] Pointer to the number of arguments - char ***argv // [in] Pointer to the argument vector -){ - MPI_Comm_create_errhandler(&fatal_error_, &MPI_ERRORS_ARE_FATAL); - MPI_Comm_create_errhandler(&no_op_error_, &MPI_ERRORS_RETURN); -// MPI_ERRORS_ARE_FATAL = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_));//{&fatal_error_}; -// static struct MPI_Errhandler_impl_ MPI_ERRORS_RETURN = {&no_op_error_}; - -// MPI_COMM_NULL = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_)); -// MPI_COMM_NULL->errhandler_ = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); -// MPI_Comm_set_errhandler(MPI_COMM_NULL, MPI_ERRORS_ARE_FATAL); - - MPI_COMM_WORLD = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_)); - MPI_COMM_WORLD->errhandler_ = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); - MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); -// MPI_COMM_WORLD->errhandler_ = MPI_ERRORS_ARE_FATAL; - - MPI_COMM_SELF = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_)); - MPI_COMM_SELF->errhandler_ = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); - MPI_Comm_set_errhandler(MPI_COMM_SELF, MPI_ERRORS_ARE_FATAL); -// MPI_COMM_SELF->errhandler_ = MPI_ERRORS_ARE_FATAL; - - return MPI_SUCCESS; -} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm - - -WEAK -int MPI_Finalize( // Terminates MPI execution environment -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm - return MPI_SUCCESS; -} - -WEAK -int MPI_Initialized( // Indicates whether MPI_Init has been called. - int *flag // [out] Flag is true if MPI_Init or MPI_Init_thread has been called and false otherwise. -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node225.htm - if (flag) *flag = true; - return MPI_SUCCESS; -} - -WEAK -int MPI_Abort( // Terminates MPI execution environment - MPI_Comm comm, // [in] communicator of tasks to abort - int errorcode // [in] error code to return to invoking environment -){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node225.htm - exit(errorcode); -// return MPI_SUCCESS; // function never returns -} - - -WEAK -void fatal_error_(MPI_Comm * comm, int * errcode, ...){ - switch(*errcode){ - case MPI_ERR_COMM : puts("[] *** MPI_ERR_COMM: invalid communicator\n[] *** MPI_ERRORS_ARE_FATAL (will now abort)"); MPI_Abort(*comm, *errcode); - } -} - -// ----------------------------------------------------------------------------- -// Chapter 8.7.2 Determining Whether MPI Has Finished -// ----------------------------------------------------------------------------- - -int MPI_Finalized( // Indicates whether MPI_Finalize has been called - int *flag // [out] true if MPI was finalized (logical) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node227.htm - - - -// ----------------------------------------------------------------------------- -// Chapter 10 - Process Creation and Management -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 10.3 Process Manager Interface -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 10.3.2 Starting Processes and Establishing Communication -// ----------------------------------------------------------------------------- - -int MPI_Comm_spawn( // Spawn up to maxprocs instances of a single MPI application - const char *command, // [in] name of program to be spawned (string, at root) - char *argv[], // [in] arguments to command (array of strings, at root) - int maxprocs, // maximum number of processes to start(int at root) - MPI_Info info, // a set of key-value pairs telling the runtime system where and how to start the processes (handle, significant only at root) - int root, // rank of process in which previous arguments are examined (integer) - MPI_Comm comm, // intracommunicator containing group of spawning processes (handle) - MPI_Comm *intercomm, // [out] intercommunicator between original group and the newly spawned group (handle) - int array_of_errcodes[] // [out] one code per process (array of integer) -); - -int MPI_Comm_get_parent( // Return the parent communicator for this process - MPI_Comm *parent // [out] the parent communicator (handle) -); - -// ----------------------------------------------------------------------------- -// Chapter 10.4 Establishing Communication -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 10.4.2 Server Routines -// ----------------------------------------------------------------------------- - -const int MPI_MAX_PORT_NAME = 128; - -int MPI_Open_port( - MPI_Info info, - char *port_name -); - -int MPI_Close_port( - const char *port_name -); - -WEAK -int MPI_Comm_accept( - const char *port_name, - MPI_Info info, - int root, - MPI_Comm comm, - MPI_Comm *newcomm -) { - return MPI_SUCCESS; -} - - -// ----------------------------------------------------------------------------- -// Chapter 10.4.2 Client Routines -// ----------------------------------------------------------------------------- - - -WEAK -int MPI_Comm_connect( - const char *port_name, - MPI_Info info, - int root, - MPI_Comm comm, - MPI_Comm *newcomm -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 10.5.4 Releasing Connections -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Comm_disconnect( // MPI_Comm_disconnect - MPI_Comm *comm // [in] communicator (handle) -){ - if(*comm == MPI_COMM_NULL) return MPI_ERR_COMM; - *comm = MPI_COMM_NULL; - return MPI_SUCCESS; -} - - - -// ----------------------------------------------------------------------------- -// Chapter 12.2 Generalized Requests -// ----------------------------------------------------------------------------- - -typedef int MPI_Grequest_query_function(void *extra_state, MPI_Status *status); -typedef int MPI_Grequest_free_function(void *extra_state); -typedef int MPI_Grequest_cancel_function(void *extra_state, int complete); - -WEAK -int MPI_Grequest_start( // Start new generalized request - MPI_Grequest_query_function *query_fn, // [in] status query callback function - MPI_Grequest_free_function *free_fn, // [in] query free callback function - MPI_Grequest_cancel_function *cancel_fn, // [in] request cancel callback function - void *extra_state, // [in] extra state - MPI_Request *request // [out] generalized request (handle) -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Grequest_complete( - MPI_Request request // [in] generalized request (handle) -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 12.3 Associating Information with Status -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Status_set_elements( - MPI_Status *status, - MPI_Datatype datatype, - int count -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Status_set_elements_x( - MPI_Status *status, - MPI_Datatype datatype, - MPI_Count count -) { - return MPI_SUCCESS; -} - -WEAK -int MPI_Status_set_cancelled( - MPI_Status *status, - int flag -) { - return MPI_SUCCESS; -} - -// ----------------------------------------------------------------------------- -// Chapter 12.4 MPI and Threads -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 12.4.3 Initialization -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Init_thread( // Initialize the MPI execution environment - int *argc, // [in] Pointer to the number of arguments - char ***argv, // [in] Pointer to the argument vector - int required, // [in] Level of desired thread support - int *provided // [out] Level of provided thread support -){// http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm - *provided = MPI_THREAD_MULTIPLE; - return MPI_SUCCESS; -} - -WEAK -int MPI_Query_thread( // The following function can be used to query the current level of thread support. - int *provided // provided level of thread support (integer) -) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm#Node303 - return MPI_SUCCESS; -} - -int MPI_Is_thread_main( // This function can be called by a thread to determine if it is the main thread (the thread that called MPI_INIT or MPI_INIT_THREAD). - int *flag // true if calling thread is main thread, false otherwise (logical) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm - - -// ----------------------------------------------------------------------------- -// Chapter 14 - Tool Support -// ----------------------------------------------------------------------------- - -// ----------------------------------------------------------------------------- -// Chapter 14.2.4 Miscellaneous Control of Profiling -// ----------------------------------------------------------------------------- - -WEAK -int MPI_Pcontrol( - int level, - ... -) { - return MPI_SUCCESS; -} - - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/fake/mpi_qmcpack_compile.h b/external_codes/mpi_wrapper/mpi3/fake/mpi_qmcpack_compile.h deleted file mode 100644 index 2ef4962b06f..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/mpi_qmcpack_compile.h +++ /dev/null @@ -1,741 +0,0 @@ -#ifndef FAKE_MPI_H -#define FAKE_MPI_H -// Fake MPI for serial execution - -#include // HOST_NAME_MAX -#include // gethostname -#include // strlen - -const int MPI_MAX_PROCESSOR_NAME = HOST_NAME_MAX; -const int MPI_MAX_INFO_KEY = 128; -const int MPI_MAX_INFO_VAL = 128; -const int MPI_MAX_ERROR_STRING = 128; - -enum : int { // error classes - MPI_SUCCESS, // No error - MPI_ERR_BUFFER, // Invalid buffer pointer - MPI_ERR_COUNT, // Invalid count argument - MPI_ERR_TYPE, // Invalid datatype argument - MPI_ERR_TAG, // Invalid tag argument - MPI_ERR_COMM, // Invalid communicator - MPI_ERR_RANK, // Invalid rank - MPI_ERR_REQUEST, // Invalid request (handle) - MPI_ERR_ROOT, // Invalid root - MPI_ERR_GROUP, // Invalid group - MPI_ERR_OP, // Invalid operation - MPI_ERR_TOPOLOGY, // Invalid topology - MPI_ERR_DIMS, // Invalid dimension argument - MPI_ERR_ARG, // Invalid argument of some other kind - MPI_ERR_UNKNOWN, // Unknown error - MPI_ERR_TRUNCATE, // Message truncated on receive - MPI_ERR_OTHER, // Known error not in this list - MPI_ERR_INTERN, // Internal MPI (implementation) error - MPI_ERR_IN_STATUS, // Error code is in status - MPI_ERR_PENDING, // Pending request - MPI_ERR_KEYVAL, // Invalid keyval has been passed - MPI_ERR_NO_MEM, // MPI_ALLOC_MEM failed because memory is exhausted - MPI_ERR_BASE, // Invalid base passed to MPI_FREE_MEM - MPI_ERR_INFO_KEY, // Key longer than MPI_MAX_INFO_KEY - MPI_ERR_INFO_VALUE, // Value longer than MPI_MAX_INFO_VAL - MPI_ERR_INFO_NOKEY, // Invalid key passed to MPI_INFO_DELETE - MPI_ERR_SPAWN, // Error in spawning processes - MPI_ERR_PORT, // Invalid port name passed to MPI_COMM_CONNECT - MPI_ERR_SERVICE, // Invalid service name passed to MPI_UNPUBLISH_NAME - MPI_ERR_NAME, // Invalid service name passed to MPI_LOOKUP_NAME - MPI_ERR_WIN, // Invalid win argument - MPI_ERR_SIZE, // Invalid size argument - MPI_ERR_DISP, // Invalid disp argument - MPI_ERR_INFO, // Invalid info argument - MPI_ERR_LOCKTYPE, // Invalid locktype argument - MPI_ERR_ASSERT, // Invalid assert argument - MPI_ERR_RMA_CONFLICT, // Conflicting accesses to window - MPI_ERR_RMA_SYNC, // Wrong synchronization of RMA calls - MPI_ERR_RMA_RANGE, // Target memory is not part of the window (in the case of a window created with MPI_WIN_CREATE_DYNAMIC, target memory is not attached) - MPI_ERR_RMA_ATTACH, // Memory cannot be attached (e.g., because of resource exhaustion) - MPI_ERR_RMA_SHARED, // Memory cannot be shared (e.g., some process in the group of the specified communicator cannot expose shared memory) - MPI_ERR_RMA_FLAVOR, // Passed window has the wrong flavor for the called function - MPI_ERR_FILE, // Invalid file handle - MPI_ERR_NOT_SAME, // Collective argument not identical on all processes, or collective routines called in a different order by different processes - MPI_ERR_AMODE, // Error related to the amode passed to MPI_FILE_OPEN - MPI_ERR_UNSUPPORTED_DATAREP, // Unsupported datarep passed to MPI_FILE_SET_VIEW - MPI_ERR_UNSUPPORTED_OPERATION, // Unsupported operation, such as seeking on a file which supports sequential access only - MPI_ERR_NO_SUCH_FILE, // File does not exist - MPI_ERR_FILE_EXISTS, // File exists - MPI_ERR_BAD_FILE, // Invalid file name (e.g., path name too long) - MPI_ERR_ACCESS, // Permission denied - MPI_ERR_NO_SPACE, // Not enough space - MPI_ERR_QUOTA, // Quota exceeded - MPI_ERR_READ_ONLY, // Read-only file or file system - MPI_ERR_FILE_IN_USE, // File operation could not be completed, as the file is currently open by some process - MPI_ERR_DUP_DATAREP, // Conversion functions could not be registered because a data representation identifier that was already defined was passed to MPI_REGISTER_DATAREP - MPI_ERR_CONVERSION, // An error occurred in a user supplied data conversion function. - MPI_ERR_IO, // Other I/O error - MPI_ERR_LASTCODE // Last error code -}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm - -struct MPI_Group{ - bool operator==(const MPI_Group&o) const { return true; } -}; - -const struct MPI_Group MPI_GROUP_EMPTY; -const struct MPI_Group MPI_GROUP_NULL; - -// MPI Info handling. -// Would not be too hard to create an std::map implementation -//struct MPI_Info{}; -//const struct MPI_Info MPI_INFO_NULL; - -enum MPI_Info { - MPI_INFO_NULL -}; - -int MPI_Info_create(MPI_Info *info); - -int MPI_Info_free(MPI_Info *info); - -int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo); - -int MPI_Info_delete(MPI_Info info, const char *key); - -int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, int *flag); - -int MPI_Info_get_nkeys(MPI_Info info, int *nkeys); - -int MPI_Info_get_nthkey(MPI_Info info, int n, char *key); - -int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen, int *flag); - -int MPI_Info_set(MPI_Info info, const char *key, const char *value); - -const int MPI_MAX_PORT_NAME = 128; - -int MPI_Open_port(MPI_Info info, char *port_name); - -int MPI_Close_port(const char *port_name); - -enum : int { // level of thread support - MPI_THREAD_SINGLE, - MPI_THREAD_FUNNELED, - MPI_THREAD_SERIALIZED, - MPI_THREAD_MULTIPLE -}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm - -typedef enum _MPI_Datatype { - MPI_DATATYPE_NULL = 0, - MPI_CHAR, - MPI_SHORT, - MPI_INT, - MPI_LONG, - MPI_LONG_LONG_INT, - MPI_LONG_LONG, - MPI_SIGNED_CHAR, - MPI_UNSIGNED_CHAR, - MPI_UNSIGNED_SHORT, - MPI_UNSIGNED, - MPI_UNSIGNED_LONG, - MPI_UNSIGNED_LONG_LONG, - MPI_FLOAT, - MPI_DOUBLE, - MPI_LONG_DOUBLE, - MPI_WCHAR, - MPI_C_BOOL, - MPI_INT8_T, - MPI_INT16_T, - MPI_INT32_T, - MPI_INT64_T, - MPI_UINT8_T, - MPI_UINT16_T, - MPI_UINT32_T, - MPI_UINT64_T, - MPI_AINT, - MPI_COUNT, - MPI_OFFSET, - MPI_C_COMPLEX, - MPI_C_FLOAT_COMPLEX, - MPI_C_DOUBLE_COMPLEX, - MPI_BYTE, - MPI_PACKED, - MPI_CXX_BOOL, - MPI_CXX_FLOAT_COMPLEX, - MPI_CXX_DOUBLE_COMPLEX, - MPI_CXX_LONG_DOUBLE_COMPLEX, - MPI_FLOAT_INT, - MPI_DOUBLE_INT, - MPI_LONG_INT, - MPI_2INT, - MPI_SHORT_INT, - MPI_LONG_DOUBLE_INT, -} MPI_Datatype; - -typedef unsigned long long MPI_Aint; - -inline int MPI_Type_dup(MPI_Datatype oldtype, MPI_Datatype *newtype) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_commit(MPI_Datatype *datatype) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_contiguous(int count, MPI_Datatype oldtype, MPI_Datatype *newtype) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_vector(int count, int blocklength, int stride, MPI_Datatype oldtype, MPI_Datatype *newtype) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_create_struct(int count, const int array_of_blocklengths[], const MPI_Aint array_of_displacements[], - const MPI_Datatype array_of_types[], MPI_Datatype *newtype) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_size(MPI_Datatype datatype, int *size) -{ - if (size) *size = 0; - return MPI_SUCCESS; -} - -const int MPI_MAX_OBJECT_NAME = 128; - -inline int MPI_Type_get_name(MPI_Datatype datatype, char *type_name, int *resultlen) -{ - if (resultlen) *resultlen = 0; - return MPI_SUCCESS; -} - -inline int MPI_Type_set_name(MPI_Datatype datatype, const char *type_name) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent) -{ - return MPI_SUCCESS; -} - -inline int MPI_Type_ub(MPI_Datatype datatype, MPI_Aint *displacement) -{ - return MPI_SUCCESS; -} - - -//struct MPI_Comm { -// bool operator!=(const MPI_Comm &o) const { return false; } -// bool operator==(const MPI_Comm &o) const { return true; } -//}; - -//const struct MPI_Comm MPI_COMM_WORLD; -//const struct MPI_Comm MPI_COMM_SELF; -//const struct MPI_Comm MPI_COMM_NULL; - -enum MPI_Comm : int{ - MPI_COMM_NULL = 0, - MPI_COMM_WORLD = 1, - MPI_COMM_SELF = 2, -}; - -static const void* MPI_IN_PLACE = reinterpret_cast(-1); - -enum MPI_Op{ - MPI_MAX, - MPI_MIN, - MPI_SUM, - MPI_PROD, - MPI_MAXLOC, - MPI_MINLOC, - MPI_BAND, - MPI_BOR, - MPI_BXOR, - MPI_LAND, - MPI_LOR, - MPI_LXOR, - MPI_REPLACE, - MPI_NO_OP, -}; - -struct MPI_Status { - int MPI_SOURCE; - int MPI_TAG; -}; -static MPI_Status *MPI_STATUS_IGNORE = 0; - -static char *MPI_ARGV_NULL[] = {}; -static int MPI_ERRCODES_IGNORE[] = {}; - -struct MPI_Message{}; - -inline int MPI_Allreduce( // Combines values from all processes and distributes the result back to all processes - const void* sendbuf, // starting address of send buffer (choice) - void* recvbuf, // starting address of receive buffer (choice) - int count, // number of elements in send buffer (non-negative) - MPI_Datatype datatype, // data type of elements of send buffer (handle) - MPI_Op op, // operation (handle) - MPI_Comm comm // communicator (handle) -) // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node117.htm#Node117 -{ - return MPI_SUCCESS; -} - -inline int MPI_Bcast( // Broadcasts a message from the process with rank "root" to all other processes of the communicator - void* buffer, // starting address of buffer (choice) - int count, // number of entries in buffer (non-negative integer) - MPI_Datatype datatype, // data type of buffer (handle) - int root, // rank of broadcast root (integer) - MPI_Comm comm // communicator (handle) -) // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node101.htm -{ - return MPI_SUCCESS; -} - -int MPI_Comm_create( // Creates a new communicator - MPI_Comm comm, // [in] communicator (handle) - MPI_Group group, // [in] group, which is a subset of the group of comm (handle) - MPI_Comm *newcomm // [out] new communicator (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm -// Determines the size of the remote group associated with an inter-communictor -int MPI_Comm_remote_size(MPI_Comm comm, int *size); -inline int MPI_Comm_size( // Determines the size of the group associated with a communicator - MPI_Comm comm, // communicator (handle) - int *size // number of processes in the group of comm (integer) -){ - if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; - *size = comm > MPI_COMM_NULL; - return MPI_SUCCESS; -} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node155.htm -int MPI_Comm_spawn( // Spawn up to maxprocs instances of a single MPI application - const char *command, // [in] name of program to be spawned (string, at root) - char *argv[], // [in] arguments to command (array of strings, at root) - int maxprocs, // maximum number of processes to start(int at root) - MPI_Info info, // a set of key-value pairs telling the runtime system where and how to start the processes (handle, significant only at root) - int root, // rank of process in which previous arguments are examined (integer) - MPI_Comm comm, // intracommunicator containing group of spawning processes (handle) - MPI_Comm *intercomm, // [out] intercommunicator between original group and the newly spawned group (handle) - int array_of_errcodes[] // [out] one code per process (array of integer) -); -// Creates new communicators based on colors and keys -int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm); -int MPI_Comm_get_name( // Return the print name from the communicator - MPI_Comm comm, // communicator whose name is to be returned (handle) - char *comm_name, // the name previously stored on the communicator, or an... - int *resultlen // length of returned name (integer) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm -int MPI_Comm_get_parent( // Return the parent communicator for this process - MPI_Comm *parent // [out] the parent communicator (handle) -); -int MPI_Get_count( // Gets the number of "top level" elements - MPI_Status *status, // [in] return status of receive operation (Status) - MPI_Datatype datatype, // [out] number of received elements (integer) - int *count // [in] datatype of each receive buffer element (han -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node51.htm -int MPI_Comm_set_name( // Sets the print name for a communicator - MPI_Comm comm, // [in] communicator to name (handle) - const char *comm_name // [in] Name for communicator (string) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm#Node179 -int MPI_Comm_create_group( // must be called by all processes in group, which is a subgroup of the group of comm - MPI_Comm comm, // intracommunicator (handle) - MPI_Group group, // group, which is a subset of the group of comm (handle) - int tag, // tag (integer) - MPI_Comm *newcomm // new communicator (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm -inline int MPI_Comm_dup( // Duplicates an existing communicator with all its cached information - MPI_Comm comm, // communicator (handle) - MPI_Comm *newcomm // copy of comm (handle) -) -{ - *newcomm = comm; - return MPI_SUCCESS; -} -inline int MPI_Get_processor_name( // the name of the processor on which it was called at the moment of the call. - char *name, // A unique specifier for the actual (as opposed to virtual) node. - int *resultlen // Length (in printable characters) of the result returned in name -){ - if(gethostname(name, MPI_MAX_PROCESSOR_NAME) > 0) - return MPI_ERR_UNKNOWN; - *resultlen = strlen(name); - return MPI_SUCCESS; -} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node210.htm#Node215 -int MPI_Group_free( // Frees a group - MPI_Group *group // group (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node153.htm -int MPI_Group_incl( // Produces a group by reordering an existing group and taking only listed members - MPI_Group group, // [in] group (handle) - int n, // [in] number of elements in array ranks (and size of newgroup ) (integer) - const int ranks[], - MPI_Group *newgroup // [in] ranks of processes in group to appear in newgroup (array of integers) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm -int MPI_Group_excl( // Produces a group by reordering an existing group and taking only unlisted members - MPI_Group group, //[in] group (handle) - int n, // [in] number of elements in array ranks (integer) - const int ranks[], // [in] array of integer ranks in group not to appear in newgroup - MPI_Group *newgroup // [out] new group derived from above, preserving the order defined by group (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm -int MPI_Group_range_excl( // Produces a group by excluding ranges of processes from an existing group - MPI_Group group, // [in] group (handle) - int n, // [in] number of elements in array ranks (integer) - int ranges[][3], // [in] a one-dimensional array of integer triplets of the form (first rank, last rank, stride), indicating the ranks in group of processes to be excluded from the output group newgroup . - MPI_Group *newgroup // [out] new group derived from above, preserving the order in group (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm -int MPI_Group_range_incl( // Creates a new group from ranges of ranks in an existing group - MPI_Group group, // [in] group (handle) - int n, // [in] number of triplets in array ranges (integer) - int ranges[][3], // [in] a one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in group or processes to be included in newgroup. - MPI_Group *newgroup // [out] new group derived from above, in the order defined by ranges (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm -int MPI_Group_rank( // Determines the rank of the calling process in the communicator - MPI_Group group, // group (handle) - int *rank // rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm -int MPI_Group_size( // Returns the size of a group - MPI_Group group, // [in] group (handle) - int *size // [out] number of processes in the group (integer) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm -int MPI_Group_compare( // Compares two groups - MPI_Group group1, // [in] group1 (handle) - MPI_Group group2, // [in] group2 (handle) - int *result // [out] integer which is MPI_IDENT if the order and members of the two groups are the same, MPI_SIMILAR if only the members are the same, and MPI_UNEQUAL otherwise -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm#Node151 -int MPI_Group_intersection( // Produces a group as the intersection of two existing groups - MPI_Group group1, // [in] first group (handle) - MPI_Group group2, // [in] second group (handle) - MPI_Group *newgroup // [out] intersection group (handle) -); -int MPI_Group_translate_ranks( // Translates the ranks of processes in one group to those in another group - MPI_Group group1, // [in] group1 (handle) - int n, // [in] number of ranks in ranks1 and ranks2 arrays (integer) - const int ranks1[], // [in] array of zero or more valid ranks in group1 - MPI_Group group2, // [in] group2 (handle) - int ranks2[] // [out] array of corresponding ranks in group2, MPI_UNDEFINED when no correspondence exists. -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm -int MPI_Group_union( // Produces a group by combining two groups - MPI_Group group1, // first group (handle) - MPI_Group group2, // second group (handle) - MPI_Group *newgroup // union group (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm -int MPI_Error_class( // Converts an error code into an error class - int errorcode, // Error code returned by an MPI routine - int *errorclass // Error class associated with errorcode -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm#Node222 -int MPI_Error_string( // Return a string for a given error code - int errorcode, // [in] Error code returned by an MPI routine or an MPI error class - char *string, // [out] Text that corresponds to the errorcode - int *resultlen // [out] Length of string -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node221.htm#Node221 -inline int MPI_Finalize( // Terminates MPI execution environment -){return MPI_SUCCESS;} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm -int MPI_Finalized( // Indicates whether MPI_Finalize has been called - int *flag // [out] true if MPI was finalized (logical) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node227.htm -inline int MPI_Init( // Initialize the MPI execution environment - int *argc, // [in] Pointer to the number of arguments - char ***argv // [in] Pointer to the argument vector -){return MPI_SUCCESS;} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm -int MPI_Init_thread( // Initialize the MPI execution environment - int *argc, // [in] Pointer to the number of arguments - char ***argv, // [in] Pointer to the argument vector - int required, // [in] Level of desired thread support - int *provided // [out] Level of provided thread support -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm -int MPI_Initialized( // Indicates whether MPI_Init has been called. - int *flag // [out] Flag is true if MPI_Init or MPI_Init_thread has been called and false otherwise. -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node225.htm -int MPI_Intercomm_create( // Creates an intercommuncator from two intracommunicators - MPI_Comm local_comm, // [in] Local (intra)communicator - int local_leader, // [in] Rank in local_comm of leader (often 0) - MPI_Comm peer_comm, // [in] Communicator used to communicate between a designated process in the other communicator. Significant only at the process in local_comm with rank local_leader. - int remote_leader, // [in] Rank in peer_comm of remote leader (often 0) - int tag, // [in] Message tag to use in constructing intercommunicator; if multiple MPI_Intercomm_creates are being made, they should use different tags (more precisely, ensure that the local and remote leaders are using different tags for each MPI_intercomm_create). - MPI_Comm *newintercomm // [out] Created intercommunicator -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node168.htm -int MPI_Iprobe( // Nonblocking test for a message - int source, // [in] source rank, or MPI_ANY_SOURCE (integer) - int tag, // [in] tag value or MPI_ANY_TAG (integer) - MPI_Comm comm, // [in] communicator (handle) - int *flag, // [out] True if a message with the specified source, tag... - MPI_Status *status // [out] status object (Status) -); -int MPI_Is_thread_main( // This function can be called by a thread to determine if it is the main thread (the thread that called MPI_INIT or MPI_INIT_THREAD). - int *flag // true if calling thread is main thread, false otherwise (logical) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm -// Returns the upper bound on the amount of space needed to pack a message -int MPI_Pack_size( - int incount, - MPI_Datatype datatype, - MPI_Comm comm, - int *size -); -int MPI_Mprobe( - int source, // rank of source or MPI_ANY_SOURCE (integer) - int tag, // message tag or MPI_ANY_TAG (integer) - MPI_Comm comm, // communicator (handle) - MPI_Message *message, // returned message (handle) - MPI_Status *status // status object (Status) -); -int MPI_Mrecv( - void* buf, // initial address of receive buffer (choice) - int count, // number of elements in receive buffer (non-negati) - MPI_Datatype datatype, // datatype of each receive buffer element (handle) - MPI_Message *message, // message (handle) - MPI_Status *status // status object (Status) -); -// Blocking test for a message -int MPI_Probe( // like MPI_IMPROBE except that it is a blocking call that returns only after a matching message has been found. - int source, // [in] source rank, or MPI_ANY_SOURCE (integer) - int tag, // [in] tag value or MPI_ANY_TAG (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Status *status // [out] status object (Status) -); -int MPI_Query_thread( // The following function can be used to query the current level of thread support. - int *provided // provided level of thread support (integer) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm#Node303 -int MPI_Send( // Performs a blocking send - const void *buf, // [in] initial address of send buffer (choice) - int count, // [in] number of elements in send buffer (nonnegat...) - MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) - int dest, // [in] rank of destination (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm // [in] communicator (handle) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node47.htm#Node47 -int MPI_Recv( // Blocking receive for a message - void *buf, // [out] initial address of receive buffer (choice) - int count, // [in] maximum number of elements in receive buffer... - MPI_Datatype datatype, // [in] datatype of each receive buffer element... - int source, // [in] rank of source (integer) - int tag, // [in] message tag (integer) - MPI_Comm comm, // [in] communicator (handle) - MPI_Status *status // [out] status object (Status) -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node50.htm#Node50 -double MPI_Wtime( // Returns an elapsed time on the calling processor -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm -double MPI_Wtick( // Returns the resolution of MPI_Wtime -); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm#Node37 - - -inline int MPI_Status_set_cancelled(MPI_Status *status, int flag) -{ - return MPI_SUCCESS; -} - -inline int MPI_Test_cancelled(const MPI_Status *status, int *flag) -{ - if (flag) *flag = true; - return MPI_SUCCESS; -} - -typedef void (MPI_User_function)(void *a, void *b, int *len, MPI_Datatype *); -inline int MPI_Op_create(MPI_User_function *user_fn, int commute, MPI_Op *op) -{ - return MPI_SUCCESS; -} - -inline int MPI_Op_free(MPI_Op *op) -{ - if (op) *op = MPI_NO_OP; - return MPI_SUCCESS; -} - - -//const int MPI_MAX_PROCESSOR_NAME = 128; - - -inline int MPI_Get_address(const void *location, MPI_Aint *address) -{ - return MPI_SUCCESS; -} - - -// Memory handling - -//#include - -inline int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr) -{ - //if (baseptr) *(void **)baseptr = malloc(size); - if (baseptr) *(void **)baseptr = 0; - return MPI_SUCCESS; -} - -inline int MPI_Free_mem(void *base) -{ - //free(base); - return MPI_SUCCESS; -} - - -// MPI Requests - - -struct MPI_Request { - bool operator!=(const MPI_Request &o) { return false; } -}; -static struct MPI_Request MPI_REQUEST_NULL; - -inline int MPI_Request_get_status(MPI_Request request, int *flag, MPI_Status *status) -{ - if (flag) *flag = true; - return MPI_SUCCESS; -} - - -inline int MPI_Cancel(MPI_Request *request) -{ - return MPI_SUCCESS; -} - -inline int MPI_Request_free(MPI_Request *request) -{ - return MPI_SUCCESS; -} - -inline int MPI_Start(MPI_Request *request) -{ - return MPI_SUCCESS; -} - -inline int MPI_Wait(MPI_Request *request, MPI_Status *status) -{ - return MPI_SUCCESS; -} - -inline int MPI_Waitall(int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[]) -{ - return MPI_SUCCESS; -} - -inline int MPI_Testsome(int incount, MPI_Request array_of_requests[], int *outcount, int array_of_indices[], MPI_Status array_of_statuses[]) -{ - if (outcount) *outcount = 0; - return MPI_SUCCESS; -} - -static MPI_Status* MPI_STATUSES_IGNORE; - - -enum -{ - MPI_IDENT, - MPI_CONGRUENT, - MPI_SIMILAR, - MPI_UNEQUAL -}; - -enum { - MPI_GRAPH, - MPI_CART, - MPI_DIST_GRAPH -}; - -enum { - MPI_PROC_NULL, - MPI_ANY_SOURCE, - MPI_ANY_TAG, - MPI_UNDEFINED, - MPI_BSEND_OVERHEAD, - MPI_KEYVAL_INVALID, - MPI_LOCK_EXCLUSIVE, - MPI_LOCK_SHARED, - MPI_ROOT -}; - -enum { - MPI_MODE_APPEND, - MPI_MODE_CREATE, - MPI_MODE_DELETE_ON_CLOSE, - MPI_MODE_EXCL, - MPI_MODE_NOCHECK, - MPI_MODE_NOPRECEDE, - MPI_MODE_NOPUT, - MPI_MODE_NOSTORE, - MPI_MODE_NOSUCCEED, - MPI_MODE_RDONLY, - MPI_MODE_RDWR, - MPI_MODE_SEQUENTIAL, - MPI_MODE_UNIQUE_OPEN, - MPI_MODE_WRONLY - -}; - - -inline int MPI_Comm_group(MPI_Comm comm, MPI_Group *group) -{ - return MPI_SUCCESS; -} - -inline int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm) -{ - return MPI_SUCCESS; -} - -inline int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result) -{ - if (result) *result = MPI_UNEQUAL; - return MPI_SUCCESS; -} - -inline int MPI_Comm_test_inter(MPI_Comm comm, int *flag) -{ - if (flag) *flag = true; - return MPI_SUCCESS; -} - - -inline int MPI_Topo_test(MPI_Comm comm, int *status) -{ - if (status) *status = MPI_UNDEFINED; - return MPI_SUCCESS; -} - -inline int MPI_Comm_rank(MPI_Comm comm, int *rank) -{ - if (rank) *rank = 0; - return MPI_SUCCESS; -} - -// const should not be there -inline int MPI_Comm_disconnect(const MPI_Comm *comm) -{ - //if (comm) *comm = MPI_COMM_NULL; - return MPI_SUCCESS; -} - -// const on last arg should not be there -inline int MPI_Comm_accept(const char *port_name, MPI_Info info, int root, MPI_Comm comm, const MPI_Comm *newcomm) -{ - return MPI_SUCCESS; -} - -// const on last arg should not be there -inline int MPI_Comm_connect(const char *port_name, MPI_Info info, int root, MPI_Comm comm, const MPI_Comm *newcomm) -{ - return MPI_SUCCESS; -} - -inline int MPI_Abort(MPI_Comm comm, int errorcode) -{ - // should call exit or something here - return MPI_SUCCESS; -} - -inline int MPI_Comm_call_errhandler(MPI_Comm comm, int errorcode) -{ - return MPI_SUCCESS; -} - -inline int MPI_Barrier(MPI_Comm comm) -{ - return MPI_SUCCESS; -} - -inline int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm) -{ - return MPI_SUCCESS; -} - - - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/fake/mpic++ b/external_codes/mpi_wrapper/mpi3/fake/mpic++ deleted file mode 100755 index b0170c6b023..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/mpic++ +++ /dev/null @@ -1,2 +0,0 @@ -c++ -I/home/correaa/prj/alf/boost/mpi3/fake $@ - diff --git a/external_codes/mpi_wrapper/mpi3/fake/mpicc b/external_codes/mpi_wrapper/mpi3/fake/mpicc deleted file mode 100755 index 2e86f25c50a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/mpicc +++ /dev/null @@ -1,2 +0,0 @@ -cc -I/home/correaa/prj/alf/boost/mpi3/fake $@ - diff --git a/external_codes/mpi_wrapper/mpi3/fake/mpirun b/external_codes/mpi_wrapper/mpi3/fake/mpirun deleted file mode 100755 index 8f1104b73c0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/mpirun +++ /dev/null @@ -1,2 +0,0 @@ -$3 - diff --git a/external_codes/mpi_wrapper/mpi3/fake/test/CMakeLists.txt b/external_codes/mpi_wrapper/mpi3/fake/test/CMakeLists.txt deleted file mode 100644 index 0e0f1635d3f..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/test/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ - -cmake_minimum_required(VERSION 3.6.0) - -enable_testing() -include(CTest) - -SET(FAKE_MPI_DIR "..") -SET(MPI_WRAPPER_DIR "../../..") - -set(TEST_SRCS - test_c_mpi_init.c - test_mpi_init.cpp - test_wrapper_mpi_init.cpp -) - -foreach(TEST_FILE ${TEST_SRCS}) - SET(TEST_EXE "${TEST_FILE}x.x") - add_executable(${TEST_EXE} ${TEST_FILE}) - target_include_directories(${TEST_EXE} PUBLIC "..") - if (TEST_FILE MATCHES ".cpp$") - target_include_directories(${TEST_EXE} PUBLIC ${MPI_WRAPPER_DIR}) - endif() - add_test(NAME ${TEST_EXE} COMMAND ./${TEST_EXE}) -endforeach() - - -# Compile-time check for multiply defined symbols -add_library(library_check ${MPI_WRAPPER_DIR}/test/library_check.cpp) -add_executable(library_main ${MPI_WRAPPER_DIR}/test/library_main.cpp) -target_link_libraries(library_main library_check) -target_include_directories(library_check PUBLIC ${FAKE_MPI_DIR} ${MPI_WRAPPER_DIR}) -target_include_directories(library_main PUBLIC ${FAKE_MPI_DIR} ${MPI_WRAPPER_DIR}) - diff --git a/external_codes/mpi_wrapper/mpi3/fake/test/test_c_mpi_init.c b/external_codes/mpi_wrapper/mpi3/fake/test/test_c_mpi_init.c deleted file mode 100644 index d9db60b13ab..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/test/test_c_mpi_init.c +++ /dev/null @@ -1,11 +0,0 @@ - -#include - -/* Test MPI initialization in C */ - -int main(int argc, char **argv) -{ - MPI_Init(&argc, &argv); - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/test/test_mpi_init.cpp b/external_codes/mpi_wrapper/mpi3/fake/test/test_mpi_init.cpp deleted file mode 100644 index a706df68b84..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/test/test_mpi_init.cpp +++ /dev/null @@ -1,11 +0,0 @@ - -#include - -// Test MPI initialization in C++ - -int main(int argc, char **argv) -{ - MPI_Init(&argc, &argv); - MPI_Finalize(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/fake/test/test_wrapper_mpi_init.cpp b/external_codes/mpi_wrapper/mpi3/fake/test/test_wrapper_mpi_init.cpp deleted file mode 100644 index fb283605789..00000000000 --- a/external_codes/mpi_wrapper/mpi3/fake/test/test_wrapper_mpi_init.cpp +++ /dev/null @@ -1,11 +0,0 @@ - -#include -#include - -namespace mpi3 = boost::mpi3; - -int main(int argc, char **argv) -{ - mpi3::environment(argc, argv); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/FILE.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/FILE.hpp deleted file mode 100644 index dfa945c9600..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/FILE.hpp +++ /dev/null @@ -1,145 +0,0 @@ -#ifndef ALF_BOOST_MPI3_FILE_HPP -#define ALF_BOOST_MPI3_FILE_HPP - -namespace boost{ -namespace mpi3{ - -struct FILE{ - MPI_File impl_; - static void delete_(std::string const& filename){MPI_File_delete(filename.c_str(), MPI_INFO_NULL);} - int amode() const{ - int ret; - MPI_File_get_amode(impl_, &ret); - return ret; - } - bool atomicity() const{ - int ret; - MPI_File_get_atomicity(impl_, &ret); - return ret; - } - void atomicity(bool flag){MPI_File_set_atomicity(impl_, flag);} - MPI_Offset byte_offset(MPI_Offset offset){ - MPI_Offset disp; - MPI_File_get_byte_offset(impl_, offset, &disp); - return disp; - } - info hints() const{ - info ret; - MPI_File_get_info(impl_, &ret.impl_); - return ret; - } - void hints(info set){MPI_File_set_info(impl_, set.impl_);} - MPI_Offset position() const{ - MPI_Offset offset; - MPI_File_get_position(impl_, &offset); - return offset; - } - MPI_Offset position_shared() const{ - MPI_Offset offset; - MPI_File_get_position_shared(impl_, &offset); - return offset; - } - MPI_Offset size() const{ - MPI_Offset ret; - MPI_File_get_size(impl_, &ret); - return ret; - } - MPI_Aint extent(boost::mpi3::type const& t) const{ - MPI_Aint ret; - MPI_File_get_type_extent(impl_, t.impl_, &ret); - return ret; - } - void view(); // int MPI_File_get_view - template::value_type, class datatype = detail::datatype> - auto read_n(ContiguousIterator O, Size count) const{ - status s; - MPI_File_read(impl_, std::addressof(*O), count, datatype{}, &s.impl_); - return O + count; - } - template::value_type, class datatype = detail::datatype> - auto read_all_n(ContiguousIterator O, Size count) const{ - status s; - MPI_File_read_all(impl_, std::addressof(*O), count, datatype{}, &s.impl_); - return O + count; - } - // MPI_File_read_all_begin - // MPI_File_read_all_end - template::value_type, class datatype = detail::datatype> - auto read_at_n(MPI_Offset offset, ContiguousIterator O, Size count) const{ - status s; - MPI_File_read_at(offset, impl_, std::addressof(*O), count, datatype{}, &s.impl_); - return O + count; - } - template::value_type, class datatype = detail::datatype> - auto read_at_all_n(MPI_Offset offset, ContiguousIterator O, Size count) const{ - status s; - MPI_File_read_at_all(offset, impl_, std::addressof(*O), count, datatype{}, &s.impl_); - return O + count; - } - // MPI_File_read_at_all_begin - // MPI_File_read_at_all_end - template::value_type, class datatype = detail::datatype> - auto read_ordered_n(ContiguousIterator O, Size count) const{ - status s; - MPI_File_read_ordered(impl_, std::addressof(*O), count, datatype{}, &s.impl_); - return O + count; - } - // MPI_File_read_ordered_begin - // MPI_File_read_ordered_end - template::value_type, class datatype = detail::datatype> - auto read_shared_n(ContiguousIterator O, Size count) const{ - status s; - MPI_File_read_ordered(impl_, std::addressof(*O), count, datatype{}, &s.impl_); - return O + count; - } - void seek(MPI_Offset offset, int whence){MPI_File_seek(impl_, offset, whence);} - void seek_set(MPI_Offset offset){seek(offset, MPI_SEEK_SET);} - void seek_current(MPI_Offset offset){seek(offset, MPI_SEEK_CUR);} - void seek_end(MPI_Offset offset = 0){seek(offset, MPI_SEEK_END);} - - void seek_shared(MPI_Offset offset, int whence){MPI_File_seek_shared(impl_, offset, whence);} - //MPI_File_set_errhandler - template::value_type, class datatype = detail::datatype> - request iread_n(ContiguousIterator I, Size count); - template::value_type, class datatype = detail::datatype> - request iread_at_n(MPI_Offset offset, ContiguousIterator I, Size count); - template::value_type, class datatype = detail::datatype> - request iread_shared_n(ContiguousIterator I, Size count); - template::value_type, class datatype = detail::datatype> - status write_n(ContiguousIterator I, Size count) const{ - status ret; - MPI_File_write(impl_, std::addressof(*I), count, datatype{}, &ret.impl_); - return ret; - } - template::value_type, class datatype = detail::datatype> - status write_all_n(ContiguousIterator I, Size count) const{ - status ret; - MPI_File_write_all(impl_, std::addressof(*I), count, datatype{}, &ret.impl_); - return ret; - } - template::value_type, class datatype = detail::datatype> - request iwrite_n(ContiguousIterator I, Size count); - template::value_type, class datatype = detail::datatype> - request iwrite_at_n(MPI_Offset offset, ContiguousIterator I, Size count); - template::value_type, class datatype = detail::datatype> - request iwrite_shared_n(ContiguousIterator I, Size count); - void preallocate(MPI_Offset size){MPI_File_preallocate(impl_, size);} - void resize(MPI_Offset size){MPI_File_set_size(impl_, size);} - void sync(){MPI_File_sync(impl_);} -}; - -int ftell(FILE* stream){return stream->position();} -int fseek(FILE* stream, MPI_Offset offset, int whence/*origin*/){ - return MPI_File_seek(stream->impl_, offset, whence); -} - -FILE* communicator::fopen(const char* filename, int amode = MPI_MODE_RDWR | MPI_MODE_CREATE){ - FILE* ret; - MPI_File_open(impl_, filename, amode, MPI_INFO_NULL, &(ret->impl_)); - return ret; -} - - -}} -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/address.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/address.hpp deleted file mode 100644 index 3d4b0ada580..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/address.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;-*- */ -//#if COMPILATION -//mpic++ -D_TEST_BOOST_MPI3_ADDRESS -x c++ $0 -o $0x -lboost_serialization&&mpirun --oversubscribe -n 4 $0x&&rm $0x;exit -//#endif -// Copyright 2018-2021 Alfredo A. Correa - -#ifndef BOOST_MPI3_ADDRESS_HPP -#define BOOST_MPI3_ADDRESS_HPP - -#include "../mpi3/detail/call.hpp" -#include "../mpi3/types.hpp" - -// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -// #include - -namespace boost{ -namespace mpi3{ - -inline address get_address(void const* location){ - address ret; // NOLINT(cppcoreguidelines-init-variables) : delayed init - // this function requires an initialized environment, TODO should be a (static?) member of environment? - MPI_(Get_address)(location, &ret); // MPI_Address is deprecated - return ret; -} - -template -address addressof(T const& t){return get_address(std::addressof(t));} - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_ADDRESS - -//#include "../mpi3/main.hpp" - -//using std::cout; -//namespace mpi3 = boost::mpi3; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// std::cout << "dsadsad" << std::endl; - -// std::vector v(10); -// mpi3::address a1 = mpi3::addressof(v[0]); -// mpi3::address a2 = mpi3::addressof(v[1]); -// assert( a2 - a1 == sizeof(int) ); - -// return 0; -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/allocator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/allocator.hpp deleted file mode 100644 index 9fd4a522083..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/allocator.hpp +++ /dev/null @@ -1,129 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ - -#ifndef BOOST_MPI3_ALLOCATOR_HPP -#define BOOST_MPI3_ALLOCATOR_HPP - -#include "../mpi3/address.hpp" - -#include - -#include -#include - -namespace boost{ -namespace mpi3{ - -struct /*__attribute__((aligned(0)))*/ bad_alloc : std::bad_alloc{using std::bad_alloc::bad_alloc;}; - -inline void* malloc(mpi3::size_t size) { - void* ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - int const s = MPI_Alloc_mem(size, MPI_INFO_NULL, &ret); - if(s != MPI_SUCCESS) {return nullptr;} //s throw bad_alloc();//"cannot allocate " + std::to_string(size) + " bytes"); - return ret; -} - -inline void free(void* ptr){ - MPI_(Free_mem)(ptr); -} - -template -struct /*__attribute__((aligned(0)))*/ allocator{ - using size_type = mpi3::size_t; - using value_type = T; - using pointer = T*; - - allocator() = default; - - // cppcheck-suppress noExplicitConstructor - template allocator(allocator const&/*other*/) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) : allocator convention - - auto allocate(size_type n) { - if(void* ptr = mpi3::malloc(n * static_cast(sizeof(T)))) { - return static_cast(ptr); - } - throw bad_alloc(); - } - void deallocate(pointer p, std::size_t /*size*/) { mpi3::free(p); } - static size_type max_size() { return std::numeric_limits::max(); } -}; - -template -struct uallocator : allocator{ - template void construct(U* /*unused*/){ - static_assert( - std::is_trivially_destructible{}, - "uallocator cannot be used with non trivial types" - ); - } -}; - -template< class T1, class T2 > constexpr -bool operator==(allocator const&/*self*/, uallocator const&/*other*/) { // TODO(correaa) check that both(?) are trivial? - return true; -} - -template< class T1, class T2 > constexpr -bool operator==(uallocator const&/*self*/, allocator const&/*other*/) { - return true; -} - -template -constexpr std::add_const_t& as_const(T& t) noexcept{return t;} - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_ALLOCATOR - -//#include "../mpi3/main.hpp" - -//#include -//#include - -//#include - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - -// std::vector> v(1000000); -// std::vector> uv(1000000); -// std::iota(v.begin(), v.end(), 0.); -// using boost::mpi3::data; -// assert( data(uv.begin()) == &*uv.begin() ); -// assert( std::accumulate(v.begin(), v.end(), 0.) == (v.size()*(v.size() - 1))/2 ); -// return 0; -// -// { -// boost::container::flat_set, mpi3::allocator > fs; -// fs.insert(5.); -// fs.insert(3.); -// auto it = fs.begin(); -// assert(*it == 3.); -// ++it; -// assert(*it == 5.); -// } -// { -// boost::container::flat_set, std::allocator_traits>::rebind_alloc> fs; -// fs.insert(5); -// fs.insert(3); -// auto it = fs.begin(); -// assert(*it == 3); -// ++it; -// assert(*it == 5); -// } -// { -// boost::container::flat_set, std::less>, mpi3::allocator>> fsp; -// fsp.insert({1.,2.}); -// fsp.insert({3.,4.}); -// auto it = fsp.begin(); -// assert(*it == std::make_pair(1.,2.)); -// ++it; -// assert(*it == std::make_pair(3.,4.)); -// } -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/buffer.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/buffer.hpp deleted file mode 100644 index f973524ee3b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/buffer.hpp +++ /dev/null @@ -1,111 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wall -Wfatal-errors -D_TEST_BOOST_MPI3_BUFFER $0x.cpp -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_BUFFER_HPP -#define BOOST_MPI3_BUFFER_HPP - -#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include -#include - -namespace boost{ -namespace mpi3{ - -template -void buffer_attach_n(T* data, std::size_t n){ - static_assert(sizeof(T)%sizeof(char) == 0, ""); - int status = MPI_Buffer_attach((void*)data, n*sizeof(T)/sizeof(char)); - if(status != MPI_SUCCESS) throw std::runtime_error("cannot attach buffer"); -} -template -void buffer_attach(T* first, T* last){ - buffer_attach_n(first, std::distance(first, last)); -} - -template -void attach_n(T* data, Size n){return buffer_attach_n(data, n);} - -template -void attach(T* first, T* last){return buffer_attach(first, last);} - -template -void attach(C& c){return buffer_attach(c);} - -std::pair buffer_detach(){ - char* buffer = 0; - int size = -1; - int s = MPI_Buffer_detach(&buffer, &size); - if(s != MPI_SUCCESS) throw std::runtime_error("cannot buffer detach"); - return {buffer, size}; -} -std::pair detach(){ - return buffer_detach(); -} - -template -struct scoped_buffer_attach{ - scoped_buffer_attach(T* data, int size){ - buffer_attach_n(data, size); - } - ~scoped_buffer_attach(){ - buffer_detach(); - } -}; - -template -struct scoped_buffer{ - T* buf_; - int size_; - scoped_buffer(int size) : size_(size){ - buf_ = new T[size]; - buffer_attach_n(buf_, size_*sizeof(T)); - } - ~scoped_buffer(){ - std::pair check = buffer_detach(); - assert(check.first == (char*)(buf_)); - // assert(check.second/sizeof(T) == size_); - delete[] buf_; - } -}; - -}} - -#ifdef _TEST_BOOST_MPI3_BUFFER - -#include "../mpi3/main.hpp" -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - std::vector a(10); - - mpi3::scoped_buffer buf(2000); - - for(int j = 0; j !=10; ++j){ - auto r = world.bsend_init_n(a.data(), a.size(), 0, 27 + j); - for(int i = 0; i != 10; ++i) a[i] = (world.rank() + 10*j)*world.size() + i; - r.start(); - // r.wait(); // requests wait automatically on destruction (they don't start automatically) - } - - std::vector b(10); - - if(world.root()) - for(int i = 0; i != world.size(); ++i) - for(int j = 0; j != 10; ++j){ - world.receive(b.data(), i, 27 + j); - for(int k = 0; k != 10; ++k) - if(b[k] != (i + 10*j)*world.size() + k) assert(0); - } - - return 0; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/cartesian_communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/cartesian_communicator.hpp deleted file mode 100644 index 8ace96f7843..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/cartesian_communicator.hpp +++ /dev/null @@ -1,348 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_CARTESIAN_COMMUNICATOR_HPP -#define BOOST_MPI3_CARTESIAN_COMMUNICATOR_HPP - -#include -#include - -#include - -namespace boost::mpi3 { - -using dimensionality_type = int; - -static constexpr dimensionality_type dynamic_extent = -1; - -template struct cartesian_communicator; - -template<> -struct cartesian_communicator : communicator { - - cartesian_communicator() = default; - - cartesian_communicator(cartesian_communicator const&) = delete; - cartesian_communicator(cartesian_communicator&&) = default; - // vvv--- this is an unusual "duplicate" constructor - cartesian_communicator(cartesian_communicator& other) : communicator{other} {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) cannot be defaulted because bug in nvcc 11 - - template - cartesian_communicator(communicator& comm_old, Shape const& s, Period const& p) { - assert(s.size() == p.size()); - using dimensionality_type = int; - MPI_(Cart_create)(comm_old.get(), static_cast(s.size()), s.data(), p.data(), /*reorder*/ true, &impl_); - // assert(impl_ != MPI_COMM_NULL); // null communicator is a valid outcome - // TODO(correaa) try with mpich, WAS: there is an bug in mpich, in which if the remaining dim are none then the communicator is not well defined. - } - - template - cartesian_communicator(communicator& old, Shape const& s) - : cartesian_communicator{old, s, std::vector(s.size(), true)} {} - - cartesian_communicator(communicator& comm_old, std::initializer_list shape) - : cartesian_communicator(comm_old, std::vector(shape)) {} - - cartesian_communicator(communicator& comm_old, std::initializer_list shape, std::initializer_list period) - : cartesian_communicator(comm_old, std::vector(shape), std::vector(period)) {} - - [[deprecated("use dimensionality() instead of dimension")]] int dimension() const { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Cartdim_get(impl_, &ret); - return ret; - } - - cartesian_communicator& operator=(cartesian_communicator const&) = delete; - cartesian_communicator& operator=(cartesian_communicator&&) = default; - // vvv nvcc 11 workaround, needs explicit definition of duplicate assigment - [[deprecated]] cartesian_communicator& operator=(cartesian_communicator& other) { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) "duplicate" assignment - if(this == std::addressof(other)) { - return *this; - } // lints cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator - communicator::operator=(other); - return *this; - } - - ~cartesian_communicator() = default; - - int dimensionality() const { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Cartdim_get)(impl_, &ret); - return ret; - } - - std::vector coordinates() const { - std::vector ret(static_cast::size_type>(dimensionality())); - MPI_(Cart_coords)(impl_, rank(), dimensionality(), ret.data()); - return ret; - } - - auto topology() const { - auto const maxdims = static_cast(dimensionality()); // TODO(correaa) use safe cast - class topology_t { - std::vector dimensions_; - std::vector periods_; - std::vector coordinates_; - friend mpi3::cartesian_communicator; - - public: - explicit topology_t(std::size_t n) : dimensions_(n), periods_(n), coordinates_(n) {} - - auto const& dimensions() const { return dimensions_; } - auto const& periods() const { return periods_; } - auto const& coordinates() const { return coordinates_; } - } ret(maxdims); - - MPI_(Cart_get)(impl_, static_cast(maxdims), ret.dimensions_.data(), ret.periods_.data(), ret.coordinates_.data()); - - assert(ret.coordinates() == coordinates()); - return ret; - } - - std::vector shape() const { return topology().dimensions(); } - std::vector periods() const { - auto ps = topology().periods(); - return {ps.begin(), ps.end()}; - } - auto num_elements() const { return size(); } - - template - auto operator()(Coord const& coord) { - int rank = -1; - MPI_(Cart_rank)(impl_, coord.data(), &rank); - return (*this)[rank]; - // return operator[](rank); - } - // int MPI_Cart_map not implemented - cartesian_communicator sub_aux(std::vector const& remain_dims) { - assert(static_cast(remain_dims.size()) == dimensionality()); - cartesian_communicator ret; - MPI_(Cart_sub)(impl_, remain_dims.data(), &ret.impl_); - return ret; - } - - template> - cartesian_communicator sub(RemainDim const& remain_dims) { - return sub_aux(std::vector(remain_dims.begin(), remain_dims.end())); - } - cartesian_communicator sub() { - assert(dimensionality() > 1); - std::vector remain(static_cast(dimensionality()), 1 /*true*/); - remain[0] = 0 /*false*/; - return sub_aux(remain); - } -}; - -enum fill_t { - fill = 0, - _ = 0 -}; - -struct circular_communicator; - -template -struct cartesian_communicator : cartesian_communicator<> { - cartesian_communicator() = default; - - cartesian_communicator(cartesian_communicator& other) : cartesian_communicator<>{other} {} - cartesian_communicator(cartesian_communicator const&) = delete; - cartesian_communicator(cartesian_communicator&&) noexcept = default; - - ~cartesian_communicator() = default; - - static std::array division(int nnodes, std::array suggest = {}) { - MPI_(Dims_create)(nnodes, D, suggest.data()); - return suggest; - } - constexpr static dimensionality_type dimensionality = D; - - explicit cartesian_communicator( - communicator& other, - std::array dims = {}, - std::array periods = std::apply([](auto... e) { return std::array{(static_cast(e), true)...}; }, std::array{}) - ) try : cartesian_communicator - <>{other, division(other.size(), dims), std::apply([](auto... e) { return std::array{e...}; }, periods)} {} - catch(std::runtime_error& e) { - std::ostringstream ss; - std::copy(dims.begin(), dims.end(), std::ostream_iterator{ss, " "}); - throw std::runtime_error{"cannot create cartesian communicator with constrains " + ss.str() + " from communicator of size " + std::to_string(other.size()) + " because " + e.what()}; - } - - auto topology() const { - struct topology_t { - std::array dimensions, periods, coordinates; - } ret = {}; - MPI_(Cart_get)( - impl_, dimensionality, - ret.dimensions.data(), ret.periods.data(), ret.coordinates.data() - ); - return ret; - } - - constexpr auto dimensions() const { return topology().dimensions; } - - cartesian_communicator& operator=(cartesian_communicator const&) = delete; - cartesian_communicator& operator=(cartesian_communicator&&) noexcept = default; - // vvv nvcc 11 workaround, needs explicit definition of duplicate assigment - [[deprecated]] cartesian_communicator& operator=(cartesian_communicator& other) { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) duplicate assignment - if(this == std::addressof(other)) { - return *this; - } // lints cert-oop54-cpp - cartesian_communicator<>::operator=(other); // NOLINT(clang-diagnostic-deprecated-declarations) - return *this; - } - - cartesian_communicator<1> axis(int d) { // TODO(correaa) return a circular_communicator - assert(d >= 0); - assert(d < D); - cartesian_communicator<1> ret; - std::array remains{}; - remains.fill(false); - remains[static_cast(d)] = true; // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index) - MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); - return ret; - } - - template auto axis() -> circular_communicator; - - auto plane(int d1, int d2) { - assert(d1 >= 0 and d2 >= 0); - - auto const d1s = static_cast(d1); - auto const d2s = static_cast(d2); - - assert(d2s < D and d1s < d2s); - - std::array remains{}; - remains.at(d1s) = true; - remains.at(d2s) = true; - - cartesian_communicator<2> ret; - MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); - return ret; - } - - template - auto plane() { - static_assert(D1 < D2); - std::array remains{}; - remains.fill(false); - std::get(remains) = true; - std::get(remains) = true; - - cartesian_communicator<2> ret; - MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); - return ret; - } - - template auto shift(int displacement) const { - std::pair source_dest; - MPI_(Cart_shift)(impl_, Direction, displacement, &source_dest.first, &source_dest.second); - return source_dest; - } - - template - auto send_receive_shift(As... as, int displacement = 1) { - send_receive(as..., shift(displacement)); - } - - using coordinates_type = std::array; - - using cartesian_communicator<>::rank; - auto rank(coordinates_type cs) const -> int { - auto const ps = periods(); - auto const s = shape(); - for(std::size_t i = 0; i != D; ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm - if(ps[i] == false) { - assert(cs[i] >= 0); - assert(cs[i] < s[i]); - } - } - return MPI_(Cart_rank)(impl_, cs.data()); - } - auto coordinates(int r) const -> coordinates_type { - coordinates_type ret; - MPI_(Cart_coords)(impl_, r, D, ret.data()); - return ret; - } - auto coordinates() const -> coordinates_type { return coordinates(rank()); } - template - auto operator()(Indices... idx) { return (*this)[rank(std::array{idx...})]; } - - cartesian_communicator hyperplane(int d) { - static_assert(D != 0, "hyperplane not possible for 0D communicators"); -#if defined(MPICH_VERSION) - static_assert(D != 1, "hyperplane not possible for 1D communicators"); // they work in openMPI but do not work in MPICH -#endif - assert(d >= 0); - assert(d < D); - - cartesian_communicator ret; - std::array remains{}; - remains.fill(true); - - remains[static_cast(d)] = false; // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index) - MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); - return ret; - } -}; - -struct circular_communicator : cartesian_communicator<1> { - circular_communicator() = default; - - circular_communicator(circular_communicator& other) : cartesian_communicator<1>{other} {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) icpc needs this definition explicitly - circular_communicator(circular_communicator const&) = delete; - circular_communicator(circular_communicator&&) noexcept = default; - - ~circular_communicator() = default; - - explicit circular_communicator(communicator& other) - : cartesian_communicator<1>{other, {}, {true}} {} - - auto operator=(cartesian_communicator const&) -> circular_communicator& = delete; - auto operator=(circular_communicator&& other) noexcept -> circular_communicator& { - cartesian_communicator<1>::operator=(std::move(other)); - return *this; - } - // vvv nvcc 11 workaround, needs explicit definition of duplicate assigment - [[deprecated]] auto operator=(circular_communicator& other) -> circular_communicator& { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) duplicate assignment - if(this == std::addressof(other)) { - return *this; - } // lints cert-oop54-cpp - cartesian_communicator<1>::operator=(other); // NOLINT(clang-diagnostic-deprecated-declarations) - return *this; - } - - auto coordinate() const { return std::get<0>(this->coordinates()); } - auto coordinate(int rank) const { return std::get<0>(this->coordinates(rank)); } - - using cartesian_communicator<1>::rank; - auto rank(int coordinate) const { return cartesian_communicator<1>::rank({coordinate}); } - - template - auto rotate(As... as, int displacement) { return this->send_receive(as..., this->shift<0>(-displacement)); } - template - auto rotate(As... as) { return this->send_receive(as..., this->shift<0>(-1)); } - - template - auto unrotate(As... as, int displacement) { return this->send_receive(as..., this->shift<0>(+displacement)); } - template - auto unrotate(As... as) { return this->send_receive(as..., this->shift<0>(+1)); } -}; - -template using torus = cartesian_communicator; - -using ring = circular_communicator; - -template template -auto cartesian_communicator::axis() -> circular_communicator { - circular_communicator ret; - std::array remains{}; - remains.fill(false); - std::get
(remains) = true; - MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); - return ret; -} - -} // end namespace boost::mpi3 -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/communication_mode.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/communication_mode.hpp deleted file mode 100644 index 2d3c44d8696..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/communication_mode.hpp +++ /dev/null @@ -1,69 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef MPI3_DETAIL_COMMUNICATION_MODE -#define MPI3_DETAIL_COMMUNICATION_MODE - -#include // if you get an error here probably you need to compile with and MPI compiler wrapper - -#include // forward - -namespace boost { -namespace mpi3 { - -struct blocking_mode {}; -struct nonblocking_mode {}; - -struct standard_communication_mode { - template static int Send(As... as) { return MPI_Send(as...); } // NOLINT(readability-identifier-naming) - template static int Recv(As... as) { return MPI_Recv(as...); } // NOLINT(readability-identifier-naming) - template static int ISend(As... as) { return MPI_Isend(as...); } // NOLINT(readability-identifier-naming) - template static int IRecv(As... as) { return MPI_Irecv(as...); } // NOLINT(readability-identifier-naming) -}; - -struct buffered_communication_mode { - template static int Send(As... as) { return MPI_Bsend(as...); } // NOLINT(readability-identifier-naming) - template static int Recv(As... as) { return MPI_Brecv(as...); } // NOLINT(readability-identifier-naming) - template static int ISend(As... as) { return MPI_Ibsend(as...); } // NOLINT(readability-identifier-naming) - template static int IRecv(As... as) { return MPI_Ibrecv(as...); } // NOLINT(readability-identifier-naming) -}; - -struct synchronous_communication_mode { - template static int Send(As... as) { return MPI_Ssend(as...); } // NOLINT(readability-identifier-naming) - template static int Recv(As... as) { return MPI_Srecv(as...); } // NOLINT(readability-identifier-naming) - template static int ISend(As... as) { return MPI_Issend(as...); } // NOLINT(readability-identifier-naming) - template static int IRecv(As... as) { return MPI_Isrecv(as...); } // NOLINT(readability-identifier-naming) -}; - -struct ready_communication_mode { - template static int Send(As... as) { return MPI_Rsend(as...); } // NOLINT(readability-identifier-naming) - template static int Recv(As... as) { return MPI_Rrecv(as...); } // NOLINT(readability-identifier-naming) - template static int ISend(As... as) { return MPI_Irsend(as...); } // NOLINT(readability-identifier-naming) - template static int IRecv(As... as) { return MPI_Irrecv(as...); } // NOLINT(readability-identifier-naming) -}; - -struct gather_mode { - template int operator()(As... as) const { return MPI_Gather(as...); } -}; -struct igather_mode { - template int operator()(As... as) const { return MPI_Igather(as...); } -}; - -struct all_gather_mode { - template int operator()(As... as) const { return MPI_Allgather(as...); } -}; - -struct reduce_mode { - template int operator()(As... as) const { return MPI_Reduce(as...); } -}; -struct ireduce_mode { - template int operator()(As... as) const { return MPI_Ireduce(as...); } -}; - -struct all_reduce_mode { - template int operator()(As... as) const { return MPI_Allreduce(as...); } -}; - -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator.hpp deleted file mode 100644 index 7e708690427..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator.hpp +++ /dev/null @@ -1,3233 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef MPI3_COMMUNICATOR_HPP -#define MPI3_COMMUNICATOR_HPP - -#include "../mpi3/communication_mode.hpp" -#include "../mpi3/error.hpp" -#include "../mpi3/generalized_request.hpp" -#include "../mpi3/group.hpp" -#include "../mpi3/handle.hpp" -#include "../mpi3/info.hpp" -#include "../mpi3/message.hpp" -#include "../mpi3/operation.hpp" -#include "../mpi3/port.hpp" -#include "../mpi3/request.hpp" -#include "../mpi3/status.hpp" -#include "../mpi3/type.hpp" - -#include "../mpi3/detail/basic_communicator.hpp" -#include "../mpi3/detail/buffer.hpp" -#include "../mpi3/detail/datatype.hpp" -#include "../mpi3/detail/equality.hpp" -#include "../mpi3/detail/iterator.hpp" -#include "../mpi3/detail/value_traits.hpp" - -#include "../mpi3/detail/package.hpp" - -#include "../mpi3/config/NODISCARD.hpp" - -//#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#define BOOST_PACKAGE_ARCHIVE_SOURCE - -#include -#include - -#include -#include -#include -//#include // must be the last header - -#include -#include -#include -#include - -#include - -#include -#include - -// use this to avoid need for linking -lserialization -#ifdef _MAKE_BOOST_SERIALIZATION_HEADER_ONLY -//#include -#if BOOST_VERSION > 106000 && BOOST_VERSION < 106600 -#include "../mpi3/serialization_hack/singleton.cpp" -#endif -#if BOOST_VERSION < 105900 -#define BOOST_ARCHIVE_DECL -#define BOOST_SERIALIZATION_DECL -#endif -// NOLINTBEGIN(hicpp-use-auto,misc-const-correctness,modernize-use-auto) external code -#include "../mpi3/serialization_hack/archive_exception.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/basic_archive.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/basic_iarchive.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/basic_iserializer.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/basic_oarchive.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/basic_oserializer.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/extended_type_info.cpp" // NOLINT(bugprone-suspicious-include) hack -#include "../mpi3/serialization_hack/extended_type_info_typeid.cpp" // NOLINT(bugprone-suspicious-include,misc-const-correctness) hack -// NOLINTEND(hicpp-use-auto,misc-const-correctness,modernize-use-auto) - -#endif - -#include "../mpi3/package_archive.hpp" - -#include -#include -#include // iterator_traits -#include -#include -#include // std::accumulate -#include -#include -#include // is_same -#include - -namespace boost { -namespace mpi3 { - -#define SAFE_MPI_(F) check_mpi_(MPI_##F) // NOLINT(cppcoreguidelines-macro-usage) : name concatenation - -#if !defined(OPEN_MPI) || (OMPI_MAJOR_VERSION < 2) -#define OMPI_COMM_TYPE_NODE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_HWTHREAD MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_CORE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_L1CACHE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_L2CACHE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_L3CACHE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_SOCKET MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_NUMA MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_BOARD MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_HOST MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_CU MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#define OMPI_COMM_TYPE_CLUSTER MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) -#endif - -// https://www.open-mpi.org/doc/v4.0/man3/MPI_Comm_split_type.3.php#toc8 -enum class communicator_type : int { - shared = MPI_COMM_TYPE_SHARED ,/*synomym*/ node = OMPI_COMM_TYPE_NODE, - hw_thread = OMPI_COMM_TYPE_HWTHREAD, - core = OMPI_COMM_TYPE_CORE , - l1_cache = OMPI_COMM_TYPE_L1CACHE , - l2_cache = OMPI_COMM_TYPE_L2CACHE , - l3_cache = OMPI_COMM_TYPE_L3CACHE , - socket = OMPI_COMM_TYPE_SOCKET , - numa = OMPI_COMM_TYPE_NUMA , - board = OMPI_COMM_TYPE_BOARD , - host = OMPI_COMM_TYPE_HOST , - cu = OMPI_COMM_TYPE_CU ,/*synomym*/ cpu = OMPI_COMM_TYPE_CU , - cluster = OMPI_COMM_TYPE_CLUSTER -}; - -enum constant { - undefined = MPI_UNDEFINED , - process_null = MPI_PROC_NULL , - any_source = MPI_ANY_SOURCE -}; - -enum key { // for attributes - tag_ub = MPI_TAG_UB, - host = MPI_HOST, - io = MPI_IO, - wtime_is_global = MPI_WTIME_IS_GLOBAL, - application_number = MPI_APPNUM, - universe_size = MPI_UNIVERSE_SIZE, - last_used_code = MPI_LASTUSEDCODE -}; - -template struct overload_priority : overload_priority{ -// using overload_priority::overload_priority; -}; -template<> struct overload_priority<0>{}; - -class environment; -class group; - -using std::optional; - -struct error_handler; - -template -struct shm_pointer; - -template -struct pointer; - -using address = MPI_Aint; -using intptr_t = MPI_Aint; -using size_t = MPI_Aint; -using ptrdiff_t = std::make_signed_t; -struct request; - -struct send_request; -struct receive_request; - -struct FILE; - -class process; - -struct ostream; - -struct message_header{ - int tag; -}; - -struct graph_communicator; -struct shared_communicator; // intracommunicator - -using std::any; -using std::any_cast; - -//class communicator_ptr{}; - -template class window; - -class communicator : protected detail::basic_communicator { // in mpich MPI_Comm == int - friend struct detail::package; - friend class window; - - protected: - bool is_null() const {return MPI_COMM_NULL == impl_;} - friend class mpi3::environment; - detail::equality compare(communicator const& other) const { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Comm_compare)(impl_, other.impl_, &ret); - return static_cast(ret); - } - - public: - communicator(communicator const& o, group const& g); - - communicator(group const& g, int tag); - explicit communicator(group const& g); - - using detail::basic_communicator::basic_communicator; - - communicator() = default; - - communicator(communicator const&) = delete;//default; - communicator(communicator& other) : basic_communicator{other} {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) intel and nvcc 11 need this (not =default) - communicator(communicator&&) = default; - - communicator& operator=(communicator const&) = delete; - [[deprecated]] auto operator=(communicator& other) -> communicator& { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) duplicate assigment - communicator tmp{other}; - operator=(std::move(tmp)); - // swap(tmp); - return *this; - } - auto operator=(communicator && other) noexcept -> communicator& { // TODO(correaa) tidy this operator - if(impl_ != MPI_COMM_NULL) { - try { - MPI_(Comm_disconnect)(&impl_); //this will wait for communications to finish communications, if it gets to this point is probably an error anyway <-- not true, it is necessary to synchronize the flow - // MPI_Comm_free(&impl_); - } catch(std::exception& e) { std::cerr<< e.what() < iterator_t& {++rank_; return *this;} -//// auto operator--() -> iterator_t& {--rank_; return *this;} -//// auto operator*() const -> reference; - -//// private: -//// communicator* commP_ = nullptr; -//// int rank_ = MPI_PROC_NULL; - -//// friend class communicator; -//// iterator_t(communicator* self, int rank) : commP_{self}, rank_{rank} {} -// }; -// using iterator = iterator_t; - -// auto begin() -> iterator {return {this, 0 };} -// auto end () -> iterator {return {this, size()};} - - auto& handle() {return impl_;} - auto get_mutable() {return impl_;} - auto get() const {return impl_;} // TODO(correaa) deprecate - impl_t& get() {return this->impl_;} - - class ptr { // cppcheck-suppress noConstructor ; bug in cppcheck 2.3 - communicator* ptr_; - - public: - explicit ptr(communicator* ptr) : ptr_{ptr} {} - operator MPI_Comm() const {return ptr_->get_mutable();} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - explicit operator communicator *() const {return ptr_;} - // explicit operator communicator const*() const{return ptr_;} - friend bool operator==(ptr const& a, ptr const& b) {return a.ptr_ == b.ptr_;} - friend bool operator!=(ptr const& a, ptr const& b) {return a.ptr_ != b.ptr_;} - }; - - ptr operator&() & {return ptr{this};} // NOLINT(google-runtime-operator) - communicator const* operator&() const& {return this;} // NOLINT(google-runtime-operator) - communicator * operator&() && {return this;} // NOLINT(google-runtime-operator) - - ~communicator() { - if(impl_ != MPI_COMM_WORLD and impl_ != MPI_COMM_NULL and impl_ != MPI_COMM_SELF) { - try { - MPI_(Comm_disconnect)(&impl_); //this will wait for communications to finish communications, if it gets to this point is probably an error anyway <-- not true, it is necessary to synchronize the flow - // MPI_Comm_free(&impl_); - } catch(std::exception& e) { std::cerr<< e.what() <(size()); } - - public: - using size_type = int; - int size() const { - if(is_null()) {return 0;} - int const size = MPI_(Comm_size)(impl_); - assert(size > 0); - return size; - } - - [[nodiscard]] // ("empty is not an action")]] - bool empty() const {return is_empty();} - bool is_empty() const {return is_null();} - - void abort(int errorcode = 0) const {MPI_Abort(impl_, errorcode);} - - explicit operator group() const; - - communicator duplicate() { // note that this function is non-const - communicator ret; - MPI_(Comm_dup)(impl_, &ret.impl_); - return ret; - } - - template - class keyval { - static int delete_fn(MPI_Comm /*comm*/, int /*keyval*/, void *attr_val, void */*extra_state*/){ - delete static_cast(attr_val); // NOLINT(cppcoreguidelines-owning-memory) - // attr_val = nullptr; - return MPI_SUCCESS; - } - static int copy_fn( - MPI_Comm /*oldcomm*/, int /*keyval*/, - void * /*extra_state*/, void *attribute_val_in, - void *attribute_val_out, int *flag - ) { - *static_cast(attribute_val_out) = static_cast(new T{*(static_cast(attribute_val_in))}); - assert(flag); *flag = 1; - return MPI_SUCCESS; - } - - public: - int impl_ = {}; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - using mapped_type = T; - - keyval() { // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) - MPI_(Comm_create_keyval)(copy_fn, delete_fn, &impl_, nullptr); - } - - keyval(keyval const&) = delete; - keyval(keyval &&) = delete; - - keyval& operator=(keyval const&) = delete; - keyval& operator=(keyval &&) = delete; - - ~keyval() noexcept {MPI_Comm_free_keyval(&impl_);} - }; - - using detail::basic_communicator::send_receive_n; - using detail::basic_communicator::matched_probe; - - template - auto send_n( - It first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - int dest, int tag - ) { - MPI_(Send)( - detail::data(first), static_cast(count), // TODO(correaa) use safe cast - mpi3::datatype::value_type>{}(), - dest, tag, impl_ - ); - } - template - mpi3::request isend_n( - It first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - int dest, int tag - ) { - return MPI_I(send)( - detail::data(first), count, - datatype::value_type>{}(), - dest, tag, impl_ - ); - } - template - void send_n( - It first, - detail::forward_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - Size count, - int dest, int tag - ) { - detail::package p(*this); - package_oarchive poa(p); - std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); - // while(count--) {poa << *first++;} - send_n(p.begin(), p.size(), dest, tag); // p.send(dest, tag); - } - template - auto isend_n(It first, Size count, int dest, int tag = 0){ - return isend_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - static_cast(count), // TODO(correaa) use safe cast - dest, tag - ); - } - template - auto isend_n( - It first, - detail::forward_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - Size count, - int dest, int tag - ) { - detail::package p(*this); - package_oarchive poa(p); - std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); - // while(count--) {poa << *first++;} - return isend_n(p.begin(), p.size(), dest, tag); - } - template static std::true_type has_dimensionality_aux(T const&); - static std::false_type has_dimensionality_aux(...); - - template struct has_dimensionality : decltype(has_dimensionality_aux(T{})) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - - template{})> > - void send_n(It first, Size count, int dest, int tag = 0) { - return send_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - dest, tag - ); - } - template - auto send( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int dest, int tag - ) { - return send_n(first, std::distance(first, last), dest, tag); - } - template - auto send( - It first, It last, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int dest, int tag - ) { - return send_n(first, std::distance(first, last), dest, tag); - } - template - auto send( - It first, It last, - detail::input_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int dest, int tag - ) { - mpi3::vector::value_type> buffer(first, last); - return send_n(buffer.begin(), buffer.size(), dest, tag); - } - template - auto send( - It first, It last, - /**/ detail::input_iterator_tag /*tag*/, - /**/ detail::value_unspecified_tag /*tag*/, - int dest, int tag - ) { - detail::package p(*this); - package_oarchive poa(p); - std::copy(first, last, package_oarchive::iterator::value_type>(poa)); - // while(first!=last) {poa << *first++;} - send_n(p.begin(), p.size(), dest, tag); // p.send(dest, tag); - } - - template - auto send_n(MultiIt first, typename MultiIt::difference_type count, int dest, int tag = 0) - ->decltype( MPI_Send (mpi3::base(first), count, mpi3::type{first}, dest, tag, impl_), first + count) { - return MPI_(Send)(mpi3::base(first), count, mpi3::type{first}, dest, tag, impl_), first + count; } - - template - auto receive_n(MultiIt first, typename MultiIt::difference_type count, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) - ->decltype( MPI_Recv (mpi3::base(first), count, mpi3::type{first}, source, tag, impl_, MPI_STATUS_IGNORE), first + count) { // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - return MPI_(Recv)(mpi3::base(first), count, mpi3::type{first}, source, tag, impl_, MPI_STATUS_IGNORE), first + count; } // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - - template - auto isend( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int dest, int tag - ) { - return isend_n(first, std::distance(first, last), dest, tag); - } - template - auto send(It first, It last, int dest, int tag = 0) { - return send( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - dest, tag - ); - } - - using rank_index = int; - - template - auto isend(It first, It last, rank_index dest, int tag = 0) { - return isend( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - dest, tag - ); - } - - bool is_intercommunicator() const { - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Comm_test_inter(impl_, &flag); - return flag != 0; - } - - communicator split(int color, int key) { - communicator ret; - MPI_(Comm_split)(impl_, color, key, &ret.impl_); - if(ret) {ret.set_name(name() + std::to_string(color));} - if(ret) {ret.attribute("color") = color;} - return ret; - } - communicator split(int color = MPI_UNDEFINED) { - return split(color, rank()); - } - - communicator keep(bool cond) {return split(cond?0:mpi3::undefined);} - - shared_communicator split_shared(int key = 0); - shared_communicator split_shared(communicator_type t, int key = 0); - - int remote_size() const {return MPI_(Comm_remote_size)(impl_);} - - communicator reversed() {return split(0, size() - rank());} - - int cartesian_map(std::vector const& dims, std::vector const& periods) const { - assert(dims.size() == periods.size()); - return MPI_(Cart_map)(impl_, static_cast(dims.size()), dims.data(), periods.data()); // TODO(correaa) use safe cast - } - int cartesian_map(std::vector const& dimensions) const { - return cartesian_map(dimensions, std::vector(dimensions.size(), 0)); - } - - pointer malloc(MPI_Aint size) const; - template void deallocate_shared(pointer p); - template void deallocate(pointer& p, MPI_Aint size = 0); - void free(pointer& p) const; - - bool similar(communicator const& o) const {return compare(o)==detail::equality::similar;} - template//, typename = typename std::enable_if{}>::type> - communicator subcomm(Vector const& v) const { - MPI_Group old_g; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Comm_group(impl_, &old_g); - MPI_Group new_g; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Group_incl(old_g, static_cast(v.size()), v.data(), &new_g); // TODO(correaa) safe cast - communicator ret; MPI_Comm_create(impl_, new_g, &ret.impl_); - MPI_Group_free(&new_g); - MPI_Group_free(&old_g); - return ret; - } - communicator subcomm(std::initializer_list l) const { - return subcomm(std::vector(l)); - } - enum class topology{undefined = MPI_UNDEFINED, graph = MPI_GRAPH, cartesian = MPI_CART}; - - int rank() const { - assert(not is_empty()); // an empty communicator doesn't have ranks - int rank; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Comm_rank)(impl_, &rank); - return rank; - } - int right() const { - int const s = size(); assert(s != 0); - return (rank() + 1) % s; - } - int left() const { - int const s = size(); assert(s != 0); - int left = rank() - 1; - if(left < 0) {left = s - 1;} - return left; - } - int next(int n = 1) const { - assert(rank() + n < size()); - return rank() + n; - } - int prev(int n = 1) const { - assert(rank() - n > 0); - return rank() - n; - } - communicator accept(port const& p, int root = 0) const { - communicator ret; - MPI_Comm_accept(p.name_.c_str(), MPI_INFO_NULL, root, impl_, &ret.impl_); - return ret; - } - [[deprecated("call non const version")]] - void barrier() const { MPI_( Barrier)(get() ) ;} - void barrier() { MPI_( Barrier)(handle()) ;} - auto ibarrier() {request ret; MPI_(Ibarrier)(handle(), &ret.impl_); return ret;} - - communicator connect(port const& p, int root = 0) const { - communicator ret; - MPI_(Comm_connect)(p.name_.c_str(), MPI_INFO_NULL, root, impl_, &ret.impl_); - return ret; - } - - bool root() const {return (not empty()) and (rank() == 0);} - bool is_root() const {return root();} - bool at_root() const {return root();} - - void set_error_handler(error_handler const& eh); - error_handler get_error_handler() const; - - auto operator[](int rank) -> reference; - - protected: - template void set_attribute(int kv_idx, T const& t) { - MPI_(Comm_set_attr)(impl_, kv_idx, new T{t}); // NOLINT(readability-implicit-bool-conversion, cppcoreguidelines-owning-memory) TODO(correaa) - } - inline void delete_attribute(int kv_idx){ - MPI_Comm_delete_attr(impl_, kv_idx); - } - void* get_attribute(int kvidx) const { - void* v = nullptr; - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Comm_get_attr)(impl_, kvidx, &v, &flag); - if(flag == 0) {assert(!v); return nullptr;} - return v; - } - bool has_attribute(int kvidx) const { - void* v = nullptr; - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Comm_get_attr(impl_, kvidx, &v, &flag); - return flag != 0; - } - - public: - template void - set_attribute(keyval const& k, TT const& t = {}) {set_attribute(k.impl_, t);} - template - inline void delete_attribute(keyval const& k) {delete_attribute(k.impl_);} - template - T const& get_attribute(keyval const& kv) const {return *static_cast(get_attribute(kv.impl_));} - template - T& get_attribute(keyval const& kv) {return *static_cast(get_attribute(kv.impl_));} - template - bool has_attribute(keyval const& kv) const {return has_attribute(kv.impl_);} - template - T& attribute(keyval const& kv) { - if(not has_attribute(kv)) {set_attribute(kv);} - return get_attribute(kv); - } - mpi3::any& attribute(std::string const& s); - - void call_error_handler(int errorcode) noexcept { - auto const s = MPI_Comm_call_errhandler(impl_, errorcode); (void)s; - assert(s == MPI_SUCCESS); - } - void error(mpi3::error const& e) noexcept { - auto const s = MPI_Comm_call_errhandler(impl_, static_cast(e)); (void)s; - assert(s == MPI_SUCCESS); - } - communicator divide_low(int n) { - assert(n != 0); - return split( - (rank() < size()/n*(n-size()%n))? - rank()/(size()/n): - n-size()%n + (rank() - (n-size()%n)*(size()/n))/((size()/n)+1) - ); - } - communicator divide_high(int n) { - int const bat=size()/n; int const residue = size()%n; - int i = 0; - for(int last = 0; ; i++) { // NOLINT(altera-unroll-loops) TODO(correaa) - last += bat + ((i < residue)?1:0); - if(rank() < last) {break;} - } - return split(i); - } - communicator operator/(int n) { - assert(n!=0); - if(n > 0) {return divide_high(n);} - return divide_low(n); - } - communicator operator%(int n) {return split(rank()%n);} - communicator divide_even(int n) { - return split(2*(rank()%n) > n?mpi3::undefined:rank()/n); - } - - communicator operator< (int n) {return split((rank() < n)?0:MPI_UNDEFINED);} - communicator operator<=(int n) {return split((rank() <= n)?0:MPI_UNDEFINED);} - communicator operator> (int n) {return split((rank() > n)?0:MPI_UNDEFINED);} - communicator operator>=(int n) {return split((rank() >= n)?0:MPI_UNDEFINED);} - communicator operator==(int n) {return split((rank() == n)?0:MPI_UNDEFINED);} - - template - void send_value(T const& t, int dest, int tag = 0) { - send(std::addressof(t), std::next(std::addressof(t)), dest, tag); - } - template - auto isend_value(T const& t, int dest, int tag = 0) { - return isend(std::addressof(t), std::next(std::addressof(t)), dest, tag); - } - template - void send_value(T(&t)[N], int dest, int tag = 0) { // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) compatibility - send(std::addressof(t[0]), std::next(std::addressof(t[N-1])), dest, tag); - } - template - request send_init_n(ContIt first, Size count, int dest, int tag = 0); - template - request send_init(ContIt first, ContIt last, int dest, int tag = 0); - - template - receive_request receive_init_n(ContIt first, Size count, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG); - template - receive_request receive_init(ContIt first, ContIt last, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG); - - template< - class InputIt, class Size, - typename V = typename std::iterator_traits::value_type, class Datatype = datatype - > - auto unpack_from_n(InputIt first, Size count, char const* buffer) { - int position = 0; - using detail::data; - MPI_(Unpack)(buffer, count*pack_size(), &position, data(first), count, Datatype{}(), impl_); - return std::next(buffer, position); - } - - template - auto pack( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - uvector& b, int pos - ) { - return pack_n(first, std::distance(first, last), b, pos); - } - template - auto pack(It first, It last, uvector& b, int pos) { - return pack( - first, last, - detail::iterator_category_t{}, - b, pos - ); - } - -#if 0 -#ifdef MPICH_NUMVERSION -#if MPICH_NUMVERSION >= 30400000 - private: - template - [[nodiscard]] auto isend_receive_replace_n(It first, Size count, It2 d_first, int dest, int source = MPI_ANY_SOURCE) - -> decltype(detail::data(first), std::declval()){ - auto const sz = size(); - assert(sz != 0); - mpi3::request r; - MPI_I(sendrecv_replace)(detail::data(first), count/sz, datatype::value_type>{}(), dest, 0, source, MPI_ANY_TAG, &*this, &r.impl_); - return r; - } - - public: -#endif -#endif -#endif - - template - auto send_receive_replace_n( - It first, Size size, - int dest, int source, // = MPI_ANY_SOURCE, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - using value_type = typename std::iterator_traits::value_type; - return send_receive_replace_n( - first, - detail::iterator_category_t{}, - detail::value_category_t{}, - static_cast(size), - dest, source, sendtag, recvtag - ); - } - template - It send_receive_replace_n( - It first, - detail::random_access_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, int dest, int source, int sendtag, int recvtag - ) { - mpi3::status const s = MPI_(Sendrecv_replace)( - detail::data(first), count, - datatype::value_type>{}(), - dest, sendtag, source, recvtag, impl_ - ); - return first + s.count::value_type>(); - } - template - auto send_receive_n( - It1 first, Size count, int dest, - It2 d_first, Size d_count, int source, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_n( - first, count, dest, - d_first, d_count, source, - detail::iterator_category_t{}, // It2??? - detail::value_category_t::value_type>{}, - sendtag, recvtag - ); - } - - template - auto send_receive_n( - It1 first, Size count, int dest, - It2 d_first, int source = MPI_ANY_SOURCE, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_n( - first, count, dest, - d_first, source, - detail::iterator_category_t{}, // It2??? TODO(correaa) - detail::value_category_t::value_type>{}, // It2??? TODO(correaa) - sendtag, recvtag - ); - } - -// private: - -// public: -// template -// auto isend_receive_replace_n( -// It first, Size size, -// int dest, int source, // = MPI_ANY_SOURCE, -// int sendtag = 0, int recvtag = MPI_ANY_TAG -// ) { -// using value_type = typename std::iterator_traits::value_type; -// return isend_receive_replace_n( -// first, -// detail::iterator_category_t{}, -// detail::value_category_t{}, -// size, -// dest, source, sendtag, recvtag -// ); -// } - - private: - template - auto send_receive_n( - It first, Size count, int dest, - It2 d_first, Size d_count, int source, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - int sendtag, int recvtag - ) { - status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init - MPI_(Sendrecv)( - detail::data( first), static_cast( count), datatype::value_type>{}(), dest , sendtag, // TODO(correaa) use safe cast - detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), source, recvtag, // TODO(correaa) use safe cast - impl_, &ret.impl_ - ); - assert( static_cast(ret.count::value_type>()) == d_count ); - return d_first + static_cast::difference_type>(d_count); - } - - template::value_type - , class V2 = typename std::iterator_traits::value_type - > - auto send_receive_n( - It1 first, Size count, int dest, - It2 d_first, int source, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - int sendtag, int recvtag - ) { - static_assert( std::is_same{}, "source and destination need to be same type" ); - status const ret = MPI_(Sendrecv)( - detail::data(first), static_cast(count), datatype{}(), - dest, sendtag, - detail::data(d_first), std::numeric_limits::max() /*unlim in receiving*/, datatype{}(), - source, recvtag, - impl_ //, &ret.impl_ // status refers to the receive operation. - ); - return d_first + ret.count(); - } - - template - auto send_receive_replace_n( - It first, - /**/ detail::forward_iterator_tag /*tag*/, - /**/ detail::value_unspecified_tag /*tag*/, - Size count, - int dest, int source, - int sendtag, int recvtag - ) { - detail::package p(*this); - package_oarchive poa(p); - auto first_copy = first; - std::copy_n(first_copy, count, package_oarchive::iterator::value_type>(poa) ); - // while(count--) {poa << *first_copy++;} // TODO(correaa) remove first_copy - auto s = p.size(); - send_receive_replace_n(&s, 1, dest, source, sendtag, recvtag); - detail::package p2(*this); - p2.resize(s); - auto st = send_receive_n( - p.begin(), p.size(), dest, - p2.begin(), p2.size(), - source, sendtag, recvtag - ); - (void)st; - package_iarchive pia(p2); - // while(p2) {pia >> *first++;} - // return first; - return std::copy_n( - package_iarchive::iterator::value_type>(pia), - count, first - ); - } - - template - auto send_receive_replace_n( - It first, - detail::forward_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, int dest, int source, int sendtag, int recvtag - ) { - uvector::value_type> v(static_cast(count)); - std::copy_n(first, count, v.begin()); - send_receive_replace_n(v.begin(), v.size(), dest, source, sendtag, recvtag); - return std::copy_n(v.begin(), v.size(), first); - } - - public: - template - auto send_receive_n( - It first, Size size, - int dest, int source, // = MPI_ANY_SOURCE, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_replace_n( - first, size, - dest, source, sendtag, recvtag - ); - } - - private: - template - auto send_receive( - It1 first, It1 last, int dest, - It2 d_first, It2 d_last, int source, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int sendtag, int recvtag - ) { - return send_receive_n( - first, std::distance(first, last), dest, - d_first, std::distance(d_first, d_last), source, - sendtag, recvtag - ); - } - - template - auto send_receive( - It1 first, It1 last, int dest, - It2 d_first, int source, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int sendtag, int recvtag - ) { - return send_receive_n( - first, std::distance(first, last), dest, - d_first, source, - sendtag, recvtag - ); - } - - public: - template - auto send_receive( - It1 first, It1 last, int dest, - It2 d_first, It2 d_last, int source = MPI_ANY_SOURCE, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ){ - return send_receive( - first, last, dest, - d_first, d_last, source, - detail::iterator_category_t{}, // It2??? - detail::value_category_t::value_type>{}, // It2??? - sendtag, recvtag - ); - } - - template - auto send_receive( - It1 first, It1 last, int dest, - It2 d_first - ){ - return send_receive( - first, last, dest, - d_first, MPI_ANY_SOURCE, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - /*sendtag*/ 0, /*recvtag*/ MPI_ANY_TAG - ); - } - - template - auto send_receive_n(It first, Size n, std::pair dest_source, std::pair send_recv_tag = {0, MPI_ANY_TAG}) { - return send_receive_n(first, n, dest_source.first, dest_source.second, send_recv_tag.first, send_recv_tag.second); - } - template - auto send_receive (It first, It last, std::pair dest_source, std::pair send_recv_tag = {0, MPI_ANY_TAG}) { - return send_receive (first, last, dest_source.first, dest_source.second, send_recv_tag.first, send_recv_tag.second); - } - - status probe(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - status s; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init - MPI_(Probe)(source, tag, impl_, &s.impl_); - return s; - } - auto iprobe(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - status s; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Iprobe)(source, tag, impl_, &flag, &s.impl_); - if(flag == 0) {return optional();} - return optional(s); - } - template - auto send_receive_replace( - It first, It last, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int dest, int source, int sendtag, int recvtag - ) { - return send_receive_replace_n( - first, std::distance(first, last), - dest, source, sendtag, recvtag - ); - } - template - auto send_receive_replace( - It first, It last, - detail::forward_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int dest, int source, int sendtag, int recvtag - ) { - mpi3::vector::value_type> buffer(first, last); - send_receive_replace_n(buffer.begin(), buffer.size(), dest, source, sendtag, recvtag); - return std::copy(buffer.begin(), buffer.end(), first); - } - template - auto send_receive_replace( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int dest, int source, int sendtag, int recvtag - ) { - return send_receive_replace_n( - first, std::distance(first, last), - dest, source, sendtag, recvtag - ); - } - template - auto send_receive_replace( - It first, It last, - int dest, - int source = MPI_ANY_SOURCE, int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_replace( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - dest, source, sendtag, recvtag - ); - } - template - auto send_receive( - It first, It last, - int dest, - int source = MPI_ANY_SOURCE, int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_replace(first, last, dest, source, sendtag, recvtag); - } - - void send_packed_n(void const* begin, int n, int dest, int tag = 0) { - std::cout<<"sending packet of size "<< n <(); - // receive_packed_n(begin, n, source, tag); - return static_cast(std::next(static_cast(begin), count)); - } - template - auto receive_n( - It dest, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - int source, int tag - ) { - status sta; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init - MPI_(Recv)( - detail::data(dest), static_cast(count), - datatype::value_type>{}(), - source, tag, impl_, &sta.impl_ - ); - assert(sta.count::value_type>() == static_cast(count)); - return dest + sta.count::value_type>(); - } - template - auto ireceive_n( - It dest, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - Size count, - int source, int tag - ) { - mpi3::request r; - MPI_(Irecv)( - detail::data(dest), static_cast(count), - datatype::value_type>{}(), - source, tag, impl_, &r.impl_ - ); - return r; - } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // MPI_Wait called on destructor of ret - template - auto receive_n( - It dest, - detail::forward_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - Size count, - int source, int tag - ){ - detail::package p(*this); - p.receive(source, tag); - package_iarchive pia(p); - return std::copy_n(package_iarchive::iterator::value_type>{pia}, count, dest); - } - - template{}, int> =0// or (not detail::is_basic::value_type>{}), int> =0 // needed by intel commpiler - > - auto receive_n(It dest, Size n, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - return receive_n( - dest, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - n, - source, tag - ); - } - template - mpi3::request ireceive_n( - It dest, Size n, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG - ) { - return ireceive_n( - dest, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - n, - source, tag - ); - } - template - auto receive( - It dest, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int source, int tag - ) { - match m = matched_probe(source, tag); - auto count = m.count::value_type>(); - m.receive_n(dest, count); - return dest + count; - } - template - [[deprecated]] auto receive( - It dest, - /**/ detail::forward_iterator_tag /*tag*/, - /**/ detail::value_unspecified_tag /*tag*/, - int source, int tag - ) { - detail::package p(*this); - p.receive(source, tag); - package_iarchive const pia(p); // TODO(correaa) investigate - while(p) {pia >> *dest++;} // NOLINT(altera-unroll-loops) deprecating - return dest; - } - template - auto receive( - It dest, - detail::forward_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int source, int tag - ) { - return matched_probe(source, tag).receive_n(dest); - } - template - [[deprecated]] auto receive(It dest, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - return receive( - dest, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - source, tag - ); - } - template - auto receive( - It d_first, It d_last, - /**/ detail::random_access_iterator_tag /*tag*/, - /**/ detail::value_unspecified_tag /*tag*/, - int source, int tag - ) { - return receive_n(d_first, std::distance(d_first, d_last), source, tag); - } - template - auto receive( - It d_first, It d_last, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int source, int tag - ) { - return receive_n(std::addressof(*d_first), std::distance(d_first, d_last), source, tag); - // return std::copy(buffer.begin(), buffer.end(), d_first); - } - - template - auto receive( - It d_first, It d_last, - /**/ detail::forward_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - int source, int tag - ) { - mpi3::uvector::value_type> buffer(std::distance(d_first, d_last)); - receive_n(buffer.begin(), buffer.size(), source, tag); - return std::copy(buffer.begin(), buffer.end(), d_first); - } -// class ir_req{ -// boost::mpi3::status query(){ -// boost::mpi3::status ret; -// ret.set_source(MPI_UNDEFINED); -// ret.set_tag(MPI_UNDEFINED); -// ret.set_cancelled(); -// ret.set_elements(0); -// return ret; -// } -// static void free(){ -// std::cout << "free" << std::endl; -// } -// static void cancel(int complete) { -// std::cout << "cancel " << complete << std::endl; -// } -// }; -// template -// struct receive_args { -// communicator* commP; -// It d_first; -// // It d_last; -// int source; -// int tag; -// MPI_Request* requestP; -// }; -// struct receive_state{ -// int cancelled = 0; -// int source = MPI_UNDEFINED; -// int tag = MPI_UNDEFINED; -// }; -// template -// inline static void* receive_thread(void* ptr) { -// receive_args* args = (receive_args*)ptr; -// args->commP->receive(args->d_first, args->source, args->tag);//, /*args->d_last,*/ ); -// MPI_Grequest_complete(*args->requestP); -// ::free(ptr); -// return nullptr; -// } -// inline static int query_fn(void* extra_state, MPI_Status *status){ -// auto* rs = static_cast(extra_state); -// /* always send just one int */ -// MPI_Status_set_elements(status, MPI_INT, 1); -// /* can never cancel so always true */ -// MPI_Status_set_cancelled(status, rs->cancelled); -// /* choose not to return a value for this */ -// status->MPI_SOURCE = rs->source; -// /* tag has not meaning for this generalized request */ -// status->MPI_TAG = rs->tag; -// /* this generalized request never fails */ -// return MPI_SUCCESS; -// } -// inline static int free_fn(void* extra_state) { -// /* this generalized request does not need to do any freeing */ -// /* as a result it never fails here */ -// ::free(extra_state); -// return MPI_SUCCESS; -// } -// inline static int cancel_fn(void* /*extra_state*/, int complete) { -// /* This generalized request does not support cancelling. -// Abort if not already done. If done then treat as if cancel failed. */ -// if(not (complete == 0)) { -// std::cerr<< "Cannot cancel generalized request - aborting program" < -// auto ireceive(It d_first, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { -// // based on http://liinwww.ira.uka.de/courses/spprakt/mpi2-html-doc/node157.html -// mpi3::request ret; /* receive_args* args = (receive_args*)::malloc(sizeof(receive_args)); args->commP = this; args->d_first = d_first; // args->d_last = d_last; args->source = source; args->tag = tag; args->requestP = &ret.impl_;*/ -// receive_state* rs = (receive_state*)::malloc(sizeof(receive_state)); -// rs->cancelled = 0; -// rs->source = source; -// rs->tag = tag; -// MPI_Grequest_start(query_fn, free_fn, cancel_fn, rs, &ret.impl_);//args->requestP); -// std::thread( // static_cast(receive_thread), args -// [this, d_first, source, tag, &ret](){ -// this->receive(d_first, source, tag); // receive_args* args = (receive_args*)ptr; // args->commP->receive(args->d_first, args->source, args->tag);//, /*args->d_last,*/ ); -// MPI_Grequest_complete(ret.impl_); // MPI_Grequest_complete(*args->requestP); // ::free(ptr); -// } -// ).detach(); // t.detach(); // pthread_t thread; // pthread_create(&thread, NULL, static_cast(receive_thread), args); // pthread_detach(thread); -// return ret; -// } - template - auto ireceive( - It d_first, It d_last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int source, int tag - ) { - return ireceive_n(d_first, std::distance(d_first, d_last), source, tag); - } - template - auto receive(It d_first, It d_last, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - return receive( - d_first, d_last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - source, tag - ); - } - template::value_type> - auto bsend_init_n( - InputIt first, Size count, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int dest, int tag - ) { - request ret; - int s = MPI_Bsend_init( - std::addressof(*first), count, datatype{}(), - dest, tag, impl_, &ret.impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot bsend init"};} - return ret; - } - template - auto bsend_init_n(It first, Size count, int dest, int tag = 0){ - return bsend_init_n( - first, count, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - dest, tag - ); - } - template - auto bsend_init( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int dest, int tag - ){ - return bsend_init_n(first, std::distance(first, last), dest, tag); - } - template - auto bsend_init(It first, It last, int dest, int tag = 0){ - bsend_init( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - dest, tag - ); - } - template::iterator_category> - auto bsend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ - return send(buffered_communication_mode{}, blocking_mode{}, It1, It2, dest, tag); - } - template::value_type> - auto dynamic_receive(InputIt first, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - // auto count = probe(source, tag).count(); - // return receive(first, first + count, source, tag); - MPI_Status status; - MPI_Message msg; // NOLINT(cppcoreguidelines-init-variables) delayed init - int count = -1; - MPI_Mprobe(source, tag, impl_, &msg, &status); - MPI_Get_count(&status, datatype{}(), &count); - using detail::data; - MPI_Mrecv(data(first), count, datatype{}(), &msg, MPI_STATUS_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - } - - template::iterator_category> - auto breceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ - return receive(buffered_communication_mode{}, blocking_mode{}, It1, It2, source, tag); - } - template::iterator_category> - auto ibsend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ - return send(buffered_communication_mode{}, nonblocking_mode{}, It1, It2, dest, tag); - } - template::iterator_category> - auto ibreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ - return receive(buffered_communication_mode{}, nonblocking_mode{}, It1, It2, source, tag); - } - - template::iterator_category> - auto ssend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ - return send(synchronous_communication_mode{}, blocking_mode{}, It1, It2, dest, tag); - } - template::iterator_category> - auto sreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ - return receive(synchronous_communication_mode{}, blocking_mode{}, It1, It2, source, tag); - } - template::iterator_category> - auto issend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ - return send(synchronous_communication_mode{}, nonblocking_mode{}, It1, It2, dest, tag); - } - template - auto isend_n( - CommunicationMode /*mode*/, - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, int dest, int tag - ) { - mpi3::request ret; - CommunicationMode{}.ISend( - std::addressof(*first), count, datatype::value_type>{}(), dest, tag, impl_, &ret.impl_ - ); - return ret; - } - template - auto issend_n(It1 first, Size count, int dest, int tag = 0) { - return isend_n( - synchronous_communication_mode{}, - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - dest, tag - ); - } - template::iterator_category> - auto isreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - return receive(synchronous_communication_mode{}, nonblocking_mode{}, It1, It2, source, tag); - } - - template::iterator_category> - auto rsend(InputIterator It1, InputIterator It2, int dest, int tag = 0) { - return send(ready_communication_mode{}, blocking_mode{}, It1, It2, dest, tag); - } - template::iterator_category> - auto rreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - return receive(ready_communication_mode{}, blocking_mode{}, It1, It2, source, tag); - } - template::iterator_category> - auto irsend(InputIterator It1, InputIterator It2, int dest, int tag = 0) { - return send(ready_communication_mode{}, nonblocking_mode{}, It1, It2, dest, tag); - } - template::iterator_category> - auto irreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { - return receive(ready_communication_mode{}, nonblocking_mode{}, It1, It2, source, tag); - } - template - auto send_category(CommunicationMode cm, BlockingMode bm, std::input_iterator_tag, - InputIterator first, InputIterator last, int dest, int tag - ); - template< - class CommunicationMode, class BlockingMode, class InputIterator, - class Category = typename std::iterator_traits::iterator_category - > - auto send(CommunicationMode cm, BlockingMode bm, InputIterator It1, InputIterator It2, int dest, int tag = 0) { - return send_category(cm, bm, Category{}, It1, It2, dest, tag); - } - - private: - template< - class CommunicationMode, class ContiguousIterator, - class Size, class ValueType = typename std::iterator_traits::value_type, - class Datatype = datatype - > - request send_n_randomaccess_contiguous_builtin( - CommunicationMode cm, nonblocking_mode /*mode*/, - std::true_type /*true*/, - ContiguousIterator first, Size n, int dest, int tag - ) { - request r; - int s = cm.ISend(detail::data(first), n, Datatype{}(), dest, tag, impl_, &r.impl_); - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot send random access iterators"};} - return r; - } - template< - class CommunicationMode, class ContiguousIterator, - class Size, class ValueType = typename std::iterator_traits::value_type, - class Datatype = datatype - > - request receive_n_randomaccess_contiguous_builtin( - CommunicationMode cm, nonblocking_mode /*mode*/, std::true_type /*true*/, - ContiguousIterator first, Size n, int dest, int tag - ) { - request r; - int s = cm.IRecv(detail::data(first), n, Datatype{}(), dest, tag, impl_, &r.impl_); - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot send random access iterators"};} - return r; - } - - template - auto all_to_all_n( - It1 first, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - Size count, - It2 d_first, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/ - ) { - // auto const sz = static_cast(size()); - // assert(sz != 0 and count % sz == 0); - using count_type = int; - MPI_(Alltoall)( - detail::data( first), static_cast(count/* / sz*/), datatype::value_type>{}(), - detail::data(d_first), static_cast(count/* / sz*/), datatype::value_type>{}(), - impl_ - ); - return d_first + count; - } - - using in_place_type = decltype(MPI_IN_PLACE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general - - template - auto all_to_all_n( - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count - ) - ->decltype(MPI_(Alltoall)( - std::declval(), 0*count/*/size()*/, - datatype::value_type>{}(), - detail::data(first), count/*/size()*/, - datatype::value_type>{}(), - impl_ - ), first + count*size()) { - // auto const sz = size(); - // assert(sz > 0); - // assert( count % sz == 0 ); - auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general // TODO(correaa) define constant globally for the library - MPI_(Alltoall)( - in_place, 0*count/*/sz*/, - datatype::value_type>{}(), - detail::data(first), count/*/sz*/, - datatype::value_type>{}(), - impl_ - ); - return first + count*size(); - } - - public: - template - auto all_to_all_inplace_n(It1 first, Size count) { - using count_type = int; - // auto const sz = static_cast(size()); // TODO(correaa) safe cast - // assert(sz > 0); - // assert( static_cast(count) % sz == 0 ); - auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general // TODO(correaa) define constant globally for the library - using count_type = int; - MPI_(Alltoall)( - in_place , 0*static_cast(count)/*/sz*/, datatype::value_type>{}(), - detail::data(first), static_cast(count)/*/sz*/, datatype::value_type>{}(), - impl_ - ); - return first + count; - } - - template - auto all_to_all_n(It1 first, Size count, It2 d_first) { - // using count_type = int; - // assert( static_cast(count) % size() == 0 ); // NOLINT(clang-analyzer-core.DivideZero) TODO(correaa) add size cache to immutable communicator - return - all_to_all_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{} - ) - ; - } - template - auto all_to_all_n(It1 first, Size count) - ->decltype( - all_to_all_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count - )) - { - assert( count % size() == 0 ); // NOLINT(clang-analyzer-core.DivideZero) TODO(correaa) add size cache to immutable communicator - return - all_to_all_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count - );} - template - auto all_to_all(It1 first, It2 d_first){return all_to_all_n(first, size(), d_first);} - template - auto all_to_all(It1 first) - ->decltype(all_to_all_n(first, size())){ - return all_to_all_n(first, size());} - - template - auto operator()(T&& t) - ->decltype(communicator::all_to_all(begin(std::forward(t))), void()){ - assert(t.size() == size()); - auto e = communicator::all_to_all(begin(std::forward(t))); - using std::end; - assert( e == end(t) ); - return std::forward(t); - } - template - [[nodiscard]] // ("do not ignore result when argument is const")]] - auto operator()(T const& t) - ->decltype(communicator::all_to_all(t.begin(), std::declval().begin()), T(communicator::size())){ - assert(t.size() == communicator::size()); - T ret(communicator::size()); - communicator::all_to_all(t.begin(), ret.begin()); - return ret; - } - - private: - template - auto scatter_builtin_q( - std::true_type /*true*/, - It1 first, It1 last, It2 d_first, int root - ) { - return scatter_builtin( - detail::iterator_category{}, - detail::iterator_category{}, first, last, d_first, root - ); - } - template< - class It1, class It2, - class Sendtype = datatype::value_type>, - class Recvtype = datatype::value_type> - > - auto scatter_builtin( - detail::contiguous_iterator_tag /*tag*/, - detail::contiguous_iterator_tag /*tag*/, - It1 first, It1 last, It2 d_first, int root - ) { - MPI_(Scatter)( - detail::data( first), std::distance(first, last), - Sendtype{}(), - detail::data(d_first), std::distance(first, last), - Recvtype{}(), - root, - impl_ - ); - } - template - auto scatter_builtin_q(std::false_type, Iterator1 first, Iterator2 last, Iterator1 d_first, int root) -// { TODO implement } - ; - - public: - template - void broadcast_value(T& t, int root = 0) {broadcast_n(std::addressof(t), 1, root);} - - template - void broadcast_value(std::vector& t, int root = 0){ - auto t_size = t.size(); - broadcast_value(t_size, root); - t.resize(t_size); - broadcast_n(t.data(), t.size(), root); - } - - template - auto ibroadcast_n( - It first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - int root - ) { - request r; - MPI_(Ibcast)( - detail::data(first), static_cast(count), datatype::value_type>{}(), - root, impl_, &r.impl_ - ); - return r; - } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // MPI_Wait called on destructor of ret - template - auto broadcast_n( - It first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - int root - ) { - MPI_(Bcast)( - detail::data(first), static_cast(count), // TODO(correaa) use safe cast - datatype::value_type>{}(), - root, impl_ - ); - return first + static_cast::difference_type>(count); - } - template - auto ibroadcast( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - int root - ) { - return ibroadcast_n( - first, std::distance(first, last), root - ); - } - template - void broadcast( - It first, It last, - detail::random_access_iterator_tag /*tag*/, - int root - ){broadcast_n(first, std::distance(first, last), root);} - - template - auto ibroadcast_n(It first, Size count, int root = 0){ - return ibroadcast_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - root - ); - } - template - auto broadcast_n(It first, Size count, int root = 0){ - return broadcast_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - root - ); - } - template - auto ibroadcast(It first, It last, int root = 0){ - return ibroadcast( - first, last, - detail::iterator_category_t{}, - root - ); - } - template - void broadcast(It first, It last, int root = 0){ - return broadcast( - first, last, - detail::iterator_category_t{}, - root - ); - } - template > - void reduce_value(T const& t, T& ret, Op op = {}, int root = 0){ - reduce_n(std::addressof(t), 1, std::addressof(ret), op, root); - } - template > - auto reduce_value(T const& t, Op op = {}, int root = 0){ - auto ret = static_cast(0); - reduce_value(t, ret, op, root); // if(rank() == root) return optional(ret); - return ret; - } - template - It2 reduce_n( - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - It2 d_first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Op /*operation*/, // TODO(correaa) why value is not used? - int root - ) { - static_assert(std::is_same::value_type, typename std::iterator_traits::value_type>{}); - static mpi3::operation::value_type, typename std::iterator_traits::pointer> const combine{Op{}}; // will leak? - MPI_(Reduce)( - detail::data(first), detail::data(d_first), static_cast(count), - datatype::value_type>{}(), &combine, - root, impl_ - ); - return rank() == root?d_first + static_cast::difference_type>(count):d_first; - } - - template > - It2 reduce_n(It1 first, Size count, It2 d_first, Op op = {}, int root = 0) { - return reduce_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - op, - // predefined_operation{}, - root - ); - } - - template > - void all_reduce_value(T const& t, T& ret, Op op = {}){ - auto* e = all_reduce_n(std::addressof(t), 1, std::addressof(ret), op); (void)e; - assert( e == std::next(std::addressof(ret)) ); - } - template, typename = decltype(T{T(0)})> - auto all_reduce_value(T const& t, Op op = {}){ - auto ret = static_cast(0); - all_reduce_value(t, ret, op); // if(rank() == root) return optional(ret); - return ret; - } - template> - bool all_reduce_value(bool t, Op op={}){ - int ret = 0; - all_reduce_value(int{t}, ret, op); - return ret != 0; - } - - template - auto max(T const& t) { - auto ret = std::numeric_limits::lowest(); - all_reduce_value(t, ret, mpi3::max<>{}); return ret; - } - template - auto min(T const& t) { - auto ret = std::numeric_limits::max(); - all_reduce_value(t, ret, mpi3::min<>{}); return ret; - } - template - auto max_loc(T const& t) { - mpi3::vlp const in{t, rank()}; - mpi3::vlp ret{std::numeric_limits::lowest(), -1}; - all_reduce_value(in, ret, mpi3::max_loc<>{}); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - return ret; - } - template - std::pair max_location(T const& t); - - template - T* max_element(T& t) { - auto const ml = max_loc(t); - if(ml.location == rank()) {assert(t == ml.value);} - return ml.location == rank()? &t : nullptr; - } - - template, typename = decltype(static_cast(detail::data(std::declval())))> - auto all_reduce_n( - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size count, - It2 d_first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Op /*operation*/ // TODO(correaa) why is not used? - ) { - static_assert(std::is_same::value_type, typename std::iterator_traits::value_type>{}); - using count_type = int; - static mpi3::operation::value_type, typename std::iterator_traits::pointer> const combine{Op{}}; // will leak? - MPI_(Allreduce)( - detail::data(first), detail::data(d_first), static_cast(count), datatype::value_type>{}(), // TODO(correaa) use safe cast - &combine, impl_ - ); - return d_first + static_cast::difference_type>(count); - } - - template< - class It1, class Size, class It2, class Op = std::plus<>, - class VC1 = detail::value_category_t::value_type>, - class VC2 = detail::value_category_t::value_type>, - class = decltype(std::declval::reference>() = std::declval()(typename std::iterator_traits::value_type{}, typename std::iterator_traits::value_type{})) - > - It2 all_reduce_n(It1 first, Size count, It2 d_first, Op op = {}) { - return all_reduce_n( - first, - detail::iterator_category_t{}, - VC1{}, - count, - d_first, - detail::iterator_category_t{}, - VC2{}, - op - ); - } - template> - It2 all_reduce(It1 first, It1 last, It2 d_first, Op op = {}){ - return all_reduce_n(first, std::distance(first, last), d_first, op); - } - - private: - template static auto data_adl(T&& t){ - using detail::data; - return data(std::forward(t)); - } - - public: - template< - class It1, class Size, class Op = std::plus<>, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(data_adl(It1{})), - class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) - > - auto all_reduce_in_place_n(It1 first, Size count, Op /*op*/) { - auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general - static mpi3::operation::value_type, typename std::iterator_traits::pointer> const combine{Op{}}; // will leak? - MPI_(Allreduce)(in_place, data_adl(first), static_cast(count), datatype{}(), &combine, impl_); - } - - template< - class It1, class Size, class Op = std::plus<>, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(data_adl(It1{})) - > - auto all_reduce_n(It1 first, Size count, Op op = {}) - ->decltype(all_reduce_in_place_n(first, count, op)) { - return all_reduce_in_place_n(first, count, op); } - - template< - class It1, class Size, class Op, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(data_adl(It1{})), - class PredefinedOp = predefined_operation - > - auto reduce_in_place_n(It1 first, Size count, Op /*op*/, int root = 0) { - auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general - (rank() == root)?MPI_(Reduce)(in_place , data_adl(first), count, datatype{}(), PredefinedOp{}, root, impl_): - MPI_(Reduce)(data_adl(first), nullptr , count, datatype{}(), PredefinedOp{}, root, impl_) - ; - } - - template< - class It1, class Size, class Op = std::plus<>, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), - class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) - > - auto reduce_n(It1 first, Size count, Op op = {}, int root = 0) { - if(rank() == root) {return reduce_in_place_n(first, count, op, root);} - return reduce_n(first, 0, nullptr, op, root); - } - template< - class It1, class Op, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), - class PredefinedOp = predefined_operation - > - auto reduce_in_place(It1 first, It1 last, Op op, int root = 0) { - return reduce_in_place_n(first, std::distance(first, last), op, root); - } - - template< - class It1, class Op = std::plus<>, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), - class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) - > - auto reduce(It1 first, It1 last, Op op = {}, int root = 0) { - return reduce_n(first, std::distance(first, last), op, root); - } - - template< - class It1, class It2, class Op = std::plus<>, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), - class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) - > - auto reduce(It1 first, It1 last, It2 d_first, Op op = {}, int root = 0) { - return reduce_n(first, std::distance(first, last), d_first, op, root); - } - - template< - class It1, class Op = std::plus<>, - class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), - class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) - > - auto all_reduce(It1 first, It1 last, Op op = {}) { - return all_reduce_in_place_n(first, std::distance(first, last), op); - } - - private: - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto reduce_n( - ReducePolicy rp, - It1 first, Size count, It2 d_first, - Op op, int root = 0 - ) { - return reduce_n_dispatch(rp, - std::integral_constant{} and detail::is_basic{} and detail::is_contiguous{} and detail::is_contiguous{}>{}, - first, count, d_first, op, root - ); - } - template - auto reduce_category( - ReducePolicy rp, std::random_access_iterator_tag /*tag*/, - RandomAccessIt1 first, RandomAccessIt1 last, It2 d_first, - Op op, int root - ) { - return reduce_n(rp, first, std::distance(first, last), d_first, op, root); - } - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto reduce_n_dispatch( - all_reduce_mode rp, std::true_type /*true*/, - It1 first, Size count, It2 d_first, - Op op, int /*root*/ = 0 - ) { - int s = rp( - detail::data(first) , - detail::data(d_first), count, datatype{}(), - op.impl_, impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot reduce");} - } - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto reduce_n_dispatch( - reduce_mode rp, std::true_type /*true*/, - It1 first, Size count, It2 d_first, - Op op, int root = 0 - ) { - int s = rp( - detail::data(first) , - detail::data(d_first), count, datatype{}(), - op.impl_, - root, impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} - } - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto reduce_n_dispatch( - ireduce_mode rp, std::true_type /*true*/, - It1 first, Size count, It2 d_first, - Op op, int root = 0 - ) { - request ret; - int s = rp( - detail::data(first) , - detail::data(d_first), count, datatype{}(), - op.impl_, - root, impl_, &ret.impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} - return ret; - } - - public: - template - auto scatter_n( - CIt1 first, Size n, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - CIt2 d_first, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - int root - ) { - auto const s = size(); - if(s == 0) {throw std::runtime_error{"invalid empty communicator for scatter_n"};} - assert( n%s == 0 ); - MPI_(Scatter)( - detail::data( first), static_cast(n/s), datatype::value_type>{}(), - detail::data(d_first), static_cast(n/s), datatype::value_type>{}(), - root, impl_ - ); - if(rank() == root) {std::advance(d_first, n);} - return d_first; - } - template - It2 scatter_n( - In1 first, Size n, detail::input_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - It2 d_first, Any2 /*unused*/, Any3 /*unused*/, - int root - ) { - auto const s = size(); - if(s == 0) { - throw std::runtime_error{"invalid empty communicator for scatter_n"}; - } - assert(n % s == 0); - vector::value_type> buff; - buff.reserve(static_cast(n)); - using std::copy_n; - copy_n(first, n, std::back_inserter(buff)); - scatter_n(buff.begin(), n, d_first, root); - std::advance(d_first, n / s); - return d_first; - } - template::value_type, class V2 = typename std::iterator_traits::value_type> - auto scatter_n(It1 first, Size n, It2 d_first, int root = 0){ - return scatter_n( - first, n, detail::iterator_category_t{}, detail::value_category_t{}, - d_first, detail::iterator_category_t{}, detail::value_category_t{}, - root - ); - } - template - auto scatter_n_from(It2 d_first, Size n, It1 first, int root = 0) { - return scatter_n(first, n*size(), d_first, root); - } - template - auto scatter( - RA1 first, RA1 last, - detail::random_access_iterator_tag /*tag*/, - It2 d_first, int root - ) { - return scatter_n(first, std::distance(first, last), d_first, root); - } - template - auto scatter(It1 first, It1 last, It2 d_first, int root = 0){ - return scatter(first, last, detail::iterator_category_t{}, d_first, root); - } - template - auto scatter_from( - RA2 first, RA2 last, - detail::random_access_iterator_tag /*tag*/, - It1 d_first, int root - ) { - return scatter_n_from(first, std::distance(first, last), d_first, root); // NOLINT(readability-suspicious-call-argument) TODO(correaa) range based passing - } - template - auto scatter_value_from(V& v, It1 first, int root = 0) { - return scatter_n_from(std::addressof(v), 1, first, root); - } - template::value_type> - V scatter(It1 first, [[maybe_unused]] It1 last, int root = 0) { - if(rank()==root) {assert(std::distance(first, last) == size());} - V v; - scatter_value_from(v, first, root); - return v; - } - template::value_type> - V scatter(It first, int root = 0) { - V v; - scatter_value_from(v, first, root); - return v; - } - template::value_type> - V scatter(Container c, int root = 0, void* /*unused*/ = nullptr) { // TODO(correaa) find name for parameter - assert( (int)c.size() == (rank()==root?size():0) ); - using std::begin; - return scatter(begin(c), root); - } - - template - auto scatter_from(It2 d_first, It2 d_last, It1 first, int root = 0){ - return scatter_from(d_first, d_last, detail::iterator_category_t{}, first, root); // NOLINT(readability-suspicious-call-argument) TODO(correaa) range based passing - } - - template - void scatterv_n( - CIt1 first, int* counts, int* displs, - detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - CIt2 d_first, Size n, - detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - int root = 0 - ) { - MPI_(Scatterv)( - detail::data( first), counts, displs, - datatype::value_type>{}(), - detail::data(d_first), n, - datatype::value_type>{}(), - root, impl_ - ); - } - template - auto scatterv_n( - CItCIt1 citcit1, CItN ns, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - CIt2 it2, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, - int root - ) { - std::vector counts(ns, ns + size()); - std::vector displs(usize()); - for(int i = 0; i != size(); ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm - displs[i+1] = detail::data(citcit1[i+1]) - detail::data(citcit1[i]); - } // adjacent_difference doesn't work here because of type incompatibility - // std::adjacent_difference(citcit1, citcit1 + size(), begin(displs) + 1, [](auto& a, auto& b){return detail::data(a) - detail::data(b);}); - int n = scatter(counts.begin(), counts.end()); - scatterv_n( - detail::data(*citcit1), counts.data(), displs.data(), detail::contiguous_iterator_tag{}, detail::basic_tag{}, - it2, n , detail::contiguous_iterator_tag{}, detail::basic_tag{}, - root - ); - return it2; - } - template - auto scatterv_n(CItIt1 citit1, CItN ns, It2 it2, int root = 0){ - scatterv_n( - citit1, ns, detail::iterator_category_t::value_type>{}, detail::basic_tag{}, - it2 , detail::iterator_category_t{}, detail::basic_tag{}, - root - ); - } - template - auto scatterv(Container const& c, It2 it2, int root = 0) { - assert( (int)c.size() == ((rank()==root)?size():0) ); - if(rank() == root) { - std::cerr<< "in scatterv " << detail::data(c[1].begin()) - detail::data(c[0].begin()) << " " << detail::data(c[2].begin()) - detail::data(c[0].begin()) << std::endl; - } - using std::begin; using std::end; - std::vector displs(c.size()); - for(std::vector::size_type i = 0; i != displs.size(); ++i) { // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) TODO(correaa) use an algorithm - std::ptrdiff_t d = detail::data(c[i].begin()) - detail::data(c[0].begin()); - assert( d <= static_cast(std::numeric_limits::max()) ); - displs[i] = static_cast(d); - } - if(rank() == root) { - std::cerr<<"in scatterv 2 "<< displs[0] <<" "<< displs[1] <<" "<< displs[2] < counts(c.size()); - std::transform( - counts.begin(), counts.end(), begin(c), counts.begin(), - [](auto& /*unused*/, auto& b){return std::distance(begin(b), end(b));} - ); - int n = scatter(counts); - scatterv_n( - detail::data(begin(*begin(c))), counts.data(), displs.data(), detail::contiguous_iterator_tag{}, detail::basic_tag{}, - detail::data(it2), n , detail::contiguous_iterator_tag{}, detail::basic_tag{}, - root - ); - return it2 + n; - } - template class id {using type = T;}; - - template=1)> > - auto gather_n(MultiIt first, Size count, MultiIt2 d_first, int root = 0) - ->decltype( MPI_Gather (mpi3::base(first), count, mpi3::type{first}, mpi3::base(d_first), count, mpi3::type{d_first}, root, impl_), d_first+count) { - return MPI_(Gather)(mpi3::base(first), count, mpi3::type{first}, mpi3::base(d_first), count, mpi3::type{d_first}, root, impl_), d_first+count; } - - using count_type = int; - - template - It2 gather_n( - It1 first, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - Size1 count, - It2 d_first, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - Size2 d_count, - int root - ) { - MPI_(Gather) - ( - detail::data( first), static_cast( count), datatype::value_type>{}(), - detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), - root, impl_ - ); - return d_first + ((rank() == root) ? static_cast::difference_type>(d_count * static_cast(size())) : 0); - } - - template - auto gather_n(It1 first, Size1 count, It2 d_first, Size2 d_count, int root) { - return gather_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_count, - root - ); - } - template{})>> - auto gather_n(It1 first, Size1 count, It2 d_first, int root = 0) { - return gather_n(first, count, d_first, count, root); - } - - template - auto scatterv_n_from(It2 d_first, Size n, It1 first, int root = 0) { - std::vector counts(usize()); - int nn = n; - gather_n(std::addressof(nn), 1, counts.data(), root); - std::vector displs(usize()); - partial_sum(counts.begin(), counts.end(), displs.begin()+1); - scatterv_n( - first, counts.data(), displs.data(), - detail::iterator_category_t{}, detail::value_category_t::value_type>{}, - d_first, n, - detail::iterator_category_t{}, detail::value_category_t::value_type>{}, - root - ); - std::advance(first, rank()==root?displs.back() + counts.back():0); - return first; - } - template - auto scatterv_from( - RA2 d_first, RA2 d_last, - detail::random_access_iterator_tag /*tag*/, It1 first, int root - ) { - return scatterv_n_from(d_first, std::distance(d_first, d_last), first, root); - } - template - auto scatterv_from(It2 d_first, It2 d_last, It1 first, int root = 0){ - return scatterv_from(d_first, d_last, detail::iterator_category_t{}, first, root); - } - - template - void all_gather_value(T const& t, It first){all_gather_n(std::addressof(t), 1, first);} - template std::vector - all_gather_value(T const& t) { - std::vector ret(size()); - all_gather_value(t, ret.begin()); - return ret; - } - - template - It gather_value(T const& t, It first, int root) { - return gather_n(std::addressof(t), 1, first, root); - } - template - std::vector gather_value(T const& t, int root = 0) { - std::vector ret((rank() == root) ? static_cast(size()) : 0); - gather_value(t, ret.begin(), root); - return ret; - } - - protected: - template - void advance(It& it, Size count) { std::advance(it, count); } - - template - void advance(It& it, Size s, int r) { std::advance(it, rank() == r ? s : 0); } - - template< - class GatherMode, - typename It1, typename Size, typename It2, - typename = std::enable_if_t< - std::is_same< - typename std::iterator_traits::value_type, - typename std::iterator_traits::value_type - >{} - >, class... Root - > - auto a_gather_n( - GatherMode gm, - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::memcopyable_tag /*tag*/, - Size count, - It2 d_first, - detail::contiguous_iterator_tag /*tag*/, - detail::memcopyable_tag /*tag*/, - Root... root - ) { - int s = gm( - detail::data(first), count*sizeof(*first), MPI_BYTE, - detail::data(d_first), count*sizeof(*d_first), MPI_BYTE, - root..., impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot gather");} - advance(d_first, count*size(), root...); - // std::advance(d_first, count); - return d_first; - } - - public: - template - auto all_gather_n( - It1 first, Size1 count, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - It2 d_first, Size2 d_count, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/ - ) { - MPI_(Allgather)( - detail::data( first), static_cast( count), datatype::value_type>{}(), // TODO(correaa) use safe cast - detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), // TODO(correaa) use safe cast - impl_ - ); - return d_first + static_cast(d_count) * usize(); - } - template - auto all_gather_n(It1 first, Size count, It2 d_first, Size d_count) { - return all_gather_n( - first, count, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_first, d_count, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{} - ); - } - template - It2 all_gather( - It1 first, It1 last, - /**/ detail::random_access_iterator_tag /*tag*/, - It2 d_first - ) {return all_gather_n(first, std::distance(first, last), d_first);} - template - It2 all_gather(It1 first, It1 last, It2 d_first) { - return all_gather(first, last, detail::iterator_category_t{}, d_first); - } - template - auto all_gatherv_n( - It1 first, Size count, - It2 d_first, - detail::output_iterator_tag /*tag*/, - CountsIt counts, DisplsIt displs - ) { - auto const s = static_cast(std::accumulate(counts, counts + size(), typename std::iterator_traits::value_type{0})); - std::vector::value_type> buff(s); - auto e = all_gatherv_n(first, count, buff.data(), counts, displs); - assert( e == std::next(buff.data(), static_cast(buff.size())) ); - using std::move; - return move(buff.begin(), buff.end(), d_first); // cppcheck-suppress returnDanglingLifetime ; cppcheck 2.3 bug - } - template - auto all_gatherv_n( - It1 first, Size count, - It2 d_first, - detail::forward_iterator_tag /*tag*/, - CountsIt counts, DisplsIt displs - ) { - return all_gatherv_n( - first, count, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - counts, displs); - } - template - auto all_gatherv_n(It1 first, Size count, It2 d_first, CountsIt counts, DisplsIt displs) { - return all_gatherv_n( - first, count, - d_first, - detail::iterator_category_t{}, - counts, displs - ); - } - template - auto all_gatherv_n( - It1 first, Size count, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - It2 d_first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - CountsIt counts, DisplsIt displs - ) { - MPI_(Allgatherv)( - detail::data( first), static_cast(count) , datatype::value_type>{}(), // TODO(correaa) use safe cast - detail::data(d_first), detail::data(counts), detail::data(displs), datatype::value_type>{}(), - impl_ - ); - return d_first + detail::data(displs)[size()-1] + detail::data(counts)[size()-1]; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic) - } - template - auto all_gather_n( - It1 first, Size count, - detail::forward_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - It2 d_first, Size d_count, - detail::forward_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/ - ) { - detail::package po(*this); - package_oarchive poa(po); - std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); - // while(count--) {poa << *(first++);} // TODO(correaa) remove comment - auto const posize = static_cast(po.size()); - std::vector sizes (usize()); - std::vector displs(usize()); - all_gather_n(&posize, 1, sizes.begin(), 1); - partial_sum(sizes.begin(), sizes.end(), displs.begin()+1); - detail::package pi(*this); - auto const total = static_cast(std::accumulate(sizes.begin(), sizes.end(), 0)); - pi.resize(total); - all_gatherv_n(po.data(), po.size(), pi.data(), sizes.data(), displs.data()); - package_iarchive pia(pi); - d_count *= size(); - return std::copy_n(package_iarchive::iterator::value_type>(pia), d_count, d_first); - } - template - auto all_gather_n(It1 first, Size count, It2 d_first) { - return all_gather_n(first, count, d_first, count); - } - template - auto all_gather_v(V const& v, It2 d_first) { - return all_gather_n(std::addressof(v), 1, d_first); - } - template - auto all_gather_as(V const& v) { - Vector ret(usize()); - all_gather_v(v, ret.begin()); - return ret; - } - template - auto all_gatherv_n(It1 first, Size count, It2 d_first) { - std::vector counts(usize() ); - std::vector displs(usize() + 1); - int c = static_cast(count); - all_gather_n(&c, 1, counts.begin()); - partial_sum(counts.begin(), counts.end(), displs.begin()+1); - return all_gatherv_n(first, count, d_first, counts.begin(), displs.begin()); - } - template - auto gatherv_n(It1 first, Size count, It2 d_first) { - std::vector counts(usize() ); - std::vector displs(usize() + 1); - int c = count; - all_gather_n(&c, 1, counts.begin()); - partial_sum(counts.begin(), counts.end(), displs.begin()+1); - return gatherv_n(first, count, d_first, counts.begin(), displs.begin()); - } - template - auto all_gather_n( - It1 first, - /**/ detail::forward_iterator_tag /*tag*/, - /**/ detail::value_unspecified_tag /*tag*/, - Size1 count, - It2 d_first, - /**/ detail::forward_iterator_tag /*tag*/, - /**/ detail::value_unspecified_tag /*tag*/, - Size2 d_count - ) { - detail::package po(*this); - package_oarchive poa(po); - std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); - // while(count--) {poa << *(first++);} - auto const posize = static_cast(po.size()); - std::vector sizes (usize()); - std::vector displs(usize()); - all_gather_n(&posize, 1, sizes.begin(), 1); - partial_sum(sizes.begin(), sizes.end(), displs.begin()+1); - detail::package pi(*this); - auto total = static_cast(std::accumulate(sizes.begin(), sizes.end(), 0)); - pi.resize(total); - all_gatherv_n(po.data(), po.size(), pi.data(), sizes.data(), displs.data()); - package_iarchive pia(pi); - d_count *= size(); - return std::copy_n(package_iarchive::iterator::value_type>(pia), d_count, d_first); - } - template - auto all_gatherv( - It1 first, It1 last, - detail::random_access_iterator_tag /*tag*/, It2 d_first - ) { - return all_gatherv_n(first, std::distance(first, last), d_first); - } - template - auto all_gatherv(It1 first, It1 last, It2 d_first) { - return all_gatherv( - first, last, - detail::iterator_category_t{}, - d_first - ); - } - - template - auto gatherv_n( - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size1 count, - It2 d_first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Itc counts, - detail::contiguous_iterator_tag /*tag*/, - Itd displs, - detail::contiguous_iterator_tag /*tag*/, - int root - ) { - MPI_(Gatherv)( - detail::data( first), static_cast(count), datatype::value_type>{}(), // TODO(correaa) use safe cast - detail::data(d_first), detail::data(counts), detail::data(displs), datatype::value_type>{}(), - root, impl_ - ); - } - template - auto gatherv_n( - It1 first, Size1 count, - It2 d_first, Itc counts, Itd displs = 0, - int root = 0 - ){ - return gatherv_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - counts, - detail::iterator_category_t{}, - displs, - detail::iterator_category_t{}, - root - ); - } - template - auto igather_n( - It1 first, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - Size1 count, - It2 d_first, - /**/ detail::contiguous_iterator_tag /*tag*/, - /**/ detail::basic_tag /*tag*/, - Size2 d_count, - int root - ) { - request ret; - MPI_(Igather)( - detail::data(first) , static_cast( count), datatype::value_type>{}(), - detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), - root, impl_, &ret.impl_ - ); - return ret; - } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) MPI_Wait called on destructor of ret - template - auto iall_gather_n( - It1 first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size1 count, - It2 d_first, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - Size2 d_count - ) { - request ret; - MPI_(Iallgather)( - detail::data(first) , static_cast( count), datatype::value_type>{}(), // TODO(correaa) use safe cast - detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), // TODO(correaa) use safe cast - impl_, &ret.impl_ - ); - return ret; - } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // wait is in request destructor - template - auto gather_n( - It1 first, - detail::input_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - Size1 count, - It2 d_first, - detail::input_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - Size2 d_count, - int root - ) { - detail::package po(*this); - package_oarchive poa(po); - std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); - // while(count--) {poa << *(first++);} TODO(correaa) remove comment - int posize = static_cast(po.size()); - std::vector sizes(rank()==root?usize():0); - gather_n(&posize, 1, sizes.begin(), 1, root); - std::vector displs(sizes.size()+1); - partial_sum(sizes.begin(), sizes.end(), displs.begin()+1); - detail::package pi(*this); - auto const total = static_cast(std::accumulate(sizes.begin(), sizes.end(), 0)); - pi.resize(total); - gatherv_n(po.data(), po.size(), pi.data(), sizes.data(), displs.data(), root); - if(rank() == root) { - package_iarchive pia(pi); - d_count *= size(); - return std::copy_n( - package_iarchive::iterator::value_type>(pia), - d_count, - d_first - ); - // while(d_count--) {pia >> *(d_first++);} - } - return d_first; - } - template - auto igather_n(It1 first, Size1 count, It2 d_first, Size2 d_count, int root){ - return igather_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_count, - root - ); - } - template - auto iall_gather_n(It1 first, Size1 count, It2 d_first, Size2 d_count) { - return iall_gather_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - d_first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_count - ); - } - template - auto igather_n(It1 first, Size1 count, It2 d_first, int root = 0) { - return igather_n(first, count, d_first, count, root); - } - template - auto iall_gather_n(It1 first, Size1 count, It2 d_first) { - return iall_gather_n(first, count, d_first, count); - } - template - auto gather( - It1 first, It1 last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - It2 d_first, int root - ) { - return gather_n(first, std::distance(first, last), d_first, root); - } - template - auto gather( - It1 first, It1 last, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - It2 d_first, int root - ) { - return gather_n(first, std::distance(first, last), d_first, root); - } - template - auto igather( - It1 first, It1 last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - It2 d_first, int root - ) { - return igather_n(first, std::distance(first, last), d_first, root); - } - template - auto iall_gather( - It1 first, It1 last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - It2 d_first - ) { - return iall_gather_n(first, std::distance(first, last), d_first); - } - template - auto gather( - It1 first, It1 last, - detail::input_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - It2 d_first, int root - ){ - mpi3::vector::value_type> buffer(first, last); - return gather_n(buffer.data(), buffer.size(), d_first, root); - } - template - auto gather(It1 first, It1 last, It2 d_first, int root = 0){ - return gather( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_first, - root - ); - } - template - auto igather(It1 first, It1 last, It2 d_first, int root){ - return igather( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_first, - root - ); - } - template - auto iall_gather(It1 first, It1 last, It2 d_first) { - return iall_gather( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_first - ); - } - template - auto gather( - It1 first, It1 last, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - It2 d_first, It2 d_last, - detail::contiguous_iterator_tag /*tag*/, - detail::basic_tag /*tag*/, - int root - ) { - return gather_n( - first, std::distance(first, last), - d_first, std::distance(d_first, d_last), - root - ); - } - template - auto gather( - It1 first, It1 last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - It2 d_first, It2 d_last, - detail::random_access_iterator_tag /*tag*/, - detail::value_unspecified_tag /*tag*/, - int root - ) { - return gather_n( - first, std::distance(first, last), - d_first, std::distance(d_first, d_last), - root - ); - } - template - auto gather( - It1 first, It1 last, - detail::forward_iterator_tag /*forward*/, - detail::basic_tag /*basic*/, - It2 d_first, It2 d_last, - detail::random_access_iterator_tag /*random_access*/, - detail::value_unspecified_tag /*value_unspecified*/, - int root - ) { - mpi3::vector::value_type> v(first, last); - return gather_n( - v.data(), v.size(), - d_first, std::distance(d_first, d_last), - root - ); - } - - template - auto gather(It1 first, It1 last, It2 d_first, It2 d_last, int root) { - return gather( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - d_first, d_last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - root - ); - } - - private: - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto gather_n_dispatch( - gather_mode g, std::true_type /*true*/, - It1 first, Size count, It2 d_first, int root = 0 - ) { - int s = g( - detail::data(first) , count, datatype{}(), - detail::data(d_first), count, datatype{}(), - root, impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} - } - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto gather_n_dispatch( - gather_mode g, std::false_type /*false*/, - It1 first, Size count, It2 d_first, int root = 0 - ) { - int s = g( - detail::data(first) , count, datatype{}(), - detail::data(d_first), count, datatype{}(), - root, impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} - } - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto gather_n_dispatch( - igather_mode g, std::true_type /*true*/, - It1 first, Size count, It2 d_first, int root = 0 - ) { - request r; - int s = g( - detail::data(first) , count, datatype{}(), - detail::data(d_first), count, datatype{}(), - root, impl_, &r.impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} - return r; - } - template::value_type, - class V2 = typename std::iterator_traits::value_type - > - auto gather_n_dispatch( - all_gather_mode g, std::true_type /*true*/, - It1 first, Size count, It2 d_first, int /*root*/= 0 - ) { - int s = g( - detail::data(first) , count, datatype{}(), - detail::data(d_first), count, datatype{}(), - impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} - } - - public: - std::string get_name() const { - std::array comm_name{}; - int len; // NOLINT(cppcoreguidelines-init-variables) : delayed initialization - MPI_(Comm_get_name)(impl_, comm_name.data(), &len); - return {comm_name.data(), static_cast(len)}; - } - void set_name(std::string const& s) {MPI_(Comm_set_name)(impl_, s.c_str());} - std::string name() const {return get_name();} - - [[deprecated]] void name(std::string const& s) {set_name(s);} - - static mpi3::communicator& parent() { - static_assert(sizeof(MPI_Comm) == sizeof(mpi3::communicator), "!"); - static_assert(std::is_same{}, "!"); - MPI_Comm* p{}; MPI_Comm_get_parent(p); assert(p); - return reinterpret_cast(*p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) : TODO(correaa) avoid reinterpret_cast - } - static communicator spawn(std::string const& argv0, int np) { - communicator intercomm; - MPI_Comm_spawn(argv0.data(), MPI_ARGV_NULL, np, MPI_INFO_NULL, 0, MPI_COMM_SELF, &intercomm.impl_, MPI_ERRCODES_IGNORE ); - return intercomm; - } - - communicator intercommunicator_create(int local_leader, communicator const& peer, int remote_leader, int tag = 0) const{ - communicator ret; - int const s = MPI_Intercomm_create(impl_, local_leader, peer.impl_, remote_leader, tag, &ret.impl_); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot create intercommunicator");} - return ret; - } - - communicator create(int local_leader, communicator const& peer, int remote_leader, int tag = 0) const{ - return intercommunicator_create(local_leader, peer, remote_leader, tag); - } - - communicator create(group const& g) const; - communicator create_group(group const& g, int tag) const; - FILE* fopen(char const* filename, int amode = unsigned{MPI_MODE_RDWR} | unsigned{MPI_MODE_CREATE}); - - inline static auto name(communicator::topology const& t) -> std::string const& { - static std::map const names = { - {communicator::topology::undefined, "undefined"}, - {communicator::topology::graph, "graph"}, - {communicator::topology::cartesian, "cartesian"}}; - return names.find(t)->second; - } - -//template -//friend auto operator,(communicator& comm, T const& t){ -// std::vector ret(comm.size()); -// comm.all_gather_n(std::addressof(t), 1, first, root); -//} - - template - friend T operator+=(communicator& comm, T const& t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator - return comm.all_reduce_value(t, std::plus<>{}); - } - - template - friend T operator&=(communicator& comm, T const& t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator - return comm.all_reduce_value(t, std::bit_and<>{}); - } - - friend bool operator&=(communicator& comm, bool t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator - bool ret = true; - comm.all_reduce_n(&t, 1, &ret, std::logical_and<>{}); - return ret; - } - friend bool operator|=(communicator& comm, bool t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator - bool ret = false; - comm.all_reduce_n(&t, 1, &ret, std::logical_or<>{}); - return ret; - } - - template - friend communicator& operator<<(communicator& comm, T const& t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator - comm.send_value(t); - return comm; - } -}; - -inline void barrier(communicator& self) { self. barrier();} -inline auto ibarrier(communicator& self) {return self.ibarrier();} - -inline communicator::communicator(group const& g, int tag){ - MPI_(Comm_create_group)(MPI_COMM_WORLD, &const_cast(g), tag, &impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with -} - -inline communicator::communicator(group const& g){ - MPI_(Comm_create)(MPI_COMM_WORLD, &const_cast(g), &impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with -} -// https://www.open-mpi.org/doc/v3.0/man3/MPI_Comm_create_group.3.php -// MPI_Comm_create_group is similar to MPI_Comm_create; however, MPI_Comm_create must be called by all processes in the group of comm, whereas MPI_Comm_create_group must be called by all processes in group, which is a subgroup of the group of comm. In addition, MPI_Comm_create_group requires that comm is an intracommunicator. MPI_Comm_create_group returns a new intracommunicator, newcomm, for which the group argument defines the communication group. No cached information propagates from comm to newcomm. - -inline communicator::communicator(communicator const& o, group const& g){ - MPI_(Comm_create)(o.impl_, &const_cast(g), &impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with -} - -inline communicator::operator group() const{ - group ret; MPI_(Comm_group)(impl_, &ret.impl_); return ret; -} -//inline group::group(communicator const& c){MPI_(Comm_group)(&const_cast(c), &impl_);} - -inline communicator communicator::create(group const& g) const{ - communicator ret; - int const s = MPI_Comm_create(impl_, &const_cast(g), &ret.impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot crate group"};} - return ret; -} - -inline communicator communicator::create_group(class group const& g, int tag = 0) const{ - communicator ret; - MPI_(Comm_create_group)(impl_, &const_cast(g), tag, &ret.impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with - return ret; -} - -template -inline void communicator::deallocate_shared(pointer /*unused*/){ -// MPI_Free_mem(p.base_ptr(rank())); -} - -template -inline void communicator::deallocate(pointer& /*p*/, MPI_Aint /*size*/) { // TODO(correaa) should be called free? -// p.pimpl_->fence(); -// MPI_Free_mem(p.local_ptr()); -// MPI_Win_free(&p.pimpl_->impl_); -// delete p.pimpl_; -// p.pimpl_ == nullptr; -} - -#if 0 -template -inline window communicator::make_window(mpi3::size_t size){ - mpi3::info inf; - void* ptr; - window ret; - int s = MPI_Win_allocate(size*sizeof(T), sizeof(T), inf.impl_, this->impl_, &ptr, &ret.impl_); - if(s != MPI_SUCCESS) throw std::runtime_error("cannot window_allocate"); - return ret; -} -#endif - -class strided_range { - int first_; - int last_; - int stride_ = 1; - - public: - strided_range(int f, int l) : first_{f}, last_{l} {} // NOLINT(bugprone-easily-swappable-parameters) - strided_range(int f, int l, int s) : first_{f}, last_{l}, stride_{s} {} // NOLINT(bugprone-easily-swappable-parameters) - - int front() const {return first_;} - int back() const {return last_ - 1;} - int size() const {return (last_ - first_) / stride_;} -}; - -template -auto operator/(Range const& r, communicator& self) // NOLINT(fuchsia-overloaded-operator) : experimental operator overloading - ->decltype(self.scatter(begin(r), end(r))) { - return self.scatter(begin(r), end(r)); } - - -inline mpi3::communicator& deref(MPI_Comm const& handle) { - return reinterpret_cast(const_cast(handle)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-type-const-cast) -} - - -inline mpi3::communicator& grip_communicator(MPI_Comm const& handle) { - return reinterpret_cast(const_cast(handle)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-type-const-cast) -} - -[[deprecated]] inline mpi3::communicator& grip(MPI_Comm const& handle) { - return grip_communicator(handle); -} - -} // end namespace mpi3 -} // end namespace boost - -//BOOST_SERIALIZATION_REGISTER_ARCHIVE(boost::mpi3::package_oarchive) -//BOOST_SERIALIZATION_USE_ARRAY_OPTIMIZATION(boost::mpi3::detail::package_oarchive) -//BOOST_SERIALIZATION_USE_ARRAY_OPTIMIZATION(boost::mpi3::detail::package_iarchive) - -//#if not __INCLUDE_LEVEL__ - -//#include "../mpi3/main.hpp" -//#include "../mpi3/version.hpp" - -//#include - -//using std::cout; -//namespace mpi3 = boost::mpi3; - -//class V{ -// mpi3::communicator comm_; -// public: -// V(mpi3::communicator const& c) : comm_(c){} -// V(mpi3::communicator&& c) : comm_(std::move(c)){} -//}; - -//int mpi3::main(int, char*[], mpi3::communicator world){ -// std::cout << mpi3::undefined << std::endl; - -// static_assert(std::is_nothrow_constructible::value, "MyType should be noexcept MoveConstructible"); - -//// auto worldcopy1 = world; -//// auto worldcopy2 = std::move(worldcopy1); -//// V v(worldcopy); -//// V v2(std::move(v)); - -// if(world.rank() == 0) cout << "MPI version " << mpi3::version() << '\n'; -//// if(world.rank() == 0) cout << "Topology: " << name(world.topo()) << '\n'; - -// cout << "MPI_ERR_COMM = " << MPI_ERR_COMM << '\n'; - -// mpi3::communicator comm; -// assert(!comm); -//// cout << comm.rank() << '\n'; - -// mpi3::communicator comm2 = world; -// assert(comm2); -// assert(comm2.size() == world.size()); -// assert(comm2 == world); -// assert(&comm2 != &world); - -// mpi3::communicator comm3 = world;//.duplicate(); -// assert(comm3); -// assert(comm3 == world); -// assert(&comm3 != &world); -// comm = comm2; -// assert(&comm != &comm2); - -//// world2 = world; - -// return 0; -//#if 0 -//// boost::mpi3::communicator newcomm = world; -// { -// int color = world.rank()/3; -// communicator row_comm; -// row_comm = world.split(color); -// world.barrier(); -// std::cout << std::to_string(world.rank()) + " " + std::to_string(row_comm.rank()) + "\n";// << std::endl; -// world.barrier(); -// } -// { -// communicator row_comm = world/3; -// world.barrier(); -// std::cout << std::to_string(world.rank()) + " " + std::to_string(row_comm.rank()) + "\n";// << std::endl; -// world.barrier(); -// } - -// world.barrier(); -// if(world.rank() == 0) cout << "prime communicator" << '\n'; -// world.barrier(); - -// { -// // group world_group(world); -// // const int ranks[4] = {2, 3, 5, 7}; -// // group prime = world_group.include(ranks, ranks + 4); -// // communicator prime_comm(world, prime); -// auto prime_comm = world.subcomm({2,3,5,7}); -// cout << world.rank() << " -> " << prime_comm.rank() << "/" << prime_comm.size() << '\n'; -//#if 0 -// if(communicator::null != prime_comm){ -// cout << world.rank() << " -> " << prime_comm.rank() << "/" << prime_comm.size() << '\n'; -// }else{ -// cout << world.rank() << " not in prime comm\n"; -// } -//#endif -// } - -// world.barrier(); -// if(world.rank() == 0) cout << "prime communicator" << '\n'; -// world.barrier(); - -// if(0){ -// auto prime = world.subcomm({2,3,5,7}); -// if(prime.is_empty()){ -// // if (communicator::null != prime){ -// cout << world.rank() << " -> " << prime.rank() << "/" << prime.size() << '\n'; -// }else{ -// cout << world.rank() << " not in prime comm\n"; -// } -// } -//#endif -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operatorators.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operatorators.hpp deleted file mode 100644 index de7322e76e7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operatorators.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpicxx -O3 -std=c++17 -Wfatal-errors -D_TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS $0x.cpp -o $0x.x && time mpirun -np 3 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP -#define BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace boost{ -namespace mpi3{ -// communicator operator/(communicator const& comm, int n){ -// int color = comm.rank()*n/comm.size(); -// return comm.split(color); -// } -}} - -#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator& world){ - auto hemi = world/2; - cout - << "I am " << world.name() << " " << world.rank() - << " also I am " << hemi.name() << " " << hemi.rank() - << std::endl - ; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operators.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operators.hpp deleted file mode 100644 index bc18b7e818c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator/operators.hpp +++ /dev/null @@ -1,36 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpicxx -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS $0x.cpp -o $0x.x && time mpirun -np 3 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP -#define BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace boost{ -namespace mpi3{ -// communicator operator/(communicator const& comm, int n){ -// int color = comm.rank()*n/comm.size(); -// return comm.split(color); -// } -}} - -#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - auto hemi = world/2; - assert(hemi); - cout - << "I am " << world.name() << " " << world.rank() - << " also I am " << hemi.name() << " " << hemi.rank() - << std::endl - ; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_fwd.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_fwd.hpp deleted file mode 100644 index 9a6cea91583..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_fwd.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -g -std=c++14 `#-Wfatal-errors` -D_TEST_BOOST_MPI3_COMMUNICATOR_FWD $0x.cpp -o $0x.x && time mpirun -np 1 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_COMMUNICATOR_FWD_HPP -#define BOOST_MPI3_COMMUNICATOR_FWD_HPP - -namespace boost{ -namespace mpi3{ - -struct communicator; - -}} - -#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_FWD - -#include -using std::cout; - -int main(){} - -#endif -#endif - - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_iterator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_iterator.hpp deleted file mode 100644 index 5b88f7736ec..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/communicator_iterator.hpp +++ /dev/null @@ -1,55 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_COMMUNICATOR_ITERATOR $0x.cpp -o $0x.x && time mpirun -n 5 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_COMMUNICATOR_ITERATOR_HPP -#define BOOST_MPI3_COMMUNICATOR_ITERATOR_HPP - -#include "../mpi3/communicator.hpp" -#include "../mpi3/process.hpp" - -namespace boost{ -namespace mpi3{ - -struct communicator_iterator { - int n_; - communicator* commptr_; - communicator_iterator() : n_(-1), commptr_(nullptr){} - communicator_iterator(void*** p) : communicator_iterator(){} -// communicator_iterator(communicator_iterator const& other) = default; - process operator*() const{return commptr_->operator[](n_);} - communicator_iterator& operator++(){n_++; return *this;} - communicator_iterator& operator--(){n_--; return *this;} - communicator_iterator& operator+=(int n){n_+=n; return *this;} - communicator_iterator& operator-=(int n){n_-=n; return *this;} -}; - -communicator_iterator next_periodic(communicator_iterator it){ - it.n_++; - it.n_ = it.n_ % it.commptr_->size(); - return it; -} - -communicator_iterator prior_periodic(communicator_iterator it){ - it.n_--; - if(it.n_ < 0) it.n_ += it.commptr_->size(); - return it; -} - -}} - -#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_ITERATOR - -#include "../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int, char*[], mpi3::communicator world){ -// for(auto&& p : world) -// std::periodic_next(p) << world.rank()*10; - return 0; -} - -#endif -#endif - - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/config/NODISCARD.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/config/NODISCARD.hpp deleted file mode 100644 index f83a9851f5d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/config/NODISCARD.hpp +++ /dev/null @@ -1,27 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4-*- -// © Alfredo A. Correa 2019-2021 -#ifndef MPI3_CONFIG_NODISCARD_HPP -#define MPI3_CONFIG_NODISCARD_HPP - -//#ifndef __has_cpp_attribute -//#define __has_cpp_attribute(name) 0 -//#endif - -//#ifndef BMPI3_NODISCARD -//#if defined(__NVCC__) -// #define BMPI3_NODISCARD(MsG) -//#elif (__has_cpp_attribute(nodiscard) and (__cplusplus>=201703L)) -// #if (__has_cpp_attribute(nodiscard)>=201907) and (__cplusplus>201703L) -// #define BMPI3_NODISCARD(MsG) [[nodiscard(MsG)]] -// #else -// #define BMPI3_NODISCARD(MsG) [[nodiscard]] -// #endif -//#elif __has_cpp_attribute(gnu::warn_unused_result) -// #define BMPI3_NODISCARD(MsG) [[gnu::warn_unused_result]] // NOLINT(cppcoreguidelines-macro-usage) : replaced by `nodiscard` in C++17 -//#else -// #define BMPI3_NODISCARD(MsG) -//#endif -//#endif - -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/core.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/core.hpp deleted file mode 100644 index 538203eefe4..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/core.hpp +++ /dev/null @@ -1,36 +0,0 @@ -// -*- indent-tabs-mode:t;c-basic-offset:4;tab-width:4; -*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef BOOST_MPI3_CORE_HPP -#define BOOST_MPI3_CORE_HPP - -#include - -#include // if you get a compilation error here it means that 1) you need to compile or defined your CXX as mpic++ or 2) have not setup the compilation flags to find MPI headers, or 3) not installed an MPI implementation (e.g. `apt install libopenmpi-dev openmpi-bin`) - -namespace boost { -namespace mpi3 { - -inline bool initialized() { - return MPI_(Initialized)(); -} - -inline bool finalized() { - return MPI_(Finalized)(); -} - -} // end namespace mpi3 -} // end namespace boost - -#if not __INCLUDE_LEVEL__ // _TEST_BOOST_MPI3_ENVIRONMENT - -#include - -namespace mpi3 = boost::mpi3; - -int main(){ - assert(not mpi3::initialized() ); -} - -#endif -#endif \ No newline at end of file diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/basic_communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/basic_communicator.hpp deleted file mode 100644 index cc42b027f73..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/basic_communicator.hpp +++ /dev/null @@ -1,336 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef MPI3_DETAIL_BASIC_COMMUNICATOR_HPP -#define MPI3_DETAIL_BASIC_COMMUNICATOR_HPP - -#include "../../mpi3/vector.hpp" - -#include "../../mpi3/detail/buffer.hpp" -#include "../../mpi3/detail/iterator_traits.hpp" -#include "../../mpi3/detail/value_traits.hpp" - -#include "../../mpi3/match.hpp" -#include - -#include - -#include - -namespace boost { -namespace mpi3 { -namespace detail { - -class basic_communicator{ - protected: - using impl_t = MPI_Comm; - impl_t impl_ = MPI_COMM_NULL; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes,misc-non-private-member-variables-in-classes) : TODO(correaa) make private - - explicit basic_communicator(MPI_Comm impl) noexcept : impl_(impl){} - - public: - basic_communicator() noexcept = default; //: impl_(MPI_COMM_NULL){} - ~basic_communicator() = default; - - auto operator=(basic_communicator const&) -> basic_communicator& = delete; - auto operator=(basic_communicator&&) -> basic_communicator& = delete; - - basic_communicator(basic_communicator const&) = delete; // communicators are not copyable, only duplicable, if you know what you are doing, use `mutable` - basic_communicator(basic_communicator& other) { - if(MPI_COMM_NULL != other.impl_) {MPI_(Comm_dup)(other.impl_, &impl_);} - } - basic_communicator(basic_communicator&& other) noexcept : - impl_{std::exchange(other.impl_, MPI_COMM_NULL)} {} - - // [[deprecated]] void swap(basic_communicator& o) noexcept {std::swap(impl_, o.impl_);} - // [[deprecated]] friend void swap(basic_communicator& self, basic_communicator& other) noexcept {self.swap(other);} - - template - int pack_size(int count, detail::basic_tag /*tag*/) { - int size = -1; - MPI_(Pack_size)(count, detail::basic_datatype{}, impl_, &size); - return size; - } - template - int pack_size(int count) { - return pack_size(count, detail::value_category_t{}); - } - template - auto pack_n( - It first, - detail::contiguous_iterator_tag /*contiguous*/, - detail::basic_tag /*basic*/, - Size count, - uvector& p, int pos // NOLINT(misc-unused-parameters) bug in clang-tidy 12 - ) { - using value_type = typename std::iterator_traits::value_type; - MPI_(Pack)( - detail::data(first), static_cast(count), detail::basic_datatype{}, - p.data(), static_cast(p.size()), // TODO(correaa) use safe cast - &pos, impl_ - ); - return pos; - } - template - auto pack_n( - It first, Size count, - uvector& b, int pos - ){ - return pack_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - b, pos - ); - } - template - auto pack_n(It first, Size count, uvector& p) { - assert(p.size() < std::numeric_limits::max()); - int const pos = static_cast(p.size()); - p.resize(p.size() + static_cast(pack_size::value_type>(static_cast(count)))); - return pack_n(first, count, p, pos); - } - template - auto pack( - It first, It second, - detail::random_access_iterator_tag /*random_access*/, - detail::value_unspecified_tag /*value_unspecified*/, - uvector& b - ) { - return pack_n(first, std::distance(first, second), b); - } - - template - auto pack( - It first, It last, - /**/ detail::input_iterator_tag /*input*/, - uvector& b - ) { - std::for_each(first, last, [this, &b](auto& e) { pack_n(std::addressof(e), 1, b); }); - return b.size(); - } - - template - auto pack(It first, It second, uvector& b){ - return pack( - first, second, - typename detail::iterator_category::type{}, - detail::value_category_t::value_type>{}, - b - ); - } - template - auto unpack_n( - uvector& b, int pos, - It first, - /**/ detail::contiguous_iterator_tag /*contiguous*/, - /**/ detail::basic_tag /*basic*/, - Size count - ){ - using value_type = typename std::iterator_traits::value_type; - MPI_(Unpack)( - b.data(), static_cast(b.size()), &pos, - detail::data(first), static_cast(count), // TODO(correaa) use safe cast - detail::basic_datatype{}, - impl_ - ); - return pos; - } - template - auto unpack( - uvector& b, int pos, - It first, It last, - detail::random_access_iterator_tag /*random_access*/ - ) { - return unpack_n(b, pos, first, std::distance(first, last)); - } - - template - auto unpack( - uvector& b, int pos, - It first, It last, - detail::forward_iterator_tag /*forward*/ - ) { - std::for_each(first, last, [&b, &pos, this](auto& e) {pos = unpack_n(b, pos, std::addressof(e), 1);}); - // while(first != last){ - // pos = unpack_n(b, pos, std::addressof(*first), 1); - // ++first; - // } - return pos; - } - template - auto unpack( - uvector& b, int pos, - It first, It last - ) { - return unpack( - b, pos, - first, last, - /**/ detail::iterator_category_t{} - ); - } - template - auto unpack(detail::buffer& b, It first, It last){ - return b.pos = unpack(b, b.pos, first, last); - } - template - auto unpack_n(uvector& b, int pos, It first, Size count){ - return unpack_n( - b, pos, - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count - ); - } - template - auto unpack_n(detail::buffer& b, It first, Size count) { - // assert(0); - b.pos = unpack_n(b, b.pos, first, count); - return b.pos; - } - template - auto send_n( - It first, - detail::contiguous_iterator_tag /*contiguous*/, - detail::basic_tag /*basic*/, - Size count, - int dest, int tag - ) { - MPI_(Send)( - detail::data(first), static_cast(count), // TODO(correaa) use safe cast - detail::basic_datatype::value_type>{}, - dest, tag, - impl_ - ); - } - template - auto send_n(It first, Size count, int dest, int tag = 0) { - return send_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count, - dest, tag - ); - } - template - auto send( - It first, It last, - detail::random_access_iterator_tag /*random_access*/, - detail::value_unspecified_tag /*value_unspecified*/, - int dest, int tag - ) { - return send_n(first, std::distance(first, last), dest, tag); - } - template - auto send( - It first, It last, - detail::input_iterator_tag /*input*/, - detail::basic_tag /*basic*/, - int dest, int tag - ) { - mpi3::uvector::value_type> buffer(first, last); - return send_n(buffer.data(), buffer.size(), dest, tag); - } - template - auto send(It first, It last, int dest, int tag = 0) { - return send( - first, last, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - dest, tag - ); - } - auto send(uvector const& p, int dest, int tag = 0) { - return send_n(p.data(), p.size(), dest, tag); - } - match matched_probe(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) const { - match m; - MPI_(Mprobe)(source, tag, impl_, &m.message::impl_, &m.status::impl_); - return m; - } - template - auto receive_n( - It dest, - /**/ detail::forward_iterator_tag /*forward*/, - /**/ detail::basic_tag /*basic*/, - Size n, - int source, int tag - ) { - mpi3::uvector::value_type> buffer(n); - receive_n(buffer.data(), buffer.size(), source, tag); - return std::copy_n(buffer.begin(), n, dest); - } - auto receive(uvector& b, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) const { - match m = matched_probe(source, tag); - auto const count = static_cast(m.count()); - auto const size = static_cast(b.size()); - b.resize(b.size() + count); - return m.receive_n(std::next(b.data(), size), count); - } - auto receive(detail::buffer& b, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) const { - return receive(static_cast&>(b), source, tag); - } - template - auto send_receive_replace_n( - It first, - detail::forward_iterator_tag /*forward*/, - detail::basic_tag /*basic*/, - Size size, Meta... meta - ) { - using value_type = typename std::iterator_traits::value_type; - mpi3::uvector buffer(size); - std::copy_n(first, buffer.size(), buffer.begin()); - send_receive_replace_n(buffer.begin(), buffer.size(), meta...); - return std::copy_n(buffer.begin(), buffer.size(), first); - } - - template - auto send_receive_replace_n( - It first, - detail::contiguous_iterator_tag /*contiguous*/, - detail::basic_tag /*basic*/, - Size size, - int dest, int source, - int sendtag, int recvtag - ) { - using value_type = typename std::iterator_traits::value_type; - status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization - int s = MPI_Sendrecv_replace( - detail::data(first), size, detail::basic_datatype{}, - dest, sendtag, source, recvtag, impl_, &ret.impl_ - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot send_receive");} - return first + size; - } - - template - auto send_receive_replace_n( - It first, Size size, - int dest, int source, // = MPI_ANY_SOURCE, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_replace_n( - first, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - size, - dest, source, sendtag, recvtag - ); - } - template - auto send_receive_n( - It first, Size size, - int dest, int source, // = MPI_ANY_SOURCE, - int sendtag = 0, int recvtag = MPI_ANY_TAG - ) { - return send_receive_replace_n(first, size, dest, source, sendtag, recvtag); - } -}; - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp deleted file mode 100644 index 03440cfb1ea..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/buffer.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef MPI3_DETAIL_BUFFER_HPP -#define MPI3_DETAIL_BUFFER_HPP - -//#include "../../mpi3/communicator_fwd.hpp" -//#include "../../mpi3/detail/iterator.hpp" -#include "../../mpi3/detail/datatype.hpp" // packed -#include "../../mpi3/vector.hpp" - - -namespace boost{ -namespace mpi3{ -namespace detail{ - -struct buffer : mpi3::uvector{ - int pos = 0; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) : make private - buffer() = default; - explicit buffer(std::size_t r) {reserve(r);} - buffer(buffer const&) = delete; - buffer(buffer&&) = delete; - buffer& operator=(buffer const&) = delete; - buffer& operator=(buffer&&) = delete; - ~buffer() = default; -}; - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_MPI3_BUFFER - -//#include "../mpi3/communicator.hpp" -//#include "../mpi3/main.hpp" - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int, char*[], mpi3::communicator world){ -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/call.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/call.hpp deleted file mode 100644 index 7edb5633702..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/call.hpp +++ /dev/null @@ -1,75 +0,0 @@ -// © Alfredo A. Correa 2019-2021 -#ifndef BOOST_MPI3_DETAIL_CALL_HPP -#define BOOST_MPI3_DETAIL_CALL_HPP - -#include "../error.hpp" -#include "../status.hpp" - -#include "../config/NODISCARD.hpp" - -#include // MPI_MAX_PROCESSOR_NAME - -#include - -namespace boost { -namespace mpi3 { -namespace detail { - -template -int call() { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - auto const e = static_cast((*F)(&ret)); - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} - return ret; -} - -template -std::string call() { - int len = -1; - std::array name{}; - auto const e = static_cast((*F)(name.data(), &len)); - assert(len >= 0); - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} - return {name.data(), static_cast(len)}; -} - -template((*F)(std::declval()...)))* = nullptr> -void call(Args... args) { - auto const e = static_cast((*F)(args...)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} -} - -template((*F)(std::declval()..., std::declval())))* = nullptr> -mpi3::status call(Args... args) { - mpi3::status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization - auto const e = static_cast((*F)(args..., &ret.impl_)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} - return ret; -} - -template((*F)(std::declval()..., std::declval(), std::declval())))* = nullptr> -[[nodiscard]] auto call(Args... args) { - std::pair ret; // NOLINT(cppcoreguidelines-init-variables) delayed initialization - auto const e = static_cast((*F)(args..., &ret.first, &ret.second)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} - // cppcheck-suppress uninitvar - return ret; -} - -template((*F)(std::declval()..., std::declval())))* = nullptr> -[[nodiscard]] int call(Args... args) { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed initialization - auto const e = static_cast((*F)(args..., &ret)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} - // cppcheck-suppress uninitvar - return ret; -} - -#define MPI3_CALL(F) detail::call // NOLINT(cppcoreguidelines-macro-usage) -#define MPI_(F) MPI3_CALL(MPI_##F) // NOLINT(cppcoreguidelines-macro-usage): name concatenation - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/datatype.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/datatype.hpp deleted file mode 100644 index adf95dd7021..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/datatype.hpp +++ /dev/null @@ -1,205 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2017-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_DETAIL_DATATYPE_HPP -#define BOOST_MPI3_DETAIL_DATATYPE_HPP - -// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#if defined(__NVCC__) -#include -#endif - -#include -#include -#include // std::byte -#include -#include -#include // pair - -namespace boost { -namespace mpi3 { - -template -struct vlp { // value location pair - T value; // NOLINT(misc-non-private-member-variables-in-classes) - int location; // NOLINT(misc-non-private-member-variables-in-classes) - bool operator<(vlp const& other) const { - // if(value == other.value) {return location < other.location;} // partial order on purpose? - return value < other.value; - } -}; - -namespace detail { - -using float_int = std::pair; -using long_int = std::pair; // NOLINT(google-runtime-int) : long <-> int64 -using double_int = std::pair; -using short_int = std::pair; // NOLINT(google-runtime-int) : short <-> int16 -using int_int = std::pair; - -using long_double_int = std::pair; - -using float_float = std::pair; -using double_double = std::pair; -using long_double_long_double = std::pair; - -using cxx_float_complex = std::complex; -using cxx_double_complex = std::complex; -using cxx_long_double_complex = std::complex; - -// using cxx_2double_complex = std::pair, std::complex>; - -using cxx_bool = bool; - -using wchar = wchar_t; - -#if(__cplusplus >= 201703L) -using byte = std::byte; -#endif - -class packed { - unsigned char t{}; - - public: - explicit packed(unsigned char t_) : t{t_} {}; - packed() = default; - - packed& operator=(unsigned char const& rhs) {t = rhs; return *this;} - - explicit operator const unsigned char&() const {return t;} - explicit operator unsigned char&() {return t;} - - bool operator==(packed const& rhs) const {return t == rhs.t;} - bool operator< (packed const& rhs) const {return t < rhs.t;} -}; - -template struct basic_datatype; - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define MPI3_DECLARE_DATATYPE(TypE, MpiiD) \ -template<> struct basic_datatype { \ -/* constexpr*/ operator MPI_Datatype() const { \ - assert(MPI_DOUBLE_COMPLEX != MPI_DATATYPE_NULL ); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert)*/ \ - assert( (MpiiD) != MPI_DATATYPE_NULL ); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) in some MPI distros this is not constexpr */ /*this system doesn't support this type*/ \ - return MpiiD; \ - } \ - auto get() const -> MPI_Datatype {return MpiiD;} \ -/* static constexpr MPI_Datatype value = MpiiD;*/ \ -} - -// basic data types http://beige.ucs.indiana.edu/I590/node100.html -MPI3_DECLARE_DATATYPE(char , MPI_CHAR); -MPI3_DECLARE_DATATYPE(unsigned char , MPI_UNSIGNED_CHAR); - -#if(__cplusplus >= 201703L) -MPI3_DECLARE_DATATYPE(byte , MPI_BYTE); -#endif -MPI3_DECLARE_DATATYPE(wchar , MPI_WCHAR); - -MPI3_DECLARE_DATATYPE(short , MPI_SHORT); -MPI3_DECLARE_DATATYPE(unsigned short , MPI_UNSIGNED_SHORT); -MPI3_DECLARE_DATATYPE(int , MPI_INT); -MPI3_DECLARE_DATATYPE(unsigned int , MPI_UNSIGNED); -MPI3_DECLARE_DATATYPE(long , MPI_LONG); -MPI3_DECLARE_DATATYPE(unsigned long , MPI_UNSIGNED_LONG); -MPI3_DECLARE_DATATYPE(float , MPI_FLOAT); -MPI3_DECLARE_DATATYPE(double , MPI_DOUBLE); -MPI3_DECLARE_DATATYPE(long double , MPI_LONG_DOUBLE); -MPI3_DECLARE_DATATYPE(long long int , MPI_LONG_LONG_INT); - -MPI3_DECLARE_DATATYPE(bool , MPI_C_BOOL); // C++ binding not used MPI_CXX_BOOL); - -// MPI_INT8_T int8_t -// MPI_INT16_T int16_t -// MPI_INT32_T int32_t -// MPI_INT64_T int64_t -// MPI_UINT8_T uint8_t -// MPI_UINT16_T uint16_t -// MPI_UINT32_T uint32_t -// MPI_UINT64_T uint64_t - -MPI3_DECLARE_DATATYPE(cxx_float_complex , MPI_C_FLOAT_COMPLEX); -MPI3_DECLARE_DATATYPE(cxx_double_complex , MPI_C_DOUBLE_COMPLEX); -MPI3_DECLARE_DATATYPE(cxx_long_double_complex, MPI_C_LONG_DOUBLE_COMPLEX); - -// MPI3_DECLARE_DATATYPE(cxx_2double_complex , MPI_2DOUBLE_COMPLEX); // not available in mpich - -// TODO(correaa) these types below probably don't behave correctly for reductions with multiplication - -MPI3_DECLARE_DATATYPE(float_float , MPI_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); -MPI3_DECLARE_DATATYPE(double_double , MPI_DOUBLE_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); -MPI3_DECLARE_DATATYPE(decltype(std::tuple{}), MPI_DOUBLE_COMPLEX); -MPI3_DECLARE_DATATYPE(long_double_long_double, MPI_DOUBLE_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); - -#if defined(__NVCC__) -MPI3_DECLARE_DATATYPE(thrust::complex, MPI_DOUBLE_COMPLEX); -#endif - -MPI3_DECLARE_DATATYPE(float_int , MPI_FLOAT_INT); -MPI3_DECLARE_DATATYPE(long_int , MPI_LONG_INT); -MPI3_DECLARE_DATATYPE(double_int , MPI_DOUBLE_INT); -MPI3_DECLARE_DATATYPE(short_int , MPI_SHORT_INT); -MPI3_DECLARE_DATATYPE(int_int , MPI_2INT); -MPI3_DECLARE_DATATYPE(long_double_int , MPI_LONG_DOUBLE_INT); - -MPI3_DECLARE_DATATYPE(vlp , MPI_FLOAT_INT); -MPI3_DECLARE_DATATYPE(vlp , MPI_LONG_INT); -MPI3_DECLARE_DATATYPE(vlp , MPI_DOUBLE_INT); -MPI3_DECLARE_DATATYPE(vlp , MPI_SHORT_INT); -MPI3_DECLARE_DATATYPE(vlp , MPI_2INT); -MPI3_DECLARE_DATATYPE(vlp , MPI_LONG_DOUBLE_INT); - -//BOOST_MPI3_DECLARE_DATATYPE(std::intptr_t , MPI_AINT); -//BOOST_MPI3_DECLARE_DATATYPE(std::size_t , MPI_AINT); -MPI3_DECLARE_DATATYPE(void* , MPI_AINT); - -MPI3_DECLARE_DATATYPE(packed , MPI_PACKED); - -// LB -// UB - -#undef BOOST_MPI3_DECLARE_DATATYPE - -template{})> -std::true_type is_basic_aux(T ); -std::false_type is_basic_aux(...); - -template -struct is_basic : decltype(is_basic_aux(std::declval())) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - -template constexpr bool is_basic_v = is_basic::value; - -} // end namespace detail - -template class datatype; - -template class default_datatype { - public: - template{}.get())> - R operator()() const { return boost::mpi3::detail::basic_datatype{}.get(); } - template{}.get())> - R get() const { return boost::mpi3::detail::basic_datatype{}.get(); } -}; - -template -auto datatype_detect(...) -> default_datatype; - -template -auto datatype_detect(U const&) -> default_datatype>; - -template class datatype : public decltype(datatype_detect(std::declval())) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) detection idiom - -template{}())> -std::true_type has_datatype_aux(T ); -std::false_type has_datatype_aux(...); - -template -struct has_datatype : decltype(has_datatype_aux(std::declval())) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - -template constexpr bool has_datatype_v = has_datatype::value; - -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/equality.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/equality.hpp deleted file mode 100644 index dd713b914a5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/equality.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* -*- indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4;-*- */ -//#if COMPILATION_INSTRUCTIONS -//mpic++ -D_TEST_BOOST_MPI3_DETAIL_EQUALITY -xc++ $0 -o $0x&&mpirun -n 4 $0x&&rm $0x;exit -//#endif -// Copyright 2018-2021 Alfredo A. Correa - -#ifndef BOOST_MPI3_DETAIL_EQUALITY_HPP -#define BOOST_MPI3_DETAIL_EQUALITY_HPP - -// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include // underglying_type - -namespace boost{ -namespace mpi3{ -namespace detail{ - -static_assert( sizeof(int)>=sizeof(MPI_IDENT), "standard requires"); - -enum equality : int{ - identical = MPI_IDENT, // same (same address) - congruent = MPI_CONGRUENT, // equal (in the value sense) - similar = MPI_SIMILAR, // same processes in different order (permutation) - unequal = MPI_UNEQUAL // not equal in any sense -}; - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_DETAIL_EQUALITY - -//namespace mpi3 = boost::mpi3; - -//int main(){ - -// mpi3::detail::equality q = mpi3::detail::identical; -// q = mpi3::detail::congruent; -// q = mpi3::detail::similar; -// q = mpi3::detail::unequal; - -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator.hpp deleted file mode 100644 index 3254de53b96..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator.hpp +++ /dev/null @@ -1,170 +0,0 @@ -//#if COMPILATION_INSTRUCTIONS -//(echo "#include\""$0"\"">$0x.cpp) && mpic++ -O3 -std=c++17 -Wfatal-errors -D_BOOST_MPI3_MAIN_ENVIRONMENT -D_TEST_MPI3_DETAIL_ITERATOR $0x.cpp -o $0x.x && time mpirun -n 1 $0x.x $@ && rm -f $0x.cpp; exit -//#endif -// Copyright 2018-2021 Alfredo A. Correa - -#ifndef MPI3_DETAIL_ITERATOR_HPP -#define MPI3_DETAIL_ITERATOR_HPP - -#include // iterator_traits - -namespace boost{ -namespace mpi3{ -namespace detail{ - -template struct is_declared_contiguous : std::false_type{}; - -#if 0 -template< - class It, - class V = typename std::iterator_traits::value_type, - typename = std::enable_if_t< - /**/std::is_convertible::const_iterator>{} - or std::is_convertible::const_iterator>{} - or std::is_convertible::const_iterator>{} - or std::is_pointer{} - or std::is_constructible::pointer, It>{} - or std::is_convertible::pointer>{} - or is_declared_contiguous{} - > -> -typename std::iterator_traits::pointer -data(It const& it){return std::addressof(*it);} -#endif - -template struct priority : priority{}; -template<> struct priority<0>{}; - -template().base())> -typename std::iterator_traits::pointer -data(It it, priority<5> /*priority*/){return it.base();} - -template().get_ptr())> -typename std::iterator_traits::pointer -data(It it, priority<0> /*priority*/){return it.get_ptr();} - -template auto data(It it) -> decltype(data(it, priority<>{})){ - return data(it, priority<>{}); -} - -template auto data(T* t){return t;} - -template -typename std::iterator_traits::value_type const* -cdata(It it){return data(it);} - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_MPI3_DETAIL_ITERATOR - -//#include "../../mpi3/allocator.hpp" -//#include "../../mpi3/main.hpp" -//#include "../../mpi3/vector.hpp" - -//#include -//#include -//#include -//#include -//#include - -//#include - -//#include -//#include -//#include -//#include -//#include -//#include - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int, char*[], mpi3::environment&){ - -// using mpi3::detail::data; -// using mpi3::detail::cdata; - -// { -// std::vector v(30); -// assert( data(begin(v)) == &v[0] ); -// assert( data(begin(v) + 10) == &v[10] ); -// } -// { -// std::array arr; -// assert( data(begin(arr)) == &arr[0] ); -// assert( data(begin(arr) + 2) == &arr[2] ); -// } -// { -// std::valarray varr(30); -// varr[0] = 11.; -// assert( data(begin(varr)) == &varr[0] ); -// assert( data(begin(varr) + 4) == &varr[4] ); -// } -// { -// std::set s; -// s.insert(13.); -// // assert( *data(begin(s)) == 13. ); // data fails here -// } -// { -// std::vector v(30); -// v[0] = 10.; -// assert( *data(v.cbegin()) == 10. ); -// } -// { -// std::array arr; -// assert( data(arr.cbegin()) == &arr[0] ); -// *data(arr.begin()) = 12.; -// assert( arr[0] == 12. ); -// } -// { -// std::array arr; -// arr[0] = 10.; -// assert( *cdata(arr.begin()) == 10. ); -// } -// { -// std::string s = "hola"; -// assert( *data(begin(s)) == 'h' ); -// assert( data(begin(s) + 2) == &s[2] ); -// } -// // all boost.containers iterator that are contiguous are particular cases of vector -// { -// boost::container::static_vector const sv(20); -// assert( *data(sv.begin()) == 0. ); -// } -// { -// boost::container::flat_set fs = {1.,2.,3.}; // flat_set::iterator is the same type as static_vector::iterator -// assert( data(begin(fs))[1] == 2. ); -// } -// { -// boost::container::flat_map fm = {{1., 10},{2.,20},{3.,30}}; assert( data(begin(fm))[1] == (std::pair(2., 20)) ); -// } -// { -// boost::container::small_vector sv(3); sv[1] = 2.;// = {1.,2.,3.}; -// assert( data(sv.begin())[1] == 2. ); -// } -// { -// boost::container::stable_vector v(10); v[1] = 5.; -// // assert ( data(v.begin())[1] == 5. ); // stable vector is not contiguous -// } -// { -// std::vector v(30, false); v[2] = true; -// assert(begin(v)[2] == true); -// // assert( data(v.begin())[1] == true ); // vector::iterator is not contiguous -// } -// { -// assert(mpi3::detail::is_contiguous::iterator>{}); -// } -// { -// std::vector> v(100); -// assert( data(begin(v)) == &v[0] ); -// assert( mpi3::detail::is_contiguous::iterator>{} ); -// assert((mpi3::detail::is_contiguous>::iterator>{}) ); -// assert( mpi3::detail::is_contiguous{} ); -// assert( mpi3::detail::is_basic{} ); -// } -// return 0; -//} -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator_traits.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator_traits.hpp deleted file mode 100644 index 7523118ed76..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/iterator_traits.hpp +++ /dev/null @@ -1,115 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef MPI3_DETAIL_ITERATOR_TRAITS_HPP -#define MPI3_DETAIL_ITERATOR_TRAITS_HPP -#pragma once - -#include "./iterator.hpp" - -#include "./just.hpp" - -#include -#include - -namespace boost { -namespace mpi3 { -namespace detail { - -template -struct iterator_traits : std::iterator_traits {}; - -struct unspecified {}; -struct output_iterator_tag {using base = unspecified;}; -struct input_iterator_tag {using base = unspecified;}; -struct forward_iterator_tag : input_iterator_tag {using base = input_iterator_tag;}; -struct random_access_iterator_tag : forward_iterator_tag { - using base = forward_iterator_tag; -}; -struct contiguous_iterator_tag : random_access_iterator_tag { // NOLINT(bugprone-forward-declaration-namespace) interferes with boost.move.detail.iterator - using base = random_access_iterator_tag; -}; -struct strided_contiguous_iterator_tag : std::random_access_iterator_tag {}; - -template struct std_translate; - -template<> struct std_translate {using type = output_iterator_tag;}; -template<> struct std_translate {using type = input_iterator_tag;}; -template<> struct std_translate {using type = forward_iterator_tag;}; -template<> struct std_translate {using type = forward_iterator_tag;}; -template<> struct std_translate {using type = random_access_iterator_tag;}; - -//template struct is_declared_contiguous : std::false_type{}; -template struct is_contiguous; - -template()))> -std::true_type is_contiguous_aux(It ); -std::false_type is_contiguous_aux(...); - -template struct is_contiguous -: decltype(is_contiguous_aux(std::declval())){}; - -template -std::true_type has_data_aux(T ); -std::false_type has_data_aux(...); - -template struct has_data : decltype(has_data_aux(std::declval())) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - -template::value> > -typename std_translate::iterator_category>::type iterator_category_aux(It); - -//template::iterator_category, std::output_iterator_tag>{}>> -//std_translate::type iterator_category_aux(It); - -// intel compiler 17 needs this specialization -template -contiguous_iterator_tag iterator_category_aux(T*); - -template{}>> -contiguous_iterator_tag iterator_category_aux(Iter); -template{}>> -strided_contiguous_iterator_tag iterator_category_aux(Iter); - -template -struct iterator_category { - using type = decltype(iterator_category_aux(std::declval>())); -}; -template -using iterator_category_t = typename iterator_category::type; - -template struct forward_iterator : just_t { - using just_t::just_t; -}; -template struct random_access_iterator : forward_iterator { - using forward_iterator::forward_iterator; -}; - -template struct contiguous_iterator : random_access_iterator { - using random_access_iterator::random_access_iterator; -}; - -template -forward_iterator category_iterator_aux(Iter&& it, forward_iterator_tag /*forward*/) { - return {std::forward(it)}; -} - -template -random_access_iterator category_iterator_aux(Iter&& it, random_access_iterator_tag /*random_access*/) { - return {std::forward(it)}; -} - -template -contiguous_iterator category_iterator_aux(Iter&& it, contiguous_iterator_tag /*contiguous*/) { - return {std::forward(it)}; -} - -template -auto category_iterator(Iter&& it){ - return category_iterator_aux(std::forward(it), typename iterator_category::type{}); -} - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -#endif \ No newline at end of file diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/just.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/just.hpp deleted file mode 100644 index 6dc08a05ba2..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/just.hpp +++ /dev/null @@ -1,159 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef BOOST_JUST_HPP -#define BOOST_JUST_HPP - -#include -#include // std::forward - -namespace boost { - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) : -#define BOOST_JUST_REPRODUCE_OPERATOR(OperatoR) \ - template \ - auto OperatorR(T&&... t) - -template -struct wrapper /*: T*/ { - using type = T; - T value; // NOLINT(misc-non-private-member-variables-in-classes) : wrapper - - template(std::declval()...)...))> - explicit wrapper(Args&&... args) : value(std::forward(args)...) {} - - template() = std::declval())> - wrapper& operator=(Args&& args) { - value = std::forward(args); - return *this; - } - - explicit operator T &() {return value;} - explicit operator T const&() const {return value;} - - template - auto operator[](Arg&& arg) const - ->decltype(value[std::forward(arg)]) { - return value[std::forward(arg)]; } - - auto operator*()->decltype(*std::declval()) {return *value;} -}; - -template -using reference = wrapper; - -template -struct just : - std::conditional< - std::is_class::value, - T, - typename std::conditional::value, - std::array::type, std::extent::value>, - wrapper - >::type - > /*no ::type here*/ { -}; - -template -using just_t = typename just::type; - -#undef BOOST_JUST_REPRODUCE_OPERATOR - -//template -//typename just::type& _(T&& t){ -// return reinterpret_cast::type&>(std::forward(t)); -//} - -//template -//typename just::type& wrap(T&& t){ -// return reinterpret_cast::type&>(std::forward(t)); -//} - - -/* -template -struct just{// : boost::reference_wrapper{ - T* t_ptr; - just(T& t) : t_ptr(&t){} - typedef just type; - just& operator=(T const& t){*t_ptr = t;} -// typedef std::reference_wrapper type; -};*/ - -/* -template<> -struct just{ - double impl_; - typedef just type; - just() : impl_(){} - just(double const& d) : impl_(d){} - operator double const&() const{return impl_;} - double& operator+=(double const& d){return impl_+=d;} -}; -template<> -struct just{ - bool impl_; - typedef just type; - just(bool const& d) : impl_(d){} - operator bool const&() const{return impl_;} -// double& operator+=(double const& d){return impl_+=d;} -}; -*/ - -} // end namespace boost - -#endif - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) : TODO(correaa) : check if this is necessary -#define BOOST_INHERIT_UNARY_CONSTRUCTOR(MyclasS, MybaseclasS) \ - template MyclasS(A& arg) : MybaseclasS(arg) {} - -// #ifdef _TEST_BOOST_JUST - -// //struct number : boost::just::type{ -// // BOOST_INHERIT_UNARY_CONSTRUCTOR(number, boost::just::type) -// //}; - -// #include -// #include -// #include - -// template -// class A : boost::just::type {}; - -// int main() { -// A a; - -// A b; -// assert( std::is_class::value == false ); -// assert( std::is_array::value == true ); - -// { -// double d=5; -// boost::wrapper n(d); -// n+=4; -// std::cout<< n < n(d); -// double aa = 6.; -// std::cout<< n <> v3; -// v3.push_back(d); -// v3.push_back(d); -// v3[0] = 4.; -// std::cout << v3[0] << " " << v3[1] << std::endl; -// } -// } -// #endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/package.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/package.hpp deleted file mode 100644 index 4bceb71817a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/package.hpp +++ /dev/null @@ -1,167 +0,0 @@ -#ifndef MPI3_DETAIL_PACKAGE_HPP -#define MPI3_DETAIL_PACKAGE_HPP - -// TODO(correaa) move from detail to root - -#include "../../mpi3/vector.hpp" - -#include "../../mpi3/detail/basic_communicator.hpp" -#include "../../mpi3/detail/buffer.hpp" -#include "../../mpi3/detail/iterator.hpp" -#include "../../mpi3/detail/iterator_traits.hpp" -#include "../../mpi3/detail/value_traits.hpp" - -namespace boost { -namespace mpi3 { - -class communicator; - -namespace detail{ - -struct package : buffer { - private: - basic_communicator& bcomm_; - - public: - explicit package(communicator& comm, buffer::size_type n = 0) - : buffer{n}, bcomm_{reinterpret_cast(comm)} { // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) TODO(correaa) break cyclic dependency of classes - reserve(n); - } - package(package const&) = delete; - package(package &&) = delete; - - package& operator=(package const&) = delete; - package& operator=(package &&) = delete; - - ~package() noexcept = default; - - template - void pack_n(It first, Size count){ - bcomm_.pack_n(first, count, static_cast(*this)); - } - template - auto pack(It first, It last){ - bcomm_.pack(first, last, static_cast(*this)); - } - template - void unpack_n(It first, Size count){ - bcomm_.unpack_n(static_cast(*this), first, count); - } - template - void unpack(It first, It last){ - bcomm_.unpack(static_cast(*this), first, last); - } - explicit operator bool() const {return pos < static_cast(size());} - - template - package& operator>>(T& t){ - unpack_n(std::addressof(t), 1); - return *this; - } - auto send(int dest, int tag = 0) { - return bcomm_.send(static_cast(*this), dest, tag); - } - auto receive(int dest, int tag = 0) { - return bcomm_.receive(static_cast(*this), dest, tag); - } - -// package const& send(int dest, int tag = 0) const; -/* package const& send(int dest, int tag = 0) const{ - comm_.send_packed_n(buffer_.data(), in_, dest, tag); - return *this; - }*/ -// package& receive(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG); -/* package& receive(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ - MPI_Status status; - MPI_Message msg; - int count = -1; - MPI_Mprobe(source, tag, comm_.impl_, &msg, &status); - MPI_Get_count(&status, MPI_PACKED, &count); - buffer_.resize(count); - MPI_Mrecv(buffer_.data(), count, MPI_PACKED, &msg, MPI_STATUS_IGNORE); - // int n = comm_.probe(source, tag).count(); - // buffer_.resize(n); - // comm_.receive_packed_n(buffer_.data(), n, source, tag); - return *this; - }*/ -/* package& broadcast(int root = 0){ // see https://www.researchgate.net/publication/228737912_Dynamically-Sized_Messages_in_MPI-3 - comm_.broadcast_value(in_, root); - buffer_.resize(in_); - comm_.broadcast_n(buffer_.data(), in_, root); - return *this; - }*/ -// package& gather(int root = 0){ -// -// } - -// template int size(int n = 1) const{return comm_.pack_size(n);} - -}; - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_MPI3_PACKAGE - -//#include "../../mpi3/communicator.hpp" -//#include "../../mpi3/main.hpp" - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - -// if(world.rank() == 0){ -// char buf[1000]; -// int i = 12; -// int j = 13; -// auto end = world.pack_n(&i, 1, buf); -// end = world.pack_n(&j, 1, end); -// world.send_packed(buf, end, 1); //world.send_packed_n(buff, std::distance(buff, end), 1); //world.send_packed_n(buff, end - buff, 1); -// world.send_packed_n(buf, 1000, 2); -// }else if(world.rank() == 1){ -// std::vector v(2); -// world.receive(v.begin(), v.end(), 0); -// assert(v[0] == 12); -// assert(v[1] == 13); -// }else if(world.rank() == 2){ -// char buf[1000]; -// world.receive_packed_n(buf, 1000, 0); -// int i = -1; -// int j = -1; -// auto end = world.unpack_n(&i, 1, buf); -// end = world.unpack_n(&j, 1, end); -// assert(i == 12); -// assert(j == 13); -// } -// world.barrier(); -//// return 0; - -// if(world.rank() == 0){ -// mpi3::package p(world); -// int i = 12; -// int j = 13; -// (p << i << j).send(1).send(2); -// // p.send(1); -// // p.send(2); -// }else if(world.rank() == 1){ -// std::vector v(2, -1); -// world.receive(v.begin(), v.end(), 0); -// assert(v[0] = 12); -// assert(v[1] == 13); -// }else if(world.rank() == 2){ -// mpi3::package p(world); -// int i = -1; -// int j = -1; -// p.receive(0) -// >> i -// >> j -// ; -// assert(i == 12); -// assert(j == 13); -// } -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/tuple_offset.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/tuple_offset.hpp deleted file mode 100644 index 1777833ded4..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/tuple_offset.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef MPI3_DETAIL_TUPLE_OFFSET_HPP -#define MPI3_DETAIL_TUPLE_OFFSET_HPP - -#include - -namespace boost::mpi3::detail { - -template -[[deprecated]] constexpr std::size_t tuple_offset_aux() { // from https://stackoverflow.com/questions/70647441/how-to-determine-the-offset-of-an-element-of-a-tuple-at-compile-time - static_assert(not std::is_reference_v>); - union u { - constexpr u() : a{} {} // GCC bug needs a constructor definition - char a[sizeof(Tuple)]; // NOLINT(cppcoreguidelines-avoid-c-arrays,cppcoreguidelines-pro-bounds-pointer-arithmetic,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) pretty low level stuff - Tuple t; - } x; - auto off = - std::find_if( - static_cast(x.a), static_cast(x.a) + sizeof(Tuple), // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic) pretty low level stuff - [&x](char const& e) { std::addressof(e) == std::addressof(std::get(x.t)); } - ) - - static_cast(x.a); - // std::size_t off = 0; - // while(static_cast(x.a + off) != std::addressof(std::get(x.t))) {++off;} // NOLINT(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-bounds-pointer-arithmetic) - return off; -} - -template -std::size_t element_offset() { - static std::size_t const ret = [] { - static_assert(!std::is_reference_v>); - union u { - constexpr u() : a{} {} // GCC bug needs a constructor definition - char a[sizeof(Tuple)]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - Tuple t; - } x; - return reinterpret_cast(std::addressof(std::get(x.t))) - x.a; // NOLINT(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-type-reinterpret-cast) - }(); - return ret; -} - -template -struct tuple_offset : std::integral_constant()> {}; - -template -constexpr std::size_t tuple_offset_v = tuple_offset::value; - -} // end namespace boost::mpi3::detail -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/value_traits.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/value_traits.hpp deleted file mode 100644 index 4661a1903bb..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/detail/value_traits.hpp +++ /dev/null @@ -1,83 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2017-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_DETAIL_VALUE_TRAITS_HPP -#define BOOST_MPI3_DETAIL_VALUE_TRAITS_HPP - -//#include "../../mpi3/detail/package_archive.hpp" -//#include "../../mpi3/communicator.hpp" - -#include - -#include - -#include -#include - -namespace boost { -namespace mpi3 { -namespace detail { - -/* -template -struct is_memcopyable : std::integral_constant{}>{}; - -template -struct is_memcopyable> : - std::integral_constant{} and is_memcopyable>{} - > -{}; - -template -struct is_memcopyable> : - std::integral_constant{} and is_memcopyable{} - > -{}; -*/ -/* -template< - class T, - typename = decltype(std::declval() << std::declval()), - typename = decltype(std::declval() >> std::declval()) -> -std::true_type is_serializable_aux(T); -std::false_type is_serializable_aux(...); -*/ - -template -struct is_serializable : decltype(is_serializable_aux(std::declval())){}; - -struct value_unspecified_tag{}; -struct nonmemcopyable_serializable_tag : value_unspecified_tag{ - using base = value_unspecified_tag; -}; -struct memcopyable_tag : value_unspecified_tag{ - using base = value_unspecified_tag; -}; -struct basic_tag : memcopyable_tag{using base = memcopyable_tag;}; - -//template{} and not is_basic{}>> -//memcopyable_tag value_category_aux(V&&); -template{} or has_datatype{}, int> =0> -basic_tag value_category_aux(V const&); - -//template{}, int> =0> -//basic_tag value_category_aux(V const&); - -template -value_unspecified_tag value_category_aux(V[N]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) : legacy compatibility -value_unspecified_tag value_category_aux(... ); - -template -struct value_category{using type = decltype(value_category_aux(std::declval()));}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) - -template -using value_category_t = typename value_category::type; - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/dummy/.dummy b/external_codes/mpi_wrapper/mpi3/include/mpi3/dummy/.dummy deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/dynamic_window.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/dynamic_window.hpp deleted file mode 100644 index 7cb22fee673..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/dynamic_window.hpp +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ -#ifndef BOOST_MPI3_DYNAMIC_WINDOW_HPP -#define BOOST_MPI3_DYNAMIC_WINDOW_HPP - -#include - -#include "../mpi3/window.hpp" - -namespace boost { -namespace mpi3 { - -template -struct dynamic_window : window{ - protected: - dynamic_window() : window{}{} - public: - explicit dynamic_window(communicator& comm){ - MPI_(Win_create_dynamic)(MPI_INFO_NULL, comm.get(), &(this->impl_)); - } - template - void attach_n(TT* base, mpi3::size_t n){MPI_Win_attach(this->impl_, base, n*sizeof(TT));} - void detach(void* base){MPI_Win_detach(this->impl_, base);} -/* void* bottom() const{ - void* base; int flag; - int s = MPI_Win_get_attr(impl_, MPI_BOTTOM, &base, &flag); - if(s != MPI_SUCCESS) throw std::runtime_error("cannot get base"); - assert(flag); - return base; - }*/ - -}; - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_DYNAMIC_WINDOW - -//#include "../mpi3/main.hpp" - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - -// mpi3::dynamic_window dw(world); -// std::vector a(1000, world.rank()); -// dw.attach_n(a.data(), a.size()); -// world.barrier(); -// dw.lock(0); -// if(world.rank() == 0){ -// std::vector v(1000); -// dw.get_n(v.data(), v.size(), 3, 0); -// assert(a[0] == 3); -// } -// dw.unlock(0); -// world.barrier(); -// dw.detach(a.data()); - -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/environment.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/environment.hpp deleted file mode 100644 index ce44ba437ec..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/environment.hpp +++ /dev/null @@ -1,315 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_ENVIRONMENT_HPP -#define BOOST_MPI3_ENVIRONMENT_HPP - -#pragma once - -#include "./communicator.hpp" -#include "./core.hpp" -#include "./wall_clock.hpp" - -#include "./detail/call.hpp" -#include "./version.hpp" - -#include - -// #include // for std::clog -#include - -namespace boost { -namespace mpi3 { - -enum class thread_level : int { - single = MPI_THREAD_SINGLE, - funneled = MPI_THREAD_FUNNELED, - serialized = MPI_THREAD_SERIALIZED, - multiple = MPI_THREAD_MULTIPLE -}; - -using thread = thread_level; - -inline void finalize() { - assert(initialized()); - assert(not finalized()); - - if(int const count = std::uncaught_exceptions()) { - std::cerr << "finalizing MPI environment with " << count << " uncaught exceptions"; - } - - std::set_terminate(&std::abort); - int const s = MPI_Finalize(); // TODO(correaa) modernize call? - if(s != MPI_SUCCESS) { - std::terminate(); - } //{throw std::runtime_error{"cannot finalize"};} -} -inline void myterminate() { - std::cerr << "myterminate handler called" << '\n'; - finalize(); - std::abort(); -} - -inline void initialize(int& argc, char**& argv) { - assert(not initialized()); // double initialize - assert(not finalized()); - - if(mpi3::version() != mpi3::Version()) { - std::cerr << "WARNING: MPI version inconsistency\n"; - std::cerr << "Compile version (" << mpi3::version() <<") and linked version (" << mpi3::Version() << ") do not agree. Likely a link error."; - } - - if([[maybe_unused]] char const* ompi_size_cstr = std::getenv("OMPI_COMM_WORLD_SIZE")) { // NOLINT(concurrency-mt-unsafe) -#ifndef OMPI_MAJOR_VERSION - if(char const* ompi_rank_cstr = std::getenv("OMPI_COMM_WORLD_RANK")) { // NOLINT(concurrency-mt-unsafe) - if(std::string{ompi_rank_cstr} == "0") { - std::cerr << "WARNING: MPI environment inconsistency?\n"; - std::cerr << "running program " << *argv << " compiled with " << boost::mpi3::library_version_short() << " but allocated with OMPI (np = " << ompi_size_cstr << "). Try running with `mpirun.mpich` or load MPICH module.\n\n"; - } - } else { - std::cerr << "WARNING: MPI environment inconsistency?\n"; - std::cerr << "running program " << *argv << " compiled with " << boost::mpi3::library_version_short() << " but allocated with OMPI (np = " << ompi_size_cstr << "). Try running with `mpirun.mpich` or load MPICH module.\n\n"; - } - using namespace std::chrono_literals; - std::this_thread::sleep_for(1s); -#endif - } - if([[maybe_unused]] char const* pmi_size_cstr = std::getenv("PMI_SIZE")) { // NOLINT(concurrency-mt-unsafe) -#ifndef MPICH_VERSION - if(char const* pmi_rank_cstr = std::getenv("PMI_RANK")) { // NOLINT(concurrency-mt-unsafe) - if(std::string{pmi_rank_cstr} == "0") { - std::cerr << "WARNING: MPI environment inconsistency?\n"; - std::cerr << "running program " << *argv << " compiled with " << boost::mpi3::library_version_short() << " but allocated with PMI (Hydra or MPICH) (np = " << pmi_size_cstr << "). Try running with `mpirun.openmpi` or load OpenMPI module.\n\n"; - } - } else { - std::cerr << "WARNING: MPI environment inconsistency?\n"; - std::cerr << "running program " << *argv << " compiled with " << boost::mpi3::library_version_short() << " but allocated with PMI (Hydra or MPICH) (np = " << pmi_size_cstr << "). Try running with `mpirun.openmpi` or load OpenMPI module.\n\n"; - } - using namespace std::chrono_literals; - std::this_thread::sleep_for(1s); -#endif - } - - MPI_(Init)(&argc, &argv); // TODO(correaa) modernize call? - - int nprocs = -1; - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - int rank = -1; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if([[maybe_unused]] char const* ompi_size_cstr = std::getenv("OMPI_COMM_WORLD_SIZE")) { // NOLINT(concurrency-mt-unsafe) - char const* ompi_rank_cstr = std::getenv("OMPI_COMM_WORLD_RANK"); // NOLINT(concurrency-mt-unsafe) - if(std::to_string(nprocs) != ompi_size_cstr and std::string{ompi_rank_cstr} == "0" and rank == 0) { - std::cerr << "WARNING: MPI size inconsistency?\n"; - std::cerr << "running program " << *argv << " in " << std::to_string(nprocs) << " processes but allocated with " << ompi_size_cstr << " processes \n\n"; - MPI_Barrier(MPI_COMM_WORLD); - using namespace std::chrono_literals; - std::this_thread::sleep_for(1s); - } - } - if([[maybe_unused]] char const* pmi_size_cstr = std::getenv("PMI_SIZE")) { // NOLINT(concurrency-mt-unsafe) - char const* pmi_rank_cstr = std::getenv("PMI_RANK"); // NOLINT(concurrency-mt-unsafe) - if(std::to_string(nprocs) != pmi_size_cstr and std::string{pmi_rank_cstr} == "0" and rank == 0) { - std::cerr << "WARNING: MPI size inconsistency?\n"; - std::cerr << "running program " << *argv << " in " << std::to_string(nprocs) << " processes but allocated with " << pmi_size_cstr << " processes \n\n"; - MPI_Barrier(MPI_COMM_WORLD); - using namespace std::chrono_literals; - std::this_thread::sleep_for(1s); - } - } -} - -inline thread_level initialize(int& argc, char**& argv, thread_level required) { - int provided; // NOLINT(cppcoreguidelines-init-variables) : delayed initialization - MPI_(Init_thread)(&argc, &argv, static_cast(required), &provided); - return static_cast(provided); -} - -inline thread_level initialize_thread(thread_level required) { - int provided; // NOLINT(cppcoreguidelines-init-variables) delayed initialization - MPI_(Init_thread)(nullptr, nullptr, static_cast(required), &provided); - return static_cast(provided); -} - -inline thread_level initialize(thread_level required = thread_level::single) { - return initialize_thread(required); -} - -// inline void throw_error_fn(MPI_Comm* comm, int* errorcode, ...) { -// char name[MPI_MAX_OBJECT_NAME]; -// int rlen; // NOLINT(cppcoreguidelines-init-variables) delayed initialization -// int status = MPI_Comm_get_name(*comm, name, &rlen); -// if(status != MPI_SUCCESS) {throw std::runtime_error{"cannot get name"};} -// std::string sname(name, rlen); -// throw std::runtime_error{"error code "+ std::to_string(*errorcode) +" from comm "+ sname}; -// } - -inline thread_level initialize_thread( - int& argc, char**& argv, thread_level required -) { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed initialization - MPI_(Init_thread) - (&argc, &argv, static_cast(required), &ret); - return static_cast(ret); -} - -inline thread_level thread_support() { - int r; // NOLINT(cppcoreguidelines-init-variables) : delayed initialization - MPI_(Query_thread) - (&r); - return static_cast(r); -} - -inline bool is_thread_main() { - int flag; // NOLINT(cppcoreguidelines-init-variables) : delayed initialization - MPI_(Is_thread_main) - (&flag); - return flag != 0; -} - -inline std::string processor_name() { return detail::call<&MPI_Get_processor_name>(); } - -inline std::string get_processor_name() { return detail::call<&MPI_Get_processor_name>(); } - -class environment { - public: - environment() { - initialize_thread(thread_level::multiple); - // std::clog << "ctor() environment" << std::endl; - named_attributes_key_f() = std::make_unique>>(); - } - explicit environment(thread_level required) { - // std::clog << "ctor(thread_level) environment" << std::endl; - initialize_thread(required); - named_attributes_key_f() = std::make_unique>>(); - } - explicit environment(int& argc, char**& argv) { // cppcheck-suppress constParameter ; bug in cppcheck 2.3 or it can't see through the MPI C-API - initialize(argc, argv); // initialize(argc, argv); // TODO have an environment_mt/st version? - named_attributes_key_f() = std::make_unique>>(); - } - explicit environment(int& argc, char**& argv, thread_level required) { // cppcheck-suppress constParameter ; bug in cppcheck 2.3 - initialize(argc, argv, required); - named_attributes_key_f() = std::make_unique>>(); - } - - environment(environment const&) = delete; - environment(environment&&) = delete; - - environment& operator=(environment const&) = delete; - environment& operator=(environment&&) = delete; - - ~environment() noexcept { // NOLINT(bugprone-exception-escape) finalizes throws as an instance of UB - // std::clog << "dtor environment" << std::endl; - named_attributes_key_f().reset(); - finalize(); // cppcheck-suppress throwInNoexceptFunction ; finalizes throws as an instance of UB - } - - inline static thread_level thread_support() { return mpi3::thread_support(); } - // static /*inline*/ communicator::keyval const* color_key_p; - // static communicator::keyval const& color_key(){return *color_key_p;} - // static /*inline*/ communicator::keyval> const* named_attributes_key_p; - static std::unique_ptr> const>& named_attributes_key_f() { - static std::unique_ptr> const> named_attributes_key_p; - return named_attributes_key_p; - } - static communicator::keyval> const& named_attributes_key() { - // static communicator::keyval> const named_attributes_key_p; - // return named_attributes_key_p; - return *named_attributes_key_f(); - } - - static inline void initialize() { mpi3::initialize(); } - static inline void initialize(int argc, char** argv) { mpi3::initialize(argc, argv); } - - static inline thread_level initialize(thread_level required) { return mpi3::initialize_thread(required); } - static inline thread_level initialize(int argc, char** argv, thread_level req) { return mpi3::initialize_thread(argc, argv, req); } - - static inline void finalize() { mpi3::finalize(); } - - static inline bool is_initialized() { return mpi3::initialized(); } - static inline bool is_finalized() { return mpi3::finalized(); } - - using wall_clock = mpi3::wall_clock; - explicit operator bool() const { return initialized(); } - - static bool is_thread_main() { return mpi3::is_thread_main(); } - - static inline communicator& get_self_instance() { - assert(initialized()); - static communicator instance = [] { - // MPI_Comm_create_errhandler(&throw_error_fn, &throw_error_); - // MPI_Comm_set_errhandler(MPI_COMM_WORLD, throw_error_); - MPI_Comm_set_errhandler(MPI_COMM_SELF, MPI_ERRORS_RETURN); - // MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); - return communicator{MPI_COMM_SELF}; - }(); - return instance; - } - - static communicator self() { // returns a copy! - MPI_Comm_set_errhandler(MPI_COMM_SELF, MPI_ERRORS_RETURN); - return communicator{MPI_COMM_SELF}; - } - static inline communicator& get_world_instance() { - assert(initialized()); - static communicator instance = [] { - // MPI_Comm_create_errhandler(&throw_error_fn, &throw_error_); - // MPI_Comm_set_errhandler(MPI_COMM_WORLD, throw_error_); - MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); - // MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); - return communicator{MPI_COMM_WORLD}; - }(); - return instance; - } - - [[nodiscard]] communicator world() { // NOLINT(readability-convert-member-functions-to-static) to force instance - communicator ret{get_world_instance()}; - ret.set_name("world"); - return ret; - } - - static std::string processor_name() { return get_processor_name(); } - static std::string get_processor_name() { return mpi3::get_processor_name(); } - - inline static auto wall_time() { return mpi3::wall_time(); } - inline static auto wall_tick() { return mpi3::wall_tick(); } - - template - static auto wall_sleep_for(Duration d) { return mpi3::wall_sleep_for(d); } -}; - -inline mpi3::any& communicator::attribute(std::string const& s) { - return attribute(environment::named_attributes_key())[s]; -} - -} // end namespace mpi3 -} // end namespace boost - -//#if not __INCLUDE_LEVEL__ // _TEST_BOOST_MPI3_ENVIRONMENT - -//#include // this_tread::sleep_for -//#include - -// namespace mpi3 = boost::mpi3; -// using std::cout; -// using namespace std::chrono_literals; // 2s - -// int main(){//int argc, char* argv[]){ -// mpi3::environment::initialize(mpi3::thread_level::multiple);//argc, argv); // same as MPI_Init(...); -// assert( mpi3::environment::thread_support() == mpi3::thread_level::multiple ); -// assert(mpi3::environment::is_initialized()); -// { -// mpi3::communicator world = mpi3::environment::get_world_instance(); // a copy -// auto then = mpi3::environment::wall_time(); -// std::this_thread::sleep_for(2s); -// // cout<< (mpi3::environment::wall_time() - then).count() <<" seconds\n"; -// } -// mpi3::environment::finalize(); // same as MPI_Finalize() -// assert(mpi3::environment::is_finalized()); -//// or better: -//// mpi3::environment env(argc, argv); -//// auto world = env.world(); -//} -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/error.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/error.hpp deleted file mode 100644 index 67cc89f55cc..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/error.hpp +++ /dev/null @@ -1,99 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2017-2022 Alfredo A. Correa - -#ifndef BOOST_MPI3_ERROR_HPP -#define BOOST_MPI3_ERROR_HPP - -#include - -#include -#include - -namespace boost { -namespace mpi3 { - -static_assert(sizeof(MPI_SUCCESS) <= sizeof(int)); - -enum class error : int { // decltype(MPI_SUCCESS) { - success = MPI_SUCCESS, - invalid_buffer_pointer = MPI_ERR_BUFFER, - invalid_count = MPI_ERR_COUNT, - invalid_datatype = MPI_ERR_TYPE, - invalid_tag = MPI_ERR_TAG, - invalid_communicator = MPI_ERR_COMM, - invalid_rank = MPI_ERR_RANK, - invalid_root = MPI_ERR_ROOT, - invalid_group = MPI_ERR_GROUP, - invalid_operation = MPI_ERR_OP, - invalid_topology = MPI_ERR_TOPOLOGY, - illegal_dimension = MPI_ERR_DIMS, - invalid_dimension = MPI_ERR_DIMS, - invalid_argument = MPI_ERR_ARG, - invalid_domain = MPI_ERR_ARG, - unknown = MPI_ERR_UNKNOWN, - truncated_message = MPI_ERR_TRUNCATE, - other = MPI_ERR_OTHER, - internal = MPI_ERR_INTERN, - in_status = MPI_ERR_IN_STATUS, - pending = MPI_ERR_PENDING, - illegal_request = MPI_ERR_REQUEST, - last_code = MPI_ERR_LASTCODE -}; - -auto inline string(enum error err) { - std::array estring{}; - int len; // NOLINT(cppcoreguidelines-init-variables) delayed initialization - MPI_Error_string(static_cast(err), estring.data(), &len); - return std::string(estring.data(), static_cast(len)); -} - -struct error_category : std::error_category { - char const* name() const noexcept override{return "mpi3 wrapper";} - std::string message(int err) const override{return string(static_cast(err));} - static error_category& instance(){ - static error_category instance; - return instance; - } -}; - -inline auto make_error_code(error err) noexcept { - return std::error_code{static_cast(err), error_category::instance()}; -} - -} // end namespace mpi3 -} // end namespace boost - -namespace std { - template<> struct is_error_code_enum<::boost::mpi3::error> : true_type{}; -} // end namespace std - -//#if not __INCLUDE_LEVEL__ // def _TEST_BOOST_MPI3_ERROR - -//#include "../mpi3/main.hpp" - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// std::error_code ec = mpi3::error::invalid_buffer_pointer; -// assert( ec == mpi3::error::invalid_buffer_pointer); -// assert( ec != std::io_errc::stream ); - -// try { -// world.broadcast_n((int*)nullptr, 0, -1); -// } catch(std::system_error const& e) { -// cout -// <<"code: " << e.code() <<'\n' -// <<"message: "<< e.code().message() <<'\n' -// <<"what: " << e.what() <<'\n' -// ; -// } - -// return 0; - -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/error_handler.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/error_handler.hpp deleted file mode 100644 index 7794096ae10..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/error_handler.hpp +++ /dev/null @@ -1,103 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2017-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_ERROR_HANDLER_HPP -#define BOOST_MPI3_ERROR_HANDLER_HPP - -#include - -#include "../mpi3/communicator.hpp" - -namespace boost { -namespace mpi3 { - -class fatal { -// explicit operator MPI_Errhandler() const {return MPI_ERRORS_ARE_FATAL;} -}; - -class code { -// explicit operator MPI_Errhandler() const {return MPI_ERRORS_RETURN;} -}; - -struct error_handler { - MPI_Errhandler impl_ = MPI_ERRORS_ARE_FATAL; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - explicit constexpr error_handler(MPI_Errhandler impl) noexcept : impl_{impl} {} - - error_handler() = default; - error_handler(error_handler const&) = delete; - friend mpi3::communicator; - - error_handler& operator=(error_handler const&) = delete; - error_handler& operator=(error_handler &&) = delete; - - private: - error_handler(error_handler&&) = default; // this is necessary in C++14 (to return from function) - - public: -// error_handler(void(*fn)(MPI_Comm*, int* err, ...)){ -// MPI_Comm_create_errhandler(fn, &impl_); -// } -// void operator()(communicator& comm, int error) const{comm.call_error_handler(error);} - ~error_handler() { - if(impl_ != MPI_ERRORS_ARE_FATAL and impl_ != MPI_ERRORS_RETURN) { - MPI_Errhandler_free(&impl_); - } - } -// static error_handler const exception; - static void exception(MPI_Comm* /*comm*/, int const* err) {//, ...){ - int len; // NOLINT(cppcoreguidelines-init-variables,-warnings-as-errors) delayed init - std::array estring{}; - MPI_Error_string(*err, estring.data(), &len); - std::string const w(estring.data(), static_cast(len)); - throw std::runtime_error{"error code"+ std::to_string(*err) +" "+ w}; -// throw boost::mpi3::exception("error code " + std::to_string(*err) + " from comm " + std::to_string(*comm) + ": " + w); -// throw std::runtime_error("error code " + std::to_string(*err) + " from comm " + std::to_string(*comm) + ": " + w); - } - - static error_handler const fatal; - static error_handler const code; -}; - -error_handler const error_handler::fatal{MPI_ERRORS_ARE_FATAL}; // NOLINT(misc-definitions-in-headers,fuchsia-statically-constructed-objects) TODO(correaa) -error_handler const error_handler::code{MPI_ERRORS_RETURN}; // NOLINT(misc-definitions-in-headers,fuchsia-statically-constructed-objects) TODO(correaa) - -inline void communicator::set_error_handler(error_handler const& eh) { - MPI_(Comm_set_errhandler)(impl_, eh.impl_); -} - -inline error_handler communicator::get_error_handler() const { - error_handler ret; - MPI_(Comm_get_errhandler)(impl_, &ret.impl_); - return ret; -} - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_ERROR_HANDLER - -//#include "../mpi3/main.hpp" - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) { - -//// error_counter_t ec; -//// mpi3::error_handler<&ehh> ehherr; -//// mpi3::error_handler newerr(eh); -//// world.set_error_handler(ec); -//// world.set_error_handler(newerr); -//// world.set_error_handler( -//// world.error(error::other); -//// cout << ec.count() << '\n'; -//// newerr(world, MPI_ERR_OTHER); - -//// auto f = world.get_error_handler(); - -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/exception.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/exception.hpp deleted file mode 100644 index e72760217d7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/exception.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef BOOST_MPI3_EXCEPTION -#define BOOST_MPI3_EXCEPTION - -#include - -namespace boost { -namespace mpi3 { - -struct exception : std::runtime_error { - using runtime_error::runtime_error; -// ~exception() override = default; -}; - -struct invalid_communicator : exception { - using exception::exception; -// ~invalid_communicator() override = default; -}; - -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/future.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/future.hpp deleted file mode 100644 index 4b27a63bfa7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/future.hpp +++ /dev/null @@ -1,49 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wall -Wextra -Wfatal-errors -D_TEST_MPI3_FUTURE $0x.cpp -o $0x.x && time mpirun -n 8 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef MPI3_FUTURE_HPP -#define MPI3_FUTURE_HPP - -#include "../mpi3/communicator.hpp" - -namespace boost{ -namespace mpi3{ - -template -struct future{ - template - future(communicator& comm, F&& f, Args&&... args){ - if(not comm.root()){ - int r = comm.rank(); - comm.send_n(&r, 1, 0); - }else{ - int r = -1; - std::vector reqs; - req - auto req = comm.ireceive_n(&r, 1); - } - } -}; - -// futures factory, todo implement launch policy -template auto async( - communicator& c, F&& f, Args&&... args -) -> future{ - return {c, std::forward(f), std::forward(args)...}; -} - -}} - -#ifdef _TEST_MPI3_FUTURE - -#include "../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int, char*[], mpi3::communicator world){ - return 0; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/generalized_request.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/generalized_request.hpp deleted file mode 100644 index e91ed99f8c5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/generalized_request.hpp +++ /dev/null @@ -1,158 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ -//#if COMPILATION_INSTRUCTIONS -//(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wall `#-Wfatal-errors` -D_TEST_BOOST_MPI3_GENERALIZED_REQUEST $0x.cpp -o $0x.x && time mpirun -n 2 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -//#endif -// Copyright 2018-2021 Alfredo A. Correa - -#ifndef BOOST_MPI3_GENERALIZED_REQUEST_HPP -#define BOOST_MPI3_GENERALIZED_REQUEST_HPP - -#include "../mpi3/request.hpp" -#include "../mpi3/status.hpp" - -#include - -#include - -namespace boost { -namespace mpi3 { - -struct default_request { - static status query() { - status ret{}; - ret.set_source(MPI_UNDEFINED); - ret.set_tag(MPI_UNDEFINED); - ret.set_cancelled(); - ret.set_elements(0); - return ret; - } - void free() {} - void cancel(int /*complete*/) {} -}; - -struct generalized_request : request{ - template - static int query_fn(void *extra_state, MPI_Status *status) { - try { - *status = reinterpret_cast(extra_state)->query().impl_; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) TODO(correaa) - } catch(...) { - return MPI_ERR_UNKNOWN; - } - return MPI_SUCCESS; - } - template - static int free_fn(void* extra_state) { - try { - reinterpret_cast(extra_state)->free(); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) TODO(correaa) - } catch(...) {return MPI_ERR_UNKNOWN;} - return MPI_SUCCESS; - } - template - static int cancel_fn(void* extra_state, int complete) { - try { - reinterpret_cast(extra_state)->cancel(complete); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) TODO(correaa) - } catch(...) {return MPI_ERR_UNKNOWN;} - return MPI_SUCCESS; - } - template - explicit generalized_request(F& f){ - int s = MPI_Grequest_start( - &query_fn, //MPI_Grequest_query_function *query_fn, - &free_fn, //MPI_Grequest_free_function *free_fn, - &cancel_fn, //MPI_Grequest_cancel_function *cancel_fn, - std::addressof(f), // void *extra_state, - &impl_ //MPI_Request *request - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot create generalized request");} - } - void complete() { MPI_(Grequest_complete)(impl_); } -}; - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_GENERALIZED_REQUEST - -//#include "../mpi3/main.hpp" -//#include - -//using std::cout; -//namespace mpi3 = boost::mpi3; - -//struct custom_request{ -// int counter = 0; -// boost::mpi3::status query(){ -// counter -= 1; -// boost::mpi3::status ret; -// ret.set_source(MPI_UNDEFINED); -// ret.set_tag(MPI_UNDEFINED); -// ret.set_cancelled(); -// ret.set_elements(0); -// return ret; -// } -// void free(){} -// void cancel(int complete){} -//}; - -//struct print_request{ -//// double* first; -//// boost::mpi3::size_type n; -//// int counter = 0; -// boost::mpi3::status query(){ -// // counter -= 1; -// std::cout << "query" << std::endl; -// boost::mpi3::status ret; -// ret.set_source(MPI_UNDEFINED); -// ret.set_tag(MPI_UNDEFINED); -// ret.set_cancelled(); -// ret.set_elements(0); -// return ret; -// } -// void free(){ -// std::cout << "free" << std::endl; -// } -// void cancel(int complete){ -// std::cout << "cancel " << complete << std::endl; -// } -//}; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// { -// custom_request c{}; -// mpi3::generalized_request gr(c); -// assert(not gr.completed()); -// gr.complete(); -// gr.wait(); -// } -// { -// custom_request c{1}; -// mpi3::generalized_request gr(c); -// gr.complete(); -// gr.wait(); -// assert(c.counter == 0); -// } -// { -// assert( world.size() == 2); - -// // int right = (world.rank() + 1) % world.size(); -// // int left = world.rank() - 1; -// // if(left < 0) left = world.size() - 1; - -// using T = double; -// std::vector buffer(10); std::iota(buffer.begin(), buffer.end(), 0); -// std::vector buffer2(10); -// // mpi3::request r1 = world.ireceive(buffer2.begin(), buffer2.end(), left, 123); -// print_request pr{}; -// mpi3::generalized_request r1(pr); -// // world.send(buffer.begin(), buffer.end(), right, 123); -// std::cout << "middle" << std::endl; -// r1.wait(); -// // assert( buffer == buffer2 ); -// } -// return 0; -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/graph_communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/graph_communicator.hpp deleted file mode 100644 index 914ac141b34..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/graph_communicator.hpp +++ /dev/null @@ -1,140 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_GRAPH_COMMUNICATOR $0x.cpp -o $0x.x && time mpirun -n 5 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_GRAPH_COMMUNICATOR_HPP -#define BOOST_MPI3_GRAPH_COMMUNICATOR_HPP - -#include "../mpi3/communicator.hpp" -#include -#include -#include - -namespace boost{ -namespace mpi3{ - -struct graph_communicator : communicator{ - using graph = boost::adjacency_list; - - std::vector neighbors(int rank) const{ - int nneighbors; - MPI_Graph_neighbors_count(impl_, rank, &nneighbors); - std::vector ret(nneighbors); - MPI_Graph_neighbors(impl_, rank, ret.size(), ret.data()); - return ret; - } - - graph topology() const{ - int nnodes; - int nedges; - MPI_Graphdims_get(impl_, &nnodes, &nedges); - std::vector index(nnodes); - std::vector edges(nedges); - MPI_Graph_get(impl_, nnodes, nedges, index.data(), edges.data()); - auto ite = std::begin(edges); - int source = 0; - graph ret; - // std::vector v_ptr; - // for(int i = 0; i != nnodes; ++i){ - // v_ptr.push_back(add_vertex(g)); - // } - // std::cout << " index: "; - // for(auto e: index) std::cout << e << " "; - // std::cout << '\n'; - - // std::cout << " edges: "; - // for(auto e: edges) std::cout << e << " "; - // std::cout << '\n'; - // return ret; - int j = 0; - for(int i : index){ - for(; j != i; ++j){ - add_edge(source, edges[j], ret); - } - ++source; - } - return ret; - } -}; - -/* -template -graph_communicator communicator::make_graph(Graph const& g) const{ - graph_communicator ret; - std::vector indx; - std::vector edges; - std::pair vs = boost::vertices(g); - for(auto vit = vs.first; vit != vs.second; ++vit){ - auto neighbors = boost::adjacent_vertices(*vit, g); - for(auto nit = neighbors.first; nit != neighbors.second; ++nit) edges.push_back(*nit); - indx.push_back(edges.size()); // http://stackoverflow.com/a/32833024/225186 - } - MPI_Graph_create(impl_, num_vertices(g), indx.data(), edges.data(), true, &ret.impl_); - return ret; -}*/ - -/* -template -int communicator::graph_rank(Graph const& g) const{ - std::vector indx; - std::vector edges; - std::pair vs = boost::vertices(g); - for(auto vit = vs.first; vit != vs.second; ++vit){ - auto neighbors = boost::adjacent_vertices(*vit, g); - for(auto nit = neighbors.first; nit != neighbors.second; ++nit) edges.push_back(*nit); - indx.push_back(edges.size()); // http://stackoverflow.com/a/32833024/225186 - } - int ret; - MPI_Graph_map(impl_, num_vertices(g), indx.data(), edges.data(), &ret); - return ret; -}*/ - -}} - -#ifdef _TEST_BOOST_MPI3_GRAPH_COMMUNICATOR - -#include -#include "../mpi3/main.hpp" -#include "../mpi3/version.hpp" -#include "../mpi3/ostream.hpp" - -#include -#include -#include - -#include -#include - -int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator world){ - if(world.size() != 5) return 1; - - typedef boost::adjacency_list graph; - graph g; - - std::default_random_engine rd; - boost::generate_random_graph(g, 5, 8, rd); - - if(world.rank()==0){ - write_graphviz(std::cout, g); - std::cout << "---\n"; - } - -// auto world_graph = world.make_graph(g); - -/* auto outg = world_graph.topology(); - - if(world.rank()==0){ - write_graphviz(std::cout, outg); - std::cout << "---\n"; - } - - assert(boost::isomorphism(outg, g)); -*/ - - return 0; - -} - -#endif -#endif - - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/group.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/group.hpp deleted file mode 100644 index 36ffc53032b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/group.hpp +++ /dev/null @@ -1,142 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;-*- -// Copyright 2018-2021 Alfredo A. Correa - -#ifndef MPI3_GROUP_HPP -#define MPI3_GROUP_HPP - -#include "detail/equality.hpp" - -#include "../mpi3/error.hpp" - -#include "../mpi3/detail/call.hpp" -#include "../mpi3/detail/iterator_traits.hpp" - -#include - -// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -namespace boost { -namespace mpi3 { - -using ptr = MPI_Group; - -class group { - MPI_Group impl_ = MPI_GROUP_EMPTY; - - public: - friend class communicator; - template friend class window; - - MPI_Group& get() {return impl_;} - MPI_Group operator&() {return get();} // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions,google-runtime-operator) access implementation as pointer - -// std::pointer_traits::element_type const* operator&() const{return impl_;} // this doesn't work because in mpich MPI_Group is not really pointer type - - group() = default; - group(group&& other) noexcept : impl_{std::exchange(other.impl_, MPI_GROUP_EMPTY)}{} - group(group const& other){MPI_(Group_excl)(other.impl_, 0, nullptr, &impl_);} - - void swap(group& other) noexcept{std::swap(impl_, other.impl_);} -// group& operator=(group other) noexcept{swap(other); return *this;} - group& operator=(group const& other) {group tmp(other); swap(tmp) ; return *this;} - group& operator=(group && other) noexcept { swap(other); return *this;} - - void clear(){ - if(impl_ != MPI_GROUP_EMPTY) { - try { - MPI_(Group_free)(&impl_); - } catch(...) {} - } - impl_ = MPI_GROUP_EMPTY; - } - ~group(){ - if(impl_ != MPI_GROUP_EMPTY) { - try {MPI_(Group_free)(&impl_);} catch(...) {} - } - } - - group include(std::initializer_list il) const { - group ret; MPI_(Group_incl)(impl_, static_cast(il.size()), il.begin(), &ret.impl_); return ret; - } - group exclude(std::initializer_list il) const { - group ret; MPI_(Group_excl)(impl_, static_cast(il.size()), il.begin(), &ret.impl_); return ret; // TODO(correaa) use safe cast - } - auto rank() const -> int {int rank = -1; MPI_(Group_rank)(impl_, &rank); return rank;} - auto root() const -> bool {assert(not empty()); return rank() == 0;} - auto size() const -> int {int size = -1; MPI_(Group_size)(impl_, &size); return size;} - - group sliced(int first, int last, int stride = 1) const { - int ranges[][3] = {{first, last - 1, stride}}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - group ret; MPI_(Group_range_incl)(impl_, 1, ranges, &ret.impl_); return ret; - } - - bool empty() const {return size()==0;} - friend auto compare(group const& self, group const& other){ - int result; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Group_compare)(self.impl_, other.impl_, &result); - return static_cast(result); - } - bool operator==(group const& other) const{ - auto e=compare(*this, other); - return e == mpi3::detail::identical or e == mpi3::detail::congruent; - } - bool operator!=(group const& other) const{return not operator==(other);} - bool friend is_permutation(group const& self, group const& other){ - return compare(self, other) != mpi3::detail::unequal; - } - friend group set_intersection(group const& self, group const& other){ - group ret; MPI_(Group_intersection)(self.impl_, other.impl_, &ret.impl_); return ret; - } - friend group set_difference(group const& self, group const& other){ - group ret; MPI_(Group_difference)(self.impl_, other.impl_, &ret.impl_); return ret; - } - friend group set_union(group const& self, group const& other){ - group ret; MPI_(Group_union)(self.impl_, other.impl_, &ret.impl_); return ret; - } - int translate_rank(int rank, group const& other) const{ - int out; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Group_translate_ranks)(impl_, 1, &rank, other.impl_, &out); - return out; - } -}; - -} // end namespace mpi3 -} // end namespace boost - -//%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -#if not __INCLUDE_LEVEL__ // def _TEST_MPI3_GROUP - -#include "../mpi3/main.hpp" - -#include - -using std::cout; -namespace mpi3 = boost::mpi3; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::communicator w1 = world; - assert( w1.size() == world.size() ); - assert( w1.rank() == world.rank() ); - - mpi3::group g1{w1}; - { - mpi3::group g2 = g1; - assert( g1 == g2 ); - } - assert( g1.rank() == w1.rank() ); - - mpi3::communicator w2 = w1.create(g1); - assert( w2.size() == w1.size() ); - assert( w2.rank() == w1.rank() ); - assert( w2.rank() == world.rank() ); - - return 0; -} - -#endif -#endif - - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/handle.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/handle.hpp deleted file mode 100644 index 2721bbf5abd..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/handle.hpp +++ /dev/null @@ -1,139 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_DETAIL_HANDLE_HPP -#define BOOST_MPI3_DETAIL_HANDLE_HPP - -#include -#include // runtime_error -#include - -#include - -namespace boost { -namespace mpi3 { -namespace detail { - -template -struct caller { - Impl& impl() {return static_cast(*this).impl_;} - Impl const& impl() const {return static_cast(*this).impl_;} - - template - void static_call(F f, Args&&... args) { - int const status = f(std::forward(args)...); // cppcheck-suppress [redundantInitialization,redundantAssignment] ; bug in cppcheck 2.9 - if(status != 0) {throw std::runtime_error{"error "+ std::to_string(status)};} - } - template void call( - char const* c1, char const* c2 - ) { - int const status = F(impl(), c1, c2); - if(status != 0) {throw std::runtime_error{"error "+ std::to_string(status)};} - } - template void call( - std::string const& s1, std::string const& s2 - ) { - return call(s1.c_str(), s2.c_str()); - } - template int call() const { - int ret = -1; - // static_call(F, impl_, &ret); - int const status = F(impl(), &ret); - if(status != MPI_SUCCESS) {throw std::runtime_error{"error " + std::to_string(status)};} - return ret; - } - template std::string call(int n) const { - std::array ret{}; - int const status = F(impl(), n, ret.data()); - if(status != 0) {throw std::runtime_error{"error "+ std::to_string(status)};} - return std::string{ret.data()}; - } - template std::pair call(std::string const& key) const { - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - int valuelen; // NOLINT(cppcoreguidelines-init-variables) delayed init - int const status = F(impl(), key.c_str(), &valuelen, &flag); - if(status != 0) {throw std::runtime_error{"error "+ std::to_string(status)};} - return {valuelen, flag}; - } - template - std::pair call(std::string const& key, int valuelen) const { - std::array value{}; - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - int const status = F(impl(), key.c_str(), valuelen, value.data(), &flag); - if(status != 0) {throw std::runtime_error{"error "+ std::to_string(status)};} - return {std::string(value.data(), static_cast(valuelen)), flag}; - } - template void call(std::string const& key) const { - int const status = F(impl(), key.c_str()); - if(status != 0) {throw std::runtime_error{"error "+ std::to_string(status)};} - } -}; - -template< - class Self, - class Impl, - int(*CreateFunction)(Impl*), - int(*DupFunction)(Impl, Impl*), - int(*FreeFunction)(Impl*) -> -// TODO(correaa) rename as `indirect` -struct regular_handle : caller, Impl> { - using caller, Impl>::call; - using impl_t = Impl; - impl_t impl_; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - regular_handle() {CreateFunction(&impl_);} - regular_handle(regular_handle&&) = delete; // TODO(correaa) : introspect if "null state" is valid - regular_handle(regular_handle const& other) { // TODO(correaa) : revise in what cases a regular const& is correct - int status = DupFunction(other.impl_, &impl_); - if(status != MPI_SUCCESS) {throw std::runtime_error{"cannot copy handle"};} - } - ~regular_handle() { - assert(impl_ != MPI_INFO_NULL); - if(impl_ != MPI_INFO_NULL) {FreeFunction(&impl_);} - } - void swap(Self& other) {std::swap(impl_, other.impl_);} - regular_handle& operator=(regular_handle const& other) { - if(this == &other) {return *this;} - regular_handle tmp{other}; - swap(tmp); - return *this; - } - regular_handle operator=(regular_handle&&) = delete; -}; - - -struct uninitialized{}; - -template -struct nondefault_handle : caller, Impl>{ - private: - using impl_t = Impl; - - public: - impl_t impl_; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - bool predefined_ = false; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - explicit nondefault_handle(Impl code) : impl_(code), predefined_(true){} - nondefault_handle() = delete; - explicit nondefault_handle(uninitialized /*unused*/){}; - nondefault_handle(nondefault_handle const&) = delete; - nondefault_handle(nondefault_handle&&) = delete; - ~nondefault_handle(){ - int fin; MPI_Finalized(&fin); // NOLINT(cppcoreguidelines-init-variables) delayed init - if(not static_cast(fin)) { - if(not predefined_) {FreeFunction(&impl_);} - } - } - void swap(nondefault_handle& other){std::swap(impl_, other.impl_);} - nondefault_handle& operator=(nondefault_handle const& other) = delete; - nondefault_handle& operator=(nondefault_handle&& other) = delete; -}; - - -} // end namespace detail -} // end namespace mpi3 -} // end namespace boost - -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/info.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/info.hpp deleted file mode 100644 index f77541e80e7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/info.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ - -#ifndef BOOST_MPI3_INFO_HPP -#define BOOST_MPI3_INFO_HPP - -#include "../mpi3/handle.hpp" - -#include // for std::for_each -#include -#include // for std::tie - -namespace boost { -namespace mpi3 { - -struct info : - detail::regular_handle< - info, MPI_Info, MPI_Info_create, MPI_Info_dup, MPI_Info_free - > -{ - using base = detail::regular_handle; - using detail::regular_handle::call; - - info() = default; - info(info const&) = default; - info(info &&) = delete; // TODO(correaa) consider relation with default constructor - - info& operator=(info const&) = default; - info& operator=(info &&) = delete; // TODO(correaa) consider relation with default constructor - - ~info() = default; - - // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3, initialize_list ctor must be implicit - info(std::initializer_list> il) { - std::for_each(il.begin(), il.end(), [this](auto const& e) {set(e.first, e.second);}); - // for(auto const& e : il) {set(e.first, e.second);} - } - -// void delete_(std::string const& key) {call<&MPI_Info_delete>(key);} - std::pair get(std::string const& key, int valuelen) const{return base::call<&MPI_Info_get>(key, valuelen);} - int get_nkeys() const{return call<&MPI_Info_get_nkeys>();} - std::string get_nthkey(int n) const{return call<&MPI_Info_get_nthkey>(n);} - std::pair get_valuelen(std::string const& key) const{return call<&MPI_Info_get_valuelen>(key);} - void set(std::string const& key, std::string const& value){call<&MPI_Info_set>(key, value);} - - void insert(std::string const& key, std::string const& value){return set(key, value);} - void erase(std::string const& key) {call<&MPI_Info_delete>(key);} - int size() const{return get_nkeys();} - std::string operator[](std::string const& key) const{ - int valuelen = MPI_MAX_INFO_VAL; - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - std::tie(valuelen, flag) = get_valuelen(key); - if(flag == 0) {throw std::logic_error{"key `" + key + "' not found in info set"};} - return get(key, valuelen).first; - }; - std::pair operator[](int n) const{ - auto key = call<&MPI_Info_get_nthkey>(n); - return {key, operator[](key)}; - } - friend std::ostream& operator<<(std::ostream& os, info const& self) { - for(int i = 0; i != self.get_nkeys(); ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm - auto p = self[i]; - os<< p.first <<" : "<< p.second <<'\n'; - } - return os; - } -}; - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_INFO - -//#include "../mpi3/main.hpp" -//#include - -//using std::cout; -//using std::endl; - -//int boost::mpi3::main(int, char*[], boost::mpi3::communicator world){ -// if(world.rank() == 0){ -// boost::mpi3::info nfo; -// nfo.set("file", "runfile.txt"); -// nfo.set("soft", "host"); -// cout << nfo.get_nkeys() << '\n'; -// cout << nfo << '\n'; -// nfo.delete_("soft"); -// cout << nfo << '\n'; -// assert( nfo["file"] == "runfile.txt" ); -// boost::mpi3::info nfo2 = nfo; -// cout << nfo2 << '\n'; -// } -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/main.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/main.hpp deleted file mode 100644 index 6800266533a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/main.hpp +++ /dev/null @@ -1,45 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4-*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef BOOST_MPI3_MAIN_HPP -#define BOOST_MPI3_MAIN_HPP - -#include "../mpi3/communicator.hpp" -#include "../mpi3/environment.hpp" -#include "../mpi3/exception.hpp" -#include "../mpi3/timed_terminate.hpp" - -#include - -namespace boost { -namespace mpi3 { - -static int main(int /*argc*/, char** /*argv*/, boost::mpi3::communicator /*world*/); // if you include this file you should define `::boost::mpi3::main`NOLINT(bugprone-exception-escape) - -} // end namespace mpi3 -} // end namespace boost - -// cppcheck-suppress syntaxError ; bug cppcheck 2.3 -auto main(int argc, char** argv) -> int /*try*/ { // NOLINT(misc-definitions-in-headers,bugprone-exception-escape) : if you include this file you shouldn't have your own `::main`, you should define `boost::mpi3::main(int argc, char** argv, boost::mpi3::communicator world)` instead - boost::mpi3::environment const env{argc, argv}; - std::set_terminate([]{ - using namespace std::chrono_literals; - boost::mpi3::timed_terminate(3s); - }); -// try { - int const ret = boost::mpi3::main(argc, argv, /*env.*/ boost::mpi3::environment::get_world_instance()); - boost::mpi3::environment::get_world_instance().barrier(); - return ret; -// } catch(std::exception& e) { -// if(boost::mpi3::environment::get_world_instance().root()) {std::cerr<<"exception message: "<< e.what() <<"\n\n\n"< - -#include "../mpi3/communicator.hpp" -#include "../mpi3/environment.hpp" -#include "../mpi3/exception.hpp" - -namespace boost { -namespace mpi3 { - -static int main(int /*argc*/, char** /*argv*/, boost::mpi3::environment& /*env*/); // NOLINT(bugprone-exception-escape) - -} // end namespace mpi3 -} // end namespace boost - -int main(int argc, char* argv[]) { // NOLINT(bugprone-exception-escape,misc-definitions-in-headers) main defined in a header file for replacement - boost::mpi3::environment env{argc, argv}; - return boost::mpi3::main(argc, argv, env); -} -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/match.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/match.hpp deleted file mode 100644 index b0734fda54d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/match.hpp +++ /dev/null @@ -1,26 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ -#ifndef MPI3_MATCH_HPP -#define MPI3_MATCH_HPP - -#include "../mpi3/message.hpp" -#include "../mpi3/status.hpp" - -#include - -namespace boost{ -namespace mpi3{ - -struct match : public message, public status { // NOLINT(fuchsia-multiple-inheritance) - friend class communicator; - template - auto receive(It dest){ - return receive_n( - dest, - count::value_type>() - ); - } -}; - -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/message.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/message.hpp deleted file mode 100644 index 734bc580c89..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/message.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ - -#ifndef MPI3_MESSAGE_HPP -#define MPI3_MESSAGE_HPP - -#include "../mpi3/detail/iterator_traits.hpp" -#include "../mpi3/detail/value_traits.hpp" - -#include - -namespace boost { -namespace mpi3 { - -class message { - public: - MPI_Message impl_; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - MPI_Message operator&() const {return impl_;} // NOLINT(google-runtime-operator) design - - template - auto receive_n( - It it, - /**/ detail::contiguous_iterator_tag /*contiguous*/, - /**/ detail::basic_tag /*basic*/, - Size count - ) { - MPI_(Mrecv)( - detail::data(it), static_cast(count), detail::basic_datatype::value_type>{}, // TODO(correaa) use safe cast - &impl_, MPI_STATUS_IGNORE - ); - } - template - auto receive_n(It it, Size count) { - return receive_n( - it, - detail::iterator_category_t{}, - detail::value_category_t::value_type>{}, - count - ); - } - template - auto receive( - It first, It last, - detail::random_access_iterator_tag /*random_access*/ - ) { - return receive_n(first, std::distance(first, last)); - } - template - auto receive(It first, It last) { - return receive( - first, last, - detail::iterator_category_t{} - ); - } -}; - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_MPI3_MESSAGE - -//#include "../mpi3/main.hpp" - -//namespace mpi3 = boost::mpi3; - -//int mpi3::main(int, char*[], mpi3::communicator world){ -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/mutex.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/mutex.hpp deleted file mode 100644 index 278e38d271c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/mutex.hpp +++ /dev/null @@ -1,379 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include \""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_MUTEX $0x.cpp -o $0x.x && time mpirun -n 8 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_MUTEX_HPP -#define BOOST_MPI3_MUTEX_HPP - -#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include "../mpi3/window.hpp" -#include "../mpi3/detail/basic_mutex.hpp" - -namespace boost{ -namespace mpi3{ - -struct mutex{ //https://gist.github.com/aprell/1486197#file-mpi_mutex-c-L61 - using window_t = mpi3::window; - static int tag_counter; - using flag_t = unsigned char; - - communicator& comm_; - int rank_; //home - flag_t* addr_; - window_t win_; - - std::vector wait_list; //[comm_.size()]; - -// int tag_; - - mutex(mutex&&) = delete; - mutex(mutex const&) = delete; - mutex& operator=(mutex const&) = delete; - mutex& operator=(mutex&&) = delete; - - mutex(communicator& comm, int rank = 0) : - comm_(comm), - rank_(rank), - addr_((comm.rank() == rank)?(flag_t*)mpi3::malloc(sizeof(flag_t)*comm.size()):nullptr), - win_(addr_, addr_?comm.size():0, comm_), - wait_list(comm.size()) - { - if(addr_) std::memset(addr_, 0, comm.size()); - // tag_ = tag_counter; - ++tag_counter; - comm.barrier(); - } - - void lock(){ - std::fill(wait_list.begin(), wait_list.end(), 0); // flag_t wait_list[comm_.size()]; - flag_t lock = 1; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); // win_.put_n(&lock, 1, rank_, comm_.rank()); - win_.get_n(wait_list.data(), comm_.size(), rank_); - win_.unlock(rank_); - for(int i = 0; i != comm_.size(); ++i){ - if(wait_list[i] == 1 and i != comm_.rank()){ - comm_.receive_n(&lock, 0/*, MPI_ANY_SOURCE, tag_*/); //dummy receive - break; - } - } - } - bool try_lock(){ - std::fill(wait_list.begin(), wait_list.end(), 0); - flag_t lock = 1; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); - win_.get_n(wait_list.data(), comm_.size(), rank_); - win_.unlock(rank_); - for(int i = 0; i != comm_.size(); ++i) - if(wait_list[i] == 1 and i != comm_.rank()) return false; - return true; - } - void unlock(){ - std::fill(wait_list.begin(), wait_list.end(), 0);// wait_list.assign(unsigned char(0)); // flag_t wait_list[comm_.size()]; - flag_t lock = 0; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); - win_.get_n(wait_list.data(), comm_.size(), rank_); - win_.unlock(rank_); - - for(int i = 0; i != comm_.size(); ++i){ - int next = (comm_.rank() + i +1) % comm_.size(); - if(wait_list[next] == 1){ - comm_.send_n(&lock, 0, next/*, tag_*/); - break; - } - } - } - ~mutex(){ - comm_.barrier(); - // if(addr_) - mpi3::free(addr_); - } -}; - -int mutex::tag_counter = 11023; - -struct rmutex{ //https://gist.github.com/aprell/1486197#file-mpi_mutex-c-L61 - using flag_t = int; - using window_t = mpi3::window; - static int tag_counter; - - - communicator& comm_; - int rank_; //home - flag_t* addr_; - window_t win_; - - std::vector wait_list; //[comm_.size()]; - -// int tag_; - - rmutex(rmutex&&) = delete; - rmutex(rmutex const&) = delete; - rmutex& operator=(rmutex const&) = delete; - rmutex& operator=(rmutex&&) = delete; - - rmutex(communicator& comm, int rank = 0) : - comm_(comm), - rank_(rank), - addr_((comm.rank() == rank)?(flag_t*)mpi3::malloc(sizeof(flag_t)*(comm.size()+2)):nullptr), - win_(addr_, addr_?(comm.size()+2):0, comm_), - wait_list(comm_.size()) - { - win_.fence(); - if(addr_){ - std::memset(addr_ + 2, 0, comm.size()); - addr_[0] = 0; - addr_[1] = -10000; - assert(comm_.rank() == 0); - } - win_.fence(); - // tag_ = tag_counter; - ++tag_counter; - comm_.barrier(); - } - - void lock(){ - win_.lock_exclusive(rank_); - int count = -1; - win_.get_n(&count, 1, rank_); - std::cout << "count " << count << std::endl; - assert(count == 0); - { - int owner = -1; - win_.get_n(&owner, 1, rank_, 1); - std::cout << "count is " << count << std::endl; - std::cout << "onwer is " << owner << std::endl; - if(count > 0) assert(owner == comm_.rank()); - ++count; - owner = comm_.rank(); - // win_.put_value(count, rank_, 0); - // win_.put_value(owner, rank_, 1); - if(count > 1) return; - } - win_.unlock(rank_); - - std::fill(wait_list.begin(), wait_list.end(), 0); // flag_t wait_list[comm_.size()]; - flag_t lock = 1; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank() + 2); // win_.put_n(&lock, 1, rank_, comm_.rank()); - win_.get_n(wait_list.data(), comm_.size(), rank_, 2); - win_.unlock(rank_); - for(int i = 0; i != comm_.size(); ++i){ - if(wait_list[i] == 1 and i != comm_.rank()){ - comm_.receive_n(&lock, 0/*, MPI_ANY_SOURCE, tag_*/); //dummy receive - break; - } - } - } - bool try_lock(){ - std::fill(wait_list.begin(), wait_list.end(), 0); - flag_t lock = 1; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); - win_.get_n(wait_list.data(), comm_.size(), rank_); - win_.unlock(rank_); - for(int i = 0; i != comm_.size(); ++i) - if(wait_list[i] == 1 and i != comm_.rank()) return false; - return true; - } - void unlock(){ - win_.lock_exclusive(rank_); - int count; - win_.get_value(count, rank_, 0); - if(count > 0){ - int owner; - win_.get_value(owner, rank_, 1); - assert(owner == comm_.rank()); - --count; - win_.put_value(count, rank_, 0); - if(count > 0) return; - assert(count == 0); - } - win_.unlock(rank_); - - std::fill(wait_list.begin(), wait_list.end(), 0);// wait_list.assign(unsigned char(0)); // flag_t wait_list[comm_.size()]; - flag_t lock = 0; - win_.lock_exclusive(rank_); - - win_.put_value(lock, rank_, comm_.rank() + 2); - win_.get_n(wait_list.data(), comm_.size(), rank_, 2); - win_.unlock(rank_); - - for(int i = 0; i != comm_.size(); ++i){ - int next = (comm_.rank() + i +1) % comm_.size(); - if(wait_list[next] == 1){ - comm_.send_n(&lock, 0, next/*, tag_*/); - break; - } - } - } - ~rmutex(){ - comm_.barrier(); - // if(addr_) - mpi3::free(addr_); - } -}; - -int rmutex::tag_counter = 11024; - - -template -struct atomic{ - int rank_; - T* addr_; - communicator& comm_; -// T counter = -999; - window win_; - - atomic(T const& value, communicator& comm, int rank = 0) : - comm_(comm), - rank_(rank), - addr_(static_cast(comm.rank()==rank?mpi3::malloc(sizeof(T)):nullptr)), - win_(addr_, addr_?sizeof(T):0, comm_) - { - if(addr_) new (addr_) T(value); -// if(addr_) *addr_ = value; - } - atomic& operator+=(T const& t){ - win_.lock_exclusive(rank_); - win_.fetch_sum_value(t, *addr_, rank_); - // win_.fetch_sum_value(t, counter, rank_); - win_.unlock(rank_); - return *this; - } - atomic& operator++(){ - win_.lock_exclusive(rank_); - T t = T(1); - win_.fetch_sum_value(t, *addr_, rank_); - // win_.fetch_sum_value(t, counter, rank_); - win_.unlock(rank_); - return *this; - } - atomic& operator--(){ - win_.lock_exclusive(rank_); - T t = -T(1); - win_.fetch_sum_value(t, *addr_, rank_); - // win_.fetch_sum_value(t, counter, rank_); - win_.unlock(rank_); - return *this; - } - - atomic& operator-=(T const& t){return operator+=(-t);} - atomic& operator*=(T const& t){ - win_.lock_exclusive(rank_); - win_.fetch_prod_value(t, *addr_, rank_); - // win_.fetch_prod_value(t, counter, rank_); - win_.unlock(rank_); - return *this; - } - atomic& operator/=(T const& t); - atomic& operator=(T const& t){ - win_.lock_exclusive(rank_); - win_.fetch_replace_value(t, *addr_, rank_); - // win_.fetch_replace_value(t, counter, rank_); - win_.unlock(rank_); - return *this; - } -// T const& load() const{return counter;} - operator T(){ - T t; - win_.lock_exclusive(0); - win_.put_value(*addr_, 0, comm_.rank()); - win_.get_n(&t, 1, 0); - win_.unlock(0); - return t; - } - ~atomic(){ - comm_.barrier(); - if(addr_) boost::mpi3::free(addr_); - } -}; - -template<> atomic& atomic::operator/=(double const& d){return operator*=(1./d);} -template<> atomic& atomic::operator/=(float const& d){return operator*=(1./d);} - -}} - -#ifdef _TEST_BOOST_MPI3_MUTEX - -#include "../mpi3/main.hpp" - -#include -#include -#include - -#include -#include - -namespace mpi3 = boost::mpi3; using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - { - mpi3::mutex m(world); - { - m.lock(); - cout << "locked from " << world.rank() << '\n'; - cout << "never interleaved " << world.rank() << '\n'; - cout << "forever blocked " << world.rank() << '\n'; - cout << std::endl; - m.unlock(); - } - } - - return 0; - cout << "=================" << std::endl; - world.barrier(); - { - mpi3::rmutex m(world); - { - m.lock(); - // m.lock(); - cout << "locked from " << world.rank() << '\n'; - cout << "never interleaved " << world.rank() << '\n'; - cout << "forever blocked " << world.rank() << '\n'; - cout << std::endl; - // m.unlock(); - m.unlock(); - } - } - cout << "=================" << std::endl; - world.barrier(); - return 0; - - - { - mpi3::atomic counter(0, world); - counter += 1; - if(counter + 1 == world.size()) cout << "Process #" << world.rank() << " did the last updated" << std::endl; - } - { - mpi3::atomic counter(0, world); - counter += 1; - world.barrier(); - { - mpi3::mutex m(world); - std::lock_guard lock(m); - cout << "on process " << world.rank() << " counter = " << (int)counter << std::endl; - } - } - { - mpi3::mutex m(world); - std::lock_guard lock(m); - - cout <<"locked from "<< world.rank() << '\n'; - cout <<"never interleaved" << world.rank() << '\n'; - cout <<"forever blocked "<< world.rank() << '\n'; - cout << std::endl; - } - - cout << "end" << std::endl; - return 0; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/communicator.hpp deleted file mode 100644 index 84149a79286..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/communicator.hpp +++ /dev/null @@ -1,367 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2022- Alfredo A. Correa - -#ifndef MPI3_NCCL_COMMUNICATOR_HPP_ -#define MPI3_NCCL_COMMUNICATOR_HPP_ - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/nccl/detail/basic_datatype.hpp" -#include "../../mpi3/nccl/detail/basic_reduction.hpp" - -#include -#include - -#include // for plus -#include - -#include - -namespace boost { -namespace mpi3 { -namespace nccl { - -namespace detail { - -template auto datatype(T const&) -> decltype(basic_datatype_t::value) {return basic_datatype;} -template auto reduction(T const&) -> decltype(basic_reduction) {return basic_reduction;} - -} - -class communicator { - static auto get_unique_id() -> ncclUniqueId { - ncclUniqueId nccl_id; - auto r = ncclGetUniqueId(&nccl_id); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0 && "likely \"Duplicate GPU detected\", for example if rank 0 and rank 1 both on CUDA device 1000"); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - return nccl_id; - } - - public: - explicit communicator(mpi3::communicator& mpi) : impl_{nullptr} { - if(mpi.empty()) {return;} - ncclUniqueId nccl_id = mpi.root()?get_unique_id():ncclUniqueId{}; - mpi.broadcast_n(reinterpret_cast(&nccl_id), sizeof(ncclUniqueId)); - // TODO(correaa) may need mpi.barrier(); here - { - ncclResult_t r = ncclCommInitRank(&impl_, mpi.size(), nccl_id, mpi.rank()); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0 && "likely \"Duplicate GPU detected\", for example if rank 0 and rank 1 both on CUDA device 1000"); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - } - } - - communicator(communicator const&) = delete; -// [[deprecated("experimental")]] - communicator(communicator& other) : impl_{nullptr} { - if(other.empty()) {return;} - ncclUniqueId nccl_id = other.root()?get_unique_id():ncclUniqueId{}; - { - ncclResult_t r = ncclBcast(&nccl_id, sizeof(ncclUniqueId), ncclChar, 0, other.impl_, NULL); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - } - cudaStreamSynchronize(NULL); - // TODO(correaa) may need mpi.barrier(); here - { - ncclResult_t r = ncclCommInitRank(&impl_, other.count(), nccl_id, other.rank()); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0 && "likely \"Duplicate GPU detected\", for example if rank 0 and rank 1 both on CUDA device 1000"); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - } - } - communicator(communicator&& other) : impl_{std::exchange(other.impl_, nullptr)} {} - // moved from communicators are left in a partially formed, since there is no assigmment it cannot be used - - auto duplicate() {return communicator{*this};} - - template< - class Op = std::plus<>, - class P1, class Size, class P2, - typename = decltype( - *thrust::raw_pointer_cast(P2{}) = Op{}(*thrust::raw_pointer_cast(P1{}), *thrust::raw_pointer_cast(P1{})), - detail::datatype(*raw_pointer_cast(P1{})) - ) - > - auto all_reduce_n(P1 first, Size count, P2 dest, Op op = {}) { - ncclResult_t r = ncclAllReduce( - thrust::raw_pointer_cast(first), thrust::raw_pointer_cast(dest), count, - detail::datatype(*raw_pointer_cast(first)), - detail::reduction(op), impl_, NULL - ); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - return dest + count; - } - - template - auto send_n(P first, Size n, int peer) { - // ncclGroupStart(); - ncclResult_t r = ncclSend(thrust::raw_pointer_cast(first), n, detail::datatype(*raw_pointer_cast(first)), peer, impl_, NULL); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - // ncclGroupEnd(); - // cudaStreamSynchronize(NULL); - return first + n; - } - template - auto receive_n(P first, Size n, int peer) { - // ncclGroupStart(); - ncclResult_t r = ncclRecv(thrust::raw_pointer_cast(first), n, detail::datatype(*raw_pointer_cast(first)), peer, impl_, NULL); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - // ncclGroupEnd(); - // cudaStreamSynchronize(NULL); - return first + n; - } - template< - class P, //typename = decltype(detail::datatype(*raw_pointer_cast(P{}))), - class Size - > - P broadcast_n(P first, Size n, int root = 0) { - // ncclGroupStart(); - using thrust::raw_pointer_cast; - ncclResult_t r = ncclBcast(raw_pointer_cast(first), n, detail::datatype(*raw_pointer_cast(first)), root, impl_, NULL); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - // ncclGroupEnd(); - // cudaStreamSynchronize(NULL); - return first + n; - } - - private: - template static constexpr bool is_numeric_complex_of = - std::is_same_v - and sizeof(Complex) == sizeof(Complex{}.real()) + sizeof(Complex{}.imag()) - ; - - public: - template static constexpr bool is_numeric_complex = is_numeric_complex_of; - template< - class P, class Size, - class PT = std::pointer_traits

, class Complex = typename PT::element_type, - class Value = typename std::conditional< - std::is_const_v, - std::add_const_t, - typename Complex::value_type - >::type, - std::enable_if_t, std::decay_t>, int> =0 - > - auto send_n(P first, Size n, int peer) { - send_n(thrust::reinterpret_pointer_cast>(first), n*2, peer); - return first + n; - } - template< - class P, class Size, - class PT = std::pointer_traits

, class Complex = typename PT::element_type, - class Value = typename std::conditional< - std::is_const_v, - std::add_const_t, - typename Complex::value_type - >::type, - std::enable_if_t, std::decay_t>, int> =0 - > - auto receive_n(P first, Size n, int peer) { - receive_n(thrust::reinterpret_pointer_cast>(first), n*2, peer); - return first + n; - } - -// template< -// class P1, class Size, class P2 -//// , typename = decltype( -//// *thrust::raw_pointer_cast(P2{}) = std::plus<>{}(*thrust::raw_pointer_cast(P1{}), *thrust::raw_pointer_cast(P1{})), -//// detail::datatype(*raw_pointer_cast(P1{})) -//// ) -// , class PT1 = std::pointer_traits -//// , class Complex1 = typename PT1::element_type, -//// class Value1 = typename std::conditional< -//// std::is_const_v, -//// std::add_const_t, -//// typename Complex1::value_type -//// >::type -//// , std::enable_if_t, std::decay_t>, int> =0, -// , class PT2 = std::pointer_traits -//// , class Complex2 = typename PT2::element_type, -//// class Value2 = typename std::conditional< -//// std::is_const_v, -//// std::add_const_t, -//// typename Complex2::value_type -//// >::type, -//// std::enable_if_t, std::decay_t>, int> =0 -// > -// auto all_reduce_n(P1 first, Size count, P2 dest, std::plus<> op = {}) { -// using Value1 = double; -// using Value2 = double; -// all_reduce_n( -// reinterpret_pointer_cast>(first), count*2, -// reinterpret_pointer_cast>(dest ), op -// ); -// return dest + count; -// } - template - auto all_reduce_n(thrust::cuda::pointer> first, Size n, thrust::cuda::pointer> dest) { - using thrust::reinterpret_pointer_cast; - all_reduce_n( - reinterpret_pointer_cast>(first), n*2, - reinterpret_pointer_cast>(dest ), std::plus<>{} - ); - } - template - auto all_reduce_n(thrust::cuda::universal_pointer> first, Size n, thrust::cuda::universal_pointer> dest) { - using thrust::reinterpret_pointer_cast; - all_reduce_n( - reinterpret_pointer_cast>(first), n*2, - reinterpret_pointer_cast>(dest ), std::plus<>{} - ); - } - - ~communicator() { - if(impl_) { - ncclCommDestroy(impl_); // call ncclCommFinalize internally if necessary - } - } - - int rank() const { // aka user_rank() - int ret; - ncclResult_t r = ncclCommUserRank(impl_, &ret); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - return ret; - } - int count() const { - int ret; - ncclResult_t r = ncclCommCount(impl_, &ret); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - return ret; - } - [[deprecated("in NCCL nomenclature `.size` is called `.count`")]] int size() const {return count();} - [[nodiscard]] bool empty() const {return not count();} - [[nodiscard]] bool is_empty() const {return not count();} - - [[nodiscard]] bool root() const {return not rank();} - - [[deprecated("using comm handle, try implementating")]] ncclComm_t operator&() {return impl_;} - - private: - ncclComm_t impl_; -}; - -struct group { - static void start() { - ncclResult_t r = ncclGroupStart(); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - } - static void end() { - ncclResult_t r = ncclGroupEnd(); - switch(r) { - case ncclSuccess: break; - case ncclUnhandledCudaError: assert(0); - case ncclSystemError: assert(0); - case ncclInternalError: assert(0); - case ncclInvalidArgument: assert(0); - case ncclInvalidUsage: assert(0); - case ncclRemoteError: assert(0); - case ncclNumResults: assert(0); - } - } -}; - -class group_block { - group g_; - - public: - [[nodiscard]] group_block(group g = {}) : g_{std::move(g)} {g_.start();} - ~group_block() {g_.end();} -}; - -} // end namespace nccl -} // end namespace mpi3 -} // end namespace boost -#endif // MPI3_NCCL_COMMUNICATOR_HPP_ diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_datatype.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_datatype.hpp deleted file mode 100644 index 722d609124e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_datatype.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2022 Alfredo A. Correa - -#ifndef MPI3_NCCL_DETAIL_BASIC_DATATYPE_HPP_ -#define MPI3_NCCL_DETAIL_BASIC_DATATYPE_HPP_ - -#include // for ncclDataType_t - -namespace boost { -namespace mpi3 { -namespace nccl { - -namespace detail { - -template struct basic_datatype_t; - -template<> struct basic_datatype_t : std::integral_constant {}; -template<> struct basic_datatype_t : std::integral_constant {}; -template<> struct basic_datatype_t : std::integral_constant {}; -template<> struct basic_datatype_t : std::integral_constant {}; - // template<> struct basic_datatype_t : std::integral_constant {}; // is redundant with some other type, int32_t? -template<> struct basic_datatype_t : std::integral_constant {}; - -template<> struct basic_datatype_t : std::integral_constant {}; - -//template<> struct basic_datatype_t : std::integral_constant {}; // not defined in NCCL 2.13.4 - -// template<> struct basic_datatype_t : std::integral_constant {}; // needs C++23 - -// template<> struct basic_datatype_t : std::integral_constant {}; - -// template<> struct basic_datatype_t : std::integral_constant {}; // needs C++23 - -template<> struct basic_datatype_t : std::integral_constant {}; - -// template<> struct basic_datatype_t : std::integral_constant {}; // needs C++23 - -template<> struct basic_datatype_t : std::integral_constant {}; - -// template<> struct basic_datatype_t : std::integral_constant {}; // needs C++23 - -template auto basic_datatype = basic_datatype_t::value; - -} - -}}} - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_reduction.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_reduction.hpp deleted file mode 100644 index 64f933b8022..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/detail/basic_reduction.hpp +++ /dev/null @@ -1,37 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2022 Alfredo A. Correa - -#ifndef MPI3_NCCL_DETAIL_BASIC_REDUCTION_HPP_ -#define MPI3_NCCL_DETAIL_BASIC_REDUCTION_HPP_ - -#include - -namespace boost { -namespace mpi3 { -namespace nccl { - -template using plus = std::plus; -template using multiplies = std::multiplies<>; -template struct min {}; -template struct max {}; -template struct average {}; - -namespace detail { - -template struct basic_reduction_t; - -template struct basic_reduction_t; - -template<> struct basic_reduction_t > : std::integral_constant {}; -template<> struct basic_reduction_t> : std::integral_constant {}; -template<> struct basic_reduction_t > : std::integral_constant {}; -template<> struct basic_reduction_t > : std::integral_constant {}; -template<> struct basic_reduction_t > : std::integral_constant {}; - -template auto basic_reduction = basic_reduction_t::value; - -} - -}}} - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/CMakeLists.txt b/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/CMakeLists.txt deleted file mode 100644 index 0b7503a1b74..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/CMakeLists.txt +++ /dev/null @@ -1,145 +0,0 @@ -# Use cmake/ctest for building and testing boost mpi3 tests -cmake_minimum_required(VERSION 3.18.4) # 3.10 for FindMPI, 3.12 for MPI_ROOT -project(boostmpi3nccl_tests VERSION 0.1 LANGUAGES CXX) - -enable_testing() -include(CTest) - -find_package(MPI REQUIRED) - -include_directories(SYSTEM ${MPI_CXX_INCLUDE_DIRS}) -link_libraries(${MPI_CXX_LIBRARIES}) - -if (NOT MPIEXEC) - set(MPIEXEC mpirun) -endif() - - -# Set BOOST_ROOT if boost is in a non-standard location -find_package(Boost REQUIRED) -if(Boost_FOUND) - set(HAVE_LIBBOOST 1) - include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) - message(STATUS "Setting Boost_INCLUDE_DIRS=${Boost_INCLUDE_DIRS}") -endif() - -#if(ENABLE_CUDA) - enable_language(CUDA) - find_package(CUDA 11.7.0 REQUIRED) -# set(CXX_COMPILER ${CMAKE_CUDA_COMPILER}) -# set(CXX_FLAGS ${CMAKE_CUDA_FLAGS}) -# string(APPEND CMAKE_CUDA_FLAGS " --forward-unknown-to-host-linker") # ${ARCH_FLAGS} -std=c++17 --expt-relaxed-constexpr --extended-lambda --Werror=cross-execution-space-call -Xcudafe \"--diag_suppress=implicit_return_from_non_void_function\"") -# add_compile_options(--forward-unknown-to-host-linker) -#endif() - -set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr --extended-lambda --Werror=cross-execution-space-call -Xcudafe \"--display_error_number\" --std=c++17") - -set(NCCL_INCLUDE_DIR $ENV{NCCL_INCLUDE_DIR} CACHE PATH "Folder contains NVIDIA NCCL headers") -set(NCCL_LIB_DIR $ENV{NCCL_LIB_DIR} CACHE PATH "Folder contains NVIDIA NCCL libraries") -set(NCCL_VERSION $ENV{NCCL_VERSION} CACHE STRING "Version of NCCL to build with") - -if ($ENV{NCCL_ROOT_DIR}) - message(WARNING "NCCL_ROOT_DIR is deprecated. Please set NCCL_ROOT instead.") -endif() -list(APPEND NCCL_ROOT $ENV{NCCL_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}) -# Compatible layer for CMake <3.12. NCCL_ROOT will be accounted in for searching paths and libraries for CMake >=3.12. -list(APPEND CMAKE_PREFIX_PATH ${NCCL_ROOT}) - -find_path(NCCL_INCLUDE_DIRS - NAMES nccl.h - HINTS ${NCCL_INCLUDE_DIR}) - -if (USE_STATIC_NCCL) - MESSAGE(STATUS "USE_STATIC_NCCL is set. Linking with static NCCL library.") - SET(NCCL_LIBNAME "nccl_static") - if (NCCL_VERSION) # Prefer the versioned library if a specific NCCL version is specified - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a.${NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES}) - endif() -else() - SET(NCCL_LIBNAME "nccl") - if (NCCL_VERSION) # Prefer the versioned library if a specific NCCL version is specified - set(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${NCCL_VERSION}" ${CMAKE_FIND_LIBRARY_SUFFIXES}) - endif() -endif() - -find_library(NCCL_LIBRARIES - NAMES ${NCCL_LIBNAME} - HINTS ${NCCL_LIB_DIR}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIRS NCCL_LIBRARIES) - -if(NCCL_FOUND) # obtaining NCCL version and some sanity checks - set (NCCL_HEADER_FILE "${NCCL_INCLUDE_DIRS}/nccl.h") - message (STATUS "Determining NCCL version from ${NCCL_HEADER_FILE}...") - set (OLD_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES}) - list (APPEND CMAKE_REQUIRED_INCLUDES ${NCCL_INCLUDE_DIRS}) - include(CheckCXXSymbolExists) - check_cxx_symbol_exists(NCCL_VERSION_CODE nccl.h NCCL_VERSION_DEFINED) - - if (NCCL_VERSION_DEFINED) - set(file "${PROJECT_BINARY_DIR}/detect_nccl_version.cc") - file(WRITE ${file} " - #include - #include - int main() - { - std::cout << NCCL_MAJOR << '.' << NCCL_MINOR << '.' << NCCL_PATCH << std::endl; - int x; - ncclGetVersion(&x); - return x == NCCL_VERSION_CODE; - } -") - try_run(NCCL_VERSION_MATCHED compile_result ${PROJECT_BINARY_DIR} ${file} - RUN_OUTPUT_VARIABLE NCCL_VERSION_FROM_HEADER - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${NCCL_INCLUDE_DIRS}" - LINK_LIBRARIES ${NCCL_LIBRARIES}) - if (NOT NCCL_VERSION_MATCHED) - message(FATAL_ERROR "Found NCCL header version and library version do not match! \ -(include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES}) Please set NCCL_INCLUDE_DIR and NCCL_LIB_DIR manually.") - endif() - message(STATUS "NCCL version: ${NCCL_VERSION_FROM_HEADER}") - else() - message(STATUS "NCCL version < 2.3.5-5") - endif () - set (CMAKE_REQUIRED_INCLUDES ${OLD_CMAKE_REQUIRED_INCLUDES}) - - message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})") - mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES) -endif() - -# This list is only a subset of files. It might be useful to divide some of the tests into -# different categories (MPI-1, MPI-2, etc., or something else) -set(TEST_SRCS - nccl_constructor.cu -) - -foreach(TEST_FILE ${TEST_SRCS}) - SET(TEST_EXE "${TEST_FILE}.x") - add_executable(${TEST_EXE} ${TEST_FILE}) - target_compile_features(${TEST_EXE} PRIVATE cxx_std_17) - -# if(ENABLE_CUDA) - set_source_files_properties(${TEST_FILE} PROPERTIES LANGUAGE CUDA) -# endif() - - target_compile_options(${TEST_EXE} PRIVATE $<$:-Wall -Wextra -Werror>) # -Wpedantic -Wunused -Wnon-virtual-dtor -Woverloaded-virtual -Wcast-qual -Wformat=2>) #-Wsign-conversion -Wshadow -Wconversion - target_compile_options(${TEST_EXE} PRIVATE $<$:-Wall -Wextra>) - target_include_directories(${TEST_EXE} PUBLIC "../../../..") - - target_include_directories(${TEST_EXE} SYSTEM PRIVATE /home/correaa/prj/alf/boost/multi/include) - - list(FIND NEED_BOOST_SERIALIZATION_SRCS ${TEST_FILE} NEED_BOOST_SERIALIZATION) - if (NOT (${NEED_BOOST_SERIALIZATION} EQUAL -1)) - target_compile_options(${TEST_EXE} PUBLIC -D_MAKE_BOOST_SERIALIZATION_HEADER_ONLY) -# target_link_libraries(${TEST_EXE} boost_serialization) - endif() - - target_link_libraries(${TEST_EXE} pthread) - - target_include_directories(${TEST_EXE} PUBLIC ${NCCL_INCLUDE_DIRS}) - target_link_libraries(${TEST_EXE} ${NCCL_LIBRARIES}) - - add_test(NAME ${TEST_EXE} COMMAND ${MPIEXEC} $ENV{MPI_OVERSUBSCRIBE} -n ${NPROC} ./${TEST_EXE}) - set_tests_properties(${TEST_EXE} PROPERTIES TIMEOUT 30) -endforeach() diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/nccl_constructor.cu b/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/nccl_constructor.cu deleted file mode 100644 index c240e17ff33..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/test/nccl_constructor.cu +++ /dev/null @@ -1,63 +0,0 @@ -#include "../../../mpi3/nccl/communicator.hpp" -#include "../../../mpi3/nccl/universal_communicator.hpp" - -#include "../../../mpi3/main.hpp" - -#include -#include -#include - -#include - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator WORLD) { - assert(WORLD.size() == 4); - - cudaSetDevice(WORLD.rank()); - - auto HEMI = WORLD / 2; - - using Communicator = mpi3::nccl::universal_communicator; - - Communicator magnesium{HEMI}; - assert(magnesium.rank() == HEMI.rank()); - - using T = thrust::complex; // int64_t; -// thust::device_vector> A(1000, world.rank()); - thrust::device_vector> A(1000, T{1.*WORLD.rank()}); - - magnesium.all_reduce_n(A.data(), A.size(), A.data()); - thrust::host_vector H = A; - - std::cout<<"[WORLD rank"<< WORLD.rank() <<" HEMI rank"<< HEMI.rank() <<"] result:"<< H[0] < V(100, 1.); - magnesium.all_reduce_n(V.data(), V.size(), V.data()); - assert( V[10] == magnesium.size() ); - - switch(magnesium.rank()) { - case 0: { - magnesium.send_n(A.data(), A.size(), 1); - } - case 1: { - thrust::device_vector> B(1000, T{}); - magnesium.receive_n(B.data(), B.size(), 0); - assert( A == B ); - } - } - -// thrust::device_vector> singleton(1); - std::vector singleton(1); - if(magnesium.rank() == 0) { singleton[0] = 99; } - magnesium.broadcast_n(singleton.data(), 1); - cudaStreamSynchronize(NULL); - assert( singleton[0] == 99 ); - - auto magnesium2{magnesium}; - assert( magnesium2.size() == magnesium.size() ); - - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/universal_communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/universal_communicator.hpp deleted file mode 100644 index 20986c0b125..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/nccl/universal_communicator.hpp +++ /dev/null @@ -1,57 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2022 Alfredo A. Correa - -#ifndef MPI3_NCCL_UNIVERSAL_COMMUNICATOR_HPP_ -#define MPI3_NCCL_UNIVERSAL_COMMUNICATOR_HPP_ - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/nccl/communicator.hpp" - -#define BMPI3_DECLRET(ExpR) ->decltype(ExpR) {return ExpR;} // NOLINT(cppcoreguidelines-macro-usage) saves a lot of typing -#define BMPI3_JUSTRET(ExpR) {return ExpR;} // NOLINT(cppcoreguidelines-macro-usage) saves a lot of typing - -namespace boost { -namespace mpi3 { -namespace nccl { - -template struct priority : std::conditional_t> {}; - -struct universal_communicator : mpi3::communicator, private mpi3::nccl::communicator { - template - universal_communicator(As&&... as) - : mpi3::communicator{std::forward(as)...} - , mpi3::nccl::communicator{static_cast(*this)} {} - - using mpi3::communicator::rank; - using mpi3::communicator::size; - - private: - template - using comm_system = typename std::conditional< - std::is_same_v::type, thrust::cuda_cub::tag>, - mpi3::nccl::communicator, - mpi3:: communicator - >::type; - - public: - template auto all_reduce_n(priority<0>, As... as ) BMPI3_JUSTRET(mpi3::communicator::all_reduce_n( as... )) - template auto all_reduce_n(priority<1>, P first, Rs... rs) BMPI3_DECLRET(comm_system

::all_reduce_n( first, rs...)) - template auto all_reduce_n( As... as ) {return this->all_reduce_n(priority<1>{}, as... );} - - template auto broadcast_n(priority<0>, As... as ) BMPI3_JUSTRET(mpi3::communicator::broadcast_n( as... )) - template auto broadcast_n(priority<1>, P first, Rs... rs) BMPI3_DECLRET(comm_system

::broadcast_n( first, rs...)) - template auto broadcast_n( As... as ) {return this-> broadcast_n(priority<1>{}, as... );} - - template auto send_n(priority<0>, As... as ) BMPI3_JUSTRET(mpi3::communicator::send_n( as... )) - template auto send_n(priority<1>, P first, Rs... rs) BMPI3_DECLRET(comm_system

::send_n( first, rs...)) - template auto send_n( As... as ) {return this-> send_n(priority<1>{}, as... );} - - template auto receive_n(priority<0>, As... as ) BMPI3_JUSTRET(mpi3::communicator::receive_n( as... )) - template auto receive_n(priority<1>, P first, Rs... rs) BMPI3_DECLRET(comm_system

::receive_n( first, rs...)) - template auto receive_n( As... as ) {return this-> receive_n(priority<1>{}, as... );} -}; - -} // end namespace nccl -} // end namespace mpi3 -} // end namespace boost -#endif // MPI3_NCCL_UNIVERSAL_COMMUNICATOR_HPP_ diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/operation.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/operation.hpp deleted file mode 100644 index 05d025c1aef..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/operation.hpp +++ /dev/null @@ -1,222 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_OPERATION_HPP -#define BOOST_MPI3_OPERATION_HPP - -#include -#include - - -#if defined(__NVCC__) -#include -#endif - -#include // std::transform_n -#include // std::forward - -namespace boost { -namespace mpi3 { - -template -struct commutative_operation; - -template -struct builtin_operation; - -template -struct operation : detail::nondefault_handle, MPI_Op, MPI_Op_free> { // user_operation, operator_ . operator is a C++ keyword - using base = typename detail::nondefault_handle, MPI_Op, MPI_Op_free>; - using typename base::nondefault_handle; - operation() = delete; - - template - // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) signature is fixed - constexpr static void combine(void const* in, void* inout, int const* len, MPI_Datatype* /*dtype*/) { // cppcheck-suppress constParameter ; signature is fixed - auto in_t = reinterpret_cast::template rebind&>(in ); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-reinterpret-cast) - auto inout_t = reinterpret_cast< P &>(inout); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-reinterpret-cast) - #if defined(__NVCC__) - thrust::transform( - #else - std::transform( - #endif - in_t, std::next(in_t, *len), inout_t, inout_t, - [] - #if defined(__NVCC__) - __host__ __device__ - #endif - (T const& a, T const& b) - #if not defined(__NVCC__) - constexpr - #endif - {return Op{}(a, b);} - ); - // for(int i = 0; i != *len; i++) { - // inout_t[i] = Op{}(std::move(inout_t[i]), in_t[i]); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic) - // } - } - - public: - template - explicit operation(F const& /*f*/) : base(detail::uninitialized{}) { - MPI_Op_create(reinterpret_cast(&combine), /*commutative*/ true, &(this->impl_)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - - template, operation>{}> > - operation(F&& f, bool commutative) : base(detail::uninitialized{}) { - MPI_Op_create( - &f, - // reinterpret_cast(&f), - commutative, - &(this->impl_) - ); - } - - operation(operation const&) = delete; - operation(operation &&) = delete; - - operation& operator=(operation const&) = delete; - operation& operator=(operation &&) = delete; - - ~operation() = default; - - MPI_Op operator&() const {return this->impl_;} // NOLINT(google-runtime-operator) - -#if 0 - enum struct code : MPI_Op{ - maximum = MPI_MAX, minimum = MPI_MIN, - sum = MPI_SUM, product = MPI_PROD, - logical_and = MPI_LAND, bitwise_and = MPI_BAND, - logical_or = MPI_LOR, bitwise_or = MPI_BOR, - logical_xor = MPI_LXOR, bitwise_xor = MPI_BXOR, - max_value_location = MPI_MAXLOC, - min_value_location = MPI_MINLOC - }; -#endif -// operation(operation::code c) : base((MPI_Op)c){} - -// static operation const sum;//(operation::code::sum); -// static operation const product; -// static operation const maximum; -// static operation const minimum; - -}; - -//operation const sum (operation::code::sum); -//operation const product(operation::code::product); -//operation const maximum(operation::code::maximum); -//operation const minimum(operation::code::minimum); - -template -using plus = std::plus; -template -using minus = std::minus; -template -using multiplies = std::multiplies; - -template struct min { - constexpr T const& operator()(T const& t1, T const& t2) const {return std::min(t1, t2);} -}; -template<> struct min{ - template constexpr decltype(auto) operator()(T1&& t1, T2&& t2) const {return std::min(std::forward(t1), std::forward(t2));} -}; - -template struct max { - constexpr T const& operator()(T const& t1, T const& t2) const {return std::max(t1, t2);} -}; -template<> struct max { - template constexpr decltype(auto) operator()(T1&& t1, T2&& t2) const {return std::max(std::forward(t1), std::forward(t2));} -}; - -template struct max_loc { // the only differences is that argument is assumed to be a pair, and second element is int - constexpr T const& operator()(T const& t1, T const& t2) const {std::max(t1, t2);} -}; -template<> struct max_loc { - template constexpr decltype(auto) operator()(T1&& t1, T2&& t2) const {return std::max(std::forward(t1), std::forward(t2));} -}; - -template struct min_loc { // the only differences is that argument is assumed to be a pair, and second element is int - constexpr T const& operator()(T const& t1, T const& t2) const {std::min(t1, t2);} -}; -template<> struct min_loc { - template constexpr decltype(auto) operator()(T1&& t1, T2&& t2) const {return std::min(std::forward(t1), std::forward(t2));} -}; - -template<> -struct operation { - private: - MPI_Op impl_; - - public: - explicit operation(std::plus<> /*op*/) : impl_{MPI_SUM} {} - explicit operation(std::plus /*op*/) : impl_{MPI_SUM} {} - - explicit operation(mpi3::min<> /*op*/) : impl_{MPI_MIN} {} - explicit operation(mpi3::min /*op*/) : impl_{MPI_MIN} {} - - explicit operation(mpi3::max<> /*op*/) : impl_{MPI_MAX} {} - explicit operation(mpi3::max /*op*/) : impl_{MPI_MAX} {} - - MPI_Op operator&() const {return impl_;} // NOLINT(google-runtime-operator) -}; - -template<> -struct operation > { - private: - MPI_Op impl_; - - public: - // explicit operation(std::plus<> /*op*/) : impl_{MPI_SUM} {} - // explicit operation(std::plus /*op*/) : impl_{MPI_SUM} {} - - // explicit operation(mpi3::min<> /*op*/) : impl_{MPI_MIN} {} - // explicit operation(mpi3::min /*op*/) : impl_{MPI_MIN} {} - - explicit operation(mpi3::max_loc<> /*op*/) : impl_{MPI_MAXLOC} {} - explicit operation(mpi3::max_loc > /*op*/) : impl_{MPI_MAXLOC} {} - - MPI_Op operator&() const {return impl_;} // NOLINT(google-runtime-operator) -}; - - -template struct predefined_operation; - -// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(CppoP, MpinamE, NamE) \ -template<> struct predefined_operation{ \ -/* constexpr*/ operator MPI_Op() const{return MpinamE;} \ -/* static constexpr MPI_Op value = MpinamE;*/ \ -}; \ -using NamE = predefined_operation // NOLINT(bugprone-macro-parentheses) - -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(std::plus<> , MPI_SUM , sum ); -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(std::multiplies<> , MPI_PROD, product ); -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(std::logical_and<>, MPI_LAND, logical_and); - -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(std::bit_and<>, MPI_BAND, bitwise_and); -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(std::bit_or<> , MPI_BOR , bitwise_or); - -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(max<>, MPI_MAX, maximum); -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(min<>, MPI_MIN, minimum); - -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(max_loc<>, MPI_MAXLOC, maximum_location); -BOOST_MPI3_DECLARE_PREDEFINED_OPERATION(min_loc<>, MPI_MINLOC, minimum_location); - -#undef BOOST_MPI3_DECLARE_PREDEFINED_OPERATION - -template -struct commutative_operation : operation { - template, operation >{}> > - explicit commutative_operation(F&& f) : operation(std::forward(f), true){} -}; - -template -struct non_commutative_operation : operation { - template, operation >{}>> - explicit non_commutative_operation(F&& f) : operation(std::forward(f), false){} -}; - -} // end namespace mpi3 -} // end namespace boost - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/ostream.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/ostream.hpp deleted file mode 100644 index aeca72d7f21..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/ostream.hpp +++ /dev/null @@ -1,198 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2020-2022 Alfredo A. Correa - -#ifndef MPI3_OSTREAM_HPP -#define MPI3_OSTREAM_HPP - -#include "../mpi3/communicator.hpp" -#include "../mpi3/process.hpp" - -#include // needs Boost.ICL - -#include -#include -#include - -#ifdef __clang__ -#define CALLABLE_WHEN(ConsumedORUnconsumed) [[clang::callable_when(ConsumedORUnconsumed)]] // NOLINT(cppcoreguidelines-macro-usage) -#define CONSUMABLE(ConsumedORUnconsumed) [[clang::consumable(ConsumedORUnconsumed)]] // NOLINT(cppcoreguidelines-macro-usage) -#define RETURN_TYPESTATE(ConsumedORUnconsumed) [[clang::return_typestate(ConsumedORUnconsumed)]] // NOLINT(cppcoreguidelines-macro-usage) -#define SET_TYPESTATE(ConsumedORUnconsumedORUnknown) [[clang::set_typestate(ConsumedORUnconsumedORUnknown)]] // NOLINT(cppcoreguidelines-macro-usage) -#else -#define CALLABLE_WHEN(ConsumedORUnconsumed) // NOLINT(cppcoreguidelines-macro-usage) -#define CONSUMABLE(ConsumedORUnconsumed) // NOLINT(cppcoreguidelines-macro-usage) -#define RETURN_TYPESTATE(ConsumedORUnconsumed) // NOLINT(cppcoreguidelines-macro-usage) -#define SET_TYPESTATE(ConsumedORUnconsumedORUnknown) // NOLINT(cppcoreguidelines-macro-usage) -#endif - -namespace boost { -namespace mpi3 { - -struct ostream : public std::ostream { // NOLINT(fuchsia-multiple-inheritance) bug in clang-tidy 12? - class streambuf : public std::stringbuf { // http://stackoverflow.com/a/2212940/225186 - communicator& comm_; - std::ostream& output; - std::string msg_; - - bool doing_table = false; - bool doing_formatting = false; - - public: - explicit streambuf(communicator& comm, std::ostream& strm = std::cout) - : comm_{comm}, output{strm} {} - - int sync() override { - // following code can be improved by a custom reduce operation - if(comm_.at_root()) { - boost::icl::interval_map messages; - messages.insert(std::make_pair(0, str())); - for(int i = 1; i != comm_.size(); ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm - match m = comm_.matched_probe(i); - msg_.resize(static_cast(m.count())); - m.receive(msg_.begin()); - messages.insert(std::make_pair(i, msg_)); - } - if( - std::all_of(messages.begin(), messages.end(), [](auto const& e) { return e.second.empty(); }) or - std::all_of(messages.begin(), messages.end(), [](auto const& e) { return e.second.empty() or e.second.back() == '\n'; }) - ) { - if(not doing_formatting) { - doing_formatting = true; - output << '\n'; - } - if(doing_table) { - doing_table = false; - output << '\n'; - } - collapse_lines(messages); - } else if(std::all_of(messages.begin(), messages.end(), [](auto const& e) { return e.second.empty() or e.second.back() == '\t'; })) { - if(not doing_formatting) { - doing_formatting = true; - output << '\n'; - } - if(not doing_table) { - output // <<"\e[1;31m" - << std::setw(18) << (comm_.name() + "↓"); - for(int i = 0; i != comm_.size(); ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm - output << std::setw(20) << ("↓" + std::to_string(i) + "↓"); - } - output //<<"\e[0m" - << '\n'; - doing_table = true; - } - headed_row(messages); - } else { - doing_formatting = false; - unformatted_one_or_all(messages); - } - } else { - comm_.send_n(str().begin(), str().size(), 0); - } - str(""); - output.flush(); - comm_.barrier(); - return 0; - } - - private: - void collapse_lines(boost::icl::interval_map const& messages) const { - for(auto const& m : messages) { // NOLINT(altera-unroll-loops) use algorithm - std::string range = comm_.name(); - if(static_cast(size(m.first)) < static_cast(comm_.size())) { - if(size(m.first) == 1) { - range += ("[" + std::to_string(lower(m.first)) + "]"); - } else { - range += ("[" + std::to_string(lower(m.first)) + "-" + std::to_string(upper(m.first)) + "]"); - } - } - output << /*"\e[1;32m"<<*/ std::setw(16) << std::left << range; - output << "→ " << /*"\e[0m"<<*/ m.second; - } - if(messages.iterative_size() > 1) { - output << '\n'; - } - } - void headed_row(boost::icl::interval_map const& messages) const { - std::size_t last_idx2 = 0; - std::size_t last_idx = 0; - while( // NOLINT(altera-unroll-loops) use algorithm - std::all_of(messages.begin(), messages.end(), [last_idx2, &messages](auto const& e) { return last_idx2 != e.second.size() and e.second[last_idx2] == messages.begin()->second[last_idx2]; }) - ) { - if(messages.begin()->second[last_idx2] == ' ') { - last_idx = last_idx2 + 1; - } - ++last_idx2; - } - output << std::setw(16) << std::left << std::setfill(' ') << messages.begin()->second.substr(0, last_idx); - - for(auto const& m : messages) { - for(auto i = m.first.lower(); i != m.first.upper() + 1; ++i) { // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) use algorithms - output - << std::setw(16) << std::setfill(' ') << std::left - << m.second.substr(last_idx, m.second.size() - last_idx - 1) // .substr(last_idx, last_idx + 5 /*m.second.size() - 4*/) - ; - } - } - output << '\n' - << std::flush; - } - void unformatted_one_or_all(boost::icl::interval_map const& messages) const { - if(messages.iterative_size() == 1) { - output << messages.begin()->second << '\n'; - } else { - for(auto const& m : messages) { - for(auto i = m.first.lower(); i != m.first.upper() + 1; ++i) { // NOLINT(altera-unroll-loops,altera-id-dependent-backward-branch) TODO(correaa) use algorithm - output << m.second; - } - } - } - output << std::flush; - } - }; - - ostream(ostream const&) = delete; - ostream& operator=(ostream const&) = delete; - - ostream(ostream&&) = delete; - ostream& operator=(ostream&&) = delete; - - private: - streambuf buffer; - - public: - explicit ostream(communicator& comm, std::ostream& os = std::cout) - : std::ostream(&buffer), buffer(comm, os) {} - ~ostream() override { flush(); } - - // template - // struct CONSUMABLE(unconsumed) - // iterator { - // using iterator_category = std::output_iterator_tag; - // using value_type = void; - // using difference_type = std::ptrdiff_t; - // using pointer = void; - // using reference = void; - // using ostream_type = ostream; - - // RETURN_TYPESTATE(unconsumed) - // explicit iterator(package_oarchive& oa) : out_{&oa} {} - - // iterator& operator=(T const& value) { *out_ << value; return *this; } - - // RETURN_TYPESTATE(consumed) CALLABLE_WHEN(unconsumed) - // auto operator*() -> iterator& {return *this;} - - // RETURN_TYPESTATE(unconsumed) - // auto operator++() -> iterator& {return *this;} - // RETURN_TYPESTATE(unconsumed) - // auto operator++(int) -> iterator& {return *this;} - - // private: - // ostream_type* out_; - // }; -}; - -} // end namespace mpi3 -} // end namespace boost - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/package_archive.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/package_archive.hpp deleted file mode 100644 index 6ffaa7e9c41..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/package_archive.hpp +++ /dev/null @@ -1,451 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2017-2022 Alfredo A. Correa - -#ifndef MPI3_PACKAGE_ARCHIVE_HPP -#define MPI3_PACKAGE_ARCHIVE_HPP - -#include "../mpi3/detail/package.hpp" - -#include -#include - -#include -#include - -#include - -#include - -namespace boost { -namespace mpi3 { - -namespace detail { - -class basic_package_iprimitive { - protected: - package& p_; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes,misc-non-private-member-variables-in-classes) TODO(correaa) - - public: - // we provide an optimized save for all basic (and fundamental) types - // this note is taken from boost serialization: - // typedef serialization::is_bitwise_serializable - // use_array_optimization; - // workaround without using mpl lambdas - struct use_array_optimization { - template - struct apply : public mpl::bool_::value> {}; - }; - template -#if(BOOST_VERSION < 106100) - void load_array(boost::serialization::array& t, unsigned int /*version*/ = 0){// for boost pre 1.63 -#else - void load_array(boost::serialization::array_wrapper& t, unsigned int /*version*/ = 0) { -#endif - p_.unpack_n(t.address(), t.count()); -} template -void load(T& t) { - p_ >> t; -} -explicit basic_package_iprimitive(mpi3::detail::package& p) : p_(p) {} -}; // namespace detail - -class basic_package_oprimitive { - protected: - package& p_; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes,misc-non-private-member-variables-in-classes) TODO(correaa) - - public: - struct use_array_optimization { - template - struct apply : public boost::serialization::is_bitwise_serializable {}; - }; - template - void save(const T& t, unsigned int /*version*/ = 0) { p_.pack_n(std::addressof(t), 1); } // p_ << t;} - template -#if(BOOST_VERSION < 106100) - void save_array(boost::serialization::array const& t, unsigned int /*version*/ = 0) { -#else - void save_array(boost::serialization::array_wrapper const& t, unsigned int /*version*/ = 0) { -#endif - p_.pack_n(t.address(), t.count()); - } -#if 0 - void save(const boost::archive::object_id_type&){} - void save(const boost::archive::object_reference_type&){} - void save(const boost::archive::class_id_type&){} - void save(const boost::archive::class_id_optional_type&){} - basic_memory_oprimitive(size_t& os) : os_(os){} -#endif - explicit basic_package_oprimitive(mpi3::detail::package& p) : p_{p} {} -}; - -template -class basic_package_iarchive : public boost::archive::detail::common_iarchive { - friend class boost::archive::detail::interface_iarchive; - using detail_common_iarchive = boost::archive::detail::common_iarchive; - - template - void load_override(T& t, /*BOOST_PFTO*/ int /*unused*/) { -#if(BOOST_VERSION < 105900) - this->detail_common_iarchive::load_override(t, 0); -#else - this->detail_common_iarchive::load_override(t); //, 0); -#endif - } - template - void load_override(T& t) { load_override(t, 0); } - - protected: - explicit basic_package_iarchive(unsigned int flags) - : boost::archive::detail::common_iarchive(flags) {} -}; - -template -class basic_package_oarchive : public boost::archive::detail::common_oarchive { - friend class boost::archive::detail::interface_oarchive; - using detail_common_oarchive = boost::archive::detail::common_oarchive; - - protected: - template - void save_override(T& t, /*BOOST_PFTO*/ int /*unused*/) { -#if(BOOST_VERSION < 105900) - this->detail_common_oarchive::save_override(t, 0); //, 0); -#else - this->detail_common_oarchive::save_override(t); -#endif - } - template - void save_override(T& t) { save_override(t, 0); } -#if 0 - void save_override(const object_id_type&, int){/* this->This()->newline(); this->detail_common_oarchive::save_override(t, 0);*/} - void save_override(const class_id_optional_type&, int){} - void save_override(const class_name_type&, int){/* const std::string s(t); * this->This() << s;*/} -#endif - - explicit basic_package_oarchive(unsigned int flags) - : boost::archive::detail::common_oarchive(flags) {} -}; - -template -class package_iarchive_impl // NOLINT(fuchsia-multiple-inheritance) follow Boost Serialization design -: public basic_package_iprimitive -, public basic_package_iarchive { - public: - template - void load(T& t) { basic_package_iprimitive::load(t); } - // empty functions follow, so that metadata is communicated - void load(boost::archive::version_type& /*version*/) {} - // void save(const boost::serialization::item_version_type&){/*save(static_cast(t));*/} - void load(boost::archive::tracking_type& /*tracking*/) { /*save(static_cast(t));*/ - } - void load(boost::archive::object_id_type& /*object_id*/) {} - void load(boost::archive::object_reference_type& /*object_reference*/) {} - void load(boost::archive::class_id_type& /*class_id*/) {} - void load(boost::archive::class_id_optional_type& /*class_id_optional*/) {} - void load(boost::archive::class_id_reference_type& /*class_id_reference*/) {} - void load(boost::archive::class_name_type& /*class_name*/) {} - - void load(boost::serialization::collection_size_type& t) { - unsigned int x = 0; - load(x); - t = serialization::collection_size_type(x); - } - void load(boost::serialization::item_version_type& /*item_version*/) {} - - void load(char* s) { - assert(0); - const std::size_t len = std::ostream::traits_type::length(s); - *this->This() << len; - p_.pack_n(s, len); - } - void load(wchar_t* ws) { - const std::size_t l = std::wcslen(ws); - *this->This() << l; - assert(0); - } - void load(std::string& s) { - // NOLINTNEXTLINE(cppcoreguidelines-init-variables) delayed initialization - std::size_t size; // *this->This() >> size; - p_.unpack_n(&size, 1); - s.resize(size); - p_.unpack_n(s.data(), size); - } - void load(std::wstring& ws) { // cppcheck-suppress constParameter ; TODO(correaa) implement? - const std::size_t size = ws.size(); - *this->This() << size; - // ++tokens_; // this->This()->newtoken(); - // os_ += ws.size()*sizeof(wchar_t);// os << s; - assert(0); - } - - package_iarchive_impl(mpi3::detail::package& p, unsigned int flags) : // size_t& os, size_t& tokens, unsigned int flags) : - basic_package_iprimitive(p), - basic_package_iarchive(flags) {} -}; - -template -class package_oarchive_impl // NOLINT(fuchsia-multiple-inheritance) follow Boost.Serialization design -: public basic_package_oprimitive -, public basic_package_oarchive { - public: - template - void save(const T& t) { basic_package_oprimitive::save(t); } - -#if(BOOST_VERSION < 106100) - void save(boost::serialization::array& /*arr*/) { -#else - void save(boost::serialization::array_wrapper& /*arr*/) { -#endif - assert(0); - } - - void save(boost::archive::version_type const& /*version*/) {} - // void save(const boost::serialization::item_version_type&){/*save(static_cast(t));*/} - void save(boost::archive::tracking_type const& /*tracking*/) { /*save(static_cast(t));*/ - } - void save(boost::archive::object_id_type const& /*object_id*/) {} - void save(boost::archive::object_reference_type const& /*object_reference*/) {} - void save(boost::archive::class_id_type const& /*class_id*/) {} - void save(boost::archive::class_id_optional_type const& /*class_id_optional*/) {} - void save(boost::archive::class_id_reference_type const& /*class_id_reference*/) {} - void save(boost::archive::class_name_type const& /*class_name*/) {} - - void save(boost::serialization::collection_size_type const& t) { - save(static_cast(t)); - // save(static_cast(t)); - } - void save(boost::serialization::item_version_type const& /*item_version*/) {} - - // string types (like char*, string, etc) have special handling - // types that need special handling - void save(char const* s) { - assert(0); - const std::size_t len = std::ostream::traits_type::length(s); - *this->This() << len; - // ++tokens_;// this->This()->newtoken(); - // os_ += len*sizeof(char);// os << s; - p_.pack_n(s, len); - } - void save(wchar_t const* ws) { - const std::size_t l = std::wcslen(ws); - *this->This() << l; - assert(0); - // ++tokens_; // this->This()->newtoken(); - // os_ += l*sizeof(wchar_t);// os.write((const char *)ws, l * sizeof(wchar_t)/sizeof(char)); - } - void save(std::string const& s) { - const std::size_t size = s.size(); - // *this->This() << size; - p_.pack_n(&size, 1); - // std::cout << " packed size = " << size << '\n'; - // ++tokens_; // this->This()->newtoken(); - // os_ += s.size()*sizeof(char);// os << s; - p_.pack_n(s.c_str(), size); - } - void save(std::wstring const& ws) { - const std::size_t size = ws.size(); - *this->This() << size; - // ++tokens_; // this->This()->newtoken(); - // os_ += ws.size()*sizeof(wchar_t);// os << s; - assert(0); - } - // using package_oarchive_impl::save_override; - -#if 1 - // Save all supported datatypes directly - template -#if(BOOST_VERSION < 106100) - void save(boost::serialization::array const& t, unsigned int /*version*/){ -#else - void save(boost::serialization::array_wrapper const& t, unsigned int /*version*/) { -#endif - assert(0); - save_override(t, boost::mpl::bool_{}); // std::true_type{}); -} -#endif - -package_oarchive_impl(mpi3::detail::package& p, unsigned int flags) // size_t& os, size_t& tokens, unsigned int flags) : -: basic_package_oprimitive(p), basic_package_oarchive(flags) { -} -}; // namespace mpi3 - -} // namespace detail - -#ifdef __clang__ -#define CALLABLE_WHEN(ConsumedORUnconsumed) [[clang::callable_when(ConsumedORUnconsumed)]] // NOLINT(cppcoreguidelines-macro-usage) -#define CONSUMABLE(ConsumedORUnconsumed) [[clang::consumable(ConsumedORUnconsumed)]] // NOLINT(cppcoreguidelines-macro-usage) -#define RETURN_TYPESTATE(ConsumedORUnconsumed) [[clang::return_typestate(ConsumedORUnconsumed)]] // NOLINT(cppcoreguidelines-macro-usage) -#define SET_TYPESTATE(ConsumedORUnconsumedORUnknown) [[clang::set_typestate(ConsumedORUnconsumedORUnknown)]] // NOLINT(cppcoreguidelines-macro-usage) -#else -#define CALLABLE_WHEN(ConsumedORUnconsumed) // NOLINT(cppcoreguidelines-macro-usage) -#define CONSUMABLE(ConsumedORUnconsumed) // NOLINT(cppcoreguidelines-macro-usage) -#define RETURN_TYPESTATE(ConsumedORUnconsumed) // NOLINT(cppcoreguidelines-macro-usage) -#define SET_TYPESTATE(ConsumedORUnconsumedORUnknown) // NOLINT(cppcoreguidelines-macro-usage) -#endif - -template -struct CONSUMABLE(unconsumed) basic_iterator { - using iterator_category = Category; - using value_type = ValueType; - using difference_type = std::ptrdiff_t; - using pointer = Pointer; - using reference = Reference; -}; - -struct package_iarchive -: public detail::package_iarchive_impl { - explicit package_iarchive(mpi3::detail::package& p, unsigned int flags = 0) - : package_iarchive_impl(p, flags) {} - - template - struct CONSUMABLE(unconsumed) - iterator { - using iterator_category = std::input_iterator_tag; - using value_type = void; - using difference_type = std::ptrdiff_t; - using pointer = T const*; - using reference = T const&; - - RETURN_TYPESTATE(unconsumed) - explicit iterator(package_iarchive& oa) : in_archive_{&oa} {*in_archive_ >> current_;} - - RETURN_TYPESTATE(unconsumed) - iterator() = default; - - // iterator& operator=(T const& value) { *out_archive_ << value; return *this; } - - // RETURN_TYPESTATE(consumed) - SET_TYPESTATE(consumed) CALLABLE_WHEN(unconsumed) - auto operator*() -> T&& {return std::move(current_);} - - RETURN_TYPESTATE(unconsumed) SET_TYPESTATE(unconsumed) - auto operator++() -> iterator& {*in_archive_ >> current_; return *this;} - // RETURN_TYPESTATE(unconsumed) - // auto operator++(int) -> iterator& {*in_archive_ >> current_; return *this;} - - bool operator!=(iterator const& other) const = delete; - bool operator==(iterator const& other) const = delete; - - // bool operator==(iterator const& other) const {return static_cast(*in_archive_);} - // bool operator!=(iterator const& other) const {return not static_cast(*in_archive_);} - - private: - package_iarchive* in_archive_ = nullptr; - T current_; - }; -}; - -struct package_oarchive : public detail::package_oarchive_impl { - explicit package_oarchive(mpi3::detail::package& p, unsigned int flags = 0) - : package_oarchive_impl(p, flags) {} - using package_oarchive_impl::operator&; - -#if(BOOST_VERSION < 106100) - package_oarchive& operator&(boost::serialization::array& /*arr*/) -#else - package_oarchive& operator&(boost::serialization::array_wrapper& /*arr*/) -#endif - { - assert(0); - return *this; - } - - template - struct CONSUMABLE(unconsumed) - iterator { - using iterator_category = std::output_iterator_tag; - using value_type = void; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = void; - using oarchive_type = package_oarchive; - - RETURN_TYPESTATE(unconsumed) - explicit iterator(package_oarchive& oa) : out_archive_{&oa} {} - - iterator& operator=(T const& value) { *out_archive_ << value; return *this; } - - RETURN_TYPESTATE(consumed) CALLABLE_WHEN(unconsumed) - auto operator*() -> iterator& {return *this;} - - RETURN_TYPESTATE(unconsumed) - auto operator++() -> iterator& {return *this;} - RETURN_TYPESTATE(unconsumed) - auto operator++(int) -> iterator& {return *this;} - - private: - package_oarchive* out_archive_; - }; -}; - -} // end namespace mpi3 -} // end namespace boost - -// maybe needed for optimization to take effect? -// BOOST_SERIALIZATION_USE_ARRAY_OPTIMIZATION(boost::archive::package_oarchive) - -//#ifdef _TEST_MPI3_PACKAGE_ARCHIVE - -//#include "../mpi3/main.hpp" -//#include "../mpi3/process.hpp" - -//#include -//#include - -// namespace mpi3 = boost::mpi3; -// using std::cout; - -// int mpi3::main(int, char*[], mpi3::communicator world) { -// assert(world.size() > 1); -// switch(world.rank()){ -// case 0: { -// mpi3::detail::package p(world); -// mpi3::package_oarchive poa(p); -// std::string s("hello"); -// int -// i = 12, -// j = 13 -// ; -// std::vector v(20, 5.); -// std::map m = {{1,2},{2,4},{3,4}}; -// poa -// << s -// << i -// << j -// << v -// << 5 -// << m -// ; -// p.send(1); -// } break; -// case 1: { -// mpi3::detail::package p(world); -// mpi3::package_iarchive pia(p); -// p.receive(0); -// std::string s; -// int -// i, -// j -// ; -// std::vector v; -// int c; -// std::map m; -// pia -// >> s -// >> i -// >> j -// >> v -// >> c -// >> m -// ; -// assert( s == "hello" ); -// assert( i == 12 ); -// assert( j == 13 ); -// assert(v.size() == 20); -// assert(c == 5); -// assert( m[3] == 4 ); -// } -// } -// return 0; -// } -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/pointer.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/pointer.hpp deleted file mode 100644 index 6669a9ffbc6..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/pointer.hpp +++ /dev/null @@ -1,205 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_POINTER $0x.cpp -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_POINTER_HPP -#define BOOST_MPI3_POINTER_HPP - -#include "../mpi3/window.hpp" -#include - -namespace boost{ -namespace mpi3{ - -template<> struct pointer{ - std::shared_ptr> winSP_; -}; - -template -struct pointer{ - std::shared_ptr> winSP_; - ~pointer(){} -// pointer() : winSP_(){} -// pointer(pointer const& other) : winSP_(other.win_){} -// pointer(pointer&& other) : win_(std::move(other.win_)){} -// pointer(pointer&& other) : ptr_(other.ptr_), pimpl_(other.pimpl_){ -// other.pimpl_ = nullptr; -// } -// pointer& operator=(pointer&& other){ -// pimpl_ = other.pimpl_; -// other.pimpl_ = nullptr; -// } -// pointer(pointer const& other) = delete; -// pointer& operator=(pointer const& other) = delete; -// ~pointer(){if(pimpl_) delete pimpl_;} - - T* local_ptr() const{ - void* base; - int flag; - MPI_Win_get_attr(winSP_->impl_, MPI_WIN_BASE, &base, &flag); - return static_cast(base); - } -/* - MPI_Aint local_size() const{ - MPI_Aint* size_p; - int flag; - int i = MPI_Win_get_attr(pimpl_->impl_, MPI_WIN_SIZE, &size_p, &flag); - assert(i==0); - assert(flag); - return *size_p/local_disp_unit(); - } - int const& local_disp_unit() const{ - int* disp_unit_p; - int flag; - int i = MPI_Win_get_attr(pimpl_->impl_, MPI_WIN_DISP_UNIT, &disp_unit_p, &flag); - assert(i==0); - assert(flag); - return *disp_unit_p; - } -*/ - void source(T& t) const{ - winSP_->lock_exclusive(0); - winSP_->get_n(&t, sizeof(T), 0, 0*sizeof(T)); - winSP_->unlock(0); - } - reference operator*() const{ - return {*this}; - } -}; - -template -struct reference{ - pointer p_; - T buffer_; - reference const& operator=(T const& t) const{ - p_.winSP_->lock_exclusive(0); - p_.winSP_->put_n(&t, sizeof(T), 0, 0*sizeof(T)); - p_.winSP_->unlock(0); - return *this; - } - operator T(){ - p_.source(buffer_); - return buffer_; - } -}; - -#if 0 -template -pointer communicator::allocate(MPI_Aint size) const{ - pointer ret; - ret.pimpl_ = new window; - int i = MPI_Win_allocate( - size*sizeof(T), sizeof(T), MPI_INFO_NULL, impl_, - &ret.ptr_, //&static_cast(ret).impl_ - &ret.pimpl_->impl_ - ); - if(size == 0) ret.ptr_ = nullptr; - if(i!=0) assert(0); - return ret; -} -#endif - -pointer communicator::malloc(MPI_Aint size) const{ - pointer ret; - void* ignore; // ??? - ret.winSP_ = std::make_shared>(); - int i = MPI_Win_allocate( - size, 1, MPI_INFO_NULL, impl_, - &ignore, //&ret.ptr_, - &(ret.winSP_->operator&()) - ); -// if(size == 0) ret.ptr_ = nullptr; - if(i!=0) assert(0); - return ret; -} - -void communicator::free(pointer& p) const{ -// p.win_.fence(); - MPI_Free_mem(p.winSP_->base()); //the window frees the memory -// MPI_Win_free(&p.win_.impl_); -// p.win_.impl_ = MPI_WIN_NULL; -} - -template -struct pgas_allocator{ - communicator const& comm_; - pgas_allocator(communicator const& comm) : comm_(comm){} - pointer allocate(std::size_t size){ - pointer ret; - void* ignore; - int local_size = size/comm_.size() + (comm_.rank() < (size % comm_.size()))?1:0; - ret.winSP_ = std::make_shared>(); - int i = MPI_Win_allocate( - local_size*sizeof(T), sizeof(T), - MPI_INFO_NULL, &comm_, - &ignore, - &(ret.winSP_->operator&()) - ); - assert(i==0); - return ret; - } - void deallocate(pointer& p, std::size_t){ - // p = pointer(); - // MPI_Free_mem(p.local_ptr()); - // p.win_.impl_ = MPI_WIN_NULL; - } -}; - - -}} - -#ifdef _TEST_BOOST_MPI3_POINTER -#include - -#include "../mpi3/main.hpp" -using std::cout; -using std::endl; - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int, char*[], mpi3::communicator world){ - { - auto p = world.malloc(world.rank()==0?100*sizeof(double):0); - if(world.rank() == 1){ - double cinco = 5; - p.winSP_->lock_exclusive(0); - p.winSP_->put_n(&cinco, sizeof(double), 0, 11*sizeof(double)); - p.winSP_->unlock(0); - } - p.winSP_->fence(); - if(world.rank() == 0){ - cout << static_cast(p.winSP_->base())[11] << endl; - cout << p.winSP_->size() << endl; - cout << p.winSP_->disp_unit() << endl; - } - return 0; - if(world.rank() == 1){ - double t; - p.winSP_->lock_exclusive(0); - p.winSP_->get_n(&t, sizeof(double), 0, 11*sizeof(double)); - p.winSP_->unlock(0); - assert(t == 5.); - } - world.free(p); - } - return 0; - if(1){ - boost::mpi3::pgas_allocator alloc(world); - boost::mpi3::pointer p = alloc.allocate(1); - if(world.rank() == 0) *p = 5.1; - p.winSP_->fence(); - if(world.rank() == 0){if(*p == 5.1) cout << "ok\n"; else cout << "BAD\n";} - if(world.rank() == 1){if(*p == 5.1) cout << "ok\n"; else cout << "BAD\n";} - if(world.rank() == 2){if(*p == 5.1) cout << "ok\n"; else cout << "BAD\n";} - p.winSP_->fence(); - // alloc.deallocate(p, 20); - MPI_Win_free(&(p.winSP_.get()->operator&())); - } - double r = 5.; - cout <<"great\n"; - - return 0; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/port.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/port.hpp deleted file mode 100644 index 4949e88553c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/port.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ - -#ifndef BOOST_MPI3_PORT_HPP -#define BOOST_MPI3_PORT_HPP - -// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include -#include - -namespace boost { -namespace mpi3 { - -struct port { - // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes) TODO(correaa) - std::string name_; // typically this will be something like tag#0$description#inspiron$port#47425$ifname#172.17.5.240$ - - port() {open();} - port(port const&) = delete; - port(port &&) = delete; - - port& operator=(port const&) = delete; - port& operator=(port &&) = delete; - - explicit port(std::string name) : name_{std::move(name)} {}; - - ~port() noexcept{ try{if(is_open()) {close();}}catch(...){} } - - void open() { - std::array name_buffer{}; - MPI_(Open_port)(MPI_INFO_NULL, name_buffer.data()); - name_ = std::string{name_buffer.data()}; - } - void open(std::string const& name) {name_ = name;} - - std::string const& name() const{return name_;} - - bool is_open() const {return not name_.empty();} - void close() { - MPI_(Close_port)(name_.c_str()); - name_ = ""; - } -}; - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_PORT - -//#include "../mpi3/environment.hpp" - -//using std::cout; -//namespace mpi3 = boost::mpi3; - -//int main(int argc, char* argv[]){ -// mpi3::environment env(argc, argv); -// auto world = env.world(); -// assert(world.size() > 2); -// switch(world.rank()){ -// break; case 0:{ -// mpi3::port p1; -// mpi3::port p2; -// world.send_value(p1.name(), 1); -// world.send_value(p2.name(), 2); -// mpi3::communicator comm1 = env.self().accept(p1, 0); -// mpi3::communicator comm2 = env.self().accept(p2, 0); -// comm1.send_value(1, 0); -// comm2.send_value(2, 0); -// }; -// break; case 1:{ -// std::string s; -// world.receive_n(&s, 1, 0); -// mpi3::port p1(s); -// mpi3::communicator comm1 = env.self().connect(p1, 0); -// int data = -1; -// comm1.receive_n(&data, 1, 0); -// assert(data == 1); -// }; -// break; case 2:{ -// std::string s; -// world.receive_n(&s, 1); -// mpi3::port p2(s); -// mpi3::communicator comm2 = env.self().connect(p2, 0); -// int data; -// comm2.receive_n(&data, 1, 0); -// assert(data == 2); -// }; -// } -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/process.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/process.hpp deleted file mode 100644 index 0eeca999550..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/process.hpp +++ /dev/null @@ -1,120 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_PROCESS_HPP -#define BOOST_MPI3_PROCESS_HPP - -#include "../mpi3/communicator.hpp" - -#include - -#include "config/NODISCARD.hpp" - -namespace boost { -namespace mpi3 { - -using std::optional; - -class process { - communicator& comm_; - int rank_; - friend boost::mpi3::communicator; - - process(communicator& comm, int rank) : comm_{comm}, rank_{rank} {} - - public: - communicator& comm() const {return comm_;} - - int rank() const{return rank_;} - template - optional operator+=(T const& t) && { - T val = comm_.reduce_value(t, std::plus<>{}, rank_); - if(rank_ != comm_.rank()) {return {};} - return optional(val); - } - - template - process& operator>>(T& t) & { - comm_.receive_n(&t, 1, rank_); - // comm_.receive_value(t, rank_); - return *this; // NOLINT(hicpp-move-const-arg,performance-move-const-arg) TODO(correaa) - } - template - process&& operator&(T& t) && { - comm_.broadcast_value(t, rank_); - return std::move(*this); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) TODO(correaa) - } -}; - -template -auto operator<<(process&& p, const T& value) -> decltype(std::move(p << value)) { - return std::move(p << value); -} - -template -auto operator>>(process&& p, T&& value) -> decltype(std::declval() >> value) { - return p >> value; -} - -template -process& operator<<(process& self, T const& t) { - self.comm().send_value(t, self.rank()); - return self; -} - -inline auto communicator::operator[](int rank) -> process {return {*this, rank};} - -//inline auto communicator::iterator::operator*() const -> process {return {*commP_, rank_};} - -template -auto operator&(communicator& comm, T&& t) -->decltype(comm.all_to_all(begin(std::forward(t))), std::forward(t)) { - assert( t.size() == comm.size() ); -// using std::begin; - auto e = comm.all_to_all(begin(std::forward(t))); - using std::end; - assert( e == end(t) ); - return std::forward(t); -} - -template -//NODISCARD("do not ignore result when second argument is const") -auto operator&(communicator& comm, T const& t) -->decltype(comm.all_to_all(t.begin(), std::declval().begin()), T(comm.size())) { - assert(t.size() == comm.size()); - T ret(comm.size()); - comm.all_to_all(t.begin(), ret.begin()); - return ret; -} - -template -auto operator||(process&& self, T& t) {self.comm().broadcast_value(t, self.rank());} - -template -communicator& operator>>(communicator& comm, T& t) { - comm.receive_n(&t, 1); -// comm.receive_value(t); - return comm; -} -template -std::vector operator|=(communicator& comm, T const& t) { - return comm.all_gather_value(t); -} - -template -std::vector operator|=(process&& self, T const& t) { - return self.comm().gather_value(t, self.rank()); -} - -template -std::pair communicator::max_location(T const& t) { - auto const ml = max_loc(t); - return std::pair{ - ml.value, - process{*this, ml.location} - }; -} - -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/request.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/request.hpp deleted file mode 100644 index 580aca37c2b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/request.hpp +++ /dev/null @@ -1,180 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2019-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_REQUEST_HPP -#define BOOST_MPI3_REQUEST_HPP -#pragma once - -#include "../mpi3/detail/call.hpp" -#include "../mpi3/detail/iterator.hpp" // detail::data - -#include "../mpi3/status.hpp" - -// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include -#include - -namespace boost { -namespace mpi3 { - -struct [[nodiscard]] request { - // in mpich MPI_Request is same as int - MPI_Request impl_ = MPI_REQUEST_NULL; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - auto handle() {return impl_;} // NOLINT(readability-make-member-function-const) MPI_Request is a handle (pointer-like semantics) - - request() = default; - [[nodiscard]] auto valid() const noexcept -> bool {return impl_ != MPI_REQUEST_NULL;} - - request(request const&) = delete; - - request(request&& other) noexcept : impl_{std::exchange(other.impl_, MPI_REQUEST_NULL)} {} - - request& operator=(request const&) = delete; - request& operator=(request&& other) noexcept { - request(std::move(other)).swap(*this); // cppcheck-suppress accessMoved ; false positive? - return *this; - } - bool completed() const { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Request_get_status)(impl_, &ret, MPI_STATUS_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - return ret != 0; - } - status get_status() const { - int ignore = -1; - return MPI_(Request_get_status)(impl_, &ignore); - } - void swap(request& other) {std::swap(impl_, other.impl_);} - void cancel() {MPI_Cancel(&impl_);} - - ~request() noexcept { // TODO(correaa) check it can be no noexcept and cancellable - try { - wait(); - if(impl_ != MPI_REQUEST_NULL) {MPI_Request_free(&impl_);} - } catch(...) { - std::terminate(); - } - } - void wait() { // TODO(correaa) make wait const - // assert(valid()); // TODO(correaa) investigate why this is failing - if(impl_ != MPI_REQUEST_NULL) { - status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization - MPI_(Wait)(&impl_, &ret.impl_); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) non-blocking call was used to create the object - } - } - status get() { - status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization - MPI_(Wait)(&impl_, &ret.impl_); - // if(s != MPI_SUCCESS) {throw std::runtime_error("cannot wait on request");} - return ret; - } - void start(){MPI_(Start)(&impl_);} - status test() const{return get_status();} -}; - -inline std::vector test_some(std::vector const& requests) { - int outcount = -1; - std::vector ignore(requests.size()); - std::vector ret(requests.size()); - int const s = MPI_Testsome( // TODO(correaa) modernize calls - static_cast(requests.size()), - const_cast(&(requests.data()->impl_)), // NOLINT(cppcoreguidelines-pro-type-const-cast) TODO(correaa) - &outcount, - ignore.data(), - &(ret.data()->impl_) - ); - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot test some"};} - return ret; -} - -inline std::vector completed_some(std::vector const& requests) { - int outcount = -1; - std::vector ret(requests.size()); - int const s = MPI_Testsome( // TODO(correaa) modernize calls - static_cast(requests.size()), - const_cast(&(requests.data()->impl_)), // NOLINT(cppcoreguidelines-pro-type-const-cast) TODO(correaa) - &outcount, - ret.data(), - MPI_STATUSES_IGNORE // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - ); - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot completed some");} - ret.resize(static_cast(outcount)); - return ret; -} - -template -void wait_all_n(ContRequestIterator it, Size n){ - MPI_Waitall(n, &detail::data(it)->impl_, MPI_STATUSES_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro -} - -template -void wait_all(ContRequestIterator it1, ContRequestIterator it2){ - wait_all_n(it1, std::distance(it1, it2)); -} - -template -void wait(Args&&... args){ - auto move_impl = [](request&& r)->MPI_Request{ MPI_Request const ret = r.impl_; // NOLINT(misc-misplaced-const) MPI_Request is a pointer itself in some MPI - r.impl_ = MPI_REQUEST_NULL; - return ret; - }; - std::vector v{move_impl(std::move(args))...}; - MPI_Waitall(static_cast(v.size()), v.data(), MPI_STATUSES_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro -} - -template -ContiguousIterator wait_any_n(ContiguousIterator it, Size n){ - int index = -1; - int s = MPI_Waitany(n, &detail::data(it)->impl_, &index, MPI_STATUS_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - if(s != MPI_SUCCESS) {throw std::runtime_error("cannot wait any");} - return it + index; -} - -template -ContiguousIterator wait_any(ContiguousIterator first, ContiguousIterator last){ - return wait_any_n(first, std::distance(first, last)); -} - -template -std::vector wait_some_n(ContiguousIterator it, Size n){ - int outcount = -1; - std::vector indices(n); - MPI_(Waitsome)(n, &detail::data(it)->impl_, &outcount, indices.data(), MPI_STATUSES_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) - indices.resize(static_cast(outcount)); - return indices; -} - -template -std::vector wait_some(ContiguousIterator first, ContiguousIterator last){ - return wait_some_n(first, std::distance(first, last)); -} - -namespace detail { - -// this doesn't work in general because MPI_Request == int in mpich -//template((*F)(std::declval()..., std::declval())))* = nullptr> -//BMPI3_NODISCARD("") mpi3::request call(Args... args) { -// mpi3::request ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization -// auto const e = static_cast((*F)(args..., &ret.impl_)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor -// if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} -// return ret; -//} - -template((*F)(std::declval()..., std::declval())))* = nullptr> -mpi3::request call_i(Args... args) { - mpi3::request ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization - auto const e = static_cast((*F)(args..., &ret.impl_)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor - if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // MPI_Wait called on destructor of ret - return ret; // ret destructor will call wait -} // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) - -#define MPI_I(F) detail::call_i // NOLINT(cppcoreguidelines-macro-usage): name concatenation - -} // end namespace detail - -} // end namespace mpi3 -} // end namespace boost - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/counter.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/counter.hpp deleted file mode 100644 index 63d1295762d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/counter.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include<"$0">" > $0x.cpp) && mpicxx -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_COUNTER $0x.cpp -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_RMA_COUNTER_HPP -#define BOOST_MPI3_RMA_COUNTER_HPP - - -#include "../../mpi3/window.hpp" - -namespace boost{ -namespace mpi3{ - -template -class counter{ - mpi3::window w_; - counter(mpi3::communicator& c, T num){ - int lnum = num / c.size(); - int lleft = num % c.size(); - if(rank < lleft) ++lnum; - - } -}; - -}} - -#ifdef _TEST_BOOST_MPI3_COUNTER - -#include "alf/boost/mpi3/main.hpp" - -using std::cout; - -int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator& world){ - -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/memory.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/memory.hpp deleted file mode 100644 index 9ab67765e9d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/memory.hpp +++ /dev/null @@ -1,266 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++17 -Wall -Wextra -Wpedantic -D_TEST_MPI3_RMA_MEMORY $0x.cpp -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -// (C) Copyright 2019 Alfredo A. Correa -#ifndef MPI3_RMA_MEMORY_HPP -#define MPI3_RMA_MEMORY_HPP - -#include "../../mpi3/window.hpp" - -namespace boost{ -namespace mpi3{ -namespace rma{ - -template class ptr; - -//template<> class ptr{}; - -template -class ptr{ - window* w_ = nullptr; - mpi3::size_t global_offset_ = 0; - ptr(std::nullptr_t) : w_{nullptr}, global_offset_{0}{} - using decay_t = T; - T operator*() const{ - T ret; - w_->get_n(&ret, 1, global_offset_/w_->size(), global_offset_%w_->size()); - return ret; - } -}; - -template class ref; - -template -class ptr{ -protected: - template friend class allocator; - ptr(window* w, mpi3::size_t global_offset = 0) : w_{w}, global_offset_{global_offset}{} - window* w_; - mpi3::size_t global_offset_; -public: -// int target() const{return global_offset_/w_->size();} -// int offset() const{return global_offset_%w_->size();} - auto address() const{ - auto d = std::div(global_offset_, w_->size()); - return mpi3::target{static_cast(d.quot), d.rem}; - }; - using difference_type = mpi3::ptrdiff_t; - ptr() = default; - ptr(std::nullptr_t) : w_{nullptr}, global_offset_{0}{} - ptr(ptr const&) = default; - ref operator*() const; - ref operator[](difference_type d) const{ptr tmp{*this}; tmp+=d; return *tmp;} - ptr& operator+=(difference_type d){global_offset_ += d; return *this;} - ptr operator+(difference_type d){ptr tmp{*this}; tmp+=d; return tmp;} - operator void*() const{ - auto target = address(); - return w_->get_group().rank()==target.rank?(w_->base() + target.disp):nullptr; - } -}; - -class mutex{ - window* w_; - mutex(mutex const&) = delete; -// mutex& operator=(mutex const&) = deleted; - -}; - -class shared_mutex{ -}; - -template -class ref : public ptr{ - ref(ptr const& p) : ptr{p}{} - friend class ptr; -public: - using decay_t = T; - operator T() const&{ - T t; - this->w_->fence(MPI_MODE_NOPUT | MPI_MODE_NOPRECEDE); - auto target = this->address(); - this->w_->get_n(&t, 1, target.rank, target.disp); - this->w_->fence(MPI_MODE_NOSUCCEED); - return t; - } - ref const& operator=(T const& t) const&{ - auto target = this->address(); - // this->w_->fence(MPI_MODE_NOPRECEDE); - this->w_->lock_exclusive(target.rank); - if(this->w_->get_group().rank() == target.rank) - this->w_->put_n(&t, 1, target.rank, target.disp); - // this->w_->fence(MPI_MODE_NOSTORE | MPI_MODE_NOSUCCEED); - this->w_->unlock(target.rank); - return *this; - } -// ref const& operator=(ref const& other) const&{ -// if(this->w_ == other.w_) -// } - template decltype(auto) - operator-=(U&& u) const&{T t{*this}; return operator=(t-=std::forward(u));} - ptr const& operator&() const&{return *this;} -}; - -template ref ptr::operator*() const{return ref{*this};} - -template > -struct allocator{ - template struct rebind{typedef allocator other;}; - using value_type = T; - using pointer = rma::ptr; - using const_pointer = rma::ptr; - using size_type = mpi3::size_t; - using difference_type = mpi3::ptrdiff_t; -private: - Alloc a_; - mpi3::communicator const* c_; - allocator() = delete; -public: - allocator(mpi3::communicator const& c, Alloc const& a = {}) : a_{a}, c_{std::addressof(c)}{} - template allocator(allocator const& o) : a_{o.a_}, c_{o.c_}{} - ~allocator() = default; - pointer allocate(size_type n, const void* /*hint*/ = 0){ - if(n==0) return pointer{nullptr}; - auto local_n = (n-1)/c_->size()+1; - return {new mpi3::window{*c_, a_.allocate(local_n), local_n}}; - } - void deallocate(pointer p, size_type n){ - assert(n == p.w_->size()); - a_.deallocate(p.w_->base(), p.w_->size()); - delete p.w_; - } - allocator& operator=(allocator const& other) = default; - bool operator==(allocator const& o) const{return a_==o.a_ and c_==o.c_;} - bool operator!=(allocator const& o) const{return not(o == *this);} - template - void construct(pointer p, As&&... as){ - void* v = static_cast(p); - p.w_->fence(); - if(v){ - T t; a_.construct(static_cast(&t), std::forward(as)...); - p.w_->put_n(&t, 1, p.w_->get_group().rank(), p.global_offset_%p.w_->size()); - } - p.w_->fence(); - } - template void destroy(P p){ - void* v = static_cast(p); - if(v) a_.destroy(v); - } -}; - -} -#if 0 -namespace shm{ - -template -struct pointer_traits : std::pointer_traits{ - static auto to_address(Ptr const& p){ - return std::addressof(*p); - } -}; - -template -struct pointer : - std::pointer_traits, - boost::dereferenceable, T*>, - boost::random_access_iteratable, T*, std::ptrdiff_t, T&> -{ - template using rebind = pointer; -// template struct rebind{typedef pointer other;}; - std::shared_ptr>> w_; - typename pointer::difference_type offset_; - pointer() = default; - pointer(std::nullptr_t) : offset_(0){} -// pointer(pointer const&) = default; - pointer& operator=(pointer const& other) = default; - template>{std::declval().w_})> - pointer(Other&& o) : w_{o.w_}, offset_{o.offset_}{} - pointer(pointer> const& o) : w_{o.w_}, offset_{o.offset_}{} - pointer& operator=(std::nullptr_t){w_ = nullptr; offset_ = 0; return *this;} - ~pointer() = default; - T& operator*() const{return *(static_cast(w_->base(0)) + offset_);} - pointer& operator+=(typename pointer::difference_type d){offset_+=d; return *this;} - pointer& operator++(){++offset_; return *this;} -// pointer operator->() const{return wSP_->base(0) + offset_;} -// reference operator[](difference_type d) const{return *((*this)+d);} - explicit operator pointer() const{return w_->base(0) + offset_;} - explicit operator bool() const{return bool{w_};} - bool operator==(pointer const& o) const{assert(w_==o.w_); return offset_==o.offset_;} - bool operator<(pointer const& o) const{assert(w_==o.w_); return offset_(pointer const& o) const{assert(w_==o.w_); return offset_>o.offset_;} - friend typename std::pointer_traits::pointer to_address(pointer const& p){ - p.w_->base(0) + p.offset_; - } -}; - -template using ptr = pointer; - -template -F for_each(pointer f, pointer l, F fun){ //TODO do a partitioning std::for_each - auto& comm = f.wSP_->comm_; assert(comm == l.wSP_->comm_); - using std::for_each; - if(mpi3::group(*f.wSP_).root()) for_each(to_address(f), to_address(l), fun); - f.wSP_->fence(); - f.wSP_->fence(); - return f; -} - -template -pointer copy_n(It1 f, Size n, pointer d){ - d.wSP_->fence(); - using std::copy_n; - if(mpi3::group(*d.wSP_).root()) copy_n(f, n, to_address(d)); - d.wSP_->fence(); - return d + n; -} - -template -pointer copy(It1 f, It1 l, pointer d){ - d.wSP_->fence(); - using std::copy; - if(mpi3::group(*d.wSP_).root()) copy(f, l, to_address(d)); - d.wSP_->fence(); - using std::distance; using std::advance; - advance(d, distance(f, l)); - return d; -} - -template -pointer uninitialized_fill_n(pointer f, Size n, TT const& val){ - using std::uninitialized_fill_n; - if(mpi3::group(*f.wSP_).root()) uninitialized_fill_n(to_address(f), n, val); - f.wSP_->fence(); - f.wSP_->fence(); - return f + n; -} - -} -#endif - -}} - -#ifdef _TEST_MPI3_RMA_MEMORY - -#include "../../mpi3/main.hpp" -#include "../../mpi3/ostream.hpp" - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::rma::allocator a{world}; - auto p = a.allocate(8); - for(int i = 0; i != 8; ++i) a.construct(&p[i], double(i)); - auto&& p2 = p[2]; - assert( p2 == 2 ); -// if(world.rank() == 1) - p[7] -= 7; - world.barrier(); - assert(p[7] == 0); - - return 0; -} - -#endif - - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/mutex.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/mutex.hpp deleted file mode 100644 index aaa31bafc69..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/rma/mutex.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++17 -Wall -Wextra -Wpedantic -D_TEST_MPI3_RMA_MEMORY $0x.cpp -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -// (C) Copyright 2019 Alfredo A. Correa -#ifndef MPI3_RMA_MEMORY_HPP -#define MPI3_RMA_MEMORY_HPP - -#include "../../mpi3/window.hpp" - -namespace boost{ -namespace mpi3{ -namespace rma{ - - -class mutex{ -// window* w_; - mutex(mutex const&) = delete; -}; - - -}}} - -#ifdef _TEST_MPI3_RMA_MEMORY - -#include "../../mpi3/main.hpp" -#include "../../mpi3/ostream.hpp" - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - - - return 0; -} - -#endif - - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/archive_exception.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/archive_exception.cpp deleted file mode 100644 index 252262c56bf..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/archive_exception.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// archive_exception.cpp: - -// (C) Copyright 2009 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. -// NOLINTBEGIN(altera-unroll-loops,cppcoreguidelines-pro-type-member-init,hicpp-member-init,misc-const-correctness) external code -#if (defined _MSC_VER) && (_MSC_VER == 1200) -# pragma warning (disable : 4786) // too long name, harmless warning -#endif - -#include -#include -#include - -#define BOOST_ARCHIVE_SOURCE -#include -#include -#include - -namespace boost { -namespace archive { - -#if BOOST_VERSION < 105900 -#define BOOST_NOEXCEPT -#endif - -BOOST_ARCHIVE_DECL -unsigned int -archive_exception::append(unsigned int l, const char * a){ - while(l < (sizeof(m_buffer) - 1)){ - char c = *a++; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic) external code - if('\0' == c) { - break; } - m_buffer[l++] = c; // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index) external code - } - m_buffer[l] = '\0'; // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index) external code - return l; -} - -BOOST_ARCHIVE_DECL -archive_exception::archive_exception( // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) external code - exception_code c, - const char * e1, - const char * e2 -) BOOST_NOEXCEPT : - code(c) -{ - unsigned int length = 0; - switch(code){ - case no_exception: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "uninitialized exception"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case unregistered_class: - length = append(length, "unregistered class"); - if(nullptr != e1){ - length = append(length, " - "); - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, e1); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - } - break; - case invalid_signature: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "invalid signature"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case unsupported_version: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "unsupported version"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case pointer_conflict: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "pointer conflict"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case incompatible_native_format: - length = append(length, "incompatible native format"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - if(nullptr != e1){ - length = append(length, " - "); - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, e1); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - } - break; - case array_size_too_short: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "array size too short"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case input_stream_error: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "input stream error"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case invalid_class_name: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "class name too long"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case unregistered_cast: - length = append(length, "unregistered void cast "); - length = append(length, (nullptr != e1) ? e1 : "?"); - length = append(length, "<-"); - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, (nullptr != e2) ? e2 : "?"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case unsupported_class_version: - length = append(length, "class version "); - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, (nullptr != e1) ? e1 : ""); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case other_exception: - // if get here - it indicates a derived exception - // was sliced by passing by value in catch - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "unknown derived exception"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - case multiple_code_instantiation: - length = append(length, "code instantiated in more than one module"); - if(nullptr != e1){ - length = append(length, " - "); - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, e1); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - } - break; - case output_stream_error: - // cppcheck-suppress [unreadVariable] ; external code - length = append(length, "output stream error"); // NOLINT(clang-analyzer-deadcode.DeadStores) external code - break; - default: - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - length = append(length, "programming error"); // cppcheck-suppress [unreadVariable] ; external code - break; - } -} - -#if BOOST_VERSION > 105900 -BOOST_ARCHIVE_DECL -archive_exception::archive_exception(archive_exception const & oth) BOOST_NOEXCEPT : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) external code - std::exception(oth), - code(oth.code) -{ - std::memcpy(m_buffer,oth.m_buffer,sizeof m_buffer); -} -#endif - -BOOST_ARCHIVE_DECL -archive_exception::~archive_exception() BOOST_NOEXCEPT_OR_NOTHROW {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code - -BOOST_ARCHIVE_DECL const char * -archive_exception::what() const BOOST_NOEXCEPT_OR_NOTHROW { - return m_buffer; -} - -BOOST_ARCHIVE_DECL -archive_exception::archive_exception() BOOST_NOEXCEPT : - code(no_exception) -{} - -} // end namespace archive -} // end namespace boost -// NOLINTEND(altera-unroll-loops,cppcoreguidelines-pro-type-member-init,hicpp-member-init,misc-const-correctness) external code diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_archive.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_archive.cpp deleted file mode 100644 index bd5ca5868a7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_archive.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// basic_archive.cpp: - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. - -////////////////////////////////////////////////////////////////////// -// -// objects are stored as -// -// class_id* // -1 for a null pointer -// if a new class id -// [ -// exported key - class name* -// tracking level - always/never -// class version -// ] -// -// if tracking -// [ -// object_id -// ] -// -// [ // if a new object id -// data... -// ] -// -// * required only for pointers - optional for objects - -#define BOOST_ARCHIVE_SOURCE -#include - -#include - -namespace boost { -namespace archive { - -/////////////////////////////////////////////////////////////////////// -// constants used in archive signature -//This should never ever change. note that is not an std::string -// string. -BOOST_SYMBOL_VISIBLE const char * -BOOST_ARCHIVE_SIGNATURE(){ - return "serialization::archive"; -} - -// this should change if the capabilities are added to the library -// such that archives can be created which can't be read by previous -// versions of this library -// 1 - initial version -// 2 - made address tracking optional -// 3 - numerous changes - can't guarentee compatibility with previous versions -// 4 - Boost 1.34 -// added item_version to properly support versioning for collections -// 5 - Boost 1.36 -// changed serialization of collections: adding version even for primitive -// types caused backwards compatibility breaking change in 1.35 -// 6 - Boost 1.41 17 Nov 2009 -// serializing collection sizes as std::size_t -// 7 Boost 1.42 2 Feb 2010 -// error - changed binary version to 16 bits w/o changing library version # -// That is - binary archives are recorded with #6 even though they are -// different from the previous versions. This means that binary archives -// created with versions 1.42 and 1.43 will have to be fixed with a special -// program which fixes the library version # in the header -// Boost 1.43 6 May 2010 -// no change -// 8 - Boost 1.44 -// separated version_type into library_version_type and class_version_type -// changed version_type to be stored as 8 bits. -// 10- fixed base64 output/input. -// 11- not changes -// 12- improved serialization of collections -// 13- simplified visibility, removed Borland, removed pfto -// 14- improved visibility, refactor map/set - -BOOST_SYMBOL_VISIBLE library_version_type -BOOST_ARCHIVE_VERSION(){ - return library_version_type(14); -} - -} // namespace archive -} // namespace boost diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iarchive.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iarchive.cpp deleted file mode 100644 index 21b9302169d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iarchive.cpp +++ /dev/null @@ -1,602 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// basic_archive.cpp: - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. -// NOLINTBEGIN(altera-id-dependent-backward-branch,altera-unroll-loops,hicpp-use-auto,hicpp-use-equals-default,modernize-use-auto,modernize-use-equals-default,misc-const-correctness) external code -#include // msvc 6.0 needs this to suppress warnings - -#include - -#include // size_t, NULL -#include -#include -#include - -#include // NOLINT(readability-duplicate-include) third party code -#if defined(BOOST_NO_STDC_NAMESPACE) -namespace std{ - using ::size_t; -} // namespace std -#endif - -#include - -#define BOOST_ARCHIVE_SOURCE -// include this to prevent linker errors when the -// same modules are marked export and import. -#define BOOST_SERIALIZATION_SOURCE -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -using namespace boost::serialization; // NOLINT(google-build-using-namespace,google-global-names-in-headers) external code TODO(correaa) avoid global namespace - -namespace boost { -namespace archive { -namespace detail { - -class basic_iarchive_impl { - friend class basic_iarchive; - library_version_type m_archive_library_version; - unsigned int m_flags; - - ////////////////////////////////////////////////////////////////////// - // information about each serialized object loaded - // indexed on object_id - struct aobject - { - void * address; // NOLINT(modernize-use-default-member-init,misc-non-private-member-variables-in-classes) external code - bool loaded_as_pointer; // NOLINT(modernize-use-default-member-init,misc-non-private-member-variables-in-classes) external code - class_id_type class_id; // NOLINT(misc-non-private-member-variables-in-classes) external code - aobject( - void *a, - class_id_type class_id_ // NOLINT(performance-unnecessary-value-param) external code - ) : - address(a), - loaded_as_pointer(false), - class_id(class_id_) - {} - aobject() : - address(nullptr), - loaded_as_pointer(false), - class_id(-2) - {} - }; - typedef std::vector object_id_vector_type; // NOLINT(modernize-use-using) external code - object_id_vector_type object_id_vector; - - ////////////////////////////////////////////////////////////////////// - // used to implement the reset_object_address operation. - struct moveable_objects { - object_id_type start; // NOLINT(misc-non-private-member-variables-in-classes) external code - object_id_type end; // NOLINT(misc-non-private-member-variables-in-classes) external code - object_id_type recent; // NOLINT(misc-non-private-member-variables-in-classes) external code - bool is_pointer; // NOLINT(modernize-use-default-member-init,misc-non-private-member-variables-in-classes) external code - moveable_objects() : - start(0), - end(0), - recent(0), - is_pointer(false) - {} - } m_moveable_objects; - - void reset_object_address( - const void * new_address, - const void *old_address - ); - - ////////////////////////////////////////////////////////////////////// - // used by load object to look up class id given basic_serializer - struct cobject_type // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) external code - { - const basic_iserializer * m_bis; // NOLINT(misc-non-private-member-variables-in-classes) external code - const class_id_type m_class_id; // NOLINT(misc-non-private-member-variables-in-classes) external code - cobject_type( - std::size_t class_id, - const basic_iserializer & bis - ) : - m_bis(& bis), - m_class_id(class_id) - {} - cobject_type(const cobject_type & rhs) : // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code - m_bis(rhs.m_bis), - m_class_id(rhs.m_class_id) - {} - // the following cannot be defined because of the const - // member. This will generate a link error if an attempt - // is made to assign. This should never be necessary - cobject_type & operator=(const cobject_type & rhs); - bool operator<(const cobject_type &rhs) const - { - return *m_bis < *(rhs.m_bis); - } - }; - typedef std::set cobject_info_set_type; // NOLINT(modernize-use-using) external code - cobject_info_set_type cobject_info_set; - - ////////////////////////////////////////////////////////////////////// - // information about each serialized class indexed on class_id - class cobject_id // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) external code - { - public: - cobject_id & operator=(const cobject_id & rhs){ // NOLINT(hicpp-use-equals-default,modernize-use-equals-default,bugprone-unhandled-self-assignment,cert-oop54-cpp) external code - bis_ptr = rhs.bis_ptr; - bpis_ptr = rhs.bpis_ptr; - file_version = rhs.file_version; - tracking_level = rhs.tracking_level; - initialized = rhs.initialized; - return *this; - } - const basic_iserializer * bis_ptr; // NOLINT(misc-non-private-member-variables-in-classes) third party code - const basic_pointer_iserializer * bpis_ptr; // NOLINT(misc-non-private-member-variables-in-classes,modernize-use-default-member-init) third party code - version_type file_version; // NOLINT(misc-non-private-member-variables-in-classes) third party code - tracking_type tracking_level; // NOLINT(misc-non-private-member-variables-in-classes) third party code - bool initialized; // NOLINT(misc-non-private-member-variables-in-classes,modernize-use-default-member-init) third party code - - explicit cobject_id(const basic_iserializer & bis_) : - bis_ptr(& bis_), - bpis_ptr(nullptr), - file_version(0), - tracking_level(track_never), // NOLINT(readability-implicit-bool-conversion) external code - initialized(false) - {} - cobject_id(const cobject_id &rhs): // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code - bis_ptr(rhs.bis_ptr), - bpis_ptr(rhs.bpis_ptr), - file_version(rhs.file_version), - tracking_level(rhs.tracking_level), - initialized(rhs.initialized) - {} - }; - typedef std::vector cobject_id_vector_type; // NOLINT(modernize-use-using) external code - cobject_id_vector_type cobject_id_vector; - - ////////////////////////////////////////////////////////////////////// - // address of the most recent object serialized as a poiner - // whose data itself is now pending serialization - struct pending { - void * object; // NOLINT(misc-non-private-member-variables-in-classes,modernize-use-default-member-init) external code - const basic_iserializer * bis; // NOLINT(misc-non-private-member-variables-in-classes,modernize-use-default-member-init) external code - version_type version; // NOLINT(misc-non-private-member-variables-in-classes) external code - pending() : - object(nullptr), - bis(nullptr), - version(0) - {} - } m_pending; - - explicit basic_iarchive_impl(unsigned int flags) : - m_archive_library_version(BOOST_ARCHIVE_VERSION()), - m_flags(flags) - {} - void set_library_version(library_version_type archive_library_version){ // NOLINT(performance-unnecessary-value-param) external code - m_archive_library_version = archive_library_version; - } - bool - track( - basic_iarchive & ar, - void * & t - ); - void - load_preamble( - basic_iarchive & ar, - cobject_id & co - ); - class_id_type register_type( - const basic_iserializer & bis - ); - - // redirect through virtual functions to load functions for this archive - template - void load(basic_iarchive & ar, T & t){ - ar.vload(t); - } - -//public: - void - next_object_pointer(void * t){ - m_pending.object = t; - } - void delete_created_pointers(); - class_id_type register_type( - const basic_pointer_iserializer & bpis - ); - void load_object( - basic_iarchive & ar, - void * t, - const basic_iserializer & bis - ); - const basic_pointer_iserializer * load_pointer( - basic_iarchive & ar, - void * & t, - const basic_pointer_iserializer * bpis, - const basic_pointer_iserializer * (*finder)( - const boost::serialization::extended_type_info & type - ) - ); -}; - -inline void -basic_iarchive_impl::reset_object_address( - void const * const new_address, // NOLINT(bugprone-easily-swappable-parameters) external code - void const * const old_address -){ - if(m_moveable_objects.is_pointer) { - return; } - - // this code handles a couple of situations. - // a) where reset_object_address is applied to an untracked object. - // In such a case the call is really superfluous and its really an - // an error. But we don't have access to the types here so we can't - // know that. However, this code will effectively turn this situation - // into a no-op and every thing will work fine - albeat with a small - // execution time penalty. - // b) where the call to reset_object_address doesn't immediatly follow - // the << operator to which it corresponds. This would be a bad idea - // but the code may work anyway. Naturally, a bad practice on the part - // of the programmer but we can't detect it - as above. So maybe we - // can save a few more people from themselves as above. - object_id_type i = m_moveable_objects.recent; - for(; i < m_moveable_objects.end; ++i){ - if(old_address == object_id_vector[i].address) { - break; } - } - for(; i < m_moveable_objects.end; ++i){ - void const * const this_address = object_id_vector[i].address; - // calculate displacement from this level - // warning - pointer arithmetic on void * is in herently non-portable - // but expected to work on all platforms in current usage - if(this_address > old_address){ - std::size_t member_displacement - = reinterpret_cast(this_address) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) external - - reinterpret_cast(old_address); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) external - object_id_vector[i].address = reinterpret_cast( // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) external - reinterpret_cast(new_address) + member_displacement // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) external - ); - } - else{ - std::size_t member_displacement - = reinterpret_cast(old_address) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) external - - reinterpret_cast(this_address); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) external - object_id_vector[i].address = reinterpret_cast( // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,performance-no-int-to-ptr) external - reinterpret_cast(new_address) - member_displacement // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) external code - ); - } - } -} - -inline void -basic_iarchive_impl::delete_created_pointers() -{ - object_id_vector_type::iterator i; - for( - i = object_id_vector.begin(); - i != object_id_vector.end(); - ++i - ){ - if(i->loaded_as_pointer){ - // borland complains without this minor hack - const int j = i->class_id; - const cobject_id & co = cobject_id_vector[static_cast(j)]; - //const cobject_id & co = cobject_id_vector[i->class_id]; - // with the appropriate input serializer, - // delete the indicated object - co.bis_ptr->destroy(i->address); - } - } -} - -inline class_id_type -basic_iarchive_impl::register_type( - const basic_iserializer & bis -){ - class_id_type cid(cobject_info_set.size()); - cobject_type co(static_cast(cid), bis); - std::pair - result = cobject_info_set.insert(co); - - if(result.second){ - cobject_id_vector.push_back(cobject_id(bis)); - BOOST_ASSERT(cobject_info_set.size() == cobject_id_vector.size()); - } - cid = result.first->m_class_id; - // borland complains without this minor hack - const int tid = cid; - cobject_id & coid = cobject_id_vector[static_cast(tid)]; - coid.bpis_ptr = bis.get_bpis_ptr(); - return cid; -} - -void -basic_iarchive_impl::load_preamble( - basic_iarchive & ar, - cobject_id & co -){ - if(! co.initialized){ - if(co.bis_ptr->class_info()){ - class_id_optional_type cid(class_id_type(0)); - load(ar, cid); // to be thrown away - load(ar, co.tracking_level); - load(ar, co.file_version); - } - else{ - // override tracking with indicator from class information - co.tracking_level = co.bis_ptr->tracking(m_flags); - co.file_version = version_type( // NOLINT(google-readability-casting) third pary code - co.bis_ptr->version() - ); - } - co.initialized = true; - } -} - -bool -basic_iarchive_impl::track( - basic_iarchive & ar, - void * & t -){ - object_id_type oid; - load(ar, oid); - - // if its a reference to a old object - if(object_id_type(object_id_vector.size()) > oid){ - // we're done - t = object_id_vector[oid].address; - return false; - } - return true; -} - -inline void -basic_iarchive_impl::load_object( - basic_iarchive & ar, - void * t, - const basic_iserializer & bis -){ - m_moveable_objects.is_pointer = false; - serialization::state_saver ss_is_pointer(m_moveable_objects.is_pointer); - // if its been serialized through a pointer and the preamble's been done - if(t == m_pending.object && & bis == m_pending.bis){ - // read data - (bis.load_object_data)(ar, t, m_pending.version); - return; - } - - const class_id_type cid = register_type(bis); - const int i = cid; - cobject_id & co = cobject_id_vector[static_cast(i)]; - - load_preamble(ar, co); - - // save the current move stack position in case we want to truncate it - boost::serialization::state_saver ss_start(m_moveable_objects.start); - - // note: extra line used to evade borland issue - const bool tracking = co.tracking_level; - - object_id_type this_id; - m_moveable_objects.start = - this_id = object_id_type(object_id_vector.size()); - - // if we tracked this object when the archive was saved - if(tracking){ - // if it was already read - if(!track(ar, t)) { - // we're done - return; } - // add a new enty into the tracking list - object_id_vector.push_back(aobject(t, cid)); - // and add an entry for this object - m_moveable_objects.end = object_id_type(object_id_vector.size()); - } - // read data - (bis.load_object_data)(ar, t, co.file_version); - m_moveable_objects.recent = this_id; -} - -inline const basic_pointer_iserializer * -basic_iarchive_impl::load_pointer( - basic_iarchive &ar, - void * & t, - const basic_pointer_iserializer * bpis_ptr, - const basic_pointer_iserializer * (*finder)( - const boost::serialization::extended_type_info & type_ - ) -){ - m_moveable_objects.is_pointer = true; - serialization::state_saver w(m_moveable_objects.is_pointer); - - class_id_type cid; - load(ar, cid); - -#if BOOST_VERSION < 107400 - if(NULL_POINTER_TAG == cid){ - t = NULL; - return bpis_ptr; - } -#else // this case is taken from https://github.com/boostorg/serialization/blob/develop/src/basic_iarchive.cpp#L430-L433 - if(BOOST_SERIALIZATION_NULL_POINTER_TAG == cid){ - t = nullptr; - return bpis_ptr; - } -#endif - - // if its a new class type - i.e. never been registered - if(class_id_type(cobject_info_set.size()) <= cid){ - // if its either abstract - if(nullptr == bpis_ptr - // or polymorphic - || bpis_ptr->get_basic_serializer().is_polymorphic()){ - // is must have been exported - char key[BOOST_SERIALIZATION_MAX_KEY_SIZE]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) external code - class_name_type class_name(key); - load(ar, class_name); - // if it has a class name - const serialization::extended_type_info *eti = nullptr; - if(0 != key[0]) { - eti = serialization::extended_type_info::find(key); } - if(nullptr == eti) { - boost::serialization::throw_exception( - archive_exception(archive_exception::unregistered_class) - ); } - bpis_ptr = (*finder)(*eti); // cppcheck-suppress [nullPointerRedundantCheck,nullPointer] ; legacy code - } - BOOST_ASSERT(nullptr != bpis_ptr); - // class_id_type new_cid = register_type(bpis_ptr->get_basic_serializer()); - BOOST_VERIFY(register_type(bpis_ptr->get_basic_serializer()) == cid); - int i = cid; - cobject_id_vector[static_cast(i)].bpis_ptr = bpis_ptr; - } - int i = cid; - cobject_id & co = cobject_id_vector[static_cast(i)]; - bpis_ptr = co.bpis_ptr; - - load_preamble(ar, co); - - // extra line to evade borland issue - const bool tracking = co.tracking_level; - // if we're tracking and the pointer has already been read - if(tracking && ! track(ar, t)) { - // we're done - return bpis_ptr; } - - // save state - serialization::state_saver w_start(m_moveable_objects.start); - - // allocate space on the heap for the object - to be constructed later - t = bpis_ptr->heap_allocation(); - BOOST_ASSERT(nullptr != t); - - if(! tracking){ - bpis_ptr->load_object_ptr(ar, t, co.file_version); - } - else{ - serialization::state_saver x(m_pending.object); - serialization::state_saver y(m_pending.bis); - serialization::state_saver z(m_pending.version); - - m_pending.bis = & bpis_ptr->get_basic_serializer(); - m_pending.version = co.file_version; - - // predict next object id to be created - const unsigned int ui = static_cast(object_id_vector.size()); - - serialization::state_saver w_end(m_moveable_objects.end); - - - // add to list of serialized objects so that we can properly handle - // cyclic strucures - object_id_vector.push_back(aobject(t, cid)); - - // remember that that the address of these elements could change - // when we make another call so don't use the address - bpis_ptr->load_object_ptr( - ar, - t, - m_pending.version - ); - object_id_vector[ui].loaded_as_pointer = true; - } - - return bpis_ptr; -} - -} // namespace detail -} // namespace archive -} // namespace boost - -////////////////////////////////////////////////////////////////////// -// implementation of basic_iarchive functions -namespace boost { -namespace archive { -namespace detail { - -BOOST_ARCHIVE_DECL void -basic_iarchive::next_object_pointer(void *t){ - pimpl->next_object_pointer(t); -} - -BOOST_ARCHIVE_DECL -basic_iarchive::basic_iarchive(unsigned int flags) : - pimpl(new basic_iarchive_impl(flags)) -{} - -BOOST_ARCHIVE_DECL -basic_iarchive::~basic_iarchive() -{} - -BOOST_ARCHIVE_DECL void -basic_iarchive::set_library_version(library_version_type archive_library_version){ // NOLINT(performance-unnecessary-value-param) - pimpl->set_library_version(archive_library_version); -} - -BOOST_ARCHIVE_DECL void -basic_iarchive::reset_object_address( - const void * new_address, - const void * old_address -){ - pimpl->reset_object_address(new_address, old_address); -} - -BOOST_ARCHIVE_DECL void -basic_iarchive::load_object( - void *t, - const basic_iserializer & bis -){ - pimpl->load_object(*this, t, bis); -} - -// load a pointer object -BOOST_ARCHIVE_DECL const basic_pointer_iserializer * -basic_iarchive::load_pointer( - void * &t, - const basic_pointer_iserializer * bpis_ptr, - const basic_pointer_iserializer * (*finder)( - const boost::serialization::extended_type_info & type_ - ) - -){ - return pimpl->load_pointer(*this, t, bpis_ptr, finder); -} - -BOOST_ARCHIVE_DECL void -basic_iarchive::register_basic_serializer(const basic_iserializer & bis){ - pimpl->register_type(bis); -} - -BOOST_ARCHIVE_DECL void -basic_iarchive::delete_created_pointers() -{ - pimpl->delete_created_pointers(); -} - -BOOST_ARCHIVE_DECL boost::archive::library_version_type -basic_iarchive::get_library_version() const{ - return pimpl->m_archive_library_version; -} - -BOOST_ARCHIVE_DECL unsigned int -basic_iarchive::get_flags() const{ - return pimpl->m_flags; -} - -} // namespace detail -} // namespace archive -} // namespace boost -// NOLINTEND(altera-id-dependent-backward-branch,altera-unroll-loops,hicpp-use-auto,hicpp-use-equals-default,modernize-use-auto,modernize-use-equals-default,misc-const-correctness) external code diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iserializer.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iserializer.cpp deleted file mode 100644 index cfad07d34f3..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_iserializer.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// basic_iserializer.cpp: - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. - -#include // NULL - -#define BOOST_ARCHIVE_SOURCE -#include - -#include - -namespace boost { -namespace archive { -namespace detail { - -BOOST_ARCHIVE_DECL -basic_iserializer::basic_iserializer( - const boost::serialization::extended_type_info & type /*eti*/ -) : - basic_serializer(type), - m_bpis(nullptr) -{} - -BOOST_ARCHIVE_DECL -basic_iserializer::~basic_iserializer(){} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code - -} // namespace detail -} // namespace archive -} // namespace boost diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oarchive.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oarchive.cpp deleted file mode 100644 index cffb81a4b7d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oarchive.cpp +++ /dev/null @@ -1,474 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// basic_oarchive.cpp: - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. -// NOLINTBEGIN(misc-const-correctness) external code -#include // msvc 6.0 needs this for warning suppression - -#include - -#include // NULL -#include - -#include - -// including this here to work around an ICC in intel 7.0 -// normally this would be part of basic_oarchive.hpp below. -#define BOOST_ARCHIVE_SOURCE -// include this to prevent linker errors when the -// same modules are marked export and import. -#define BOOST_SERIALIZATION_SOURCE -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifdef BOOST_MSVC -# pragma warning(push) -# pragma warning(disable : 4251 4231 4660 4275) -#endif - -using namespace boost::serialization; // NOLINT(google-build-using-namespace,google-global-names-in-headers) TODO(correaa) remove this global using namespace - -namespace boost { -namespace archive { -namespace detail { - -class basic_oarchive_impl { - friend class basic_oarchive; - unsigned int m_flags; - - ////////////////////////////////////////////////////////////////////// - // information about each serialized object saved - // keyed on address, class_id - struct aobject - { - const void * address; // NOLINT(misc-non-private-member-variables-in-classes, modernize-use-default-member-init) external code - class_id_type class_id; // NOLINT(misc-non-private-member-variables-in-classes) external code - object_id_type object_id; // NOLINT(misc-non-private-member-variables-in-classes) external code - - bool operator<(const aobject &rhs) const - { - BOOST_ASSERT(nullptr != address); - BOOST_ASSERT(nullptr != rhs.address); - if( address < rhs.address ) { - return true; } - if( address > rhs.address ) { - return false; } - return class_id < rhs.class_id; - } -//fatal error: definition of implicit copy constructor for 'aobject' is deprecated because it has a user-declared copy assignment operator [-Wdeprecated-copy] in clang -// aobject & operator=(const aobject & rhs) -// { -// address = rhs.address; -// class_id = rhs.class_id; -// object_id = rhs.object_id; -// return *this; -// } - aobject( - const void *a, - class_id_type class_id_, // NOLINT(performance-unnecessary-value-param) external code - object_id_type object_id_ // NOLINT(performance-unnecessary-value-param) external code - ) : - address(a), - class_id(class_id_), - object_id(object_id_) - {} - aobject() : address(nullptr){} - }; - // keyed on class_id, address - typedef std::set object_set_type; // NOLINT(modernize-use-using) external code - object_set_type object_set; - - ////////////////////////////////////////////////////////////////////// - // information about each serialized class saved - // keyed on type_info - struct cobject_type // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) external code - { - const basic_oserializer * m_bos_ptr; // NOLINT(misc-non-private-member-variables-in-classes) external code - const class_id_type m_class_id; // NOLINT(misc-non-private-member-variables-in-classes) external code - bool m_initialized; // NOLINT(misc-non-private-member-variables-in-classes) external code - cobject_type( - std::size_t class_id, - const basic_oserializer & bos - ) : - m_bos_ptr(& bos), - m_class_id(class_id), - m_initialized(false) - {} - // cppcheck-suppress uninitMemberVar - explicit cobject_type(const basic_oserializer & bos) // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) external code - : m_bos_ptr(& bos) - {} - cobject_type( // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code - const cobject_type & rhs - ) : - m_bos_ptr(rhs.m_bos_ptr), - m_class_id(rhs.m_class_id), - m_initialized(rhs.m_initialized) - {} - // the following cannot be defined because of the const - // member. This will generate a link error if an attempt - // is made to assign. This should never be necessary - // use this only for lookup argument - cobject_type & operator=(const cobject_type &rhs); - bool operator<(const cobject_type &rhs) const { - return *m_bos_ptr < *(rhs.m_bos_ptr); - } - }; - // keyed on type_info - typedef std::set cobject_info_set_type; // NOLINT(modernize-use-using) third-party code - cobject_info_set_type cobject_info_set; - - // list of objects initially stored as pointers - used to detect errors - // keyed on object id - std::set stored_pointers; - - // address of the most recent object serialized as a poiner - // whose data itself is now pending serialization - const void * pending_object; // NOLINT(modernize-use-default-member-init) third-party code - const basic_oserializer * pending_bos; // NOLINT(modernize-use-default-member-init) third-party code - - explicit basic_oarchive_impl(unsigned int flags) : - m_flags(flags), - pending_object(nullptr), - pending_bos(nullptr) - {} - - const cobject_type & - find(const basic_oserializer & bos); - const basic_oserializer * - find(const serialization::extended_type_info &ti) const; - -//public: - const cobject_type & - register_type(const basic_oserializer & bos); - void save_object( - basic_oarchive & ar, - const void *t, - const basic_oserializer & bos - ); - void save_pointer( - basic_oarchive & ar, - const void * t, - const basic_pointer_oserializer * bpos - ); -}; - -////////////////////////////////////////////////////////////////////// -// basic_oarchive implementation functions - -// given a type_info - find its bos -// return NULL if not found -inline const basic_oserializer * -basic_oarchive_impl::find(const serialization::extended_type_info & ti) const { - #ifdef BOOST_MSVC - # pragma warning(push) - # pragma warning(disable : 4511 4512) - #endif - class bosarg : - public basic_oserializer - { - bool class_info() const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return false; - } - // returns true if objects should be tracked - bool tracking(const unsigned int /*flags*/) const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return false; - } - // returns class version - version_type version() const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return version_type(0); - } - // returns true if this class is polymorphic - bool is_polymorphic() const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return false; - } - void save_object_data( - basic_oarchive & /*ar*/, const void * /*x*/ - ) const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - } - public: - explicit bosarg(const serialization::extended_type_info & eti) : - boost::archive::detail::basic_oserializer(eti) - {} - }; - #ifdef BOOST_MSVC - #pragma warning(pop) - #endif - bosarg bos(ti); - cobject_info_set_type::const_iterator cit // NOLINT(hicpp-use-auto,modernize-use-auto) - = cobject_info_set.find(cobject_type(bos)); - // it should already have been "registered" - see below - if(cit == cobject_info_set.end()){ - // if an entry is not found in the table it is because a pointer - // of a derived class has been serialized through its base class - // but the derived class hasn't been "registered" - return nullptr; - } - // return pointer to the real class - return cit->m_bos_ptr; -} - -inline const basic_oarchive_impl::cobject_type & -basic_oarchive_impl::find(const basic_oserializer & bos) -{ - std::pair cresult = - cobject_info_set.insert(cobject_type(cobject_info_set.size(), bos)); - return *(cresult.first); -} - -inline const basic_oarchive_impl::cobject_type & -basic_oarchive_impl::register_type( - const basic_oserializer & bos -){ - cobject_type co(cobject_info_set.size(), bos); - std::pair - result = cobject_info_set.insert(co); - return *(result.first); -} - -inline void -basic_oarchive_impl::save_object( - basic_oarchive & ar, - const void *t, - const basic_oserializer & bos -){ - // if its been serialized through a pointer and the preamble's been done - if(t == pending_object && pending_bos == & bos){ - // just save the object data - ar.end_preamble(); - (bos.save_object_data)(ar, t); - return; - } - - // get class information for this object - const cobject_type & co = register_type(bos); - if(bos.class_info()){ - if( ! co.m_initialized){ - ar.vsave(class_id_optional_type(co.m_class_id)); - ar.vsave(tracking_type(bos.tracking(m_flags))); - ar.vsave(version_type(bos.version())); // NOLINT(google-readability-casting) third-party code - (const_cast(co)).m_initialized = true; // NOLINT(cppcoreguidelines-pro-type-const-cast) third-party code - } - } - - // we're not tracking this type of object - if(! bos.tracking(m_flags)){ - // just windup the preamble - no object id to write - ar.end_preamble(); - // and save the data - (bos.save_object_data)(ar, t); - return; - } - - // look for an existing object id - object_id_type oid(object_set.size()); - // lookup to see if this object has already been written to the archive - basic_oarchive_impl::aobject ao(t, co.m_class_id, oid); - std::pair - aresult = object_set.insert(ao); - oid = aresult.first->object_id; - - // if its a new object - if(aresult.second){ - // write out the object id - ar.vsave(oid); - ar.end_preamble(); - // and data - (bos.save_object_data)(ar, t); - return; - } - - // check that it wasn't originally stored through a pointer - if(stored_pointers.end() != stored_pointers.find(oid)){ - // this has to be a user error. loading such an archive - // would create duplicate objects - boost::serialization::throw_exception( - archive_exception(archive_exception::pointer_conflict) - ); - } - // just save the object id - ar.vsave(object_reference_type(oid)); - ar.end_preamble(); -// return; -} - -// save a pointer to an object instance -inline void -basic_oarchive_impl::save_pointer( - basic_oarchive & ar, - const void * t, - const basic_pointer_oserializer * bpos_ptr -){ - const basic_oserializer & bos = bpos_ptr->get_basic_serializer(); - std::size_t original_count = cobject_info_set.size(); - const cobject_type & co = register_type(bos); - if(! co.m_initialized){ - ar.vsave(co.m_class_id); - // if its a previously unregistered class - if((cobject_info_set.size() > original_count)){ - if(bos.is_polymorphic()){ - const serialization::extended_type_info *eti = & bos.get_eti(); - const char * key = nullptr; - if(nullptr != eti) { - key = eti->get_key(); } - if(nullptr != key){ - // the following is required by IBM C++ compiler which - // makes a copy when passing a non-const to a const. This - // is permitted by the standard but rarely seen in practice - const class_name_type cn(key); - if(cn.size() > (BOOST_SERIALIZATION_MAX_KEY_SIZE - 1)) { - boost::serialization::throw_exception( - boost::archive::archive_exception( - boost::archive::archive_exception:: - invalid_class_name) - ); } - // write out the external class identifier - ar.vsave(cn); - } - else { - // without an external class name - // we won't be able to de-serialize it so bail now - boost::serialization::throw_exception( - archive_exception(archive_exception::unregistered_class) - ); } - } - } - if(bos.class_info()){ - ar.vsave(tracking_type(bos.tracking(m_flags))); - ar.vsave(version_type(bos.version())); // NOLINT(google-readability-casting) third-party code - } - (const_cast(co)).m_initialized = true; // NOLINT(cppcoreguidelines-pro-type-const-cast) external code - } - else{ - ar.vsave(class_id_reference_type(co.m_class_id)); - } - - // if we're not tracking - if(! bos.tracking(m_flags)){ - // just save the data itself - ar.end_preamble(); - serialization::state_saver x(pending_object); - serialization::state_saver y(pending_bos); - pending_object = t; - pending_bos = & bpos_ptr->get_basic_serializer(); - bpos_ptr->save_object_ptr(ar, t); - return; - } - - object_id_type oid(object_set.size()); - // lookup to see if this object has already been written to the archive - basic_oarchive_impl::aobject ao(t, co.m_class_id, oid); - std::pair - aresult = object_set.insert(ao); - oid = aresult.first->object_id; - // if the saved object already exists - if(! aresult.second){ - // append the object id to he preamble - ar.vsave(object_reference_type(oid)); - // and windup. - ar.end_preamble(); - return; - } - - // append id of this object to preamble - ar.vsave(oid); - ar.end_preamble(); - - // and save the object itself - serialization::state_saver x(pending_object); - serialization::state_saver y(pending_bos); - pending_object = t; - pending_bos = & bpos_ptr->get_basic_serializer(); - bpos_ptr->save_object_ptr(ar, t); - // add to the set of object initially stored through pointers - stored_pointers.insert(oid); -} - -} // namespace detail -} // namespace archive -} // namespace boost - -////////////////////////////////////////////////////////////////////// -// implementation of basic_oarchive functions - -namespace boost { -namespace archive { -namespace detail { - -BOOST_ARCHIVE_DECL -basic_oarchive::basic_oarchive(unsigned int flags) - : pimpl(new basic_oarchive_impl(flags)) -{} - -BOOST_ARCHIVE_DECL -basic_oarchive::~basic_oarchive() // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code -{} - -BOOST_ARCHIVE_DECL void -basic_oarchive::save_object( - const void *x, - const basic_oserializer & bos -){ - pimpl->save_object(*this, x, bos); -} - -BOOST_ARCHIVE_DECL void -basic_oarchive::save_pointer( - const void * t, - const basic_pointer_oserializer * bpos_ptr -){ - pimpl->save_pointer(*this, t, bpos_ptr); -} - -BOOST_ARCHIVE_DECL void -basic_oarchive::register_basic_serializer(const basic_oserializer & bos){ - pimpl->register_type(bos); -} - -BOOST_ARCHIVE_DECL library_version_type -basic_oarchive::get_library_version() const{ // NOLINT(readability-convert-member-functions-to-static) external code - return BOOST_ARCHIVE_VERSION(); -} - -BOOST_ARCHIVE_DECL unsigned int -basic_oarchive::get_flags() const{ - return pimpl->m_flags; -} - -BOOST_ARCHIVE_DECL void -basic_oarchive::end_preamble(){ -} - -#if BOOST_VERSION > 105900 -BOOST_ARCHIVE_DECL helper_collection & -basic_oarchive::get_helper_collection(){ - return *this; -} -#endif - -} // namespace detail -} // namespace archive -} // namespace boost -// NOLINTEND(misc-const-correctness) external code -#ifdef BOOST_MSVC -#pragma warning(pop) -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oserializer.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oserializer.cpp deleted file mode 100644 index b3dc101a006..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/basic_oserializer.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// basic_oserializer.cpp: - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. - -#include // NULL - -#define BOOST_ARCHIVE_SOURCE -#include -#include - -namespace boost { -namespace archive { -namespace detail { - -BOOST_ARCHIVE_DECL -basic_oserializer::basic_oserializer( - const boost::serialization::extended_type_info & type_ /*eti*/ -) : - basic_serializer(type_), - m_bpos(nullptr) -{} - -BOOST_ARCHIVE_DECL -basic_oserializer::~basic_oserializer(){} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) - -} // namespace detail -} // namespace archive -} // namespace boost diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info.cpp deleted file mode 100644 index 536879eb866..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info.cpp +++ /dev/null @@ -1,194 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// extended_type_info.cpp: implementation for portable version of type_info - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. -// NOLINTBEGIN(altera-id-dependent-backward-branch,altera-unroll-loops,misc-const-correctness) external code -#if (defined _MSC_VER) && (_MSC_VER == 1200) -# pragma warning (disable : 4786) // too long name, harmless warning -#endif - -#include -#include // NULL -#include -#include - -#include - -#include -#if defined(BOOST_NO_STDC_NAMESPACE) -namespace std{ using ::strcmp; } -#endif - -#include // msvc needs this to suppress warning - -#include - -// it marks our code with proper attributes as being exported when -// we're compiling it while marking it import when just the headers -// is being included. -#define BOOST_SERIALIZATION_SOURCE -#include -#include -#include -#include - -#ifdef BOOST_MSVC -# pragma warning(push) -# pragma warning(disable : 4511 4512) -#endif - -namespace boost { -namespace serialization { -namespace detail { - -struct key_compare -{ - bool - operator()( - const extended_type_info * lhs, - const extended_type_info * rhs - ) const { - // performance shortcut - if(lhs == rhs) { - return false; } - const char * l = lhs->get_key(); - BOOST_ASSERT(nullptr != l); - const char * r = rhs->get_key(); - BOOST_ASSERT(nullptr != r); - // performance shortcut - // shortcut to exploit string pooling - if(l == r) { - return false; } - // for exported types, use the string key so that - // multiple instances in different translation units - // can be matched up - return std::strcmp(l, r) < 0; - } -}; - -typedef std::multiset ktmap; // NOLINT(modernize-use-using) external code - -#ifdef BOOST_MSVC -# pragma warning(push) -# pragma warning(disable : 4511 4512) -#endif - -class extended_type_info_arg : public extended_type_info // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) external code -{ - /*virtual*/ bool - is_less_than(const extended_type_info & /*rhs*/) const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return false; - }; - /*virtual*/ bool - is_equal(const extended_type_info & /*rhs*/) const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return false; - }; - /*virtual*/ const char * get_debug_info() const override { - return get_key(); - } - /*virtual*/ void * construct(unsigned int /*count*/, ...) const override { // NOLINT(cert-dcl50-cpp) external code - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return nullptr; - } - /*virtual*/ void destroy(void const * const /*p*/) const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - } -public: - explicit extended_type_info_arg(const char * key) : - extended_type_info(0, key) - {} - - ~extended_type_info_arg() override { // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code - } -}; - -#ifdef BOOST_MSVC -# pragma warning(pop) -#endif - -} // namespace detail - -BOOST_SERIALIZATION_DECL void -extended_type_info::key_register() const{ - if(nullptr == get_key()) { - return; } - singleton::get_mutable_instance().insert(this); -} - -BOOST_SERIALIZATION_DECL void -extended_type_info::key_unregister() const{ - if(nullptr == get_key()) { - return; } - if(! singleton::is_destroyed()){ - detail::ktmap & x = singleton::get_mutable_instance(); - detail::ktmap::iterator start = x.lower_bound(this); // NOLINT(hicpp-use-auto,modernize-use-auto) external code - detail::ktmap::iterator end = x.upper_bound(this); // NOLINT(hicpp-use-auto,modernize-use-auto) external code - // remove entry in map which corresponds to this type - for(;start != end; ++start){ - if(this == *start){ - x.erase(start); - break; - } - } - } -} - -BOOST_SERIALIZATION_DECL const extended_type_info * -extended_type_info::find(const char *key) { - BOOST_ASSERT(nullptr != key); - const detail::ktmap & k = singleton::get_const_instance(); - const detail::extended_type_info_arg eti_key(key); - const detail::ktmap::const_iterator it = k.find(& eti_key); // NOLINT(hicpp-use-auto,modernize-use-auto) - if(k.end() == it) { - return nullptr; } - return *(it); -} - -BOOST_SERIALIZATION_DECL -extended_type_info::extended_type_info( - const unsigned int type_info_key, - const char * key -) : - m_type_info_key(type_info_key), - m_key(key) -{ -} - -BOOST_SERIALIZATION_DECL -extended_type_info::~extended_type_info(){ // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code -} - -BOOST_SERIALIZATION_DECL bool -extended_type_info::operator<(const extended_type_info &rhs) const { - // short cut for a common cases - if(this == & rhs) { - return false; } - if(m_type_info_key == rhs.m_type_info_key){ - return is_less_than(rhs); - } - if(m_type_info_key < rhs.m_type_info_key) { - return true; } // NOLINT(readability-simplify-boolean-expr) external code - return false; -} - -BOOST_SERIALIZATION_DECL bool -extended_type_info::operator==(const extended_type_info &rhs) const { - // short cut for a common cases - if(this == & rhs) { - return true; } - if(m_type_info_key != rhs.m_type_info_key){ - return false; - } - return is_equal(rhs); -} - -} // namespace serialization -} // namespace boost -// NOLINTEND(altera-id-dependent-backward-branch,altera-unroll-loops,misc-const-correctness) external code diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info_typeid.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info_typeid.cpp deleted file mode 100644 index d7d295c8747..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/extended_type_info_typeid.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// extended_type_info_typeid.cpp: specific implementation of type info -// that is based on typeid - -// (C) Copyright 2002 Robert Ramey - http://www.rrsd.com . -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -// See http://www.boost.org for updates, documentation, and revision history. -// NOLINTBEGIN(altera-id-dependent-backward-branch,altera-unroll-loops,misc-const-correctness) external code -#include -#include // NULL -#include -#include - -#include - -#include - -// it marks our code with proper attributes as being exported when -// we're compiling it while marking it import when just the headers -// is being included. -#define BOOST_SERIALIZATION_SOURCE -#include -#include -#include - -namespace boost { -namespace serialization { -namespace typeid_system { - -#define EXTENDED_TYPE_INFO_TYPE_KEY 1 // NOLINT(cppcoreguidelines-macro-usage,modernize-macro-to-enum) external code - -struct type_compare -{ - bool - operator()( - const extended_type_info_typeid_0 * lhs, - const extended_type_info_typeid_0 * rhs - ) const { - return lhs->is_less_than(*rhs); - } -}; - -typedef std::multiset< // NOLINT(modernize-use-using) external code - const extended_type_info_typeid_0 *, - type_compare -> tkmap; - -BOOST_SERIALIZATION_DECL bool -extended_type_info_typeid_0::is_less_than( - const boost::serialization::extended_type_info & rhs -) const { - // shortcut for common case - if(this == & rhs) { - return false; } - return 0 != m_ti->before( // NOLINT(readability-implicit-bool-conversion) - *(static_cast(rhs).m_ti) // NOLINT(cppcoreguidelines-pro-type-static-cast-downcast) external code - ); -} - -BOOST_SERIALIZATION_DECL bool -extended_type_info_typeid_0::is_equal( - const boost::serialization::extended_type_info & rhs -) const { - return - // note: std::type_info == operator returns an int !!! - // the following permits conversion to bool without a warning. - ! ( - * m_ti - != *(static_cast(rhs).m_ti) // NOLINT(cppcoreguidelines-pro-type-static-cast-downcast) external code - ) - ; -} - -BOOST_SERIALIZATION_DECL -extended_type_info_typeid_0::extended_type_info_typeid_0( - const char * key -) : - extended_type_info(EXTENDED_TYPE_INFO_TYPE_KEY, key), - m_ti(nullptr) -{} - -BOOST_SERIALIZATION_DECL -extended_type_info_typeid_0::~extended_type_info_typeid_0() // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) external code -{} - -BOOST_SERIALIZATION_DECL void -extended_type_info_typeid_0::type_register(const std::type_info & ti){ - m_ti = & ti; - singleton::get_mutable_instance().insert(this); -} - -BOOST_SERIALIZATION_DECL void -extended_type_info_typeid_0::type_unregister() -{ - if(nullptr != m_ti){ - if(! singleton::is_destroyed()){ - tkmap & x = singleton::get_mutable_instance(); - tkmap::iterator start = x.lower_bound(this); // NOLINT(hicpp-use-auto,modernize-use-auto) external code - tkmap::iterator end = x.upper_bound(this); // NOLINT(hicpp-use-auto,modernize-use-auto) external code - BOOST_ASSERT(start != end); - - // remove entry in map which corresponds to this type - do{ - if(this == *start) { - x.erase(start++); - } else { - ++start; - }}while(start != end); - } - } - m_ti = nullptr; -} - -#ifdef BOOST_MSVC -# pragma warning(push) -# pragma warning(disable : 4511 4512) -#endif - -// this derivation is used for creating search arguments -class extended_type_info_typeid_arg : // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) external code - public extended_type_info_typeid_0 -{ - /*virtual*/ void * construct(unsigned int /*count*/, ...) const override { // NOLINT(cert-dcl50-cpp) external code - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - return nullptr; - } - /*virtual*/ void destroy(void const * const /*p*/) const override { - BOOST_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) external code - } -public: - explicit // lints google-explicit-constructor,hicpp-explicit-conversions - extended_type_info_typeid_arg(const std::type_info & ti) : - extended_type_info_typeid_0(nullptr) - { - // note absense of self register and key as this is used only as - // search argument given a type_info reference and is not to - // be added to the map. - m_ti = & ti; - } - ~extended_type_info_typeid_arg() override { - m_ti = nullptr; - } -}; - -#ifdef BOOST_MSVC -# pragma warning(pop) -#endif - -BOOST_SERIALIZATION_DECL const extended_type_info * -extended_type_info_typeid_0::get_extended_type_info( // NOLINT(readability-convert-member-functions-to-static) - const std::type_info & ti -) const { - typeid_system::extended_type_info_typeid_arg etia(ti); - const tkmap & t = singleton::get_const_instance(); - const tkmap::const_iterator it = t.find(& etia); // NOLINT(hicpp-use-auto,modernize-use-auto) - if(t.end() == it) { - return nullptr; - } - return *(it); -} - -} // end namespace typeid_system -} // end namespace serialization -} // end namespace boost -// NOLINTEND(altera-id-dependent-backward-branch,altera-unroll-loops,misc-const-correctness) external code diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/singleton.cpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/singleton.cpp deleted file mode 100644 index 8ceb3826d9d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/serialization_hack/singleton.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/////////1/////////2/////////3/////////4/////////5/////////6/////////7/////////8 -// singleton.cpp -// -// Copyright (c) 201 5 Robert Ramey, Indiana University (garcia@osl.iu.edu) -// Use, modification and distribution is subject to the Boost Software -// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) -// - -// it marks our code with proper attributes as being exported when -// we're compiling it while marking it import when just the headers -// is being included. -#define BOOST_SERIALIZATION_SOURCE -#include -#include - -namespace boost { -namespace serialization { - -BOOST_SERIALIZATION_DECL bool & singleton_module::get_lock(){ - static bool lock = false; - return lock; -} - -#if BOOST_VERSION < 106400 -BOOST_SERIALIZATION_DECL void singleton_module::lock(){ - get_lock() = true; -} -BOOST_SERIALIZATION_DECL void singleton_module::unlock(){ - get_lock() = false; -} -BOOST_SERIALIZATION_DECL bool singleton_module::is_locked(){ - return get_lock(); -} -#endif - -} // namespace serialization -} // namespace boost diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_communicator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_communicator.hpp deleted file mode 100644 index fae4189e7a1..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_communicator.hpp +++ /dev/null @@ -1,139 +0,0 @@ -// -*- indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- -// © Alfredo A. Correa 2018-2021 - -#ifndef MPI3_SHARED_COMMUNICATOR_HPP -#define MPI3_SHARED_COMMUNICATOR_HPP - -#include "../mpi3/communicator.hpp" -#include "../mpi3/environment.hpp" // processor_name - -//#include "/usr/src/kernels/4.18.16-300.fc29.x86_64/include/linux/getcpu.h" -//#include -//#include // sudo dnf install numactl-devel - -#include -#include - -namespace boost { -namespace mpi3 { - -template -struct shared_window; - -struct shared_communicator : communicator { - shared_communicator() = default; - - shared_communicator(shared_communicator const&) = delete; - shared_communicator(shared_communicator &&) = default; - shared_communicator(shared_communicator &) = default; - - explicit shared_communicator(mpi3::group const& g) : communicator(g) {} - shared_communicator(mpi3::group const& g, int tag) : communicator(g, tag) {} - - private: - template friend struct shared_window; - explicit shared_communicator(communicator&& c) : communicator(std::move(c)) {} - explicit shared_communicator(communicator& comm, int key = 0) { - MPI_(Comm_split_type)(&comm, MPI_COMM_TYPE_SHARED, key, MPI_INFO_NULL, &impl_); - set_name(comm.name() +":"+ mpi3::processor_name()); - } - shared_communicator(communicator& comm, mpi3::communicator_type t, int key = 0) { - MPI_(Comm_split_type)(&comm, static_cast(t), key, MPI_INFO_NULL, &impl_); - boost::uuids::uuid const tag = boost::uuids::random_generator{}(); static_assert(sizeof(unsigned int)<=sizeof(boost::uuids::uuid), "!"); - auto utag = reinterpret_cast(tag); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) TODO(correaa) - this->broadcast_n(&utag, 1, 0); - auto Tag = std::to_string(utag); - std::string const& base_name = comm.name(); - - // switch-case doesn't work here because in some MPI impls there are "repeats" in the cases - if(communicator_type::shared == t) { - #if __linux__ - set_name(base_name +":shared/pu"+ std::to_string(::sched_getcpu())); // same as ::getcpu() TODO(correaa) - #else - set_name(base_name +":shared/pu"+ Tag); - #endif - } else if(communicator_type::core == t) { - #if __linux__ - set_name(base_name +":core/pu"+ std::to_string(::sched_getcpu())); // same as ::getcpu() TODO(correaa) - #else - set_name(base_name +":core/pu"+ Tag); - #endif - } - else if(communicator_type::hw_thread==t) {set_name(base_name +":hw_thread"+Tag);} - else if(communicator_type::l1_cache ==t) {set_name(base_name +":l1_cache" +Tag);} - else if(communicator_type::l2_cache ==t) {set_name(base_name +":l2_cache" +Tag);} - else if(communicator_type::l3_cache ==t) {set_name(base_name +":l3_cache" +Tag);} - else if(communicator_type::socket ==t) {set_name(base_name +":socket" +Tag);} - else if(communicator_type::numa ==t) {set_name(base_name +":numa" +Tag);} - else if(communicator_type::board ==t) {set_name(base_name +":board" +Tag);} - else if(communicator_type::host ==t) {set_name(base_name +":host" +Tag);} - else if(communicator_type::cu ==t) {set_name(base_name +":cu" +Tag);} - else if(communicator_type::cluster ==t) {set_name(base_name +":cluster" +Tag);} - } - friend class communicator; - - public: - shared_communicator& operator=(shared_communicator const&) = delete; - shared_communicator& operator=(shared_communicator &&) = default; - shared_communicator& operator=(shared_communicator &) = default; // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) - - ~shared_communicator() = default; - - shared_communicator * operator&() & {return this;} // NOLINT(google-runtime-operator) - shared_communicator const* operator&() const& {return this;} // NOLINT(google-runtime-operator) - shared_communicator * operator&() && {return this;} // NOLINT(google-runtime-operator) - - inline shared_communicator split(int key) {return split_shared(key);} - auto split(int color, int key) { - return shared_communicator{communicator::split(color, key)}; - } - - template - shared_window make_shared_window(mpi3::size_t size); - template - shared_window make_shared_window(); -}; - -inline shared_communicator communicator::split_shared(int key /*= 0*/) { - return shared_communicator{*this, key}; -} - -inline shared_communicator communicator::split_shared(communicator_type t, int key /*= 0*/) { - return shared_communicator{*this, t, key}; -} - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_MPI3_SHARED_COMMUNICATOR - -//#include "../mpi3/main.hpp" -//#include "../mpi3/operation.hpp" -//#include "../mpi3/shared_window.hpp" - -//#include - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int, char*[], mpi3::communicator world) { -// auto numa = world.split_shared(communicator_type::numa); // fallback to world.split_shared() if OMPI is not available -// auto win = numa.make_shared_window(numa.rank()?0:1); -// assert(win.base() != nullptr and win.size() == 1); -// win.lock_all(); -// if(numa.rank() == 0) { -// *win.base() = 42; -// win.sync(); -// } -// for(int j=1; j != numa.size(); ++j) { -// if (numa.rank()==0) {numa.send_n((int*)nullptr, 0, j, 666);} -// else if(numa.rank()==j) {numa.receive_n((int*)nullptr, 0, 0, 666);} -// } -// if(numa.rank() != 0) {win.sync();} -// win.unlock_all(); - -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_main.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_main.hpp deleted file mode 100644 index f13cad961de..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_main.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include<"$0">" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_SHARED_MAIN $0x.cpp -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_SHARED_MAIN_HPP -#define BOOST_MPI3_SHARED_MAIN_HPP - -#include - -#include "../mpi3/environment.hpp" -#include "../mpi3/shared_communicator.hpp" - -namespace boost{ -namespace mpi3{ - -int main(int argc, char* argv[], boost::mpi3::shared_communicator node); - -}} - -int main(int argc, char* argv[]){ - boost::mpi3::environment env(argc, argv); - return boost::mpi3::main(argc, argv, env.world().split_shared()); -} - -#ifdef _TEST_BOOST_MPI3_SHARED_MAIN - -#include "../mpi3/version.hpp" -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::shared_communicator& node){ - if(node.rank() == 0) cout << mpi3::version() << '\n'; - return 0; -} - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_mutex.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_mutex.hpp deleted file mode 100644 index a476a7f95bb..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_mutex.hpp +++ /dev/null @@ -1,124 +0,0 @@ -#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_SHARED_MUTEX $0x.cpp -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_SHARED_MUTEX_HPP -#define BOOST_MPI3_SHARED_MUTEX_HPP - -#include "../mpi3/shared_window.hpp" -//#include "../mpi3/detail/basic_mutex.hpp" -#include - -namespace boost{ -namespace mpi3{ - -template -struct shared_mutex{ //https://gist.github.com/aprell/1486197#file-mpi_mutex-c-L61 - static int tag_counter; - using flag_t = unsigned char; - - shared_communicator& comm_; - int rank_; //home - boost::interprocess::offset_ptr addr_; - WindowT win_; - -// std::vector wait_list; //[comm_.size()]; - boost::interprocess::offset_ptr wait_list; - -// int tag_; - - shared_mutex(shared_mutex&&) = delete; - shared_mutex(shared_mutex const&) = delete; - shared_mutex& operator=(shared_mutex const&) = delete; - shared_mutex& operator=(shared_mutex&&) = delete; - - shared_mutex(shared_communicator& comm, int rank = 0) : - comm_(comm), - rank_(rank), - addr_((comm.rank() == rank)?(flag_t*)mpi3::malloc(sizeof(flag_t)*comm.size()):nullptr), - win_(addr_, addr_?1:0, comm_), - wait_list((comm.rank() == rank)?(flag_t*)mpi3::malloc(sizeof(flag_t)*comm.size()):nullptr)//comm.size()) - { - if(addr_) std::memset(addr_.get(), 0, comm.size()); - // tag_ = tag_counter; - ++tag_counter; - comm.barrier(); - } - - void lock(){ - std::fill_n(wait_list, comm_.size(), 0); // flag_t wait_list[comm_.size()]; - flag_t lock = 1; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); // win_.put_n(&lock, 1, rank_, comm_.rank()); - win_.get_n(wait_list.get(), comm_.size(), rank_); - win_.unlock(rank_); - for(int i = 0; i != comm_.size(); ++i){ - if(wait_list[i] == 1 and i != comm_.rank()){ - comm_.receive_n(&lock, 0/*, MPI_ANY_SOURCE, tag_*/); //dummy receive - break; - } - } - } - bool try_lock(){ - std::fill_n(wait_list, comm_.size(), 0); - flag_t lock = 1; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); - win_.get_n(wait_list.get(), comm_.size(), rank_); - win_.unlock(rank_); - for(int i = 0; i != comm_.size(); ++i) - if(wait_list[i] == 1 and i != comm_.rank()) return false; - return true; - } - void unlock(){ - std::fill_n(wait_list, comm_.size(), 0);// wait_list.assign(unsigned char(0)); // flag_t wait_list[comm_.size()]; - flag_t lock = 0; - win_.lock_exclusive(rank_); - win_.put_value(lock, rank_, comm_.rank()); - win_.get_n(wait_list.get(), comm_.size(), rank_); - win_.unlock(rank_); - - for(int i = 0; i != comm_.size(); ++i){ - int next = (comm_.rank() + i +1) % comm_.size(); - if(wait_list[next] == 1){ - comm_.send_n(&lock, 0, next/*, tag_*/); - break; - } - } - } - ~shared_mutex(){ - // comm_.barrier(); - if(addr_) mpi3::free(addr_.get()); - } -}; - -}} - -#ifdef _TEST_BOOST_MPI3_SHARED_MUTEX - -#include "../mpi3/main.hpp" - -#include -#include // lock_guard - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - -// mpi3::shared_mutex m(world); - { -// mpi3::shared_mutex& m = *segment.construct(world); - } - -/* { - std::lock_guard lock(m); - cout << "locked from " << world.rank() << '\n'; - cout << "never interleaved " << world.rank() << '\n'; - cout << "forever blocked " << world.rank() << '\n'; - cout << std::endl; - }*/ - return 0; -} -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_window.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_window.hpp deleted file mode 100644 index ec26547f5f5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shared_window.hpp +++ /dev/null @@ -1,110 +0,0 @@ -// -*- indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- -// © Alfredo A. Correa 2018-2020 - -#ifndef MPI3_SHARED_WINDOW_HPP -#define MPI3_SHARED_WINDOW_HPP - -#include "../mpi3/dynamic_window.hpp" -#include "../mpi3/group.hpp" -#include "../mpi3/shared_communicator.hpp" - -#include -#include -#include - -#include - -namespace boost { -namespace mpi3 { - -template -struct shared_window : window { - shared_window(shared_communicator& comm, mpi3::size_t n, int disp_unit = alignof(T)) //: //sizeof(T)) : // here we assume that disp_unit is used for align - // window{}//, comm_{comm} - { - void* base_ptr = nullptr; - MPI_(Win_allocate_shared)(n*static_cast(sizeof(T)), disp_unit, MPI_INFO_NULL, comm.get(), &base_ptr, &this->impl_); - } - explicit shared_window(shared_communicator& comm, int disp_unit = alignof(T)) : - shared_window(comm, 0, disp_unit) - {} - - - shared_window(shared_window const&) = default; // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3 - shared_window(shared_window &&) noexcept = default; // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3 - - shared_window& operator=(shared_window const&) = delete; - shared_window& operator=(shared_window &&) = delete; - shared_window& operator=(shared_window &) = delete; - - ~shared_window() = default; - - group get_group() const{group r; MPI_(Win_get_group)(this->impl_, &(r.get())); return r;} - struct query_t{ - mpi3::size_t size; - int disp_unit; - void* base; - }; - struct query_t query(int rank = MPI_PROC_NULL) const{ - query_t r; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init - MPI3_CALL(MPI_Win_shared_query)(this->impl_, rank, &r.size, &r.disp_unit, &r.base); - return r; - } - template - mpi3::size_t size(int rank = 0) const{return query(rank).size/sizeof(TT);} - int disp_unit(int rank = 0) const{return query(rank).disp_unit;} - template - TT* base(int rank = 0) const{return static_cast(query(rank).base);} -}; - -template -shared_window shared_communicator::make_shared_window(mpi3::size_t size){ - return shared_window(*this, size); -} - -template -shared_window shared_communicator::make_shared_window(){ - return shared_window(*this);//, sizeof(T)); -} - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_MPI3_SHARED_WINDOW - -//#include "../mpi3/main.hpp" -//#include "../mpi3/ostream.hpp" - -//namespace mpi3 = boost::mpi3; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// mpi3::ostream wout{world}; - -// mpi3::shared_communicator node = world.split_shared(); - -// mpi3::shared_window win = node.make_shared_window(node.root()?node.size():0); -// mpi3::shared_communicator node_cpy{win.get_group()}; -//// mpi3::shared_communicator node_cpy{win.get_group(), 0}; -// assert( node_cpy == node ); - - -// assert( win.base() != nullptr ); -// assert( win.size() == node.size() ); - -// win.base()[node.rank()] = node.rank() + 1; -// node.barrier(); -// for(int i = 0; i != node.size(); ++i) assert(win.base()[i] == i + 1); -// { -// mpi3::shared_window win = node.make_shared_window(0); -// // assert( mpi3::shared_communicator(win.get_group()) == node ); -// } -// { -// mpi3::shared_window win = node.make_shared_window(node.root()?node.size():0); -// mpi3::shared_window win2 = std::move(win); -// } -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/allocator.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/allocator.hpp deleted file mode 100644 index f63e90ca102..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/allocator.hpp +++ /dev/null @@ -1,135 +0,0 @@ -//-*-indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4;-*- -// © Alfredo A. Correa 2019-2020 - -#ifndef MPI3_SHM_ALLOCATOR_HPP -#define MPI3_SHM_ALLOCATOR_HPP - -//#include "../../mpi3/shared_window.hpp" -#include "../shm/ptr.hpp" - -namespace boost { -namespace mpi3 { -namespace shm { - -template -struct allocator { - template struct rebind{using other = allocator;}; - using value_type = T; - using pointer = shm::ptr; - using const_pointer = shm::ptr; - using size_type = typename pointer::difference_type; - using difference_type = typename pointer::difference_type; -private: - mpi3::shared_communicator* comm_; -// allocator() = delete; - - public: - explicit allocator(mpi3::shared_communicator* comm = std::addressof(static_cast(mpi3::environment::get_self_instance()))) : comm_{comm} {} // NOLINT(cppcoreguidelines-pro-type-static-cast-downcast) TODO(correaa) check - // cppcheck-suppress noExplicitConstructor - template allocator(allocator const& o) : comm_{o.comm_} {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) common practice - - allocator(allocator const&) = default; // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3 - allocator(allocator &&) noexcept = default; // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3 - - ~allocator() = default; - pointer allocate(size_type n, const void* /*hint*/ = nullptr) const{ - pointer ret; - ret.offset_ = 0; - ret.wP_ = new shared_window{comm_->make_shared_window(comm_->root()?n:0)}; // NOLINT(cppcoreguidelines-owning-memory) TODO(correaa) use unique_ptr - ret.wP_->lock_all(); - return ret; - } - void deallocate(pointer p, size_type /*size*/) { - p.wP_->unlock_all(); - delete p.wP_; // NOLINT(cppcoreguidelines-owning-memory) TODO(correaa) use unique_ptr - } - - allocator& operator=(allocator const&) = default; - allocator& operator=(allocator &&) noexcept = default; - - bool operator==(allocator const& other) const{return comm_==other.comm_;} - bool operator!=(allocator const& other) const{return not(other == *this);} -#if 0 - template - void construct(pointer p, As&&... as){ - if(comm_->root()) ::new((void*)raw_pointer_cast(p)) value_type(std::forward(as)...); - } - template void destroy(P p){ - using V = typename std::pointer_traits

::element_type; - p->~V(); - } -#endif -// void destroy(pointer p){if(comm_->root()) raw_pointer_cast(p)->~value_type();} - template - auto alloc_uninitialized_fill_n(ptr it, Size n, TT1 const& t){ - if(comm_->root()) {std::uninitialized_fill_n(raw_pointer_cast(it), n, t);} // TODO(correaa) implement with with for_each in parallel - it.wP_->fence(); - it.wP_->fence(); - comm_->barrier(); - } - template - auto alloc_uninitialized_copy_n(InputIt first, Size count, ptr d_first){ - // return alloc_uninitialized_copy_n(first, count, raw_pointer_cast(d_first)); - if(comm_->root()) {std::uninitialized_copy_n(first, count, raw_pointer_cast(d_first));} // TODO(correaa) implement with with for_each in parallel - d_first.wP_->fence(); - comm_->barrier(); - } -// template{}, int> = 0> -// auto alloc_uninitialized_copy_n(ptr first, Size count, ForwardIt d_first){ -// first.w_->fence(); -// if(comm_->root()) std::uninitialized_copy_n(raw_pointer_cast(first), count, d_first); // TODO(correaa) implement with with for_each in parallel -// comm_->barrier(); -// } -// template -// auto alloc_uninitialized_copy_n(ptr first, Size count, ptr d_first){ -// first.w_->fence(); -// if(comm_->root()) std::uninitialized_copy_n(raw_pointer_cast(first), count, raw_pointer_cast(d_first)); // TODO(correaa) implement with with for_each in parallel -// d_first.w_->fence(); -// comm_->barrier(); -// } - template::element_type> - auto alloc_destroy_n(Ptr first, Size count) { - first.wP_->fence(); - if(comm_->root()) { - std::for_each_n(first, count, [](auto const& e) {raw_pointer_cast(&e)->~TT();}); - // for( ; count > 0; --count, ++first) {raw_pointer_cast(first)->~TT();} - } - first.wP_->fence(); - comm_->barrier(); - return first + count; - } -}; - -} // end namespace shm -} // end namespace mpi3 -} // end namespace boost - -//#if not __INCLUDE_LEVEL__ // TEST BELOW -//#include "../../mpi3/main.hpp" -//#include "../../mpi3/shm/allocator.hpp" - -//namespace mpi3 = boost::mpi3; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// mpi3::shared_communicator node = world.split_shared(); - -// mpi3::shm::allocator A1(&node); -// mpi3::shm::ptr data1 = A1.allocate(80); - -// using ptr = decltype(data1); -// std::pointer_traits::pointer pp = data1; -//// double* dp = std::addressof(*data1); -//// double* dp2 = mpi3::shm::pointer_traits::to_address(data1); - -// if(node.root()) data1[3] = 3.4; -// node.barrier(); -// assert( data1[3] == 3.4); - -// A1.deallocate(data1, 80); - -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/managed_shared_memory.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/managed_shared_memory.hpp deleted file mode 100644 index a56d19434af..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/managed_shared_memory.hpp +++ /dev/null @@ -1,187 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include<"$0">" > $0x.cpp) && time mpicxx -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_SHM_MANAGED_SHARED_MEMORY $0x.cpp -o $0x.x -lrt -lboost_system && time mpirun -np 6 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif - -#ifndef BOOST_MPI3_SHM_MANAGED_SHARED_MEMORY_HPP -#define BOOST_MPI3_SHM_MANAGED_SHARED_MEMORY_HPP - -#include -#include "../../mpi3/shared_window.hpp" -#include -#include - -int rand(int lower, int upper){ - static std::random_device rd; - static std::mt19937 rng(rd()); - static std::uniform_int_distribution uni(lower, upper); - return uni(rng); -} -int rand(int upper = RAND_MAX){return rand(0, upper);} - -namespace boost{ -namespace mpi3{ -namespace shm{ - -struct managed_shared_memory{ - communicator& comm_; - shared_window sw_; - using size_type = mpi3::size_t; - using difference_type = mpi3::size_t; - boost::simple_segregated_storage storage_; - managed_shared_memory(communicator& comm, mpi3::size_t n) : - comm_(comm), - sw_(comm.make_shared_window(comm.rank()?0:n)) - { - if(comm_.rank() == 0) storage_.add_block(sw_.base(0), sw_.size(0), n); - } - template - T* malloc(){ - std::intptr_t diff; - win_.lock_exclusive(0); -// win_.put_n(&lock, 1, rank_, comm_.rank()); -// win_.get_n(wait_list.data(), comm_.size(), rank_); - if(comm_.rank() == 0){ - auto loc = (T*)storage_.malloc_n(1, sizeof(T));(T*)storage_.malloc_n(1, sizeof(T)); - diff = loc - sw_.base(0); - } - sw_.put_value(diff, - - win_.unlock(0); - return - } - template - T* malloc_n(mpi3::size_t n){ - sw_.lock_all();//exclusive(0); - sw_.sync(); - auto ret = (T*)storage_.malloc_n(1, n*sizeof(T)); - sw_.unlock_all(); - } - void* allocate(mpi3::size_t n){return storage_.malloc_n(1, n);} - template - void deallocate(T* ptr){storage_.free_n(ptr, 1, sizeof(T));} - - template - void free_n(T* ptr, mpi3::size_t n){ - storage_.free_n(ptr, 1, n*sizeof(T)); - } - template - T* construct(Args&&... args){ - sw_.lock_all();//exclusive(0); - sw_.sync(); - T* p = malloc(); - assert(p); - if(comm_.rank() == 0) new(p) T(std::forward(args)...); - // std::this_thread::sleep_for(std::chrono::seconds(rand(10))); - // sw_.unlock(0); - sw_.unlock_all(); - return p; - } - template - void destroy_ptr(T* ptr){ - sw_.lock_exclusive(0); - if(comm_.rank()==0) ptr->~T(); - storage_.free_n(ptr, 1, sizeof(T)); - sw_.unlock(0); - } - template - void destroy(T& t){destroy_ptr(&t);} -}; - -template struct allocator{ - managed_shared_memory& msm_; - - using value_type = T; - using pointer = T*; - using const_pointer = T const*; - using reference = T&; - using const_reference = T const&; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - - allocator(managed_shared_memory& msm) : msm_(msm){} - - template - allocator(allocator const& other) : msm_(other.msm_){} - - pointer allocate(size_type n, const void* hint = 0){ - return pointer(msm_.malloc_n(n)); - } - void deallocate(pointer ptr, size_type n){msm_.free_n(ptr, n);} - bool operator==(allocator const& other) const{ - return msm_ == other.msm_; - } - bool operator!=(allocator const& other) const{ - return not (other == *this); - } -}; - -namespace container{ - -using string = boost::container::basic_string, mpi3::shm::allocator>; - -template -using vector = boost::container::vector>; - -} - -}}} - -#ifdef _TEST_BOOST_MPI3_SHM_MANAGED_SHARED_MEMORY - -#include - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/mutex.hpp" -#include -#include - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - mpi3::communicator node = world.split_shared(0); - mpi3::shm::managed_shared_memory msm(node, 20000); // share 10000 bytes - - { - node.barrier(); - double& d1 = *msm.construct(5.1); - // double& d2 = *msm.construct(7.8); - node.barrier(); - assert( d1 == 5.1 ); - // assert( d2 == 7.8 ); - node.barrier(); - } - mpi3::shm::allocator ca(msm); - auto& s = *msm.construct("Hello!", ca); -// if(node.rank() == 1) assert( s[1] == 'e' ); - node.barrier(); - return 0; - s[node.rank()] = 'a' + node.rank(); - node.barrier(); - if(node.rank()==0){ - cout << s << std::endl; - cout << boost::typeindex::type_id_runtime(s.data()).pretty_name() << std::endl; - } - node.barrier(); - msm.destroy(s); - - return 0; - node.barrier(); - { - mpi3::shm::allocator da(msm); - using Vector = boost::container::vector>; - Vector& v = *msm.construct(node.size(), 1., da); - // v[world.rank()] = world.rank(); - world.barrier(); - // assert( v[8] == 8. ); - msm.destroy_ptr(&v); - } - node.barrier(); - return 0; -} -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/memory.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/memory.hpp deleted file mode 100644 index a3b02226ef5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/memory.hpp +++ /dev/null @@ -1,162 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 `#-Wfatal-errors` -D_TEST_BOOST_MPI3_SHM_MEMORY $0x.cpp -o $0x.x -lrt && time mpirun -np 2 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_SHM_MEMORY_HPP -#define BOOST_MPI3_SHM_MEMORY_HPP - -#include -#include "../../mpi3/shared_window.hpp" -#include - - -namespace boost{ -namespace mpi3{ -namespace shm{ - -struct shared_memory_object{ - shared_communicator& comm_; - std::unique_ptr swUP_; - shared_memory_object(shared_communicator& c) : comm_(c){} - shared_memory_object(shared_communicator& c, mpi3::size_t n) : comm_(c){ - truncate(n); - } - shared_memory_object(shared_memory_object const&) = delete; - void truncate(mpi3::size_t n){ - swUP_ = std::make_unique(comm_.make_shared_window(comm_.rank()==0?n:0)); - } -}; - -struct managed_shared_memory{ - shared_communicator& comm_; - shared_window sw_; - boost::simple_segregated_storage storage_; - managed_shared_memory(shared_communicator& comm, mpi3::size_t n) : - comm_(comm), - sw_(comm.make_shared_window(comm.rank()?0:n)) - { - // if(comm_.rank()==0) - storage_.add_block(sw_.base(0), sw_.size(0), n); - } - void* malloc(){return storage_.malloc();} -#if 0 - array_ptr allocate(mpi3::size_t n){ - array_ptr ret; - ret.swP_ = new shared_window(comm_.make_shared_window(comm_.rank()==0?n:0)); //(comm_.rank()==0?n:0); - // ret.swSP_ = std::make_shared(comm_.make_shared_window(comm_.rank()==0?n:0)); - return ret; - } - void deallocate(array_ptr ptr){ - if(not ptr.swP_) return; - delete ptr.swP_; - } - template - allocator get_allocator(); -#endif -// template -// T* construct(Args&&... args){ -// if(comm_.rank()==0){ -// T* p = storage_.construct(std::forward(args)...); -// } -// } - -}; - - -template -allocator managed_shared_memory::get_allocator(){ - return allocator(*this); -} - -struct mapped_region{ - shared_memory_object& smo_; - mapped_region(shared_memory_object& smo) : smo_(smo){} - mapped_region(mapped_region const&) = delete; - void* get_address(){return smo_.swUP_->base(0);} - mpi3::size_t get_size(){return smo_.swUP_->size(0);} -// mpi3::size_t get_free_memory() const; -// void zero_free_memory() const; -// bool all_memory_deallocated(); -// bool check_sanity(); -}; - -}}} - - -namespace boost{ -namespace mpi3{ -namespace shm{ - -template - pointer uninitialized_fill_n(Alloc const& a, pointer f, Size n, TT const& val){ - assert(0); - using std::uninitialized_fill_n; - if(mpi3::group(*f.wSP_).root()) uninitialized_fill_n(to_address(f), n, val); - first.wSP_->fence(); - first.wSP_->fence(); - return first + n; -} - -}}} - -#ifdef _TEST_BOOST_MPI3_SHM_MEMORY - -#include "../../mpi3/main.hpp" -#include "../../mpi3/mutex.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - - - - { - mpi3::shm::shared_memory_object mpi3shm(world); - mpi3shm.truncate(100); - mpi3::shm::mapped_region mpi3region(mpi3shm); - if(world.rank() == 0){ - std::memset(mpi3region.get_address(), 1, mpi3region.get_size()); - } - world.barrier(); - if(world.rank() == 1){ - char* mem = static_cast(mpi3region.get_address()); - for(int i = 0; i != mpi3region.get_size(); ++i) assert(mem[i] == 1); - } - } - { - mpi3::shm::managed_shared_memory mpi3mshm(world); - - mpi3::shm::array_ptr ptr = mpi3mshm.allocate(100); - mpi3mshm.deallocate(ptr); - - ptr = mpi3mshm.allocate(200); - mpi3mshm.deallocate(ptr); - - { - mpi3::shm::allocator a = mpi3mshm.get_allocator(); - mpi3::shm::array_ptr ptr = a.allocate(10); - if(world.rank() == 0) std::fill_n(ptr, 10, 5); - world.barrier(); - if(world.rank() == 1) for(int i = 0; i != 10; ++i) assert(ptr[i] == 5); - a.deallocate(ptr); - } - { - std::allocator a; - int* ptr = a.allocate(100); - a.deallocate(ptr, 100); - } - } - - world.barrier(); - - { - mpi3::shm::managed_shared_memory mpi3mshm(world); - std::atomic& atomic = *mpi3mshm.construct>(0); - } - - return 0; -} -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/mutex.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/mutex.hpp deleted file mode 100644 index f60fba42258..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/mutex.hpp +++ /dev/null @@ -1,69 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef BOOST_MPI3_SHM_MUTEX_HPP -#define BOOST_MPI3_SHM_MUTEX_HPP - -#include - -namespace boost { -namespace mpi3 { -namespace shm { - -class mutex { - mpi3::shared_communicator& scomm_; - using allocator_type = mpi3::shm::allocator; - allocator_type alloc_; - mpi3::shm::ptr f_; - public: - explicit mutex(mpi3::shared_communicator& scomm) : scomm_(scomm), alloc_{std::addressof(scomm_)}, f_(alloc_.allocate(1)){ - if(scomm_.root()) {std::allocator_traits>::construct(alloc_, &*f_, false);} - scomm_.barrier(); - } - mutex(mutex const&) = delete; - mutex(mutex&&) = delete; - - mutex& operator=(mutex const&) = delete; - mutex& operator=(mutex &&) = delete; - void lock() { - // spin - while(f_->test_and_set(std::memory_order_acquire)) {}; // NOLINT(altera-unroll-loops) - } - void unlock(){f_->clear(std::memory_order_release);} - ~mutex() { // noexcept(false) { - try { - if(scomm_.root()) {std::allocator_traits::destroy(alloc_, &*f_);} - scomm_.barrier(); - alloc_.deallocate(f_, 1); - } catch(...) {} - } -}; - -} // end namespace shm -} // end namespace mpi3 -} // end namespace boost - -//#if not __INCLUDE_LEVEL__ // TEST_BELOW -//#include "../../mpi3/main.hpp" -//#include // sleep_for -//#include // lock_guard - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int argc, char* argv[], mpi3::communicator world){ -// mpi3::shared_communicator node = world.split_shared(); -// -// mpi3::shm::mutex m(node); -// using namespace std::chrono_literals; -// { -// std::lock_guard guard(m); -// cout << "I am rank "; -// std::this_thread::sleep_for(2s); -// cout << node.rank() << '\n'; -// } - -// return 0; -//} -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/pool.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/pool.hpp deleted file mode 100644 index a27cff52ec7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/pool.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include<"$0">" > $0x.cpp) && mpicxx -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_SHM_POOL $0x.cpp -o $0x.x -lboost_system && time mpirun -np 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_SHM_POOL_HPP -#define BOOST_MPI3_SHM_POOL_HPP - -#include - -#include "../../mpi3/shared_window.hpp" -#include - -namespace boost{ -namespace mpi3{ -namespace shm{ - -}}} - -#ifdef _TEST_BOOST_MPI3_SHM_POOL - - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - mpi3::shared_window sw = world.make_shared_window(world.rank()?0:10000); - int* shared = static_cast(sw.base(0)); - - boost::simple_segregated_storage storage; - storage.add_block(sw.base(0), sw.size(0), 256); - - int *i; - *i = static_cast(storage.malloc()) - (char*)&i; - - if(world.rank() == 2) *i = 5; - world.barrier(); - if(world.rank() == 3) assert(*i == 5); -// sw.unlock(); - - return 0; -} -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/ptr.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/ptr.hpp deleted file mode 100644 index d8c22eaca00..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/ptr.hpp +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2019-2022 Alfredo A. Correa - -#ifndef MPI3_SHM_POINTER_HPP -#define MPI3_SHM_POINTER_HPP - -#include "../../mpi3/shared_communicator.hpp" -#include "../../mpi3/shared_window.hpp" - -#include // dereferenceable, random_access_iteratable - -namespace boost { -namespace mpi3 { -namespace shm { - -template -struct pointer_traits : std::pointer_traits { - static auto to_address(Ptr const& p){ - return std::addressof(*p); - } -}; - -//template class allocator; - -template -class ptr -: boost::dereferenceable, T&> -, boost::random_access_iteratable, T*, std::ptrdiff_t, T&> { - public: - using raw_pointer = RawPtr; - using pointer = ptr; - using element_type = typename std::pointer_traits::element_type; - using difference_type = typename std::pointer_traits::difference_type; - using reference = element_type&; - template using rebind = ptr; - using value_type = typename std::iterator_traits::value_type; - using iterator_category = typename std::iterator_traits::iterator_category; - - private: - using window_type = mpi3::shared_window>; - window_type* wP_{}; - difference_type offset_{}; - template friend class ptr; - template friend class allocator; - - public: - ptr() = default; - - // cppcheck-suppress noExplicitConstructor - ptr(std::nullptr_t) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - - template//, typename = decltype(mpi3::shared_window*(std::declval>().wP_))> - // cppcheck-suppress noExplicitConstructor - ptr(ptr const& o) : wP_{o.wP_}, offset_{o.offset_} {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) TODO(correaa) make it conditionally implicit - - ptr(ptr const&) = default; // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3 - ptr(ptr &&) noexcept = default; // cppcheck-suppress noExplicitConstructor ; bug in cppcheck 2.3 - - ptr& operator=(std::nullptr_t) {wP_ = nullptr; offset_ = 0; return *this;} -// ptr& operator=(ptr const&) = default; - template ptr& operator=(ptr const& other) { - wP_ = other.wP_; - offset_ = other.offset_; - return *this; - } - - ptr& operator=(ptr const& other) { - if(this == &other) {return *this;} // lints bugprone-unhandled-self-assignment,cert-oop54-cpp - wP_ = other.wP_; - offset_ = other.offset_; - return *this; - } - ptr& operator=(ptr&& other) noexcept { - wP_ = std::move(other.wP_); - offset_ = other.offset_; - return *this; - } - - ~ptr() = default; - auto raw_pointer_cast() const {return std::next(wP_->base(0), offset_);} - friend auto raw_pointer_cast(pointer const& self) {return self.raw_pointer_cast();} - reference operator*() const {return *raw_pointer_cast();} - ptr& operator+=(difference_type d) {offset_+=d; return *this;} - ptr& operator-=(difference_type d) {offset_-=d; return *this;} - ptr& operator++() {++offset_; return *this;} - ptr& operator--() {--offset_; return *this;} - friend auto operator-(pointer const& self, pointer const& other) { - assert( self.wP_ == other.wP_ ); - return self.offset_ - other.offset_; - } - raw_pointer operator->() const {return raw_pointer_cast();} - reference operator[](difference_type d) const {return raw_pointer_cast()[d];} - explicit operator raw_pointer() const {return raw_pointer_cast();} - explicit operator bool() const {return wP_;} - bool operator==(pointer const& o) const {assert(wP_==o.wP_); return offset_==o.offset_;} - bool operator< (pointer const& o) const {assert(wP_==o.wP_); return offset_< o.offset_;} - bool operator> (pointer const& o) const {assert(wP_==o.wP_); return offset_> o.offset_;} - auto to_address() const {return to_address(raw_pointer_cast());} -/* template - auto copy_n(Size n, ForwardIt d_first) const{ - w_->fence(); - using std::copy_n; - if(d_first.w_->get_group()->root()) copy_n(raw_pointer_cast(*this), n, d_first); // TODO implement with with for_each in parallel - barrier(d_first.w_->get_group()); - }*/ -/* template - auto copy_n(Size n, ptr d_first) const{ - w_->fence(); using std::copy_n; - if(d_first.w_->get_group().root()) copy_n(raw_pointer_cast(*this), n, raw_pointer_cast(d_first)); // TODO implement with with for_each in parallel - d_first.w_->fence(); - barrier(d_first.w_->get_group()); - } -*/ -}; - -#if 0 -template())).w_->fence())> -auto copy(InputIt first){ - return [=](InputIt last, auto d_first){ - (&(*first)).w_->fence(); - if((&(*first)).w_->get_group().root()) - for( ; first != last; ++first, ++d_first) *d_first = *raw_pointer_cast(first); - (&(*d_first)).w_->fence(); - }; -} - -template -std::true_type is_a_ptr_(ptr const&); -std::false_type is_a_ptr_(...); - -template struct is_a_ptr : decltype(is_a_ptr_(Ptr{})){}; - -template -std::true_type is_a_ref_(ref const&); -std::false_type is_a_ref_(...); - -template struct is_a_ref : decltype(is_a_ref_(Ref{})){}; - -template -class ref{ - ptr pimpl_; - ref(ptr p) : pimpl_{std::move(p)}{} - template friend struct pointer; -public: - ref(ref&& o) : pimpl_(o.pimpl_){} // this is needed in C++14 and below - using decay_type = typename std::decay_t::reference>; - friend decltype(auto) raw_reference_cast(ref&& r){return *raw_pointer_cast(r.pimpl_);} - ptr operator&()&&{return pimpl_;} - template - [[deprecated("shm slow access")]] - auto operator=(TT&& t)&& - ->decltype(std::declval()=std::forward(t), std::declval()){ - pimpl_.w_->fence(); - *raw_pointer_cast(pimpl_) = std::forward(t); - pimpl_.w_->fence(); - pimpl_.w_->fence(); - return std::move(*this); - } - ref&& operator=(ref&& o) &&{ - o.pimpl_.w_->fence(); - std::move(*this).operator=(static_cast(*raw_pointer_cast(o.pimpl_))); - return std::move(*this); - } -/* template - [[deprecated("shm slow access")]] - auto operator=(ref&& t)&&{ - std::move(*this) = 1.; - // t.pimpl_.w_->fence(); - // pimpl_.w_->fence(); - // *raw_pointer_cast(pimpl_) = std::forward(t); - // pimpl_.w_->fence(); - // pimpl_.w_->fence(); - return std::move(*this); - }*/ - [[deprecated("shm slow access")]] - operator decay_type()&&{ - decay_type ret; - pimpl_.w_->fence(); - pimpl_.w_->fence(); - ret = *raw_pointer_cast(pimpl_); - return ret; - } - template - [[deprecated("shm slow access")]] - friend auto operator==(ref&& r, TT&& tt) - ->decltype(raw_reference_cast(std::move(r))==std::forward(tt)){ - r.pimpl_.w_->fence(); - return raw_reference_cast(std::move(r))==std::forward(tt);} -}; - - -template -F for_each(pointer f, pointer l, F fun){ //TODO do a partitioning std::for_each - auto& comm = f.wSP_->comm_; assert(comm == l.wSP_->comm_); - using std::for_each; - if(mpi3::group(*f.wSP_).root()) for_each(to_address(f), to_address(l), fun); - f.wSP_->fence(); - f.wSP_->fence(); - return f; -} - -template -pointer copy_n(It1 f, Size n, pointer d){ - d.w_->fence(); -// mpi3::communicator c(d.w_->get_group()); - if(d.w_->get_group().root()){ - using std::copy_n; - copy_n(f, n, to_address(d)); - } - d.w_->fence(); -// c.barrier(); -// d.wSP_->fence(); -// using std::copy_n; -// if(mpi3::group(d.w_).root()) copy_n(f, n, to_address(d)); -// d.wSP_->fence(); - - return d + n; -} - -template -pointer copy(It1 f, It1 l, pointer d){ - d.wSP_->fence(); - using std::copy; - if(mpi3::group(*d.w_).root()) copy(f, l, to_address(d)); - d.wSP_->fence(); - using std::distance; using std::advance; - advance(d, distance(f, l)); - return d; -} - -/* -template -pointer uninitialized_fill_n(pointer f, Size n, TT const& val){ - using std::uninitialized_fill_n; - if(mpi3::group(*f.wSP_).root()) uninitialized_fill_n(to_address(f), n, val); - f.wSP_->fence(); - f.wSP_->fence(); - return f + n; -}*/ -#endif - -} // end namespace shm -} // end namespace mpi3 -} // end namespace boost - -//#if not __INCLUDE_LEVEL__ -//#include "../../mpi3/main.hpp" -//#include "../../mpi3/ostream.hpp" - -//namespace mpi3 = boost::mpi3; - -//int mpi3::main(int, char*[], mpi3::communicator world){ -// using p = mpi3::shm::ptr; -// using cp = std::pointer_traits

::template rebind;//::other; -//// whatis(); -// p P; -// cp CP = P; -// return 0; -//} - -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/vector.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/vector.hpp deleted file mode 100644 index ac25e54b9d5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/shm/vector.hpp +++ /dev/null @@ -1,195 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -fpermissive -std=c++14 -Wall -Wfatal-errors -D_TEST_BOOST_MPI3_SHM_VECTOR $0x.cpp -o $0x.x && time mpirun -np 10 $0x.x $@ && rm -f $0x.x $0x.cpp; exit -#endif -#ifndef BOOST_MPI3_SHM_VECTOR_HPP -#define BOOST_MPI3_SHM_VECTOR_HPP - -//#include "../../mpi3/shm/memory.hpp" -//#include -#include "../shared_window.hpp" -#include "../shm/allocator.hpp" - -namespace boost{ -namespace mpi3{ -namespace shm{ - -template -using allocator = boost::mpi3::intranode::allocator; - -template -struct vector : std::vector>{ - using std::vector>::vector; - using alloc_traits = std::allocator_traits>; - vector(std::size_t n) = delete; - auto& start(){return this->_M_impl._M_start;} - auto& finish(){return this->_M_impl._M_finish;} - auto& get_stored_allocator(){return this->_M_impl;} - vector(std::size_t n, shm::allocator const& alloc) : vector(alloc){ - start() = alloc_traits::allocate(this->_M_impl, n); - finish() = start() + n; - uninitialized_construct(); - } - vector(std::size_t n, const T& value, boost::mpi3::shm::allocator const& alloc) : vector(alloc){ - start() = alloc_traits::allocate(get_stored_allocator(), n); - finish() = start() + n; - uninitialized_construct(value); - } - template - vector(It first, It last, shm::allocator const& alloc) : vector(alloc){ - start() = alloc_traits::allocate(get_stored_allocator(), std::distance(first, last)); - finish() = start() + n; - uninitialized_copy(first); - } - void resize(std::size_t n){ - vector tmp(this->begin(), this->begin() + n, get_stored_allocator()); - swap(start(), tmp.start()); - swap(finish(), tmp.finish()); - } - ~vector(){destroy();} - protected: - template - auto uninitialized_construct(Args&&... args){ - using std::for_each; - for_each( - start(), finish(), - [&](auto&& e){ - alloc_traits::construct( - get_stored_allocator(), std::addressof(e), - std::forward(args)... - ); - } - ); - } - template - auto uninitialized_copy(It first){ - using std::for_each; - for_each( - start(), finish(), - [&](auto&& e){ - alloc_traits::construct( - get_stored_allocator(), std::addressof(e), - *first - ); - ++fist; - } - ); - } - void destroy(){ - using std::for_each; - for_each( - start(), finish(), - [&](auto&& e){ - alloc_traits::destroy(get_stored_allocator(), std::addressof(e)); - } - ); - } -}; - -}}} - -#ifdef _TEST_BOOST_MPI3_SHM_VECTOR -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::shared_communicator node = world.split_shared(); - mpi3::shm::vector v(100, node); - for(int i = 0; i != node.size(); ++i) - assert(v[i] == 0.); - node.barrier(); - v.resize(90); - node.barrier(); - v[node.rank()] = node.rank()*10.; - node.barrier(); - node.barrier(); - for(int i = 0; i != node.size(); ++i) - assert(v[i] == i*10.); - return 0; -} - -#if 0 -#include -#include // generate -#include -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/mutex.hpp" -#include -#include -#include -#include - -int rand(int lower, int upper){ - static std::random_device rd; - static std::mt19937 rng(rd()); - static std::uniform_int_distribution uni(lower, upper); - return uni(rng); -} -int rand(int upper = RAND_MAX){return rand(0, upper);} - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - mpi3::shm::managed_shared_memory mpi3mshm(world); - - using elem = double; - - mpi3::shm::vector v(10, mpi3mshm.get_allocator()); - assert(not v.empty() and v.front() == 0 and std::equal(std::next(v.begin()), v.end(), v.begin()) ); - - mpi3::mutex m(world); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - { - std::lock_guard lock(m); // m.lock(); - for(int i = 0; i != 10; ++i){ - v[i] = world.rank(); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - } // m.unlock(); - } - world.barrier(); - - if(world.rank() == 0){ - for(int i = 0; i != 10; ++i) - cout << v[i] << " "; - cout << std::endl; - } - assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); - - v.resize(15); - - return 0; -#if 0 - - mpi3::mutex m(world); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - { - std::lock_guard lock(m); - // m.lock(); - for(int i = 0; i != 10; ++i){ - v[i] = world.rank(); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - } - // m.unlock(); - } - - world.barrier(); - - if(world.rank() == 0){ - for(int i = 0; i != 10; ++i) - cout << v[i] << " "; - cout << std::endl; - } - assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); -// v.resize(2); -#endif -} -#endif - -#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/status.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/status.hpp deleted file mode 100644 index 863c7abfc8b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/status.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef BOOST_MPI3_STATUS_HPP -#define BOOST_MPI3_STATUS_HPP - -//#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include "../mpi3/detail/datatype.hpp" - -#include - -namespace boost { -namespace mpi3 { - -struct [[nodiscard]] status { - MPI_Status impl_; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - status() = default; - status(status const&) = default; // TODO(correaa) check - status(status &&) = default; // TODO(correaa) check - - status& operator=(status const&) = default; - status& operator=(status &&) = default; - - ~status() noexcept = default; // TODO(correaa) use MPI_Status_free - //{ - // if(impl_ != MPI_STATUS_NULL) - // MPI_Status_free(&impl_); - //} - - template // = char> - int count() const { // entries received of datatype T - int ret = -1; - MPI_Get_count(&impl_, datatype{}(), &ret); // can't use MPI_(Get_count) - return ret; - } - - template - int elements() const { - int ret = -1; - int const s = MPI_Get_elements(&impl_, datatype{}(), &ret); // TODO(correaa) modernize calls - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot elements"};} - return ret; - } - template - void set_elements(int count) { - int const s = MPI_Status_set_elements(&impl_, datatype{}(), count); // TODO(correaa) modernize calls - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot set elements"};} - } - - int source() const {return impl_.MPI_SOURCE;} - void set_source(int s) {impl_.MPI_SOURCE = s;} - - int tag() const {return impl_.MPI_TAG;} - void set_tag(int t) {impl_.MPI_TAG = t;} - - void set_cancelled(bool flag = true) { - int const s = MPI_Status_set_cancelled(&impl_, flag?1:0); // TODO(correaa) modernize calls - if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot set cancelled"};} - } -// bool cancelled() const { -// int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init -// int s = MPI_Test_cancelled(&impl_, &ret); -// if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot test cancelled"};} -// return ret != 0; -// } -// constexpr static auto const ignore = MPI_STATUS_IGNORE; -}; - -} // end namespace mpi3 -} // end namespace boost -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/timed_terminate.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/timed_terminate.hpp deleted file mode 100644 index 8bcd73db358..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/timed_terminate.hpp +++ /dev/null @@ -1,32 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4-*- -// Copyright 2018-2022 Alfredo A. Correa - -#ifndef BOOST_MPI3_TIMED_TERMINATE_HPP -#define BOOST_MPI3_TIMED_TERMINATE_HPP - -#include "../mpi3/environment.hpp" - -namespace boost { -namespace mpi3 { - -template -[[noreturn]] void timed_terminate(Duration d, mpi3::communicator& comm = mpi3::environment::get_world_instance()) { - auto rbarrier = comm.ibarrier(); - auto const t0 = mpi3::wall_time(); - // now spin - while(not rbarrier.completed() and (mpi3::wall_time() - t0) < d) {} // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) investigate (w/clang-tidy 14) - - if(rbarrier.completed()) { - if(comm.root()) {comm.abort(911);} - } else { - std::cout<<"MPI program terminated from rank "<< comm.rank() <<" after timeout of "<< std::chrono::duration_cast(d).count() <<" seconds, not all processes failed within that time."< -#include - -#if defined(__NVCC__) -#include -#endif - -#include -#include // for std::invoke -#include -#include - -namespace boost { -namespace mpi3 { - -template< - class MultiIt, - class Size = typename MultiIt::difference_type, - class Stride = typename MultiIt::stride_type, - std::enable_if_t<(MultiIt::dimensionality >= 1), int> = 0, - typename Element = typename MultiIt::element, typename DataType = detail::basic_datatype, - std::enable_if_t{}, int> = 0> -typename MultiIt::element_ptr base(MultiIt first) { return first.base(); } - -template T* base(T* p) { return p; } - -struct committed_type { - private: - MPI_Datatype impl_; - explicit committed_type(MPI_Datatype dt) : impl_{dt} {} - friend struct type; - - public: - auto get() const -> MPI_Datatype {return impl_;} - explicit operator MPI_Datatype() const { return impl_; } -}; - -struct type { - explicit type(MPI_Datatype const& dt) noexcept : impl_{dt} { // NOLINT(bugprone-exception-escape) TODO(correaa) improve this global initialization - if(mpi3::initialized()) { // cppcheck-suppress[throwInNoexceptFunction]; TODO(correaa) improve this global initialization - MPI_(Type_dup)(dt, &impl_); - } - } - - template - explicit type(detail::basic_datatype bd) : impl_(bd) { - } - - // template::value_f())> - // explicit type(T const* /*unused*/) { MPI_Type_dup(detail::basic_datatype::value_f(), &impl_); } - - template< - class T, - std::enable_if_t{}, int> = 0, - std::enable_if_t{} and (not detail::is_basic{}), int> = 0> - explicit type(T const* /*type*/) - : type{type{MPI_BYTE}.contiguous(sizeof(T))} {} - - template< - class MultiIt, class Size = typename MultiIt::difference_type, class Stride = typename MultiIt::stride_type, std::enable_if_t = 0, - typename E = typename MultiIt::element, typename = decltype(detail::basic_datatype::value_f()), - std::enable_if_t{}, int> = 0> - explicit type(MultiIt first) : type{type{first.base()}.vector(1, 1, first.stride() * sizeof(E)).resize(0, first.stride() * sizeof(E))} {} - - private: - template - static decltype(auto) apply_impl(F&& f, Tuple const& t, std::index_sequence /*012*/) { - return std::forward(f)(std::get(t)...); - } - template - static decltype(auto) apply(F&& f, Tuple const& t) { - return apply_impl( - std::forward(f), std::forward(t), - std::make_index_sequence{}>{} - ); - } - - public: - template< - class MultiIt, class Stride = typename MultiIt::stride_type, - std::size_t D = MultiIt::dimensionality, std::enable_if_t<(D >= 2), int> = 0, - typename E = typename MultiIt::element, typename = decltype(detail::basic_datatype::value_f()), - std::enable_if_t{}, int> = 0> - explicit type(MultiIt first) : type{first.base()} { - auto const strides = apply([](auto... e) { return std::array{static_cast(e)...}; }, first->strides()); // NOLINT(altera-id-dependent-backward-branch) TODO(correaa) investigate - auto const sizes = apply([](auto... e) { return std::array{static_cast(e)...}; }, first->sizes()); // NOLINT(altera-id-dependent-backward-branch) TODO(correaa) investigate - for(Stride i = 1; i != Stride{strides.size()} + 1; ++i) { // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) TODO(correaa) use an algorithm - (*this) = this->vector(sizes[sizes.size() - i], 1, strides[strides.size() - i]).resize(0, strides[strides.size() - i] * sizeof(E)); - } - (*this) = this->vector(1, 1, first.stride() * sizeof(E)).resize(0, first.stride() * sizeof(E)); - } - - MPI_Datatype impl_ = MPI_DATATYPE_NULL; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) - - type() = default;// {std::clog << "ctor type()" << std::endl;} - type(type const& other) { MPI_Type_dup(other.impl_, &impl_); } - type(type&& other) noexcept : impl_{std::exchange(other.impl_, MPI_DATATYPE_NULL)} {} // TODO(correaa) consider not making it default constructible or movable - - type& operator=(type const& other) { - type tmp(other); - swap(tmp); - return *this; - } - type& operator=(type&& other) noexcept { // TODO(correaa) consider not making it default constructible or movable - type tmp(std::move(other)); - swap(tmp); - return *this; - } - - void swap(type& other) { std::swap(impl_, other.impl_); } - explicit operator MPI_Datatype() const& { - MPI_Type_commit(const_cast(&impl_)); // NOLINT(cppcoreguidelines-pro-type-const-cast) TODO(correaa) - return impl_; - } - - auto get() const -> MPI_Datatype {return impl_;} - - auto operator&() const& -> type const* { return this; } // NOLINT(google-runtime-operator) - auto operator&() && -> MPI_Datatype { // NOLINT(google-runtime-operator) - MPI_Type_commit(const_cast(&impl_)); // NOLINT(cppcoreguidelines-pro-type-const-cast) TODO(correaa) - return impl_; - } - auto operator&() & -> MPI_Datatype { // NOLINT(google-runtime-operator) - MPI_Type_commit(const_cast(&impl_)); // NOLINT(cppcoreguidelines-pro-type-const-cast) TODO(correaa) - return impl_; - } - - committed_type commit() && { - MPI_Type_commit(const_cast(&impl_)); // NOLINT(cppcoreguidelines-pro-type-const-cast) TODO(correaa) - return committed_type{std::exchange(impl_, MPI_DATATYPE_NULL)}; - } - template void commit_as(T const& /*unused*/) { return commit_as(); } - ~type() noexcept { - // std::clog << "dtor type" << std::endl; - try { - if(mpi3::initialized() and not mpi3::finalized()) { // TODO(correaa) types defined statically will generally leak on exit - if(impl_ != MPI_DATATYPE_NULL) { - MPI_Type_free(&impl_); - } - } - } catch(...) { - } - } - - type contiguous(int count) const { - type ret; // NOLINT() delayed init - int const s = MPI_Type_contiguous(count, impl_, &ret.impl_); // TODO(correaa) modernize calls - if(s != MPI_SUCCESS) { - throw std::runtime_error{"cannot build contiguous type"}; - } - ret.set_name("(" + name() + ")[" + std::to_string(count) + "]"); - return ret; - } - type vector(int count, int block_length, int stride) const { // element units, hvector for bytes - type ret; - MPI_Type_vector(count, block_length, stride, impl_, &ret.impl_); - using std::to_string; - ret.set_name("(" + name() + ")[" + to_string(count) + "," + to_string(block_length) + ":" + to_string(stride) + "]"); - return ret; - } - type resize(MPI_Aint lower_bound, MPI_Aint extent) const { - type ret; - MPI_Type_create_resized(impl_, lower_bound, extent, &ret.impl_); - return ret; - } - type stride(int stride) const { return resize(0, static_cast(stride) * size()); } - - // MPI_Type_struct is deprecated - static type struct_(std::initializer_list il) { // NOLINT(readability-identifier-naming) meta - type ret; - std::vector blocklen(il.size(), 1); - std::vector disp; - disp.reserve(il.size()); - // std::vector array_of_types; - // array_of_types.reserve(il.size()); - MPI_Aint current_disp = 0; - std::string new_name = "{"; - std::for_each(il.begin(), il.end(), [&il, &disp, ¤t_disp, &new_name](auto const& e) { - disp.push_back(current_disp); - current_disp += e.size(); - new_name += (&e != il.begin() ? ", " : "") + e.name(); - // array_of_types.push_back(e.impl_); - }); - - MPI_Type_create_struct( - static_cast(il.size()), - blocklen.data(), - disp.data(), - &il.begin()->impl_, - &ret.impl_ - ); - - ret.name(new_name); - return ret; - } - - type operator[](int count) const { return contiguous(count); } - type operator()(int stride) const { - // assert( stride == 2 ); - return vector(1, 1, stride); - } - int size() const { - int ret; // NOLINT(cppcoreguidelines-init-variables) delayed initialization - MPI_Type_size(impl_, &ret); - return ret; - } - - std::string name() const { - std::array name{}; - int namelen; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Type_get_name(impl_, name.data(), &namelen); - return {name.data(), static_cast(namelen)}; - } - void name(std::string const& s) { set_name(s); } - void set_name(std::string const& s) { MPI_Type_set_name(impl_, s.c_str()); } // NOLINT(readability-make-member-function-const) this is not really const - - MPI_Aint extent() const { - MPI_Aint lb; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Aint ext; // NOLINT(cppcoreguidelines-init-variables) delayed init - int const s = MPI_Type_get_extent(impl_, &lb, &ext); // TODO(correaa) modernize calls - if(s != MPI_SUCCESS) { - throw std::runtime_error{"cannot extent"}; - } - return ext; - } - MPI_Aint lower_bound() const { - MPI_Aint lb; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Aint ext; // NOLINT(cppcoreguidelines-init-variables) delayed init - int const s = MPI_Type_get_extent(impl_, &lb, &ext); // TODO(correaa) modernize calls - if(s != MPI_SUCCESS) { - throw std::runtime_error{"cannot lower bound"}; - } - return lb; - } - MPI_Aint upper_bound() const { - MPI_Aint lb; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_Aint ext; // NOLINT(cppcoreguidelines-init-variables) delayed init - int const s = MPI_Type_get_extent(impl_, &lb, &ext); // TODO(correaa) modernize call - if(s != MPI_SUCCESS) { - throw std::runtime_error{"cannot lower bound"}; - } - return lb + ext; - } - type operator,(type const& other) const { - type ret; - int const count = 2; - std::array blocklen = {1, 1}; - std::array disp = {0, this->size()}; - std::array array_of_types = {impl_, other.impl_}; - MPI_Type_create_struct(count, blocklen.data(), disp.data(), array_of_types.data(), &ret.impl_); - std::string const newname = name() + ", " + other.name(); - MPI_Type_set_name(ret.impl_, newname.c_str()); - return ret; - } - // static std::map registered; -}; - -// vvv TODO(correaa) -// vvv this will work in clang-tidy 14 https://clang.llvm.org/extra/clang-tidy/ -// NOLINTBEGIN(fuchsia-statically-constructed-objects) -static type const char_{MPI_CHAR}; // NOLINT(fuchsia-statically-constructed-objects) -static type const unsigned_char{MPI_UNSIGNED_CHAR}; -static type const& unsigned_char_ = unsigned_char; // NOLINT(fuchsia-statically-constructed-objects) -static type const short_{MPI_SHORT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const unsigned_short{MPI_UNSIGNED_SHORT}; -static type const& unsigned_short_ = unsigned_short; // NOLINT(fuchsia-statically-constructed-objects) -static type const int_{MPI_INT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const unsigned_int_{MPI_UNSIGNED}; -static type const& unsigned_int = unsigned_int_; -static type const& unsigned_ = unsigned_int_; // NOLINT(fuchsia-statically-constructed-objects) -static type const long_{MPI_LONG}; // NOLINT(fuchsia-statically-constructed-objects) -static type const unsigned_long{MPI_UNSIGNED_LONG}; -static type const& unsigned_long_ = unsigned_long; // NOLINT(fuchsia-statically-constructed-objects) -static type const float_{MPI_FLOAT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const double_{MPI_DOUBLE}; // NOLINT(fuchsia-statically-constructed-objects) -static type const long_double_{MPI_LONG_DOUBLE}; -static type const& long_double = long_double_; // NOLINT(fuchsia-statically-constructed-objects) -static type const long_long_int{MPI_LONG_DOUBLE_INT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const float_int{MPI_FLOAT_INT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const long_int{MPI_LONG_INT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const double_int{MPI_DOUBLE_INT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const short_int{MPI_SHORT_INT}; // NOLINT(fuchsia-statically-constructed-objects) -static type const int_int{MPI_2INT}; -static type const& _2int = int_int; // NOLINT(fuchsia-statically-constructed-objects) -static type const long_double_int{MPI_LONG_DOUBLE_INT}; // NOLINT(fuchsia-statically-constructed-objects) -// NOLINTEND(fuchsia-statically-constructed-objects) - -template auto make_type() -> type; - -template<> inline auto make_type() -> type { return type{MPI_CHAR}; } -template<> inline auto make_type() -> type { return type{MPI_UNSIGNED_CHAR}; } - -template<> inline auto make_type>() -> type { return type{MPI_CXX_FLOAT_COMPLEX}; } -template<> inline auto make_type>() -> type { return type{MPI_CXX_DOUBLE_COMPLEX}; } - -// template > > inline auto make_type() -> type { return type{MPI_CXX_FLOAT_COMPLEX}; } -// template > inline auto make_type>() -> type { return type{MPI_CXX_DOUBLE_COMPLEX}; } - -#if defined(__NVCC__) -template<> inline auto make_type>() -> type { return type{MPI_CXX_FLOAT_COMPLEX}; } -template<> inline auto make_type>() -> type { return type{MPI_CXX_DOUBLE_COMPLEX}; } -#endif - -template<> inline type make_type() { return mpi3::double_; } -template<> inline type make_type() { return mpi3::int_; } - -// // TODO(correaa) remove this? -// template class datatype { // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) -// public: -// template{}()}[2].get())> -// R operator()() const { -// static auto ret = std::invoke([]() { -// assert(boost::mpi3::initialized()); -// return boost::mpi3::type{mpi3::datatype{}()}[2].commit().get(); -// }); -// return ret; -// } -// }; - -// template class datatype> { -// public: -// template{}()}[D].get())> -// R operator()() const { -// static auto ret = std::invoke([]() { -// assert(boost::mpi3::initialized()); -// return boost::mpi3::type{mpi3::datatype{}()}[D].commit().get(); // cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays -// }); -// return ret; -// } -// }; - -template struct struct_; - -template struct struct_> : struct_ {}; - -template struct struct_ { - static auto make() { - struct dummy { - T1 t1_; - }; - const int nitems = 1; - std::array blocklengths = {1}; - std::array types = {datatype{}.get()}; - std::array offsets { - offsetof(dummy, t1_) - }; - mpi3::type ret; - MPI_Type_create_struct(nitems, blocklengths.data(), offsets.data(), types.data(), &ret.impl_); - return ret; - } - auto operator()() const { - static auto ret = make().commit(); - return ret.get(); - } -}; - -template struct struct_ { - static auto make() { - struct dummy { - T1 t1_; - T2 t2_; - }; - const int nitems = 2; - std::array blocklengths = {1, 1}; - std::array types = {datatype{}.get(), datatype{}.get()}; - std::array offsets { - offsetof(dummy, t1_), - offsetof(dummy, t2_) - }; - mpi3::type ret; - MPI_Type_create_struct(nitems, blocklengths.data(), offsets.data(), types.data(), &ret.impl_); - return ret; - } - auto operator()() const { - static auto ret = make().commit(); - return ret.get(); - } -}; - -template struct struct_ { - static auto make() { - struct dummy { - T1 t1_; - T2 t2_; - T3 t3_; - }; - const int nitems = 3; - std::array blocklengths = {1, 1, 1}; - std::array types = { - datatype{}.get(), - datatype{}.get(), - datatype{}.get() - }; - std::array offsets { - offsetof(dummy, t1_), - offsetof(dummy, t2_), - offsetof(dummy, t3_) - }; - mpi3::type ret; - MPI_Type_create_struct(nitems, blocklengths.data(), offsets.data(), types.data(), &ret.impl_); - return ret; - } - auto operator()() const { - static auto ret = make().commit(); - return ret.get(); - } -}; - -} // end namespace mpi3 -} // end namespace boost - -// #if not __INCLUDE_LEVEL__ - -// #include "../mpi3/main.hpp" -// #include "../mpi3/communicator.hpp" -// #include "../mpi3/process.hpp" - -// #include - -// namespace mpi3 = boost::mpi3; -// using std::cout; - -// struct A{ -// double d[6]; -// long l[20]; -// }; - -// struct B{ -// double d[7]; -// long l[9]; -// }; - -// template -// void serialize(Archive& ar, A& a, const unsigned int){ -// ar & boost::serialization::make_array(a.d, 6); -// ar & boost::serialization::make_array(a.l, 20); -// } - -// int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - -// assert(world.size() >= 2); -// { -// int value = -1; -// if(world.rank() == 0){ -// value = 5; -// world[1] << value; // world.send_value(value, 1); -// }else{ -// assert(value == -1); -// world[0] >> value; //world.receive(&value, &value + 1, 0); -// assert(value == 5); -// } -// } -// { -// int buffer[100]; std::fill(&buffer[0], &buffer[100], 0); -// if(world.rank() == 0){ -// std::iota(&buffer[0], &buffer[100], 0); -// world[1] << buffer; // world.send_value(buffer, 1); -// }else{ -// assert(buffer[11]==0); -// world[0] >> buffer; // world.receive_value(buffer, 0); -// // assert(buffer[11]==11); -// } -// } -// { -// // auto Atype = ( -// // mpi3::double_[6], -// // mpi3::long_[20] -// // ); -// // Atype.commit_as(); - -// A particle; -// particle.d[2] = 0.; -// if(world.rank()==0){ -// particle.d[2] = 5.1; -// world[1] << particle; -// }else{ -// assert(particle.d[2]==0.); -// world[0] >> particle; -// } -// } -///* { -// auto Btype = ( -// mpi3::double_[6], -// mpi3::long_[20] -// ); -// Btype.commit_as(); -// B b; -// b.d[2] = 0.; -// if(world.rank()==0){ -// b.d[2] = 5.1; -// world[1] << b; -// }else{ -// assert(b.d[2]==0.); -// world[0] >> b; -// } -// }*/ -// return 0; -// #if 0 -// { -// std::vector v(100); -// mpi3::type d100 = mpi3::type::double_[100]; -// d100.commit(); -// if(world.rank()==0){ -// v[5] = 6.; -// // world.send_n(v.data(), 1, d100); -// }else{ -// // world.receive_n(v.data(), 1, d100); -// assert(v[5] == 6.); -// } -// } -// #endif -// return 0; -//} - -// #endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/types.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/types.hpp deleted file mode 100644 index 15e2762107b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/types.hpp +++ /dev/null @@ -1,13 +0,0 @@ -//#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include - -#include // make_signed_t - -namespace boost { -namespace mpi3 { - -using size_t = MPI_Aint; -using address = std::make_signed_t; - -} // end namespace mpi3 -} // end namespace boost diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/vector.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/vector.hpp deleted file mode 100644 index bd9e40ee1b7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/vector.hpp +++ /dev/null @@ -1,123 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ - -#ifndef BOOST_MPI3_VECTOR_HPP -#define BOOST_MPI3_VECTOR_HPP - -#include "../mpi3/allocator.hpp" - -#include - -#include - -namespace boost{ -namespace mpi3{ - -template -using vector = std::vector>; - -template -using uvector = std::vector>; - -//template -//struct uvector_iterator : std::vector>{}; - -/* -template -struct uvector : std::vector>{ - using std::vector>::vector; - uvector(std::vector const& other) - : std::vector>(reinterpret_cast>&>(other)){} - uvector(std::vector&& other) - : std::vector>(std::move(reinterpret_cast>&>(other))){} - operator std::vector&&() &&{ - return std::move(reinterpret_cast&>(*this)); - } - operator std::vector&() &{ - return reinterpret_cast&>(*this); - } - operator std::vector const&() const&{ - return reinterpret_cast const&>(*this); - } - -};*/ - -} // namespace mpi3 -} // namespace boost - -//#ifdef _TEST_BOOST_MPI3_VECTOR - -//#include - -//#include "../mpi3/main.hpp" -//#include "../mpi3/detail/iterator.hpp" - -//using std::cout; -//namespace mpi3 = boost::mpi3; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// mpi3::vector v(100); -// std::iota(v.begin(), v.end(), 0); -// assert( std::accumulate(v.begin(), v.end(), 0) == (v.size()*(v.size() - 1))/2 ); - -// mpi3::uvector uv(100); -//// assert( std::accumulate(uv.begin(), uv.end(), 0) == 0 ); - -// std::vector v2(uv.begin(), uv.end()); -// assert(v2.size() == 100); -// assert(uv.size() == 100); - -// std::vector v3(uv.begin(), uv.end()); uv.clear(); -// assert(v3.size() == 100); -// assert(uv.size() == 0); - -// using boost::timer::cpu_timer; -// using boost::timer::cpu_times; - -// auto test_size = 100000000; // 100MB -// cout << "test size " << test_size << " chars \n"; -// using vector = std::vector>; -// cout << "std::vector> (pinned, initialized)\n"; -// { -// cpu_timer t; -// t.start(); -// cpu_times allocation_start = t.elapsed(); -// vector v(test_size); -// cpu_times allocation_end = t.elapsed(); -// cpu_times firstuse_start = t.elapsed(); -// for(std::size_t i = 0; i != test_size; ++i) v[i] = 'l'; -// cpu_times firstuse_end = t.elapsed(); -// cpu_times seconduse_start = t.elapsed(); -// for(std::size_t i = 0; i != test_size; ++i) v[i] = 'z'; -// cpu_times seconduse_end = t.elapsed(); -// cout -// << "\tallocation (+initialization if any) " << (allocation_end.wall - allocation_start.wall)/1.e6 << " milliseconds \n" -// << "\tfirst use " << (firstuse_end.wall - firstuse_start.wall)/1.e6 << " milliseconds \n" -// << "\tsecond use " << (seconduse_end.wall - seconduse_start.wall)/1.e6 << " milliseconds \n" -// ; -// } -///* -//output: -//test size 100000000 chars (mpic++ -O3) -//std::vector (no pinned, initialized) -// allocation (+initialization if any) 33.3729 milliseconds -// first use 4.78287 milliseconds -// second use 6.60647 milliseconds - -//test size 100000000 chars -//std::vector> (pinned, initialized) -// allocation (+initialization if any) 0.006804 milliseconds -// first use 31.1551 milliseconds -// second use 4.82273 milliseconds - -//std::vector> (pinned, uninitialized) -// allocation (+initialization if any) 0.007946 milliseconds -// first use 30.7246 milliseconds -// second use 4.92651 milliseconds -//*/ -// return 0; -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/version.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/version.hpp deleted file mode 100644 index e39b838f40f..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/version.hpp +++ /dev/null @@ -1,108 +0,0 @@ -/* -*- indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4;-*- */ -// © Alfredo A. Correa 2018-2020 - -#ifndef BOOST_MPI3_VERSION_HPP -#define BOOST_MPI3_VERSION_HPP - -#include - -#include "detail/call.hpp" - -#include -#include -#include // tie - -#define BOOST_MPI3_MAJOR_VERSION 0 // NOLINT(cppcoreguidelines-macro-usage,modernize-macro-to-enum) -#define BOOST_MPI3_MINOR_VERSION 81 // NOLINT(cppcoreguidelines-macro-usage,modernize-macro-to-enum) -#define BOOST_MPI3_PATCH_VERSION 0 // NOLINT(cppcoreguidelines-macro-usage,modernize-macro-to-enum) - -#define BOOST_MPI3_VERSION_STRING "Boost.MPI3/0.81" // NOLINT(cppcoreguidelines-macro-usage) - -#define BOOST_MPI3_VERSION (BOOST_MPI3_MAJOR_VERSION * 100 + BOOST_MPI3_MINOR_VERSION * 10) - -namespace boost { -namespace mpi3 { - -struct version_t { - int major; // NOLINT(misc-non-private-member-variables-in-classes) - int minor; // NOLINT(misc-non-private-member-variables-in-classes) -// version_t() = default; -// constexpr explicit version_t(int major, int minor = 0) : major{major}, minor{minor} {} // NOLINT(bugprone-easily-swappable-parameters) - friend std::ostream& operator<<(std::ostream& os, version_t const& self) { - return os << self.major << '.' << self.minor; - } - constexpr bool operator<(version_t const& other) const { - return std::tie(major, minor) < std::tie(other.major, other.minor); - } - constexpr bool operator>(version_t const& o) const { return o < *this; } - - constexpr bool operator==(version_t const& o) const { return not operator<(o) and not operator>(o); } - constexpr bool operator!=(version_t const& o) const { return not operator==(o);} - - constexpr bool operator>=(version_t const& o) const { return operator>(o) or operator==(o); } - constexpr bool operator<=(version_t const& o) const { return operator<(o) or operator==(o); } -}; - -inline constexpr auto Version() -> version_t { return {MPI_VERSION, MPI_SUBVERSION}; } // NOLINT(readability-identifier-naming) -static constexpr auto VERSION = version_t{MPI_VERSION, MPI_SUBVERSION}; - -inline auto version() { - version_t ret{}; - MPI_(Get_version)(&ret.major, &ret.minor); - return ret; -} - -inline auto library_version() -> std::string { - std::array mpi_lib_ver{}; - int len = 0; - MPI_(Get_library_version)(mpi_lib_ver.data(), &len); - return {mpi_lib_ver.data(), static_cast(len)}; -} - -inline auto library_version_short() -> std::string { - std::string ret = library_version(); - { - auto found = ret.find('\n'); - if(found != std::string::npos) {ret = std::string(ret.c_str(), found);} - } - { - auto found = ret.find(','); - if(found != std::string::npos) {ret = std::string(ret.c_str(), found);} - } - return ret; -} - -} // namespace mpi3 -} // namespace boost - -//#ifdef _TEST_BOOST_MPI3_VERSION - -//#include "../mpi3/main.hpp" -//#include - -// namespace mpi3 = boost::mpi3; -// using std::cout; - -// int mpi3::main(int, char*[], mpi3::communicator world){ - -// assert(( mpi3::version() == mpi3::Version() )); -// assert(( mpi3::version() == mpi3::version_t{MPI_VERSION, MPI_SUBVERSION} )); -// assert(( mpi3::version() == mpi3::version_t{3, 1} )); -// assert(( mpi3::version() < mpi3::version_t{3, 2} )); -// assert(( mpi3::version() > mpi3::version_t{3, 0} )); -// if(world.rank() == 0){ -// cout -// <<"mpi Version : "<< mpi3::Version() <<'\n' -// <<"mpi version : "<< mpi3::version() <<'\n' -// <<"mpi3 library version : "<< mpi3::library_version() <<'\n' -// <<"mpi3 library version short : "<< mpi3::library_version_short() <<'\n' -// <<"mpi3 wrapper version : "<< BOOST_MPI3_VERSION <<'\n' -// <<"mpi3 wrapper version string: "<< BOOST_MPI3_VERSION_STRING <<'\n' -// ; -// } -// assert( BOOST_MPI3_VERSION >= 071 ); -// return 0; - -//} -//#endif -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/wall_clock.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/wall_clock.hpp deleted file mode 100644 index 65eb9b801a3..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/wall_clock.hpp +++ /dev/null @@ -1,68 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#ifndef BOOST_MPI3_WALL_CLOCK_HPP -#define BOOST_MPI3_WALL_CLOCK_HPP - -#include - -#include - -#include - -namespace boost { -namespace mpi3 { - -inline auto wall_time() { return std::chrono::duration(MPI_Wtime()); } -inline auto wall_tick() { return std::chrono::duration(MPI_Wtick()); } - -struct wall_clock { - using rep = double; - using period = std::ratio<1>; // one second - using duration = std::chrono::duration; - using time_point = std::chrono::time_point; - - static time_point now() noexcept { return time_point{wall_time()}; }; - static duration tick() noexcept { return duration{wall_tick()}; } -}; - -template> -void wall_sleep_for(Duration d) { - auto then = wall_clock::now(); - // spin now - while(wall_clock::now() - then < d) { // NOLINT(altera-unroll-loops) spin loop - } -} - -class wall_timer { - mpi3::communicator comm_; - std::string title_; - wall_clock::time_point start_; - - public: - explicit wall_timer(mpi3::communicator comm, std::string title = "") - : comm_{std::move(comm)}, title_{std::move(title)}, start_{wall_clock::now()} {} - - wall_timer(wall_timer const&) = delete; - wall_timer(wall_timer&&) = delete; - - wall_timer& operator=(wall_timer const&) = delete; - wall_timer& operator=(wall_timer&&) = delete; - - ~wall_timer() { // NOLINT(bugprone-exception-escape) TODO(correaa) maybe it should be able to throw - auto const diff = wall_clock::now() - start_; - auto const min = comm_.min(diff.count()); // cppcheck-suppress unreadVariable ; bug in cppcheck 2.3 - auto const max = comm_.max(diff.count()); // cppcheck-suppress unreadVariable ; bug in cppcheck 2.3 - auto const total = (comm_ += diff.count()); - auto const avg = total / comm_.size(); - auto const speed_up = max / total; - if(comm_.root()) { - std::cerr << "# " << title_ << " timing " << min << "[" << avg << "]" << max << " sec, speed up = x" << speed_up << std::endl; - } - } -}; - -} // end namespace mpi3 -} // end namespace boost - -#endif diff --git a/external_codes/mpi_wrapper/mpi3/include/mpi3/window.hpp b/external_codes/mpi_wrapper/mpi3/include/mpi3/window.hpp deleted file mode 100644 index 00bcb768a05..00000000000 --- a/external_codes/mpi_wrapper/mpi3/include/mpi3/window.hpp +++ /dev/null @@ -1,301 +0,0 @@ -/* -*- indent-tabs-mode: t -*- */ -// © Alfredo A. Correa 2018-2020 - -#ifndef BOOST_MPI3_WINDOW_HPP -#define BOOST_MPI3_WINDOW_HPP - -#include "../mpi3/communicator.hpp" -#include "../mpi3/error.hpp" -#include "../mpi3/type.hpp" - -#include "../mpi3/detail/call.hpp" -#include "../mpi3/detail/datatype.hpp" - -#include - -namespace boost{ -namespace mpi3{ - -struct target{ - int rank; - mpi3::size_t disp; -}; - -template class panel; - -template class window; - -template<> -class window { - public: - MPI_Win impl_ = MPI_WIN_NULL; // NOLINT(misc-non-private-member-variables-in-classes,-warnings-as-errors) TODO(correaa) - - void clear() { - try{if(impl_ != MPI_WIN_NULL) {MPI_(Win_free)(&impl_);}} catch(...) {} - assert(impl_ == MPI_WIN_NULL); - } - - protected: - window() = default; - - public: - template - window(communicator const& c, T* b, Size n = 0) { - MPI3_CALL(MPI_Win_create)(b, n*sizeof(T), alignof(T), MPI_INFO_NULL, c.get(), &impl_); - assert( alignof(T) == sizeof(T) ); // to see in what type it is different - } - window(window const&) = delete;// see text before §4.5 in Using Adv. MPI - window(window&& o) noexcept : impl_{std::exchange(o.impl_, MPI_WIN_NULL)} {} - window& operator=(window const&) = delete; // see cctor - window& operator=(window&& other) noexcept {// self assignment is undefined - clear(); swap(*this, other); return *this; - } - friend void swap(window& a, window& b){std::swap(a.impl_, b.impl_);} - ~window(){clear();} - - template::value_type> - void accumulate_n(It1 first, Size count, int target_rank, int target_disp = 0) { - using detail::data; - int target_count = count; - MPI_(Accumulate)(data(first), count, detail::basic_datatype{}, target_rank, target_disp, target_count, detail::basic_datatype{}, MPI_SUM, impl_); - } -// void attach(void* base, MPI_Aint size){MPI_Win_attach(impl_, base, size);} -// void call_errhandler(int errorcode); - void complete() const{MPI_Win_complete(impl_);} -// void create_errhandler(...); -// void create_keyval(...); -// void delete_attr(...); - void fence(int assert_mode = 0 /*MPI_MODE_NOCHECK*/) { // NOLINT(readability-make-member-function-const) TODO(correaa) - MPI_Win_fence(assert_mode, impl_); - } -// void free_keyval(...); - - void flush(int rank){MPI_Win_flush(rank, impl_);} // NOLINT(readability-make-member-function-const) TODO(correaa) - void flush_all(){MPI_Win_flush_all(impl_);} // NOLINT(readability-make-member-function-const) TODO(correaa) - void flush(){return flush_all();} - void flush_local(int rank) {MPI_Win_flush_local(rank, impl_);} // NOLINT(readability-make-member-function-const) TODO(correaa) - void flush_local_all() {MPI_Win_flush_local_all(impl_);} // NOLINT(readability-make-member-function-const) TODO(correaa) - void flush_local() {return flush_local_all();} - - void* base() const{ - void* base; // NOLINT(cppcoreguidelines-init-variables) delayed init - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Win_get_attr)(impl_, MPI_WIN_BASE, &base, &flag); - assert(flag); - return base; - } - mpi3::size_t const& size() const { - MPI_Aint* size_p; // NOLINT(cppcoreguidelines-init-variables) delayed init - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Win_get_attr)(impl_, MPI_WIN_SIZE, &size_p, &flag); - assert(flag); - return *size_p; - } - int const& disp_unit() const { - int* disp_unit_p; // NOLINT(cppcoreguidelines-init-variables) delayed init - int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Win_get_attr)(impl_, MPI_WIN_DISP_UNIT, &disp_unit_p, &flag); - assert(flag); - return *disp_unit_p; - } - - group get_group() const { - group ret; MPI_(Win_get_group)(impl_, &ret.impl_); return ret; - } - - void lock(int rank, int lock_type = MPI_LOCK_EXCLUSIVE, int assert = MPI_MODE_NOCHECK) const { - MPI_(Win_lock)(lock_type, rank, assert, impl_); - } - void lock_exclusive(int rank, int assert = MPI_MODE_NOCHECK) const { - MPI_(Win_lock)(MPI_LOCK_EXCLUSIVE, rank, assert, impl_); - } - void lock_shared(int rank, int assert = MPI_MODE_NOCHECK) const { - MPI_(Win_lock)(MPI_LOCK_SHARED, rank, assert, impl_); - } - void lock_all(int assert = MPI_MODE_NOCHECK) const {MPI_Win_lock_all(assert, impl_);} -// void post(group const& g, int assert = MPI_MODE_NOCHECK) const{ -// MPI_Win_post(g.impl_, assert, impl_); -// } -// void set_attr(...) -// void set_errhandler(...) -// void set_info(...) -// void set_name(...) -// void shared_query(...) delegated to child class -// void start(group const& g, int assert = MPI_MODE_NOCHECK){ -// MPI_Win_start(g.impl_, assert, impl_); -// } - void sync() const{MPI_Win_sync(impl_);} -// void test(...) - void unlock(int rank) const{MPI_Win_unlock(rank, impl_);} - void unlock_all() const{MPI_Win_unlock_all(impl_);} - void wait() const{MPI_Win_wait(impl_);} -// void fetch_and_op(T const* origin, T* target, int target_rank, int target_disp = 0) const{ -// MPI_Fetch_and_op(origin, target, detail::datatype{}, target_rank, target_disp, , impl_); -// } -// template, > -// void fetch_and_op(T const* origin, T* target, int target_rank, int target_disp = 0) const{ -// MPI_Fetch_and_op(origin, target, datatype{}, target_rank, target_disp, , impl_); -// } -// void fetch_exchange(T const* origin, T* target, int target_rank, int target_disp = 0) const{ -// MPI_Fetch_and_op(origin, target,detail::datatype{}, target_rank, target_disp, MPI_REPLACE, impl_); -// } -// maybe this goes to a pointer impl - template - void fetch_sum_value(T const& origin, T& target, int target_rank, int target_disp=0) const{ - MPI3_CALL(MPI_Fetch_and_op)(&origin, &target, detail::basic_datatype{}, target_rank, target_disp, MPI_SUM, impl_); - } - template - void fetch_prod_value(T const& origin, T& target, int target_rank, int target_disp = 0) const{ - MPI3_CALL(MPI_Fetch_and_op)(&origin, &target, detail::basic_datatype{}, target_rank, target_disp, MPI_PROD, impl_); - } - template - void fetch_replace_value(T const& origin, T& target, int target_rank, int target_disp = 0) const{ - MPI3_CALL(MPI_Fetch_and_op)(&origin, &target, detail::basic_datatype{}, target_rank, target_disp, MPI_REPLACE, impl_); - } - template< - class CI1, class CI2, - class DatatypeT = detail::basic_datatype::value_type> - > - void fetch_replace(CI1 it1, CI2 it2, int target_rank, int target_disp = 0) const { - MPI3_CALL(MPI_Fetch_and_op)(std::addressof(*it1), std::addressof(*it2), DatatypeT{}, target_rank, target_disp, MPI_REPLACE, impl_); - } - template - void blocking_put_n(ContiguousIterator it, int count, int target_rank, int target_offset = 0){ - lock_shared(target_rank, 0); - put_n(it, count, target_rank, target_offset); - unlock(target_rank); - } - template - void put_n(ContiguousIterator it, std::size_t n, int target_rank, int target_disp = 0) const{ - using detail::data; - MPI3_CALL(MPI_Put)( - data(it), /* void* origin_address = a + i*/ - n, /*int origin_count = 1 */ - detail::basic_datatype::value_type>{}, - target_rank, /*int target_rank = 1*/ - target_disp, /*int target_disp = i*/ - n, /*int target_count = 1*/ - detail::basic_datatype::value_type>{}, - impl_ - ); - } - template - void put(ContiguousIterator begin, ContiguousIterator end, int target_rank, int target_disp = 0) const{ - return put_n(begin, std::distance(begin, end), target_rank, target_disp); - } - - template - void put_value(Value const& t, int target_rank, int target_disp = 0) const{ - put_n(&t, 1, target_rank, target_disp); - } - template - ContiguousIterator get_n(ContiguousIterator it, Size n, int target_rank, int target_disp = 0) const{ - using detail::data; - MPI3_CALL(MPI_Get)( - data(it), /* void* origin_address = b + i*/ - n, /*int origin_count = 1 */ - detail::basic_datatype::value_type>{}, - target_rank, /*int target_rank = 1 */ - target_disp, /*int target_disp = size1 + i*/ - n, /*int target_count = 1 */ - detail::basic_datatype::value_type>{}, - impl_ - ); - return it + n; - } - template - ContiguousIterator get(ContiguousIterator it1, ContiguousIterator it2, int target_rank, int target_disp = 0) const{ - return get_n(it1, std::distance(it1, it2), target_rank, target_disp); - } - template - void get_value(Value& t, int target_rank, int target_disp = 0) const{ - get_n(&t, 1, target_rank, target_disp); - } - panel<> operator[](int rank) const; -}; - -template -class window : public window { - protected: - window() = default; - - public: - template - window(communicator const& c, T* b, Size n = 0) : window{c, b, n} {} // cppcheck-suppress missingReturn ; bug in cppcheck 2.5 - T* base() const {return static_cast(window::base());} - mpi3::size_t size() const {return window::size()/sizeof(T);} -}; - -template -class panel { - window& w_; - int rank_; - panel(window& w, int rank) : w_(w), rank_(rank) {} -// friend window; -}; - -template struct reference; - -template -struct shm_pointer : window<> { - // TODO(correaa) in C++20 this functions can return std::span - T* local_ptr(int rank) const { - mpi3::size_t size; // NOLINT(cppcoreguidelines-init-variables) delayed init - int disp_unit; // NOLINT(cppcoreguidelines-init-variables) delayed init - void* baseptr; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Win_shared_query)(window::impl_, rank, &size, &disp_unit, &baseptr); - return static_cast(baseptr); - } - mpi3::size_t local_size(int rank) const { - mpi3::size_t ret; // NOLINT(cppcoreguidelines-init-variables) delayed init - int disp_unit; // NOLINT(cppcoreguidelines-init-variables) delayed init - void* baseptr; // NOLINT(cppcoreguidelines-init-variables) delayed init - MPI_(Win_shared_query)(window::impl_, rank, &ret, &disp_unit, &baseptr); - assert(ret % disp_unit == 0); - return ret/disp_unit; - } - reference operator*() const; -}; - -} // end namespace mpi3 -} // end namespace boost - -//#ifdef _TEST_BOOST_MPI3_WINDOW - -//#include "../mpi3/main.hpp" -//#include - -//namespace mpi3 = boost::mpi3; -//using std::cout; - -//int mpi3::main(int, char*[], mpi3::communicator world){ - -// std::vector darr(world.rank()?0:20); -// std::iota(darr.begin(), darr.end(), 0); - -// mpi3::window win{world, darr.data(), darr.size()}; -// if(world.rank() == 0){ -// std::cout << win.size() << std::endl; -// assert( win.size() == 20 ); -// assert( win.base()[13] == 13 ); -// }else{ -// assert(win.size() == 0); -// assert(win.base() == nullptr ); -// } -// win.fence(); -// if(world.rank() == 0){ -// std::vector a = {5., 6.}; -// win.put(a.begin(), a.end(), 0); -// } -// mpi3::communicator{win.get_group()}.barrier(); -// win.fence(); -// std::vector b(2); -// win.get(b.begin(), b.end(), 0); -// win.fence(); -// assert( b[0] == 5. and b[1] == 6. ); -// return 0; -//} - -//#endif -#endif - diff --git a/external_codes/mpi_wrapper/mpi3/pre-push b/external_codes/mpi_wrapper/mpi3/pre-push deleted file mode 100755 index 5a7462af8f5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/pre-push +++ /dev/null @@ -1,17 +0,0 @@ -export MPIOVER="" # --oversubscribe" - -# -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- - (mkdir -p .build.g++ && cd .build.g++ && CXX=g++ cmake .. -DCMAKE_CXX_CPPCHECK="cppcheck;--force;--enable=all;--inline-suppr;--language=c++;--suppress=missingIncludeSystem;--suppress=syntaxError;--suppress=unmatchedSuppression;--std=c++17;--error-exitcode=666;-UEXCLUDE_CPPCHECK" && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 - (mkdir -p .build.clang++-13 && cd .build.clang++-13 && CXX=clang++ MPI_OVERSUBSCRIBE=${MPIOVER} cmake .. -DCMAKE_BUILD_TYPE=Debug && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 - (mkdir -p .build.clang++.tidy && cd .build.clang++.tidy && CXX=clang++ MPI_OVERSUBSCRIBE=${MPIOVER} cmake .. -DCMAKE_CXX_CLANG_TIDY="clang-tidy" -DCMAKE_BUILD_TYPE=Debug && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 - (mkdir -p .build.nvcc && cd .build.nvcc && CUDACXX=/usr/local/cuda/bin/nvcc MPI_OVERSUBSCRIBE=${MPIOVER} cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_CUDA=1 && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 - (mkdir -p .build.clang++20 && cd .build.clang++20 && CXX=clang++ MPI_OVERSUBSCRIBE=${MPIOVER} cmake .. -DCMAKE_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Debug && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 - (mkdir -p .build.g++20 && cd .build.g++20 && CXX=clang++ MPI_OVERSUBSCRIBE=${MPIOVER} cmake .. -DCMAKE_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Debug && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 - -# vvv don't try this test connected to vpn -# (mkdir -p .build.g++mpich && cd .build.g++mpich && CXX=g++ cmake .. -DMPI_EXECUTABLE_SUFFIX=.mpich && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 6 --output-on-failure) || exit 666 -# vvv valgrind and mpich do not work well together in WSL2 -#(mkdir -p .build.g++mpichvalgrind && cd .build.g++mpichvalgrind && CXX=g++ VALGRIND_EXE="valgrind --leak-check=full --track-origins=yes --show-leak-kinds=all --error-exitcode=1" cmake .. -DMPI_EXECUTABLE_SUFFIX=.mpich && (make -j $(($(nproc) - 2)) || make VERBOSE=1) && ctest -j 2 --output-on-failure --timeout 60) || exit 666 - -#. /opt/intel/oneapi/setvars.sh - (mkdir -p .build.icpx && cd .build.icpx && CXX=/opt/intel/oneapi/compiler/latest/linux/bin/icpx MPICXX=/opt/intel/oneapi/mpi/latest/bin/mpicxx cmake .. && make -j $(($(nproc) - 2)) VERBOSE=1 && ctest -j 12 --output-on-failure) || exit 666 diff --git a/external_codes/mpi_wrapper/mpi3/pres/.gitkeep b/external_codes/mpi_wrapper/mpi3/pres/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/external_codes/mpi_wrapper/mpi3/pres/QMCPack_Alfredo_MPI3Wrapper_slides.pdf b/external_codes/mpi_wrapper/mpi3/pres/QMCPack_Alfredo_MPI3Wrapper_slides.pdf deleted file mode 100644 index ea1fdb4c1d96b012e3fbd5e21ed0f2009cd03285..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 138556 zcmd421#l!wk}WD`W@ct)W@ct)YB4i2w3u7W%+O+{7BjP2%u-8w>U-zT&hE_HH*4S4 z?3bxRkr`PTY+L=3Be14LRk>HOP6#eJS&Zdq8^kO!jGm4lR+nbm|eO~J1>}YCe3+2A*l6s(y z64??OW;ZJACc)`UA< zq3c$2|Fdfg{u2KR!(wlH79boT7{DMkfGxsu5%nt$K;HH%H~@1Gn$6`2;r_PG<-t#S z5LY-xMlzsnn8Tl1zoZQyAOWBs0l)#CE|XO&x3L^dRQGmnm9<0hm}e!|-pxo94+tn= zjvowHoTxD9YiQln(JeXg-zpSc=e)05suTdS%uZTqd+Lde37>Y*>du1h23I(d3z(U+ zchK&HHL&X_Kz}H~)c73e9@pxVMb&{B-dTE;@O>jic?5V`$ShLX(rt%ZXH`f4 zb)6~*jAKP<@pEZMQDVS$B}*XD0pN0^>BOU38W_Y3E@AIXemQ;J6I8s+^1aHqa&{qa z#+8n<55N98Ur4|9L+_ch^8-B%QO_)+eb5YE$>B!)Egq#9&-)qk+n3A{VS_o_*r_9{ zJE+>_HW|R&jT}z-^DGKRCVDu;{A&p_9^~}r0+6kXIFsNRrv(RP?RMpwUF%WkXZQnp z9Pf%sfk%Z(73+gT>9}(+@UT9nfL|1O^MeUuIIseFx+Fw8d~JjZ%GuJhWH+jo9z5f+ zgC4>91_Zmj=?R>G7N~_T>xaVfFH8#aig`ckJPpEj*ufnav-RYQ1xKKA>8MRKxLjVG z+z5DZ_%&kA&5Rh!Cq`M<3#65AGpLI{7ib}7`qUQu~+nRM#uc|mvQGRFPS(Sx_j3;#t@8a49W8ld)&95D+uF`>p$nEj{6MlRqD0^a#n&Q}e_+#4C7XJ5Js6(3!ZLLp<+LU4#fr z%EEhjkrK^|%VI8izt{>5LrSDss+>_bfY3>dst&5}m`Nx_h6L!V@t z{#i4#qlfW({yN-HFozZ56ePpb+=|3|O>f<;1w2AxER;l34hrsjB==d=qbbga z$u!X;n>e<0XX*!Xa+0uEN<<6Bq#seSsxKlnH4fF~nw*ihU4twi@n-;h1n%>08|l9A zDi=RgNh;mZ56nc3?d;(2F#1(dkWAWqNjah$4m(`&9q z0)2&AzwcTMqB04*Gqxb-DY`i@!7yiCX6E@v#LYQZ*TEv@A`Vaca}%D1kr=LJDb&nQR00+YiPMZM%Pn2%q+1ArBIEzf2V00A(foV*m7x8V3JZ{@ zf6y(UQXdB~0smu@$A_}xHfB|8P)kxjM+#D-2K~o_*py>jry$jE*DGkTymotD=**ao z4gyDa*FEZF?`->dri%K&MQ!xjiUR8<0*G7GJN31~cWO!)Tn5AhIY>CJ?(6#q^k<{v z6*)|oqzvc8#`0aN zFlrV$;2l62#stX(yn;YG|)@g7Oex1#H$Ir%8(~ z9-HeYUu+9zfoq@#!i3LKY9!!vm4RP4Sqtwz$;DpO>l zmoDK6@#3oaNZRPUDvA&Ebutri_P&tD=SHvf%-{k_H!lSzN#E%?-Xy_|>vXAQ@zjpP zt>POT6|0Pm*DUlrH^!H0*Zl`WZL;1W?u40O0l=u-`R|m`RI33yHSxOPKF6?X)KgBa zMUzUkOFt;edEQ0$)U@u=mOZ{yzXb9P&}`qq8-=W%aJd*thxm*RPf=a$kjWRmv?6>h zLNY73XNgYiieh15{1m3d>-nMeEwI)MGQa6$n zwu;9dyrLVU4{6hje%?#0duG-bz>yEh6SIp$fk=5b(OsH!Xud zWO=0|ZxBXfx#IuW#j3RAB}wvBukI_z)$D*)2&YA~!=3D=hIVQWDwa z{HnySIw=$%NsCbQ1nGfU8btxXsproJnMyh5&o9dvXmM>1@+xT8K*T(Y5A`Zg#BWqG zM+aX<9|9L^x;2Rk7}v$c2`u5!UoB?*^+hbb^fUS*Wo)gFUZ*tQqorKyy7!Z`r1=mR zDPT=1Bk1Y*{J*oAQ|;hcHue2&4FXKmJC6h0e#?}MFLtN5ty z%`oHdAGE7~B6THIkH*yrwl8jVlP-?}p0=Fe+)C1BH!cmVzkY<>;(@UWA$=~&M?TQ9 zHm3o&!Dk{J%3_l!h;gKJsgY}X@LY|q?4$DvvWi!+?(>mx%<4t2u(f_06+tD_HX{d) zUzkc3P;Q_)0SnQ)6_Hq)Rejs>v;lBrjwEhGhw~ksnG;)~Cs^r9dpW~M8Hd~Rk$T$Q~e@)DYBp3S8T2MT09VIyM`$f!bneKZ)MjexaoQOVlD(-&M{2c% zY}Pq6d1rdx_ArYGQTXe!4_PdI#;8fMdBk}b;0CW5(<-lDRTw2ie48rX`A8G`ci+<~ z53SWx$=eM~N)qT0!x8rB4mKV3K-t4EX>S<96|75hup2UD-bZ~jT9h-npYBOU>qFu) zOz|P0a(V0t#SijVP(Fph{v8a0cf?;3q|n%O3>vVdHBXM!I48jgsl0ZraM1_+vA((G zq+8TFtD!s$x*QXk6qoefrYG)ulTQm(NKb#y6VGO7&yhA({a7$K@c=|wQF5dTzk>BL z8VS+Pcoup_Z!YhezD3t0#6Dv~bvu|58k&+`#V6J;1y*$`+Sw$ckSY$LgzWSc-H4eQ zOI+4_fujC3YDy+n#u>Iu#t7Fi>Z&3nS?@D_@3AqinXrPa9=iJ&IjSa}^8f`Eg<2BC zz>XH89rj~x{8@)nZ#3$PEyeA^*ftvpvlRPbzdtA`ZX77Jh1~$ZY}0GH>884;k9xtU zBa3=sUE^~}WpVx0BIuMm2QyUcyKTc9rMf%J;U%F$(WmUxX8zG0o=T6{_oP-yhFRIT z6S3wJ%v`}Xtj@0!>7DjPyG~tvQJqJxJqxwBtmgkuyI2nABCoZ-7 zAh8x41($D->HG5jU9hfN*Dn9|OR3(%2EvSND;k28l1NXGFuJ=n22wEb2 z5+fUz;{>nb6fTuM7|=_38SVM)_gzspZYVw{0X|%e?oP^ajJa#Vh_xGwWcj}X~{3WzRpsNe(TyW0COXjD`%w>C0A^I1L zMRvSU7I5786S8%wqYGqL-Dl5-B^`nueR^~h629*|;nLnN;hM$vWb>~Zg~O*T#b~cW zBvXfG1W2<_&8isf`(NV}B!}Q0moIBzq_v(@IA$F3Ug954B=7S1N>lz!Z@z9)$gWNs zlAW&ry`o%4H`f>5zl&Y6(4_8Rls(j};E&tOqshhoffv)$dcI}vcFN(-jsk(pa^XR< zfdd8YOfwR#qo*c4VBEPB{)+$>Ns<_>Y&9PY$7_H_py>?U<7=Q#$vQzOvzq1G9wvI+ zc(m(=yozKL;mi&cJp0>M2NCIWbl@w?LcRdrrA;H89qs zwD9$5evR*(LJa%`&PZ$TJDN+bbY6@35bK#Y`5%=-6ns@#Lv__SCCKSf=`0mktS@!= zoXITfs@nKY`jBSx`%~@sDpGL~UShV(@JZf14q|hs5jza9165*`MJNiR`LO9h5U`Dq z5FaH2*cQfy4ZFYm0_EYK+-l$RDNna9KTi2J{hUAi%DMyC@ZSA0GyvxOF1Vh*pL_>! z6MkZaGPN`L8x+0pXzBF4{)7zqAoKd(?Sb+UJHG&Xf2 zVC4Kft@L-~gOT-5+=GDN|BZai!A~hrOl<=JfWHA={j(fQ{}s&ncjVys9}B?rTc-aV z0hm7T{EukVzas!68v!HR|L`|pV*cL}f#vtF`4|6n9lRf>{>je-9gwma~)!q;uWDiou^4kNT zKhn=cNLMbzAhcRg8~b>owAn;$RYpq^@G{K?lZsLAu#-{RR2pv>YdTFn9xuAGA6L3% ztMw1#s|2m$oWy!S&}k|?O)M}FvalT9)+AxvJX@A}$S+Exm6!nftn(nGb$mk@i!6s-noJhGt&IzTaR#ifi zab}(W{Hezi$|U-W(iFzM7e0(818;rE6W5GBWMeyLJB(V zH^5S8$LQekSp@{{M&j|Ns~o5X_5=h%U1g0Ks|5^53r7Jay4i&8#ERU(9}fcZ(0ZK4 zEV0>3Kl07K-3th@C%%6uxE?MqW>I(xClGp@IA0^%#tJcKcDQ^aF_|u!sZAp2>mQ;0 z)qi_5vBR6+y7U&;ceWOlP;;*NTGjsG(DUBU5ceuc^NW~^L@%qN2|!jm4Y{Q&Gr%l$yCk`lOHD20QAKhQOZ>FDgLZ1Idjf7W^RJ=`if&v7DnwmFCI6h=&wh zO_4CLv=kJ2CYiuMBiwM*OC72eX~tt40b}$tOOK=m*~54>Gs;7g;)2Hrn$xOF4F#K) z2)58rUVaxtLF^#<8XxO_JqdLp44|Y5bu>Lwvm%l_LK?!VxgkUZ7CSMPNkyFzf(UQqzRERpz&XQoKo#8Qy94BWa{W6WiJj*h>?8@S$m{6qJ()^FbT9f~K~jtf zzuooJvi90|h8LP@BwAvhJFrc8f88MP>wb1zp$u^m0v*HaS&{1<)whXKy%~lbtb64# z_hi%s{0|G2en-Cz1N9ciJXJiBZ`4YO*wOqh;ASYk7WgTVe$$zvU$sQ)ZaW(LUyLuV zElvV)M1{V|iB>Pq8Ab`Ig6r`YxFPXPANz))ccIYV;LwXDbz+R)7_m=A`Q}^}a+*ECJrXr=;$o{BrWy2K2{dn>F_xQq znP4ywTQ%iS&W)8lV;;wI^8Uiils~QZ;&p$V=yU!maOaOZKU?d#)GQcAj>SccYQVxi z^c~`52MAaNuqcPB$poGHr%_o`D{;Pfkumz^iv3*6_iiQT^~$@u4VetfhNq`5Y6j*j z6Ys6=K1b5!1;|&!ygH~7EI`{9#I+D2)XxW=3p88y<|_Ne^d1&*ExyTSZCATtMtotuxh`)7hIZYnX>9CHUjs+JC1ENgPB?rlMhWyKiI&Og zEM?x9F(%o)v}x)sNjDZymDpGF(PmD%77sL9O7hd&)if_fxo{h*^*CRX8=Vwu@7kGh zknPgC0v(J~fN{?l@$BR`#k;o?@h1v>pKsAlv>lbl)0o)`i5=Kwrh2@#X9YE@{{FSG z>YECjc+$1F`Kx38_kJ(^_^%j|dWL08UH2RENxiQN$GefQ+VE`}Ux-g)^Qq^CsNnUW zZAA^Ga*7yeqig53L`fl3?qbG44EZDj)Nct2H<9{xU+-T$C+SDWbAjaTuOl7th~S|@ zyIbJuG41)Bm9O=aD}mH=ogbDpH`yt%wLhNr+lLL~wB8NDJq!pRXRf2rdqrEuU;6JC zCb!1F_qi|Fuvgv6QB)k-$ASCviI#xtoou}}f~O7eu4ugx7{on;?%3rO_Kqhqaw^oZ zw4NlE%Ocl89sqK8^PtqT*g*w2&QB=-FT*|C0_Q)0Z)0O>{EK+a@(;1%kM!aHns@wT zf}x-yszLXgo{=-OH6@^+qqniKv!Qb^F{6Z{|6@$S(A-qc<&(#8{GE^daZ1w8%wELO z*qMO!Z`sFxBr+@+J%Ws-U)ZJNJ z+4+;3A^3e{$neKMVZq;X85@4KziAyORsy!)*@)8T(O*LSah*FG!6(!6=fFSkKdc-a z|IhR%OZKavBQ_%p$mJ_kw?C19Kf%Sbe{Q^zs<$H5!BgUqIao)c&#EmQD4dyq4^BZ0 z+4-ec#~aT1+bzSnH7o8$N%NMi)by7x&Ej(_!?i}4vPE&$ZV>q6>fBEg3l}B=MRH)(j}n?Fmlymgg3@S+ zw@>gk>DpT$5g&~D)tiVo-bJau!urs1=w!u?ap0ASB%Z1yHq^Q)(mycg@JPB9nK0(! zUsiG$B5KSOiMZ8F60VrI;BySM(}6`DoJWRs()XKts=UVx(3LUf$#<|xx9B!csn+HV z8mRD`2;dwI%nSKOS%0lD3FqRpzHsJB2UbExLU0ubmn~Qi$q!jS85M7>YZZ1;@@B<5 zZfP9GEEyq5w<>}SQbdBDs~j-KkdTnOR{l4@y7dX8e{bBJA zAFlU})$=VY{`ba~z4Iq{AfJ2PfA+;cjPkF%?Z5cq-_kg?|BH$K>l)$H#u*6y8{hiv zJAcmehp&BB8U*wzj)rzl4u+1VcE%o1^unKKL`+>RjZKxrh5qoW|FIwb_AmuQ=TAD- z?ho&C;3Q!CGxhx(`Sb5DS~ZJWWd>e?hEkGtOj&|XZHB&9g+gvhc}AW^L4y8}Nr6Fu zp>C$(lc!~3Q)ghEX<%SysZ*hlQkB?)+6Mx-M1VVo`WIULJ=tGc{Y#C1TU7q`z5Xhj z{#=m$)EXxNE92)Pq@nC=>OjE6zyL+B`5#>sOB+i&^S?f(U}$OQ{MW%>rPrTT*B>n{ z105q1#~-)*&)WN2h4<&8#K=g%#`Zb)@8#-0>xF>c{c}*n9*W>!iTB?u;C~U@od0@V z`~REKFwk+b{jq-k(fPX;K0ALm5M~7ifWB|epl>3;A$LY}2-5>X)3XE1Bg;Cy^8(X- z>oWsN`4p;zG0*_B0Fc)IN?pHI{9mns{SUSOOVa;u{_)?};{VBF{tR3H-&~9BHyZGV zA^cr(>is|L|I^&n|MCZ>Ic6qR238dYrdj3&7L^&$*_6XWDA6(~OPv560I*Id^MCdR z_TPc;FZuuU{r_IJ-%AK99V7FnKm2b0hea;??`rygrB3JkN5JH-BA1<&{ogNgZS6mc z-1i*)ULVY#tY%Mij{W{Fn#zqD8G~Qy;Lj9V>k}~=io{Zo_qQ%3B0Zy$IM7xb0j(m% zH?J?+_O3^eoC6>{yTrL@LsEpiKv4o)jWnDhGzJ(QW#|!4HAjaYM~5{cTqRGnRBfYAWN)UMA*ffwh3V| zn}*koWriyZFfJh;h4->Z6}k*ya*2ACq$oqHykqR3+9(PLk}Dek`5}e(VHEs8D%6Ck zSQTXrGlrxEALIN6)$t=E&PqHVc8H;D6ywedr)Pe$(f|ad_m1z0(uT;62EYW_6>%}I z2rSV*B~UV;kCFPmWZqY)IRa#Sm9Fk*<4?=V@Y}DsT`<7_Cw-wIy0Ugeirg#2m}q5^ z`lAceyA~UBb0E^lqEIhGCl&(CN}hQ_?D5?`6xL48mU0N)jR}G}CiBamu%{RLMa7<_>iS5c6w`VMV;uV`lts+N1N3gv)E{xeafjn3t4 zc4S4_<#dorX{KZs4hnpyNs^`B$yzGeo(gjV1cYZuYcNAiwZUQz;k@^Fk;eD?T{q*p zdKJIUEkB2+>iI4rI)DcV-ywoMPyJe(&s#5&5yQGK-d8M^inS@Mf*04TSCh-_aOE>D zIuzj~r)Bv>hyfA~0ZEqhd_S#l*vqS;Yzr3-?`~Ar6TR^VgqY~=$uR78ntY!r;9$LqV*(5+~u2AFM zZOQ!SWYf4?#$%u76210-&YQ`4$ba}ztI zH}GRgl7QQ{FdbB=KPEdy-*~D?NiXT%Nm16U)!{#+#FHqV6r1xx?jUnaMT`hzz$Y!C zkqgkO(-iRx-VC|v)O_qs{A6R7BG94t_#?ML^9{|$Z2vL z7o1h|a*BP#l6w~wf4!5v%!N(s^}w_N8H3O_iJQ#_gb<&)<|NLZNq5Yv%X*{>mjv39 zL7s>5mYJPpL&P25q6(UJ-mJ+6x+Y$~^fJ@Ni*r+|LeSk=)Nu1q(>FZ%AveFpIz8&w%m z^)T<29WlR5N{M6JP&$NgSJ1<6-klQZ?1UZG-WbAQ>l(3*^d@SO#$Z`jefISFOj9nh zJpHwQlvZ%5unzt*%C35)0&kH(JI^v+$8D5hthwQFdYuNh`N_Or$?hH9@6lcnIDyw; zI_thQ>744&qU-M)tD)z+LrvT864OMcNyFok;!U7ZI647ejU@3B@2;jj#LajWcABy# zt`*mFn15ZZ11B6tKz!j^Sr>?_-^KQ`Q1U2yV2Si$WxR{Eod(E#EtQva+`uOSkdkst z4Gyk>kQ}&lprl_>q_N`IdqYT8I38AlNQfW052%nFFfT<$UyV6S{34eYlk1_m*A12U`j^wAg3gB_44sk z(X<(*;|~6?s;}?B1~CRIB4#~22YFe&x$&=frz}^Hi^r{jH%94)=R?Cz(Pw+ywFC=F4J#IHy(gev-#d4iHw;o z9MtvtoURr=ERiyQe-cp_8#5YaNIOq~h-JD`As9P{v073&v7+D>T!uRill(l6aPlim z$(}taE5*A)d2`{bsV%j82$Pr|3mUObt&Ec}hE`O~U zJh8W};wDgHfM2QmoDbE#_{?}iU@`iO35|OmI{~e~GiM-HLX_hh@n>@q<6!yQ)n|{= zaHM|;5C2fJxa?D;%oA9W|@{8se)J4klWOu*C| zryx=-e9?Aes=(S{kL=s-V2gLZSH`a7qrGX(!W{|nay6!J{IGrTNU{3E1^FWI=N=T!$^*#G+%q&SmR`E>qN z%o_QuyD(9u9^^LQZGukq2+Z|0WuwjrrbS?cD~|a{_@>B?Wd+1FEaNUS%dPQV+7en)DoeJ#6JdUQ%K{#;(q4=RbJgkA+u4<-#}&LN?QC`1ySoXf}oeMB?ds1yX7`RFGdp}pQ7Rc zn=Ab*t5-Y#-z{uc4bBR)h2748eM2@ox>BOsxrMn)*qO7ZlL|hBLX-@14V&omCO3F{ zne}v;4#HCzgN>sVR%-I_AtQu9Sas|o21{bi2*Vz{wD7LfI79=?m*q!&J>aRSj5MVj zjy_K!qokExwo!1UoDSxqAcfZ8;>OY73G&Cm>H5y{w{J)MV@4+~T7uAst`*9rLkx_r zVQ^2Z7G*7cF^?)dA=oCA1xpU>%;6XZq(uCDy;x(GFzo6g^%IPON)jI5oNz)>)|W5( z?n+leQFqi`HJ7y)xuQmcYbKxk?V@K!oAl*SRUnjvVk3oWz>($xM!@e}n6_+R!Ro;? ziB|LDU22N9*+NmJo%=0)VCB8jmu!#2L>M@=J!DB-7CX2?+C?Mij&s!5`7mi1p4fUe zNh6mIm~tmSs=Uo*h88w|ptV|>7wC`ws<+V!kMev!=yA_CjJd=$3eXObc44frJ8X#i zD(TLKx4Dt(nf~EJAOK0&bAf9>2+Fax5vPCq;;3i$a0L9JbUf)Jy&MCSd5?B;SajX3 zqi0#eH2j?o;ukQH&WVQ0MTkk2o*lKEji2{4G@C5NlKR(f^hSdX=oYk`y=tymzL ztny*OyaIL*q{b>Nh@i)KzH?Dkyiowmv;3_G?{u*O2+g~VhZ>yv3>+hxI~$5&*fESE z&a23Wv|z|do7!hoajVi9*~Z6u2^Pm`Ubvp-9>Gq5<-I!{j9L^OuWzk3Z+TRpeR@`H zDCIiJj&7Bn+Un9`KBh~9v#eIrXV{mrzHb2g3@i3{>?rIrIu0?8p`8$DC!K64pc1bevbdrLSPYOlJ;tSm}I>|>I zWpPl4nB&I0#yJvHxTr}H8Df}YLfr-BMtdn8xkMxJYaBtXQ4|LbN^cm$oMX;u&drWr65Sd@ZAeM8P#zvfT0dZm}%aBQ%aTSF+b zBFP~Y8au&6_pq+{q_sq(l$awmR;0w}lCm7V=m9RwlNE%qEyr-+QPUC-C?*U{iq9;b zXcP_11HScZr0gE10oIr=y!_@HClRKVPS8hvuQfHt>eLZTtK=yO+_U;f+aOzbD#vY? z<=ZLIr3*{tcVnT%cMeio2ir(V4>Cy#(*6C1_o_u))Gg6!>;?`DSiDkr^tf^{7G2KV z9)G#y6Sa*X;ZeEuf)N&#z`p&%AJOfjU^SIlQae4 z^n<=GOeYcw0*od*627T-W5T8%4jO&B>~-YHz1Pp?Uvqpti(|p7*>1^xIP(C-4xd7} z#FM2)VuDg48)BFzJy4$tse;C+%;@ddA*W|ig`&vX%a3H`+)YSfk0+6V8({|w(ggvY zb5D`A3o8Jt7!RV=*}v}+0vIYVwktoi7+H+3(5xO=;+6tVIa$!Sm~v_49=CX~>r9(2 zE?9ANGi&D_KUi3}kdXy=ypo1i)gA}Gl?E?EPIPbx`+|IG1fr+82b!$v`0d1ir=kw< zoW597ZcV78*a`?Phbdx9k`lKKEGnw9Y<{ncH8FlGF9p56VO&uEX7iijtAps83Nq)S zgD-dz7~)pdfvyh*yZ_)cq@xBRaEJF25bsCH@(JFB1q;a)-GxSU+xUV(e3q4Wn+cB! zu{y`J71s~JVzP=+>F_*b!%%Yh<{M}WqD1^*4tUQ`QNo>hEaR589)y+}M_hv$xZ zYco=E2r3rn=MT1-Yn{3DWc{J>Tq#l_fc)!i7~cdl$xVE!Gollz?=o!7GJkOlDLg9O^Kp&KA&n`?vK?UG`XnU|icSRAvxy}I^L zS-R-7Z^ZCey4E6`%lPm<4hEE~aq--SIaO_Rs5~Ce>Yy-d!mSx}`8Xsk+Iq2E+8(x< z?czYlkzBUzC|l?E_L7r~34*|fJN|C+36hlOSb+GipIfI}qAn!A`9UOxW>@)gYDKqa zW(upCKJ-pbvL-)lPP};r`KJKQbDg-Tmte-)r9^Rh$~oj#duW7a@Zr+r56O_>cUZNY z3fdCzJhPnRfVBE=0@BvdUM^1Mqwi?{C`ZSQGk@wX@d|gBuDU=Vq%DcZ@Qzj{7pcDd zUR4sX(LK$MP>hq(1Xc@qijOOg`fR#+;|ISAq^tlP0J)xs4tujAPfqBP%|fuDN_<|{ z;;SFyzlH|&23LlH`_qWYk2&owNPfwa0i0JOEkJkgxPQSUD<&0z-SBW>`w$F>iKk|2 z@+GX3{rlidW70#>7I5k(95>EjQ@>8nh*|yvjITBE> zwP_-Y7@s8_J2nNtcm{K8TAMurwBzW7%2ujHI<&AMvy?# z2nP9N4U80*iK>-=w3Nq*{a7dx6cH1o5qM;gQAcf-UJkf8tRYh+U>{-D9|Dpf{UR7b z@gSx!RwA`zl7UI_!F5G}Qj&e^3iM0Yri6@$IwD3_?nK3)>_o*eKS3N=6>48=tpQ`T z+fqZ5!_)fQeZk=!*WqXPCK+(`0fose(9pTjbx_gwD7zc?(NeYw(OmVdTVpM==ZK9c zRSZ^8h(-Gg0JBpVfKHr?d;@Zq>uC9VCmqbWj{E_{NpSN#I`^cEgP0Q&wy)nF?>qRX zRy}z#TaL`n8}TlG8Lv$6tKqL6HlA0QzP7zg|Fmg&{{l11_+*TP*V)7GjCwCJU(@cE zUFwH$nNP|BDL@M^YK6a$lyqp(%z+m1P>50%*#<>=n!3?>Ms7aY2rBVv$z}_$cfME0 zZEFU7t*7RYB(hIQ(+O7@~kpT7U z$b46gdMqS^-NNHqhzH@4Yv`@ief^RStrE}>38>}0HO*C=mO2EaCce^p-dC?Viun3H zck)*!>PZfkLE4M$mMr->62<)*{?M*NV|vqFYE|-rQl*NBPDF-{Yr};jhBfrN{edZ* z8-Xmra>wQ#iE>`?&r2L?+kP(zngVDdVhXz)%^YMf=P>R}gBN z89$F$coKTWf|oqrOS%_JuPOaem0Kq-7VC$OX-nH3cxTjFJE|J0%Dyn-?R(g-+KZh* z6YVDfR5k(b3vTG{60AJoaK;(vGMnDqu%f!_{0t8=NqapOcL zN6R#l+j(2NZL;K=4a3TduA;J=(g&^&rq7O(YJ-az4lcHiy-?eXOUw$K@_N6M-(%ZA z#rh8fUahx>-4{3;z%18<)RotFCasEqpX%ZANFme?;pmJaQrmQPFQUaI^{Z=DgMKMi z>fBflB+!6GZGNT}%tIW2)J7aopXrxEfR8bNTn~-%Z_<%pBB;P<-Xr2uVbp^%syP1a z3abC8@nIDr99f?%meH=~81&uQIE z48gLirWd%3df?hAVw|)attSnX9KV%FqH)l_Dy>qg^wbTyN1<-{Q!k2uYj^MVp6dCA zESPij-VHyfrmzm%PUs=?WO$f)$X9%|iCv54Br9#j|JuEo!;?qbhJJtb7DsLDm<{Km z_3UHg4&IPfd8Ww-b}B3XYw}{&L47iN;3LKb-4G>hV`?8S#PxWub+;?Rip(AyS=M;o+N#meR6nD`4=~huc+sP}sl|agaY=THrs7Op7l{q$o5;aqS;;zexl8CBs>R99GNlF< z{gQMXaE^LlVm21(?(~zeI|RFPiHDhhOSUKD!+Qpk3QbAyOKqiL{gAtM$9wGcp&Sjq z?n_fyxw{2Clqz%(AF_1ewLPQ^%}g&j&Y`(&oML_E(e%kt;4}B~&MD)QSIQ;NndU5r zFEJlc=HG|JVv-u6AS?)uM*6ce5RJqvH9^5x@Yy-P*O`lFTCp1wfu~I3XBE79`WoR$ zM~Tc^C=>d1G@c1liA;FRL?RuAGU;RP>uAG7n#gxcJQLt0qzwbZr;eIeD!tU2q*O=rr8qqV2f6Yj<)ISJhB=ZX zptv=6%i-i5>=KGHQ<3acX8ULug*bDg^34HF%srGQ%!=i`h*hCJ;5KY*2 zSyo_)BTUX1r#lt+PFYrPi6ctRn5H{b*yllLi6c(Vn8)YHZdq11x$>1vZv*Hw8jto> zBx$KefK4y;svjC-F`m&xhz}X#;VRk(DErnF{=dk5KGT^0pt1iBJbbdvf4u(nc87o1 zN3eRXrTxkNqWDhN5U|jrNq6)!Sg+mq6F~~0C{s1!6FVx=yQkui~sHW&HrMyb8%@KpYj-m{`vqp zz*o3`_*GUI8dD>!iiP@{|79I(J7_Xc9N-v?Bn{VUq>wTfZ#lNG<+4hEB(ZB0$^GPi zMgPYC2LI-NA^+rmSqHS8Lu03CO3+H582hL-3j71*ihDrbB?+L6qs5rx%_wPN=Quw3 zUt;hv)KC7mgcN@vR){*ggVoUv$D(XXGH1RxS6i?lzj>SILCP2x;63Q2eq%W?;1 z+DnQcQ^+Jy*0D|f1 zw51Gq3Jb_=4YSW>3VjSVs(5zmI~%`E7;dh7SCU=BuLW-RC?zMnZaGHqFsxPzrqce+ z|C*8hC;m6+4U!Zk?vb)5KVq9nNoehp|9!2Mz@kKX`Q(4U0*H=FD6mu_Ud)w>=83Qd zg*R0d9&?RRwmhP3`|3inS`$Y^6juWw?*oNUV5eVmVerMEETi19KWLnvMGIYAKFu)V1}C!zkN4a?7M;jxF<5(#S2j?#gs%l7hJ-xF~Ra)z8lK!we2kXotL!hITll(S6ul=v1#7ezUx28T4AK<@Fv?U733i)Um_XA< zPa3~kY(S=;r#KtK+*NA#|4avrkSZ9i{T z^d$&MVD1+#?Gct(=Uh}fWMWlH(9XbH(C>D{bXvo|?E>MdFXyr|@TSPPGSh|~4qUW1 zSQ0~+E+5H4`>hpkmWc9o*<6FZxEZGK_A&lgJ}wmJKr4W6Th-$nobA3_a6n6*xb*1M zF?+UZ;7@H_oZKZdmX$Dr^(pHbv_$p*&IPz7qMRu)SATK?tO&K z)HbkXX^pytUFiX-EBL|tGc(s;*kKC%d80EIf-!C4l`8)gs`ZPgJtI#j;bf0s%3DQr zqJ!QKa2IRSZHc~zD#$~Gh=K(ApPgckHP%MkFqyRk4+n1vwc2@XL9x^tq3T9*Hae7t z%I(Gz=X*GbMpf`ZBnc8%oHUX}WFam6}~UteT5LMvS$!~BIJti+qKkxu=m zMpzHmc|7@Xa8#G&efsQ~r`6n^y{)++@@2+^Q8^LFWVQ>zNoc=dl3G`qeqnnnSs?B3 z;jJlykH~H}MV|2S#mt@p@ochqo&zW`F32}QD2OmFX}3`JYiRny(={GitwrCk;@+K6 zIG$gTb$LF=|5h+gh}%QM)fyOez2UsjMNWeewFG6yD{CEku1fsLnycG{ zwrIF7iM{751*m9Pw%d~d*yTsT-A=}w#Wi?$X3!jp{-Qthbq+xi!jA=ZW{^BTz^Jym z9DKYc?al{TUC_?0IG|fY;iL=Ffbk=Btn- zbiKF(yJl`RH|m{yFlY0Nv7>Xi6wGN5kE|c~8herW*+xccpod>2cKv-NSYF{gZtC&C z5$<4pv^y^7^xH;Q&C{Dby)2eKU6=6+{Ev&X?|z`^46hgRAvrtZFdp)eMqh)FIgH{3 zbIkX4z6d#2avR7iBa#qkBP()6nPPk}WKWem0T+%B4)2mE=;|OKlS=6Jm?1hhcUe(^r6Ua~gaf-Lm{Tsxq*WCtEZdhH`E1G*>ErQH;^WbgKv=4Dgi zeMASa29g&w0-IG98;my7JHKV`1{nM@c6t zAYHX)h*$^OZ=SZhl%)TdPjR;PQed;zROYoBwJrJR258%_Hd#rjBC%siDgpo^xB>)` zbYe5kdYj+#BHo7J*dz>NwiG!jH0z5*3O3Bl*to04r|5F00gd{gU7OHI#pz03KzS0b z7^*wm(_!iC$4sCL0F{xTh^hVk2+~5m+ps76zzSgINaCjD13Ymu_o0*_&*Oe)3r-qM zQG*gwfKd-CsnJBW=4A#;w$bEp#ql;AI4%sE>gaW@Sn($*&@5s-fasxPf~jeRL+WUU zk;XL9?70mD&J`(HT9Nk43$I5teu9B0661yPDScM^1QNg(f?J#GZkAAb2LeNBcSMQh z|FHIsL6WuWwszUJZQFKr*|xfDn_aFhSC@@0+qP}nw!iB4U2Ctk&tCDxJ`pD}B6G(4 zn~^!@bw6W_`vZ7N4&jG-6HRRTTuzR*x!c!~gD#m^fRmt!VCk_i4Z(J(v1Y!BcY$)J zWbS&{uY-bz1+0AUtLZ$mwqXG=_Uf+)Qp0u=S-P>8WMa3lN!@+Cp{y7;W8*1_RPjc= zkg=uz;BfwU@7@#b;}vy|8`Hb^_Wk+%(BeedtE!SGm%d)r*sOaT)+Zo-4c!PB=_;qPAydcHuRjVRc+eka#BP-UCSi0swJ0M3iHYoNw=G)7JCn`sVnyzUB ztC*elY9XD;)q9D#A?Zx(E~1 z8*)qXAMw|0BvmA<4i25eY!BE7CvM^2obpB#r(pX;86GZa>^sxn4BgHeKJGF;hnHL1 zBES_1_s|?^syFfr&f8w!C|MWuJ>GfDS=|?+JuP8Y7P}wgH1`_8E5f2yl;rr6s2^-Ki;1I?#9=FyD<83 z84oW`^mt@>Bx+lMPmcQ&G$g9gZVD?9b~{Ul4)xW*drJ`0&wr}BY8e19@K^O+oshG9 zu3+JYO;Ih@ZbD5H5k}$axfg3hwfW$;86mg6r;qUF|!z(;-f6+(HOVIhU zu#FhoqVDY%r8~FKSb=J6PS3)INUnH4^K0fda3YSV~xmkkYd1yJv&#I3)>_T zVzr=Hy%_cLq3xkY-kOH<+PUJA8|lwJ%gMe%@6<`Lfjjfn|m>J4mhzsGb?VK$pci6Kc{gfC;*NNAO=rF5>>2D z$a3V_&;7EX?vVB#d%5oy8+0q#-*F(uNs42jHKp1R+2i2Q{z9XajM+Pp+xfmYGaG4P zTkda(Z+u69spUbS#U9J&tCV!oRKOMXj6ob2JVdce{xvUAb)s98OxO2Fgw{>MQE&TY zT5ESZU-T8Yl&PM7=UzS?jffq5D0@}ZS>4bFhq(zHGoMdz;BWe`|JU5U?(k^L0NJxfivT*Y5bEC=S8T{?EoNqpA{l?L1HnG zMl{!$5jlcY1#FM&mU7+`^uR|~lJzdOaO~QzD>6vL0F0SAi+m8C36upI1vi=$WibXg zSeSCO(eG-35t* zkzQp7Z7?Z8+myll_j93@oioBy(SMls zNnNN-GUB+ckm1~?!9gZ2ac~8z-Qmkt^E^GwJUs2Jh$%QhPngA`B=X(PZ+h=7 z;BDDmVeIg|GE`26nhc6$#Bvf(z>{9%Xm?S|aGPs7(iWo&o+*d%}bUc6D@ z_4MsII%%~Io6CJbSjQi!OF`QzTac8n1XLbo=Y!3kwcxCc>ywuz^x6|Ox2dIs%DgjO zjR#Q!xf@2AB=$W!TVS`xFJPqdTnHO|%D-@%M^i(!Uv{)>nhJYAlb%j~qa`GTnE)~( zFq0>YtNfQqBENB_q!QR{<7b&P5EfUtY}!PKTCZfrx6>eY(O!zZAB1+=t<$qYmm^In z+B~7>qKtYlsDODXL`bZebo&*KW9fQ5OTnZd7)n?f8#*FBtTkw-#amoB{nmXd9ImL zDN}x@#o48>slNVMirLsSKt!l=xL)@8VBH7~nw@g~W&8q^2a~ahz+qV;2j>aQTGCXd zG^1o!i4V~fMBd2@=^ZvKY6Gq^q%zxhI+w4IEmaP#{}$qQ`7q>|nS9DV_%s9jh@EEx zxm_Hqk=6pc;FPR292se5mQAUx-7r5C-SXU+h(- zN0gb!GAxl%c~-G0U|w+w$}gtl7fOvC*{i`|{T_AoOtxRMe(HNO^msK@wu-`9dR8}> z4Z_cjo6@ot327zcRu?E6+k1Tl-r_A948W6m*=8_5Wp&=b&Hp~Ib(QI{)~s`;3(3y1 zB7;VMdt5P`)N)e8>jHJZUsuo*jwWTZE(~J4$DMSvB?5D2=g`D(Z}{28 zj|lVoM>l~jcwh?D20jL8x!-uh3qA<8pq*4ypL*<%Q>pqwA$)|*x*s|PJ`f$_;ew29 z)Goms;2n-{*9Nk9Ct!=I4Rs zs)({D6Rj0?8I&J$O6BIVv`pAPSPG1GyI4c5aGW0%f;?tHow+T;S`{in#QVmHS0Cki z6e{`&U&sN_yT>g)>jpE`6ngn8yuxbp29{8t8+d-oaobru31>N}9krtwt(4b2Qwtqz z$0Ue{WtsK)vaq4s-ETEZtXd&6I78b2hEb-8|Pgpt76v4n|}#4k)>4rjkASbUym`xPw^nVc~QUBYp-=# zzQ^PHht<5iZ&xNT;IM1B4ZNN+$f}(qjvPn_=ozGm_n_Fwu!G1utVr5qOgG9Wph2@v z(f@oh_{yjK8~pqoviXlkvz(0V|8+FW$j1D?ZXZ2QUA6goatOJ6MsfVL_OrG2um|FV z#95O-CU6OK228AbNaREe5%`zndgCSkoP;@?0IGPElb}G)P1T#3i)m9Te1r2Z7!9>6 z8RK?BS-d<_TUxQgt^Jm~l>L^Xu}DVwFt-Wun!+IvujS{F7n*NOpmE)XvR&*W?*QEi zz;Jf07(7@w)dtNwVT0nr1P4ToMAfb-M-1eB>KaSC-$hur1z4Ei92|bS?pakE{zx}E z>JylV=*Achrcp!crY|Ejb>JK+7ph3mUN)}M1ZPJ|8kbx%zS-EKEPf>@6;Xb}2tXM! zOX7_6f@C6BVAydSK|uh)L4l%M9x)Xr44qt7426pw3jH3pWGqf7l}&&JJ6bNFuwcEd zy+@=5CHezk3CA8F(29}VxDT^5s$YWOS6u8Hs`#*2^A4qbEVX}JvU7JcN*|hHV%(Dw zQvuCsPn+@<3byd_ieh#3P<0>1$+BQzR31+HnBWU>&j6YHgxG<|NdIWMNPi^PFDN5U z80P+*pBSR@UM#42xd;qK*uRo&AgJWE?79&l_N#FRAjNzCf7-0qwM;dpvqGbLSJ z{+J;q!(ghrLN2b~skOcvE^|2aXz%i@MVGp8yU)9${ina1H|gnH_MrpW7t`7K+2m5e z=}!c958Est0XQmggp#3XkVkHNioJ7VcSz}&mGgE&!9ZJ4_+^BGC zKS28?U#yp2Ek_Z@T8b?!Mki?gwxo~F$Ff)Fjzd6{9Ab+OTIT|qrEtC{PHPz&sENXN z8T8p}Nq~>qNoih$1v9ib~AW47Rg|reso1m7vB3o4O zIkDmAoJip^ZM7$vXMB3+2ng+SPUMIPZF*M@tAz-2#g;=rN_*IYba`$oL7J$)b*CTR zR_A`ZC?T%c8a!MTz9F4;;aOe^CJh{rI)Bl|{>LK+RWjg5tFsbA3zMZyM`vThW@pWoh+eIf)U@v)_p-lveq$+}$4c1B2s;Ye z9$AucC*X!TjVu*|*lDa@m$v(B4_)@{<&6A!!+lnF;>Sm;MyIxm_3OJ=N6KR5%cHBS zdT4gcO7=E;J^jo_Slaj4Q>(Wqy-@nWLfI!P^T*n!^tETB?}wGA$+$PQWj`)n=yvX! zyc<*ApSvS|Ou-6$pDcF&SiO|+6x13YoC0bX$QF zZq!Dol=W@G!^?m>pNhq$S00z_#28r0$i4{RbKV(1(~+|Rp{cNy7_?a=7hHL~54(Fz zI}{YQf9J@ij2g77b^jP0?Ifwl8)m0qX>GoDTAq~U4#KhF=gB!JYsEWy+wJ4E)^kQ( zXvS#iw*&RVA|`HbIoYH&Tc00N-Yv*#jTn9K@s%<1!EI5g0s$ICYPy9AA9b;>ufTTB z(Gf}x_#JKH0Qw_Hq+bSbdLF|ml8X$tHrj-hEH7_9)>*kd-$U{~?u2w+>UXj-eW|W$ zdzpvEea*>!;>PpE3YytKxg<74@@Xj7(}2zAXQSvA8cew_7>1{#gv+S+by>2-vdQTI z@0e|RcL!$BQ)8Imnl(J+tj(+{_)oQKPr}Ny+V31FXs|zea_tSL!$ir^GSr>^d2xrP z!BSMN;nZ5vIGOv}CD(AR^p?r-f8A(1arflV$0V=P-U9|4b)(S#1qVuVo8xEl ztIGjs6q1I#D*c}qZJB=dF& z0?j0AQ+AE1gF4w)D}5}*e2l}coq%Qr#%#puh{~6Se;IvYyH>2sN`NVGf1==U7AH`| zg)90@Tqi`ZA<;x6!xY`#j41R56zv?D6OqKrGSJwuu8Zw(aA-2oD#r4~mGXz>>4?iQ z|8B1btML9g9cWy3C(vL+0^&|Pqq_ z=NgX%Ee);Ra2N-Wa*Erwv4oSs)dLJR2t7Q&n1!g4iWR@uCANfemLr*k{Cq|zh;Q#9 z#$9q>xL+lIPQX_ug3rdMlBIo%3~|UHNr)P4av|fe@70Tjuk0hf~LHh z)6P$cr!ox!12H2Z1!Inld1d18kDyG^$UAYU%CGT6>sOdLzsa^C{V)xWVY-Vr>vXCN za5zI13BGRz2?|weB4OW-`{CRS`?42dQ7!%LW^Y+?tFKKlObbU#!J&PaClwUpz-{i^ zT>VI%n+_s+)byE#VTaQOqt(emk9i+_y1ok3nR(ug_TfMW5t&^Kn68r9lxmEj)(P_v#J=nd@J zbc40jBug7oh0C>vSEKm-h-SUZD%Fwfhsm)n?CEPj=asib+?qDCHb;Aq@Xny$0L1O9 z{lOB0^3ensLtj^677}V6citYW2nH@yLU&#Yf9JkC=-!GL%YZP;$BH}z1^`7t14B`D zHb7hJ2!N-C+EjiD@ zr*)Z<9dIgTWSP^OIjMBk;wxCRVj#EmPz=gL&u%pt_!sY*|INFF|IWL+U%ZQYabHF6 zZaZCN#F1xh7+UiNbdosn>an|4pJs0lDyb-AODhlWP!0kiR?eO9{KwUc2Wxu}QTWBX z$d~6{jOL|HZ|i^a?#0jO{c2Dq(jP70tjH}G5Qo`Jb%gyprs!6d^UkoONxNUXOMsZu ztWby8@x{AR-z7&4WU7++z)Op_3en)~c2~v9&J29Y;HxZO{^H#g{4|{|E1`|INF*)@YSC#}(sAEf-(Bi#+}p@4|iYuG#;|yYv5zcU}Ly2kWIAyVvJ4HR-ts@P0o*h@Gkfl?{0tbE@PI@7w>ZY#k;(- zgm-EGlfzjz7-oL|S*T}*10 z5KWbg0ZB3pD!lIfkiGci@ykKF_wKftTP+<;rgPUGm76rcI%9zEkUnt;J^(AbWhyhv z6?v-2El#+tgUQ^qn^!s!fdK~Y>%uXe&txq}g&6IbUmoP%Qj;b)86l^Q_>QagS+f{y zFz--losHc}3~7OC$B8w|Djh!54XZB?a@~&xn__-_wSb@do}#{f^uhtwO`Or>K%1m+ zK{Xv_#QXU;(_Z4!`^2UU{l0jP7W4n}(SVuZKeVeba(=Zy{;!QV|4aaFDG{=wjc<>$ zz@~!9_yY(U4IW9wftyfs3pw}2t!LA|Y;y0>HUCgO6mtP)5$Idjn6P^qw~N+cc96k% zehy+4>q4|^6NDlba>eya zh(YtN1ol2`2U*z(-mh%wlRcAKm6#xCptL0Bhmu6X1gBIX#ki6i7P)zJHk7!30yR)- z$d?UgNx&bhw$`m09F~ zC%ThUF(x(5 >VPrihG7pv+F3a=C1K+s8wck&BPsPeOBZSrqI55G}-wZ7>;_YRHs z^s>XdhrkCJ=KXQuoZiO1Hk7?<%BNMM{kSCjw-PaFLvtcVL}QQH#LDK)jvJa4u)s5*;#$1nP=LK^192WfP5g!=*aD zsi2xL#j~UF?VV?to{w6oxf0dw*QpW&1RnW&Ty`&QkaKs?dw!ZjwMh62$wjdpimnZ0 zWR{kqVw)sWDj>m}GJ@dJwl+$P&)xww~Mk6cX(mz&fJj&e+Cw zwY+@?H{KI?kj3^zZyS%^eFvV_2as9MK@`t>EZ=Q=zTHIzBL{ZPhfTeXHN=wp^_W|= zr&Zb4;~F4hygdXO4k)yr0tpy8@AiWzWb5Ccmm35a53WLm=-k|@x)qo7?dcd(k8rZA zd|a^`u%3TPs{GuO=6^m5uB+!fYM$uhoOEzk23=RCt1S%84JkU_oKND1 zU@l|?xy>^SAt9FWUv_K2WWt-HdU(7$t|qjMgirv2kSS+i_3yocA~IezoGj)LrwvfGLDmp=3np47}2oAXwF-as$)s*~%^A-GW?`fT-9$zAC`O8O2LM$CI_jZeEQe>BsSvEQbYQ#P zW$RLdv<^Rw;9)0{qO5^j`=H-8ARhg!*2dm%ZbkPDs1_2Vu-F^;G!^RG-K+i zGBBzSbj5yk{Z_^FD>BDd6uFk8IXJ27DPC;{(fAE*;-BpVHXw`6kihH#NNgKj_A4d?^M;elc0W z#bAsGO43Y0FI8h;(Aq4%I!aou(^w-V?RJV;&+t&WV-*)cUAYI7Bcqh$Zr^m+N>)HA zD34=yNZ^6yAeuU_aV#T2=NVKD^m2R^Mwdn@2&lndZZBxUn04A#tJ1DxMu|%&oI0pm zRyrYQ3<7Txv!Pi&S|gX`0O|l%d6*4TP**qW&IYw5j4_Q>mw;0SR`Vq?ZrX~=;SSGY zHVn<+kdBG$wJ}iq)N&RF89>y#s7fMvLfdErgN}Hg=SzIL>Y%nSDt3ObQ(qLfJ}kbK z$y~h63as_9Zp|BZm|MDb2@^e8Mal|_Ik^%5{C0t!ovr0p(-B$?kD5he8nEQbBjASB zb)2fCZVtfj^(!$N8M3~hP|3266#*;yypV?muIAj{F8QHF3kSx|y%Po4 z%uMSmpVhrZ6m&$(e-FO)AhwL5W&X<qd7TK`S;c}hw0`^y*38#M=g7sa-8-WO z;$rI+3fsw9dQlB7O{IS0{?rB+tEv6%rI+`q{>8y(eVOy)algJJa^qC^2r)w=m)^Wx zYIV&?^YdbZ^~NU8ro^;&eFV+9jSnLpfx;(}O4V*eaItg=xCLF3^|j_jbU4MQzT~)Z zd*9)#pvrRdy4>1BQZomhYwWmT=A|>(2<|i10p%kG=4PeteY!tKo1KR1zhzLkGp0B4>#M+a zov%od&KXy!k2dsdHEqM!=sM(F69&aE_;+w7zehsakxPPidX(*(*|?o=eTp3PZu^nr zBDe2-8{iQim-BP9HgKd7MXjnWs8uq*iVi#qd$fbmZ-Mu?^$C9c>2r^~k1k(uh$TP1 zar&muT-L3brp$3lv;N3~(`PzgkqG%W=sE?kh5sPv%>O>S{;yN*|A&H3%q>JqnREtb z{gvq@bRFic<65ji&ITm9m~aT`wGDAROO2K48LAX7V+4x!+htY6KulGlM%fNg+DaW0 z-`hEI3Nt^9b%^_9H};YI-Dn0+%4*}K@zoabRLzF2otN&e3y+3C=VxI9;a2Bo9G_?D zP{)!{HzX2gry5J&SF4RI)$B6nzMo+r0D`&zA;=*MT-Zjq@l(mc>4dYKq(3IUIe-<4 z3d8u@$W@k5uG54Dlc0HK&#YVG0mN@R6p3bnZ1b;BuRknGF~fkQ%0Tg^DL|2dX#k=^ zU%6|yoRV44EkG698BBd~foNPGqMcMM5e5An_*HV(szg&+r%5JmIJXolbkc~K91)?A z$NqXxW#Nj%);~hMyuV%qVXV0|Qx*0jr`;|CqTyJLfOM8H<;D0 zB4>PeGV`WJGDrWE2v>f){#U5?_NSSf9L2(4p}Tj9xz>c#(WU>-P;b}M zSE#rDE7XhW_jB-B=lY~#hNjr2??9i$GU=_&=>kJl<@D6@qh79ZmJ$yl>?d|zwSX`3 z0RExJt1#+|{+Z9hx2V)aBcmw6W!yI}AEz53KlVRe5x3Aha^aFas8|Bj4j-{?=j~Xu zXuYEfz64Dd?H7)-*mP*S#|7jm8{CCUrHfC{F}IkVKQPrA^F%-2pGRq9HP_!ce1kjB z>+~k58FmEWx&!j`0U<@grswfpr443i>wj89qZq-(Q0xW#0LP{YhDAF>$_tyg9yV=c zHzNhn9rG(4@Q%N9H$SVY2izScC555~8 zN2a~g>$E|%o89v(&*--%>puAThJijTRE64JT@6<1H)3b~s@t-YZ6zQRZA`WHF)|Za z#^NdPD9$vm5+{<7w(s8$xtow+Th@Ui?<}g)J18(2zV5{NtX`A;*p<~V84~arXY7?sN8D$5a z7sL>OW6;yi z)k``Xgp~#lFI|_WW>9<4xwKjV>9JyJ&WNtv9=F=4yK=QY@%}Yl$=C1%^rnF7^mKcUS-tHkh)p+a2U7HqOn3kGvFJulJBm3qpToDzU2rnM@BIt*}Ly; z8?TL;Tb#9t}p#V*Wvp9=z2u`U%LLWkP2VqYgMI6ON#XmT{r)it~>qF_1ZtW z-t(pFQeV3M_}_J%i{C~}4tMBF*QaN2NdBqoE`RAdQuTl7I`0zQ{L58Gsq>evyZEZB z{iW+s*^9TE?SJWdX@7C`?l@A!Tq7YLB>w!iKTyINt$<)7fQdJv5k3{4#u!L7ySv{N zi{CD4QPm8IPg7bV+&^`l@k`ffOV;S&YR2vD($L!esq1E=f9pCqIrfs;KXje%@o!yU zwf;-jBbLv3{^&a8U%Fo8>#TER>elT3FI|`TTh~|q*7c0Pb-jhV^WqHW^Y!MNp4jC* z;ho$4Jzd30KR7`PI*)`84VU__KgOANYl% zbL*pZ$xmw6gRLkXg*lb}s`ZVM?#reIZHxl)9|ZkBc66{Z{;!Ak^BR9gWgaWH@*=DT1bYSv7Gz7PY`|uWk{XR3pAkTSl}n4Hk*GsQ8W`J zyz%7(#&!?sw}C>D3SFy1OS+914WO1dI&gBXBfXq39`wD-NqUR6%+?;xoUnJ4tC-Jt zeLqKeq%fn1&}ZxcjX;%y6t2S<4j-odWmO%Im?3~9CgmU}zn65BLM;r-Wve(68X7Dc zOn|e5k%$rcxR)R;tDs1p|I85PSS1b!UOqumcqIqK$T;v_(T^?vilG81CX_T1=*y#4 zvQ*LFAgu*;3gJ;$k^Oi%3Xqh<-1R3V1K%yzPlgHxm~FIFPz^@U`V$EQ(`r~rWDNTg zv$rxbCbsLV`YF8Dw9rV@Iv>Y%8xn5@wGdP@-iRb*3{KKEpVT`~o?j~a-aodmn#KIL zzD-#S7*C!I2vAGr-RF;*^=O-h-9+>h%sErBG=85+U30U^a@lcFEPi1{gid{_T{`OCob(=oWg@!LN09Pm-E`iza|_4s#Hv%tW;KJ}ZnA541Qx{PnHnxk9un=tky!4(L*()D(%5%Sf3{(#m^~q$SPIHh#es?z6;3 zEknwihJ-2GfM^3?Ak4tlopl{hx`p*n7k!vZnhISwA4VQ0E#J3^&!ZFT;+Rx zGz6O>;rna8AALv;Q=L9|^C!HPgZ@z(uUt=GmUbr^j1XCp@c3J)Vsc639#RkiNrw&s zUlB8R1?!t*X*<-6u189_yqDUv&WReY_RJ6G4;OI>JdXn$T;^4V>;}vie|DoQ70RnJ z``ta(B@`m67QQa8W|!UjHtn(-P}#U6!x85>hBrx4+ERxk74~STo?FTYI@Mxu$SCeJ{Hp@33+}(=qN6f(5YX1`%6%^NfrZn61trbYq+Zpq* z6R!??I;hf#nss)oWZ9wegRG>}@tQUVDO|O>(6VDlw1Y(2$qUQsf_Ik zMerhS^hSj3m&1VfgJ5*Qi)5=CTfOTnEeXt}JjYanagcDdzqyA}GH-I2GH=-Q>Lx*PfPd5ZX_=3K z3N9GT%5`^Osst}dKS)ro0L3ZKmDIty%xX%#W?d&Emi@_)Y6tIJ{7XNj5(~>OjP!eB z%jUYYnS7Bu?}HMAyzXmsM2!(t&TKRN>7>w#>H8M>QN6HZMo+#YEM%>8{s@(vGOAB^fZ5x^-o=p8vkfbuQAEIuu!i#I8z zVDx5h2)4W~@|M%ST`9BIZyIm3FF3wR2v4>MMn~V7%*7zpN=mX7V9^VYX$rXZIO_~%6+qgvX6*n>QK80sHOfa+yn&Lc%Fnc);=LG#K=vBZ|_p_E5tEnRP-NO1DRO=y*2Rv z*T=;AFCWv_X6FC>@K?tcg&oB^OV1vEyWi;br8Zn1n57O$G*rIC8bxK)RD?K5r%vPh z%cbYV**a;$6kS?@Mq+H-(r)!6FJDLIwZcHFP?A?D3Px~I+|Z#!M>Z>0czy0&0_J&a ziAXb?a;dhsLo_c{Sh`&OHDf%qstu}x0>SiVL8Rbt95q~H9I+bUHnP}CI6vfjTQcAG zeG>030?lJ$^lo-OldBVZsOJ;AM5<~NZ4;@p+%?xe-s`ilz+^3H98gx29pwU2lD$&E#ebt<;(k*2ue_?u|&;V_mU}aG>xg3TN^g{VyDM*wI>1L5|zM*1~*a&N{ zVU#?1+F-+QrZ&Yf)Y)>hnVQ5B`5Goph}bq!{54scnRx-qAtwY5Qu15b1SVm#DrwSh zQiRgbURAQo?y-G}7x&QLx2b6TzXCor&c%LYMnIe@=EKTmfmRLj_BjGk-wM!YG^ha8 zB3DU?1eYXT1OQxy0(V631quSd_KGaJldHStYhfmpHIg9OEbE#L)Jn*XdNCl|P0Dd@ zAjZ#lIV}lUJjKQ#1z^-7)fPCW!5^P5Z9ee*?NLA#O!l z2|XCi5**4?=aMhLmlA6Hz~ml+=J_7FPUHs2h#>Jo2=;^O9U&08=wK`RYgJkpcu;}a z+{fE!r84YZq7z%F$ZTW$u({*qhM7yjL$?)E8h$5c6!#3GoOpyNdsdvDu8(W8n^PMG zEtiA0@GRbVAv709_5(yImh&895+=8s#-Y!!DtZlFsV))yjJ>=6v-_CVrJGeKYSLL) z2VCm;?6Z~E8toCt8Xlg)Yg?=Klt<0vr`Igxak5U}ur&I*lf8fQgS^aZS!+srMoepj zkL#lhzl+&n-fOKC7b0GI8>PAh4}nO=ELiC6^j&V&7GwQ40~T<)r32D|Z6_|_l=u&S z3Z8xcCK`>=Y?C=A^I!EenhPes%96s9QRpdk72ERNB(LLkNc*M2lTqoZbyeHi`d-t# z-n~*Ep3WmgeczwFUVfWjP9B)8wr0DSkRqtwdK-WR^=x1g(7DYfIPM zaIiOgE9y=B8DlzW7Bts(YkYU{w*TFpFYTa}=)zf~BI4{N;?V#ZC{bnUr}UUw&x!O) zSlthB@N@fDr)QC)aux`{?M35UcEp&KD3;=^Iels(h^VN#9fhbiMz)<; zmoZ{&_Xw(3l*n`fn-ml+#O{YA^xJRwFuM1aMALI9w!snxO1E5U)F`pZ*J2_&(Z2-ZnkHhQB)yt($wD*i@->*c zW!X54HwJdL&MXw|Es2)%F>HQ7ph%X|ba$igpBqC)Z@WC+Kw5jhP^iSZ3c{_S%RN+mt1n4Rh!8banj^cEo`VZD|{Y!EqT}2tH}p zyMVppVJ$r{8Dh^rb9gji8!^3?=ss=WePBARvk@#CBW_!?ySZ75fc*+U~kSD{F4?ikhCLo~xk;ZS{!6U{mL&t+S#o67a?9hLsla5dCQ zDqWSf-(NRQkYZHc+U?R!?9!FY9{U`UysOil^=J+&l1)7)kJt3Mz-KvACc2&A3LP0T z0*`T|4}HzJ_E8#hj@PPA;babfh$(%*o>$%{Wz}V?>705ly3a@1oC+-a?g=dW0D@D+ zG#AuTYyO^fRFx_!Sd6QO(G;VNPYORonWW5A8oa@4pQ{M4hDmx}ocq_c6U5{~9iS z)x*J)H}wqs(pw|vIKd2Kd2-A-$rH7i8_}=zuskZUfjgl6(BbH)a-AP0?=Q*DvFi7K zK2;Xm?Xo}Z`aZ*HiZ_v=TW&uGY|aH;t($E0h1NGx<<+Nq| z{sUS4#}We8|BLp+enHK^@Pf&V-Z3wrUR)NEBq|&zQ&q`k4ZrO)Qj&A zsw*O=CuQa6pi{wA^u^c(tt22%S_FfJKo>w=1hJ~~+uPqtf>{`njFAo!u^@3LNZL=r z7*QEfwDkIoi69s}Erbwl+?6lWN6ASRXf$e$r0AO;#%>2H2%Iq%^fk;zl%8Q>%@;98 z+qO>^#S$cq28yKy3U8SYfsYi+@-STeVm z`W7f3K%6ZFh?ubW0K8P$^-7>@Su|E7%*(9Im`o|B14a<8yAK(-6YVGgMBO0u>b5ms)Rr;$kr zJ+;C!*pO26VAjTHGdqmoFEPT&XFFBeeA=;|mw&N#T39^7FEJ;%bIA1!Y#IsgiE0*L zP1L!m*I;y)1%1SRnhyF3Y{2)=a* zQzw4^{*{h&uOlLu9wRUuPfXVq=Zx|!!nlYjSFPoYOTe@vXCSUnNU6#Anke=iA_`98 z5Unra^sh6DZzR-DUJ_7C95uM%#rWXeqA#T8t!<#XIW7#+cByC^ssuPF*?9th~#o`w)XhM z(GY4I0ws``CGXu{-Em>t^J=3c=T&c6xG5(%g7dsQqmwNkBl`3lZPvzUg_zDd9&YhA zMFD%_l<_``sm9A?S#3TCbwqZ4Du zn#~rlzE9KBANsUGUrYg6WQffU`iYa+Fi-R~)~{iU ztMAQPsaiU%(3brOPUh>pk$P^Uxhyr$q;0>^ci&H@Z!1o9Q81QDh9_yR+KUk+_GpUy zFv~P#dRJBUc5!j$G6QSaC9k+#oty%DJuFjcVk(HId<_rf1{|yHaIYP~pte<^zR-mNf>(uD_;jb%jnF0B zyH$7KEIUZUP{e}A5{%mjo8T`arW3&gecQKzjSgr zces9WKMdfC9OWy>@BV9WBEX-cmxd^E8pmT5`_q3hGWWCpSEV04` zLzhx)tVx@-0Re)l4|I?3jp?gi3ImRRvBrUQ3HCtr0HcAr^#HRb6g0y>#b)wQ&*57b z@%e7wVPU4_@hWX1GGT zqDouC{)V)A4yLWfxA_5-Df&x?Ly{) zud1DQk;-r~#{d*3CZ7CWv}Rf6pN3oaHB| z@bBx6#3Lt&F=234$;Y9!LwZ$=AD;lROx^JRQ&7GZ8~s~u7juC(QM${O8SAlNNmL#s?=9X3Ho`_k#^GOWW#DrJas?+1-g z$+4D^@jUJ*iR2Uk@p1}!9Else-Be&RA%?maE-d4@VP@QKZfoRV*Bl>*RgyN9Qp+4IoEgZ%Oik*-Q#Q zq+du=E$dbaDT?Em$0-g|s6doKxRKL%v{SBS4)Xg$T>)w-*UvAr=tpf_0 z=+b}m+Q8;Qy|&(SS?~n2p@m3}2Mcx-0gkN z8}RLGuSw}%(cF8!RVc`6i_}ii?8%6}Xj{R^@~HsC&x&-dru999u$E!cID`1qyF_Vk zG~u+^BrIw=g@rc@J6Hj`}N8%^T!)0sAs^ z7c~b0BWVV#*X)}a%pr0?hLz9oJ*UZsUyDs@4w5ZauKjrP(tda#A$|J6*wc=wgv;-L zOCcrbZ6ULlCaoSpq(FvaGeI%R&XO0pcFO?534x#^CauHu7jA+}%a0Yxp(hCtgOjf- z0}<@S+vbW=1o?)-Ty<0um)km2q~8>Y)!GuEC|Qy6s+;M_DCQV={P6?~k|S7SWS!Ih+f3g&3W@fv-uz z1f&B16GRpr?|g1-sx`<;_dDYaf0@m~FY@CQzgK)26R}SmfrA*GwOYt9`V4U_DXe@( zJNK8*F8SZxxpvd#!YOCtCm&}qVIs=&!iG(+ubS3qMj$0MY4}w-XOAOg9r#^4C=LUr zJ{H8YQzdoR5|dZ=+xEb?_x8sOlYM-1>31(NX2GoLb)cI0}_hasgB;qM!tqrq7pG_y|Frj2{EE47){4A1`C*WUg4j}x)9wJD z!DI8KR{hs6xj?db;$>V{BI04?Ak)2#y5(-Iw9==v+^X?v&u`eY#^iK1C#S9TX%qC2 z9a;P&(&Xgz&l)HvxD9s0z^ux~PH@o0eZX4W+?l{_S=j3(H zSP{!B_Cl@m5*LO$(LSZw()MnTb%#Lq*sn>o#*@Z5{BP;{u=QzV_QU9@c2AP*8S|=v zpT-5%n+6e`7a0;HJ-xrRd#=8&Nw&z&Ez659Qbj~X);1P#cN5sm5yR9JZ5AOR>(<+S z@-CZ{4Fn9=q8b8ebbYL#sSGgSVC(ym8DT_~XXmBns$`D&T8C$rP;#|%Puuz6-YXvr zAApSDH=YG;?9G=Rp2~dN0KX&ie1%J5^`MLi=&$;$$SZ3W5lP(e zU4lkOO<6f&8N@r&GFa4FJ9o>s=Bs?N5s_xYuP~0tl8Y+#7bnA|Dd?6S#M|}0i}ncd zGio{Fqpc~2oa=+c&ve2Yl)G@q+y&O4D&{}n1lDL?Tr>iDj>P5RRf6s;4cycNhPN@x z5w+xz<2K`_<}e-O5of1nF-?lGHvQHAZIzx%um6{0S@8E!R{^^9L~h7b1z>YI`1T%Z zy#&(oMZ@{lnlxiA8n$_@>Q;t+ceNxl<|DK4{d1#_rhB@rOHO*r&YWIPQfG1UrynP= zt&?&iecWEBCvz$rK^uN!-t@ie10lHk6tS&(nY%y#=L8@jX^&j zF{8TBPGMl*EJPa=43dF>kj~xQhc46e>8O_D};ogYe*kq@urZo7!P|`XK zVQh8q$pFwPw^R56+jf`jF56XIwr$&Hmu*{Jwr#u1 zwvDOZp4tDH_+}sMFY@Hg$g_;dT+dq1y01&3t+MzU-o%hM5ghjuBJeZ2^*#RHqZ>E${#bR610J-&we4;g6ijA)1URlZvW)m{ zJ&DKtti+l|B8RI5XF|<+#@~9`%VThm!hE@ja@K&S4SfWnh!Q6$O-c__`Ul~kY~=NL zi8S9!i#G5jHF}%O(w?F#X~FRRN6Ap)!LSPB`@BI+|1%PWh?A9=47YPof|1LU&@?6_ z0snhyl(x7JB9IJWP{x2Y{_BP2pKOmc)i=XGiut#{o7`qxz3YfMTA2^i!T<57$%Ff1 zhf$3)Ntt}hVThaNIYbrUszAq5?S%fWr^jfD?R~dG|rifj+Cbr`n3w*kH0y3DYA^Tc5Hve zb~#o&nWPmGcoj>##d9yh<4lxH%`I|sIPsWgqLh)sHsK1pyQnQ4t&C+6dHW~y=$KF6 zq1Yhu;4;qK14#93C3|gX{a;f2Gg?!wx{crnP9m~)w#IdJb)&s5N7ias8%ns&vF5oe z$$_3dox(a8mcA+eboy8zcqXjh;U0C9*q&g)W-QAfz}q*oXUGTWvQ}CGk5d01ke*L! zVHhg>+c(7`5fG_TEHKL;c8nn($scY3juazJIij7YnDK$J*ub6pdPR$wN*gXm*t7ZGvx@rJ z;C;-WWl@BxhNa|_s7TnrdMvGNMBP95zK$*!2*Oxi{P?bo5{x-9C67DOWibMYxKUp` z{B2W?0*Mk0<7@2wB#Mo@G~WJXD06C5jeDJLRbDlXVZQr3%xB|ox>x|$lcKZngNfDK z6~Gn41V0~4wq;e8ys*-u*USG=qZ{WY(56SPctE6K7(MqP7F(Vld@ut?PaFr56Xi6E zkfb~L`@uuS*|uHG2?KJ}CkUmtXYzbiuEnX@WD)Em_6+ayxkb$J*-s=-3usg@inQwqRqgeLyscs-4qd4~i{WqgY>e}22D8CE8d!nxYAOCWE0<^2A@f7ywpmmz(Bl*d{$Tmb6}Sln zds&Km+Z(eL#6lTs-ld{652x<|b4R&z@lt#-!UM{MfIdGdLk_;+LFj_ubG1yU#Ty+F zl7zc-Lq+Aw_Y1pLZ;D`w^=Bs86PayAe6#W>0cjxql!EewzO*(Ueq6Sw0@*p`Ng19@ zLTpGIJ)8`*C)!B?0t2k~R_gw{6He~Fb-K4Cj!_PrMqg`joql8235fcJ@Jg8O(NX8f zViJO8oEnyIBAA|ABwXp1>h+yduZuolc`*v&ggtWdpA2gq?) z<6lT|qok)YSl%BFdvZ+P6n0P!LuA3}%f*MMQdhi+yEQntL8&hNbz){vAWHxwak5w8dqL z8i$v_>>M?>`kqNUXpgRgIC>K#{8b`2pS)_6nUu<{ zG_ii@O5ZJ7)Ci*_7e8e6(s!mk*4chw+P3vuEH5_r_|xlfr?qFQTey2TJ)T~aJ*Oe3 z1_HgVyn~q(xRd*mTjtd7osJD44UgaID^4wc-pY^-8nCTTkh_Jt?TKydpLvl7EVF%k z=fwYhD*=u#E_|KcX;SV|?=Y&z?RFsoiSqW~p5)V=E)PG7k@y;0t~67zTi^z3Bt%ss z52>ML=gKXdza{grG`;KFy$r@|KUfDSEg^x;&}h&-oW0*;PMmm)n*Hb^`MBjQ9VQ72 zUZP>S22DEV$rce3OO7lO=$`NvH+xiX^!4w$$n-pAzMT9r4c(aqZM!~}Ld~?%UGJbs z-^QMX#|JB+HwXERM%#0?oNztDWJ|n8%v1PqZD@2zcz0^OKWkQ8#Cow(Zkq}|;eL6- z!Rt%3-uvtva(w~lWFUm{=4f{M5=0jf-2oFkV1(PDd_WXBVtoIhkhA|M78Ntoe{Vxo z`)j+$2J1Vmn2}D2>-2W-zy(9lB;SJvcv(9gY)5c~Wu6mAV;6b{Wl6}&w9n?psz{2k9FvKqoB z>f!z7P}ygL;3iX+{WCET5KIjfx)tA)mA14yu8fcj=&+-I({Dwx78bdYHs!ZO$cc4pTvFhhG0xz}aft@TI!L zK4C1iDTj3goc@^@VAw;)><`Jba;20^WpkXsM;7LO&!fLRs96%+gPnL}hZz)x%s@bL zAu8cr%JvvnI0ihJ89b5Q^kUtZ*=I}N9v&{$t36tL39#6bf5meZJEqR23CfAU`_D{E z7IM|uZ88~8bs)O6;D$=#3(g9e0<`}2dH*|}TY+<{>YzY>d#kU!5V?4LRUt>;@Ui+d zQK2K#t@EIVoXc@PyCK-dQ2O=Ups;V)c_4C3vquLe(6nua#hFG>dK6iYkaAaV4#s@B&DqBQ z@&t0X>z|C}=G-H4Bb$+EY}sRdv0!sSQ{G2T9fbZ#Fb3eQ-T$vtf#wrnb}pBLrlc@zFrnp!4c(4Imj=pCz3m@vF3P0 z2tBjqMBvs`IA(O?-wO3kf145XtahFolya>ii`wRD8cw7C>C^A7iM#K2;Ej;g;d!LJ zAbt0e-{Na`0&>_amEt_(ULq7nc4SJ9sj*OpvRAXQ5tMw~0G7-%X~$e595QdeZ09GJ=?GcV^`inQ&Of_Rx8W~B(I zeUaMq9tNq^G(<}?ibz`^YX^or{y-+u%wwa5yL^%^h}qFty&G<+n*0W&hwt7i69vz{ zsCePmO$grmpmjfxcfMcU^fOi?$bjbqEy$fJ=@q?Bj_*wr04dw$1+TZJHD&8`vykmd&w^l*xaq={;Xx5&!o z(?hV3-0M!2sb z*ixu<4N^0v$u1*w{d*e$U#0G8$j8ivKJM}EJ>oJkClEZOTpNbVxOl7=^ILRB#huMs zk3#d=9Y1k~&T6Jo=rbODx;=Pxc^p$)E*n1by%&5QCq$WjWHTN|z1UJs_v@2TtJw1H zP$S4h0p%ioGsoGSKbbDV%?TdmJm+~CT=AW|GrNyN?tD(J(&c!~|Dpa6sjIUuAeAS_ zMz3=sZ=H>{OTXP17-A&j{^(pgFd*Y$cWIVG+*Q9heRo{vk)J8;T++fF+m+N{ zHcaR%hy#-VzQ{csxiHt^Arr5rhBp~zmxqhibah88F7AP;6}Fu|K(EHmeyXT< zpcTSTuMJoDxG;%P>`Vn3&hut<<4>GqfZVotGTLr`Ii%8uQg^1YD3WLEYsn2zz{czKHQH(f`*Gv7nxA>tHmk=n9zK5Mdq-$u)E ziCQk#07cZZ9c**Uz#kz|@5IAj3}4-*d3Zv@>IV2rQeFym_^(U4;26<8{$ptq11z{BcLoy`J5bQ@ml-VdZ-G2U3YK_5k}~}N@2f&*CSEy3IRybq)I4A| z%a+g9$an;bJs&naLQO7x$H`!DE%9vrzInhuUXKNUxke_MfqEMC~( z55CkMPn}nGo1sap*LQVe^R~5cmpod{<}_mbMe8ZOK#%lUg8LZzbaHoh6+HIE5WIVY zoFu@`+af%8y1Vx*d-?ck`}j5s<&b?dqhLtWyU~7``oNSfmpHh>EmQ(<(6kByES2mQ zXa0yr6afDT@n;3C9B$(**HQQv>*+p<=?u$wiFvHjG*Nw;qCQ*RWJzI~sy@Bycj%WK^A@2`u=%OvXs8(f9^8T}i4k zin>JMWC~4lMPyn*VWlAS;6$=lc=M01nzOg+)@tS>_w2V-pYAu_(=l1PHgu-}ga%Q) zYC0q_25^c;w(4qOV1B~mJxEZVsJ1rh0pkEZU`%J%*hzoHm1hjdU^s==(&_jyk+?@l zSgTz|pnP6_QY;j>-%z0ZG5mzS5X4r7J@MGkvp}A>wNU;7@S<4(wS6nGz5opC`wz64 zLUSNtrd33)40turK&vp(U?5K_b^xp>*IcP|4_KZ6A^UkVa(Xe47Tq{z%+cKohXn&f zXzj;jV=aJS7aWiw06zi`cG5F3=Lf=y`QC*CUjW|%&Yx+Egb~zxptwrt_cIj{E{M|% z9-%nU3Sf=Uva$jcO@sIe-t-3E1MFA=f@nWLaBx^>3N3JL#grr}6rire0UPiE1U>)? zBxcEltJ*UNtdz-P7Df(>pjF14->ohb$?1T?<1ujIyJu~D0 zAJoRc=lw~D!9a)z84_XyfCb_Qq6P5H_pwHSO@W6q+c-sr*`fCVV;&Rhferv>`J_2Q zv7v#r*bt&1zgVx^gNcxVehTja`{eS08v3Y+{UzMX_^sdv2L+G=9^)tGPks4tyAzcV z71J|Wk$XO;-J_*!LU%#`x#jwaq}`2=1lTcC)#mZ`_7f%sA!ULJd+v-vK}*NkOOeJ& z^#|-nT8 za}Kw(3u4`RM;yjR_H!Z+lY&V-7-4T-kJ?Q|g#Af`Yj#Ky_yY;jci>mrY`_cx6hzUX z6{N88y{9I1YRHV~ez7S(aH$m=E36PdR*|WmpAJvC)K7|3GG@QwUPufXpcyn00C?Zf z#0JeMpht6}#e}ww$gkwHAqvp3g%BC00P}5s%AcLlm1gognojSY6x;i=TB>E576kwG zH~WLlFBed#zovnhE@7o-Z?&kF7YwZ1+${eEK6){)nmvat4eai9ET^M)3^G2(;UG&pSYb6`&2l9tdO zHJs|h58!g^W$iTGz{yW??E+iS4DF}ZNm&j`1!d=6%R#i&&$qtKl`CBr>^k4Y<|a)^ z7htB>+A52y#HqkT(!r6}L6U5p*Vd26>)X_H31+CKG`X(!9#l!GnF6kz9}Xw%nY%9B z>Z#KjEeYhDjW)cfxU)1CThVJsg{ zE_>OMNwe0uO3CG0oAoMc8^uJY!Wp#kdOBuvobT3-=1=pkKJc^9R*(7~%WEe8Hd9nJ z$a<103W-!zs${~pUX8!m>D0^@ip_Yj-i>1s>HaGC+#8V@_{B!tx5VEZKWVja|yY@gEK1{t)L#Yr=WQ*~P+G_R|k1naa z23nrg_|l9*fTUR^z2paV`bu|+!Id>fEJ&tN9U4aaG4dyx!qLe(Vi~DS(s`l!M^

ctctX@w>IeM zwLVyB+8vSe7W)MHxdt||oM5*toBVXl&cmrKAp4mV`&O*(4=prHHYE~#Er{MSo=^B>6)j)e?xP=P zC&gBl&d64LsDg7PJx86uTf(C#t|~ygc$xKBQ0jC_?Ia3FE}>FD>#l9RD<5k(nt-x1 zvuItD5~tlYMXywatoCH1F24-KVb7p=pQ~D7a54W2_t$3ee4*Xl!kV7KquWXfV<!N#U4`seRKfcr_SeV~o$x`eHIH~gSZPxzht=&K4c)RCa|xKu zmuQtHkhfxuRZ9IyYA7b5_ynOK$s1ES(ORlCLG40UdLzvu+v@7dVL;40g42r!H9m;c zk2y6o#nbS`6o_~_7HC_ku3FACOX?@pKSdEveiN!6LTrpahM(bt^aNpYJM&#!ZYq{(x#h&fzPL%4L2em!<9KJFkwNvqn-(m34TtDIR;lv5S&=#N4MT z2K3d=(`pV{#NTRdAk__9<+w$4ydd{+5UK2??n|@=rVWoJj8RtVX$&mJ3MZY`LKhap zpX#%nl*gBLlVr#f1?EN%no1s9+ITo!zm|&<4W~J2KBMA|-3Yr$HO62ox-OsBFbw=Z zigdDFU^dSV*KEjA&YI{+->@DCVNK(BaxiUv5LF3^-$afV%y$xrWL~77G-sj`;Zq+! zNC-tcjw%krrB~ptNp)TKKkU&+1zdRXxW-n?D_DiDd03JNwr!W=J(1OTF5-1iJz5PL znI6oUI`5g^;kU|^6tfksHvfu>Pt<}JTMZYQ|6Yn%FO$pOnuvT%Q(CDgbiXnvKolBm zo-dD;ze1>+nxcB-pCv>dhWg@ zw^5Z(q;l%VMpEqk*qK5fwBl#E^3pmxng^>~lna446BbHYJ27cG;r(*YBZpwGdSzJ_ zR$dh1`Xe~{*c0<4RbmQ-CR;M*1pE4j%iV=O+E$HZ`+2?rcKIfJwX->24zkTKuD0L{Z(*c;nTGrl4fpb98;um8|PasJ<-zyCd|{HKRvWBhlk#6-yUy_EXzcmH<}#m>a=-}g|i5K0N_ zb8K^kl4t@NO8g$k>Ec(Nu0`O;goyF6!jcqoT#{EZl=;M+Qi|na6e7QVDioGdbl!o* z9Bq1De$_m9Q(KZ<`5dOLGJ9sPp21zQk|-h>Q;Z=M`wQ+0_v8~|`IUEdv0;J&{0Bq? z2pOq#beK8df!=URb{qZlYwb`K4*4+4F)=~KdKL&dfaM8-L60|t06n{a+C}g7rJ}e)YJI0zkqb&gIPWl{x^-svM8>fs|udM{vD808jvsYXH$7&=gN`3Rn=r zb9jcuB@oPf{gLhw4_~ApAYXV006N$g^#;M!9lzbat!Y>S$`}Zoz-wO|9l(?W7@T2Y z24(PNI|vAo^ga_(Ku;djHw*(b+K&SnvRg3}z>@eI2vIQD_q6|~{x1XLL zZ^?eWUV5@hIm5qX>-23F^=aRd#?Cj1r2-v{aIoG~42T0oTu@tV9`xT{Nar^)q5hg; z=d|}=yVzWSQ)R!oGw*5oM6zx)b|JSDZ(vi?ZA3#EnzmzzCVV_{RL9J+BJN%&jV@V-br9-fJ3crvK6MW0<#| zY}9%w<3*PnKZI^zRk#GlNP9WSk}Fs8wmLWG?BXipTcYWOhEutRR(GF34m_Mow?3Tr z!~Rd7!RbxW-aEO@nR=8@bh@o^2;sP)NHcGirF z6lraEOSndZgmkiJ-}FZIZqUlNHP1iKQ`H1IyZ1LLniDF#W;b!492`?O#L##kyCWWL%NVFgQ#9-7 zu1^DpJZg*3iYN{sNhOP4*Y9>*sR!B!<-y{aV-+kMmv(r*b+VaY47q~KyQA4y+R2TW zHm-4}-WxJ9mDlRe=P81YZThj%L7_}$+XKQKK9v%&t!)j-733GWDILaD#s1XUC8hZ-rSt=SKTp`(=k#Q2n$Qzju4BX)k=TTZ<;wZObB zR^h%#Z7f+oZkR@!QoN1xDRa<7m>_rxZt7HSHFQjUWc_yg!^{5mfNSK4=Bx^0nM#3} ztT50Hgtl&V8_n$YVjwJ~FGj0(L_TO_WAK!V&%h({Z&v%OA&_o0+32`7+@DBg0UCbuX&zpdY&RRS4g!qe!CbFQEKM+BFu29% zb*w{Zg>jUc_<-`&6b>x-!KT%x6YQn`J3Pw%wAAz(v4e<(rI*3M|1!f^yHSk~JE_1W z@s2yaa8fL9%U!GDa3+0yN!t4z@ZcAz4vAckvZ*LvWtd3~tN6qD0gO}#MW2HLw*h~zQ}rY|fQ4NxsU zK^JJjH>pf~)5oigc^lHQn*$H?+-)46GjsaPxNCDOcgxBlEL@;iMl{i{A$i=vUKgj6 zL7IaF){fqr0#~qX)DxBCU9s8oATRfUw z&BQ~hyJws=w3|xu*Hk!s9IZmIUDC!rM_c9{wyi=C+T0MbLQfah;pAdqoMB_YpK5=H>MDZ#8_ z>7d(iIiUI&`|8Hn!~_OPYY}#Bq&w|m)HS3V>Z;1jWGKsNs0Ypxm%G#znUJA*-CfUaQOt zXZfjCK0Lew0R=%85mo?sdTBxrxFq9O-JqnqM6wjGBnt)0};G8twE zlfy~8E7FmfeK&(wDUU=1+x(Gyfj8Gx`=|qw)2BFxZuALNDkU9&>6(0dlS#c@cwbch z2ww_4I|sZ3d5bUmv$&016?d!bR^-L2)Rp=x?ov_BdM#bRU{90WuOojY)ASE=zT)(d zGe6zH-{DJmDEeBHUO6}pQzbCQ;utPV7+FoO`cafK=DM&L$m^ze4-MIz^Owqw&DfmE zzoPq>jT~&%9nM*@RJ1_wQ8zY%cqM=KZFanwRm3_0xqZ2XTzl+GJnn{9!yzTjYNRm* zHnro0(J*{$&8D-=3euHl7u^QA8eLZ9-ET-;H*qe4L=VV9({G)nWa&k^y8(MP>@-}C zi%W&72JUQP@9cNgtY!lfj|c-S**e-;Gg8To-Ev^8vh%ZtWwx?s9rMFq;Yox21CGJw zHNS?_St)1Sqq6s#?A9F1(&<&tdqaas{5JuoDQ9w=(!rysy$EPRl%>r>iDK=h-bn96 zFdi7`n|nIwb_rIge!sM=d?JuOU@a+cLn?bm=%eV7L)2RFW+6T$yN2EX&Sf5U@8j#J^7X zO*OKj9wX=NL1x zem2Ac83{ZLs7;Z2QGKqy<(1g0txZkS-sI`Ri!d^yCXs4so-WCWh>yu)cU@>{K+(bt zH5b$J3?wGZR!zd)p;mAgERlc@k$NSWE9tKSMNLNCftlAG!65`(N%&j+f$vq4STFlj zpB#Nxra^wGmwQ(AEmz73V`ypauo^p)aHP-P;`JySSH48zs$wzJE%b79@7S}s6NexP zW{81`nln_IOy3vr665}PeKCETGrO>_qBuH6K8HW{Jk6!+D4u5xUbl?F45B_78ZWs% zQ%~D|l7h3U#41GxMuy`xQsXtldPWa3)A=kE(Iw>WV%xCXtN&+G23%amEBM%EvOl+8 z@eWz1SBTOjLr(L7YW!{#CEVe|QcBdG+YGrGLjaGbVl%77j>zorkut+HY8it;YV_K7_&kiqOL3uVis2<(Kl2=<|)j1(x|Ck+x9){n*s_ zME+SL@-df1MS}E}3lS{;E~ob6vO>WK^22R-cI4*8W{)x*qUR|R|2C*PS?ohSK18Jh zBCh8=DUH~^j%lNE6qy^R7i9OrzS|`~of5^#^P%D+cVry!#NSC&RI@d%GvBqCcva_l z`D!lH(@ES8B};k15#Lf1)Yx8kRp9<39aBJ{dCH;%BhH!qF;sIf0K9%xX&gdBAAdyj zSUF*IOsLXP(prC(94Cg5+?AnR((Hv-WWVQ@#(akQp;3JJx0?QpXSP2r0P>^-!`&HCHyZ3OO7OnG3+Xhn83t9txJK8a}a z#wbB3{39iX-Ta+z)1s~sj%jW|q3LFpWG##j-j1JOajSz% z=)wkA(fXfdreAE59;1l$PMRpV59~j^JO?K$0%{&7j7fwSLaXV5$qRJSIepS>QnV_~ z%=m23q|6oFK4q4N6+7zPE`Re}LLEh(F9soe3MKD#$mjL}y%;|T7eySR&cZRab4ttn zk>=$qNfyMiGVU|eYco-5GPqK_dM>C1AdF`!$LTu1-i-yiRe~%r5`G8B?d(^FcRWqr&QE>5O~ z`cg^fuXoHfk1IoX&5GdpZi7+p{bkK#nIv%I@%cq+X>98{F6y*@Zm*XtM1(!98&R0| zb}8fJZ#ExUF-3Ov(;?yCw^6RCYT0My>4;g~PX|6ATlU$dx_%}GjN#MhL3=(|hZ^l@ z{%QC@FT~^SwW~aJD=~H$SMlYB zMx27Tht&LmfZ|Xujtf>B2-?d!Hy4KQLv{VF5e%C+*(L5{tHH5Lb+Cd%|Co{LrVccA z(>XeAF!-f;cEAGjAp~W3gh^sg?-xlh#=6b^=QY1lw*|@LQ9hAUfoPvADee26JN9`HT9M!ez z)7#EGx%>$01S>{BePJ|U?T}!NNNlTquy}RMb_QK6-Xy!)SyNG^SkYVBq0M!7-N**{ zoL4NK-;Lz^Ki|anldgq_c%pl7%&eg&zcIrXjgPtEaOY_8HFq9Bp2~MkJszGdPFruW zZGqg_Ni*f`BudKVDie_KG@9VS^m{Jww=`S&$H~>ADt#;k&f#5(nBEqL>dmbjULM5a zkH+fAn>gdJT*Fdlc3|oPJdCEnn$a!AEhZfqiIQzy)r5p9`-<~ZTaL_+%1$T35aUBH zEk4getuy4jwD+l=2KJ#cF{}F)ciesoY{pKzvw~9)F6E?vmz5+*lNWR8zF48PACB;S zcVOMA7fznVKv6uZu4G_J{Vy^%qYT)YO;3?)bi9Ul$aluz5C#16%r@?SPnbm>0*$pF z&#k(!^Qw*oqu^pN$zmSdaA!bK{w(CLV@^W7~CVnJWSj%AgI5;`OkiG1mbDb<5|(<113a1e2QNT?20KmL6* zr+Tz^t1x@YWWD*(>MlfcDjT&h`jTawb-Tt@F)Qcff#=gD@dugVW;InpLx%9=HZKtu zIm?c=Q((5Tc(5t?3u{5_TnXd3JY1wC)r2VDo}@)>C?T<^WO`B6we3+!Nab=${WJ7> zuSfQ4@(8Xzu>e+j-GpEIO#;ewFvKBcPeaO9|6-OnKMKQt3*1-$_>XSFVw4gbsgw0- z3403KqU}4A0Q^;vj&6o)H)B?JR-uuG-EoZ&tzA=J;ApLZj&98~geg(a0c5 z>8U&lCG+^Lgln9x&ugF8A?IxG$gX}4dQpP>c3|^TZ}f_icmEyFUzb7}0|Kdxu_`_C zzBmp#@IK$u;bABDe5o~jj7uaEDa;rf2xu^9F<4n9W35ZM5e~;)LTK_{RN!say|1PD z(;c^u3BxIM_k^xDzrQnp zXHKzwDYP-8Y(a3kP=%KVgx;(9fTMAB@$~zkNxfG`tv|Q;S3L}C!X*LOe4ffmKT8#a za^5djsrxE2Io$;PR-I~h0*e>xINTbFIm5}P37{m`bf4`kCb4Gf%}B_ea9?+|sL9)u z6XPOUwYCA?PP+)KL*P@!pcHV;o&Ku||sExf{K_KcqTg z3af8*Hbjd$)P8#+2JpstZ41QYeaI-3Vs!RxHCwj6i%T?A%-1F%s$fTP`G4q>*0r8P zJMv!=-)c&k-$kgh?vEw&`zbwRTEJQfENX*oiX#(UWS4LmHiipG zh2A1cvX}YC#g7-S$O#_qoS8zsh_Bog8hmwYB$SYpxM*+f_4^hCl0qo76t;u?NveHptx>+Mn|(G*zK#4y*EF?l1%O}Fed zEVcFuH$I;BZ?;R*aTrI>lbLh+o1Xl*$H=U3RU`K&JI=(|x|&EKHM~)#(})*qoQqML zHZ9x_EX7pB+2dDm%dxn@-^kSlNwHl5YoAo?5pg~$qt+9_HVq~z{b~-9S`Lfd5iTbd z{|HOSF*W(HxP0y7Q#jO-To&nkkI#nXG0j`N>(_O6-=_%$A50vCSZC2A_lzITdsen!lxM6wAp;zfF9tT-s}eg{KbBChn3JELFLO$uE!VYp zNI!VRUDWH#dp&r?zbJ+)^WGt`u2p|1oHpJURVucbHq~e(o-j?KTjs$*JW!rxvY2N zK~Fr`C+0Ow^Mo}?SCdWhN!40F=~JizPnK{!+kl3J=>m3nVM|*6bwlx3cd9%RY1QR= z-FCBcDqa%_?&EXTfkM1w593aZCHNu}l}|wyZC>ZW#k1A{^9f5Pi*Vyn1;lx=Xk^_Y z<^#_{!1rBDc>26JR+!^>SJUSEe}+qourGe>Ga0ZPlgnUsNjJ642xltagU4imF7lh| zdl(Z{9AY6LYh{tB^PcCxqb8Mgr{F^I8>60DcQ%E-rnt>XE6bwG?tTmjrG+f6U@q?9 z)8m>I9-j@fjlZU_QU4Tr0P%Kc*r?1A@w8{|rIQdxz!JQWOBcCJwV-R8tF}|j2saXJ zMTOSdT6V#$89-3K+JtKO8!St5S?!IC;IPOLqyB}&6yKz-RzIpPr9a&r+w_%{F~4o1 z+<7WUjnqpV`iIxM`6rDveSAX%mhCVxzHAXwEXBy0Z>-mIr^{CEEwZS;J-v-u);iu9 zKhhRGu4Q~*Y@_ZQamDm&gw=+XYcxxa|30igC9S=pf;31&-IQxtdpMW*_E`vg6f8A- z6nP4>_tA~3)$nY1HLEaPIan;U(fz@x%H=77{8e@1-$;VuFwi%-l#_RbK%GAWLFnpIZUg*ksY!*x*sIv1K&bA+~+@DOlF{ zYb$6+B(@P?pcrN*;D{4g90M__{4)W!m+$}=SpZiIfU5?8gX0(Y`}Y@L2w^RNe{Q0G z2|0HFI2M8vNL${Flz(bdPty{IT|E2C3v5Px24v;s#XarQ0V0eOxVH)2L>55C%#M>w zLRt`<2Eq?wqz~hC_gRDJB*~$N57u52()Jjs*k+2+=>V3T$Q+ZFl$52nrO0a1{vKAp(GA znmfNW6yb}*0Q6}N@0S()v1Rw=_62QpcArw877R8ryE-wtzBG!hXM6?#EfuG(YjJO7 z1e(aC@x@B#YzZLzG{HA%f;Txt4D2q=04f|@0GPx=f0a!y0$ZJ(ACB!>oLyIlF6rtM zS|>ZyB?35q0%vV#$@|q+bWbtRqc6|w`G|48bbdS|Kxtc6G)H&Nj zj7fYNf*|F+C#!?}1#DPZS$SDy0&-yh$Hw~ouhjm<&CJ*IWN-A0h2Xxrrm@UD&p^I&X*U3e{Fa*JiCeQo2Dyt{m=2$Mn)H4^ zI7aU)KvvzKpC2ZmX$(@a34u?xciqvc$|{nI3d->}Iq_f05s`r%fUXJdbASoF4I6-u zkIye9gJAn>OJ8Zy>MQRiv?4PB4YKEpT~j>cvwvkT2PyZ>l*vwexcJ)N-W__Lzgys61f6LGRe?`@=&`-du7NcE z$dGl=vR{_vAn%@zNq)&qm(gDpNsi2cG=ZrJjs4xf8=RAx9)LB<*V@!q-g)SK#~XdD z(*vteexdDKyq(zqG1S-p`G|q9(?NfXed|YKO7t!Bi@kWcN>q?VCmh$c9O|6}VE_r7 z?2um}tRLu|gu6BlJue`eIjdm=udUu}e)Y&EE#JX|>A`+t0mi>x)xS5l^*qts#(2M61CmXi zu>@hNfBbusTl~<4DqDVl1tHeIO2H{MM>f~N&Yr?Rg2&P>=FLT4R>3{qEx;Odr(dK|md{K=F z&`np-S9})nMGwZio-qteYRD)B?AQ}GV_v)hYGViuQ>T)n+A2(F_SHKsv`yYtHTV!i z53?ZY27j{mj+xiJ*~lyQ|J*B`49ssn`dZ<)lZ3B%CbIEZ0$&m<#J~(L0nF^FG|XHL zV(kFVN#KRwAvzsxY8RLf zx7QL=CjAR*zzd*8cnPF&dhXcNu)5?YWnTSb0!6Os=0Zq#=WJy-O69HN1PD)%03;T@ zpt2gd4eFlV`c(&hXl2Yl_B2?qywf&)E;M{s!~oFy1kL$IgvOJs(zcIvciq?ifzf@Lzs4 z-3}OBbpxZr@cKM^rNqRv_`3qol;Ewy?mLD6txZ|Wi>8)RG;0D+C(!I2Q*KwlN9u7< zraQaSv_ltd2~FG3(mm$Z@G8!!)T)2e(jrtrrFnmG+R(SN2%8xj{OZPwDpgR%U;^_4 zfo%LbG$&%Y>%d+o>9K24UoOruQR`kD2-}KjDX17=>;Q7*&?3P{!jvrkbvG|-^5YTi z``H-7M#vt_(7IV-AJHx}f$GyKP=2WMv9?paUHpVq4>O{5>Tt~wnpEn$_=;&K{6$Oy z`+WBb$34e8*tM;1dUdaz_gjCt6U=69!GGS1S$$IM?3$JBoxV4-O9P>@0Z;3(OS7LBGcfgr)iZ>Rbz2X)oeSqwU`-v@haBm@y=_==?5EnN^i)1u3 zs`S**>7F!muK@?3K z3vgnym~4<6o$Cs=DB=K^r$Nr55f8jc7Zj=FBex2gcQ zAS%R^j;2;NRFSp)u69Br%;_Pyfk|bCdn!BP795bydZt;17$!KSFv%j;Kw0?hLbm$TOyH%{zLrg!^tEXQJ#w8esi01o#WZ?PSPP%pRktk6_m{1DZ zQgs{gdniIFo>%TCAy7oq&RjX%eg*14eu;vY7b#ikZ>|A77*L=}>{;rnWzSqCHk8h$ z6*y7&EN&L5gBdB`V4{Kedjp{^QwfwOV}}#dpnWu&o1Xk3^h^r&1#Y&$Hn~hC|MF1u zi3*ag1V~na>5e5*@rH!-rXpiVL3-S_PMu6kq4ENRT?selvr=N@#zR~yFCCEctmKg^ z_F_W2drkZ#d>EtEViOUza}ilW-$&PeGLN*bb(ch%`U@dDjz%qSParqbw%5w*d-7KZ z5dr5T6_s({)Y|7gHf@j!DVnrne~4g6X0^pkG1~fQoN!muem^q%B@=t?^};ow&1Gce z!lxk~@|JM3J>3SC5_CV{o%}^mNz~M$3TeK0UJ$Q0`~qtrntO8RgQD=l^q#@vv-(y# zjsy@)*^1sUyN#_%R?Mx9Ev@|?KqxES(hdrQ$2{ir(g#qLvhloeEa+OJN9|PbwpBQM zZKI?HrH~eD^ceDs{f-D*|3zpI4$rzqgdA#R9#pz9$PFlaJS|I@)_zjk6P%>0U>_&b zqj5yq=OHsR#ew#uRu(9l))sh>FAwRBE4kW=H1U~5zGW;2eMCYBr!C*wCe)XoSVLc{ z7`ZG=0@bfkF%0^l5Ifwo+Mu#5gP(tU*9lv~tD-s#a;jl}9CiqfP1pFHLpK2%r!J1c z9)~OxGazluzk7nVQemt$rJCi9smkwhTj{1kO%*0jWN@`;zbbXo7T!VGko5e^j8UCWMQ7JI*bo zeZ)i7BPm%u5*#-MY4r!j`_dc=m7@FTAK8cw$9ejAOj4737q-l6j7!JQ+S79nK?@?6 zNP>;3_&AJ%gy~LZ-)ULiv%-rGNu9`KZ!FKJyZvWcyPlaiOqUN6emN$DZ#^PL+MZ-S zZK+Do>W6ftj!l$j>#gF*oz@ez#;!dE4u6Wi&0!E*S0>N8u1%gZlCx54{h+7HQ!$CBwJNU-zGMyvkA%mIBFW91v385k)J=%lXECGd>x z*dd@z`>F*<+F`HZi49rv_K3=D8{~#oB}&ab*sBCxRsm^Jr>A)Y_I$R2BF(;)g|0e@d#v>!8a3U)p<=^i3b|=Ne%Ou;4^igB2D2| z;6?S$x18xSvoa5#SGG*6mE@qzTHK^(i>-7(1EE9c(F!(Djb12Ynv{j>M{I~DuQd z{&L{&nCA9`d{%}hZ!E%Dsjq?*P2y*QRi@FZq`zKa=I&9Di!Mgn5yR zwPmNHNIiddI$d#pdJE*1n}O9lGJt8F0|6>keUD{1L)Z0hpy~5c=fMqXQ{U1DzJY4; zCUUy{3FkrHw+hswaemdpfLaOVCuXLs?B>gHj7(~u^^I~b15xcm$mRm(dDPk7Cj5lB z8h1DJ02)zB-uDqO)tnaLrZw@7&|!y* zcR4(PFpqw{ScPF1zq$#o1+70mHi^IbLRReVGTnwXNg8H}(9P$HeI)D^K2Jnh_8t3F za;HwFML^1a3JN(=e6FxaP!Q|UTyhuF4zV8~ zjb2xYdsbZ(Zu^R6cf3t>W|>%gBOV3wvPa`t)juLUV_YEYNOh9gMcJKTl6mKh$FhK( z%-u_EQGU+Olza>{FTK?ci!;<67o~W<7a%)K#5$FV8N12uJBeMfzZ?vh=iK~x{FB|R z++@IIw1U4lnd<30Po&O}$QCvv)JP|=-3tURvElFkEr?>Z!13!77N+t~DA;PvpSsa9JZw3C7V0<{8(gs^|%7C7I z2)g{VjX(A$;cNDQx|R9;h;@ z(=IWJ#Q+`sZQ3cWn&tNhMG$e@f0P{(Vuc3a*k$;dz$@?N{PQ5B*3n6N+`2^l5S5Mk z#Kg^mS1fmCjX>wl^z-$zd{Ul|M7FO?&_XogYm-h^<=agGnJ5-oUj`6ITg_u|@twkK z2IeewkO5GaT9G{nbX8Om>9sxA(hw#~{NY3lAt(HaBFVy%PLR~eG1YFN^Q}xcDJp{A z(_6$t!J>`6@mANii)wgbi*7ghyb_hYMJm8C)9XSu^(=}=F&i32t!diaGS;+*T5kpg z`>%mRskd6)a6*c^BLYPr8_|^e;0h0A*})^I>5vRv+?!X!6%2NiLI5f&!?32rXTElt zSEk?{SHB5s=c7pi?hb7Y$Fwfy=h7_=0f9*y+6OFYk8?2RAG3m%5Y9t7;nfX^=jRqi z(GuYy60>MwF}SxzA@)?1nydwxgF1(~73_g?(f9Bsqc~ymLKJ8t9Fe$ZfCY?YOSm|~ zqKJJCg0KnWfyAC5S(-pZb!%k`r$bJ{X-GqtoG0@A+20#?&#Dx%_k^l(*7L06)T7w>dy>@Do4rlMm8an8*s%ve5*DXg za=3rl^Uy{@i!Ak$?!G%rD+lgq>hEqDg?Q|MT#W}R96J#S?%jl{-YfS&+gl%D9Qgx( zaR8TT8?)Yt^<*F=bq%D{DWy7M4A$@CB^r)dmbh7$Ms>KF6k1CN>XrX02vJbP6q5L4 zC-!>jy#4F0cbg42HPObpP?YQUjK`T2+?YX<13G9A;tbh*#AUh5-nb2AlqB=SSS7$% zVx_WknPMDEK+`S2knV6VmZkeBY!tmY3!!lH0o<-}FnP5f{(V^3_^e8IO%OSF3 zs+ynOqO1NprP7e4LS5_{DIQ3*hSXY*b4-+6K2xx^6EZZn?URCSx?}Py*ta6(>poZP z(99I1{vpjv^eL&JD-S3F#}`HPdhCWiB13 zN-E=C+~Ym4B9P8udtz6rYM%b9n&K5!)U9(7cqO*|VPI)Jt#_sncC#R=efCh~DVG=h zFBIxDFD`pUKqXs_zlIdI0t#8@Q=N|wofl}bmPl`6aG^}~!vT*;m7a{{?18t7$gZP* zgOrPMn48znjPEPhZ;<+#ZJNx)QH?sFUL6{bZnV^y#D$Pdyv_NLuqtJDRl~?AvHZ4- zlL`qYw9_)@_PeZ`P%073l`8a=Y!kA4>mBXx6t%Y$WX}q}mGNwAdf)#!hW%5W`?gFN z#>KOPyH&i$sgOwB0m7L8O`F+;m-Ub!N7f-QQXU#%1CI~|kwQg^G?hUs*HhMR5VGvS zOA(!B&o!yF(z~rb>7=L^lWN$f}a@HI8jnAkY57_2(?E)g0&22Q$1;~hdiA{Yf zZ4>+(K(T^~8uAHDP$a5D zNeTz4!aL4|F>xhHVw-3%K7)o#V^6r%lck4TJ#QFXkYn0;=Gmz1jnLk&>>?=c*2E6W zbGC<*WS||5Axrl_5tjQ=0Txp>nY9<_US$!XNdt z2r*{vI>ucT#Od~9WDMux)c(7USB6N)scvXxdSgS4PBs%bfl92*%M_>{UvS~Hl{gOX z7>Ck8n<`SMWSuKngb4s72uj!YPJCr^;B5u-(BQU>QHD{s5o`qQ<6r~bR1FRU+17Ig znPpSkufOv778)JBwjGa^fQS8h!JV=4lw|ofFL$$>2{)-oBu!%vbkApSE2^p0?v&H7 zQ`AOQ8&ZA=DM7@3g!6v*oPS=vIF!9lUm9DSlgj1=1qj=xAMXp~Ja z2snh?TO?#DdV2_61xKp=y*my2%-$ltSI{|8gH?R>_ikj2Jtb%@M2Bfl1PCaibruA# z)=21Z^wq~!kQT{MIj8?av+>?OO%m@yo6nn8eFEha!Y&sp(fij~UrMfdN@b_)ml+oU zm&(W-F?dyft^WPIZY~Fv)Zp)akM+8r!~o(Q4t*#EZESa*zaI583kC+sLKOqsGnmi@ zuMJ<^>3eL%?6=U8amGbY7@4a=Q@wf{BbP`2dxkJNsjFLMZIgz^)2fsv={^U;E>L9| zoCw?=lJK@fg<0|6?;zKpo2ILY=TXV z>H(b}QGw>vgh#JfB*Kkw$6ekaZU#Fb@U~s}dNjXg7K$;KiS!6Pw_)CzO!%Wgs@j%1b`i<=L7X z1d&a}uEKv4Fb6fQxbFB;mvqjvl*4oK0KlE8cergtx;QB+UT}XZeN?c+k!zk@50K6w zA)x?e+$4jJ{8w0|SK6gV32TBZCiW&>u~uToUwD5PaZ$w74UCsz0A%uDh>GiFDUn_%KAuqPxskEu=dSK`>O{K!?e8f%3J7`?f< z`aNsg&}N(A&57|DxmS4@76aKH#TQA^1L`a3;GL@WBp+0-fMKJxNFz_RKEXPWb_ZW^ z*08)!2RmVAPnG3J)E?v;>15 zBtdLEH2-32T5Ldc;5cfPPcz*(?KDI(aAVtJurN7~>HiU(=;_)K{e8Q6O`k~OG4wsx z?`<_+EY<7D`O!-MF#%H8P#xq1m7GRoRtacid$f0HDMtP>wk zZ*@mY^O_X;`pxC6tSWY_rL&q;HYVSfrhW#g1d59}iv6roKF#3J2hEr^kyH)6@R}t+EU*8+C zp_UzDIMVOp+J_1npIB6c8-L5g+~RZIobAU19}oh%$?_^lY?*tW6{qyi1zY0!5LIi{ z#e)Chl@v#XWw+_9u<=K5=g*!SW9CZxK)i~Uikl>9yQsXb{kxH10YK~_78ljkUUQIT zw5xtkXphFZXf24uso7nz zskBi}!ZitLhGv-QRf2YhdHWAzvq{p()Z0`insBU#?%_<&Fy5DxD~g2Y8jCMHsJDr# z)t({yfVKG}X9k7rXgxyr3lGx?n)A=Y4H@Z*6B$C9@e{m$^XR;o6P5{p6YfLOmYP}g z1EzMp8ilvNXWX?{AP7&ghGed1vFqi}mXeYefta_G|99?%t%_qI^{Z|{^i*@>EKgg< ziBN3AsBc+2@$ycPR{?O$G6R0~lYZsj&xY~~McEboV6RxbYf*^%2q1$c1rylPVm*O1kG zVD*{OtD)u$*6P!1gHQUGi{uPQl}>wdD`?iIfxX0|H!)a+uzZV|=n_4{{=5?V_n9|4 znwnjO$-DZJVSNp3^*^FpM2g|I`ZTJQxQz0LHUBoB z)cSg~f4B?LNGHMKTsvnZBgQclffjBBSraX8^u|I;DnsqM6+>YTlE6VZh9!5Xg8>4i zr()q~E-yc_64AAuJe>XZwb5Czr<(U1tO#~Tm84@9MhbP!Nc}o3eF0*VtA=3FxhOBr zK(l}}b|^{Rx7OZ^R?e}wqY5XjU2@r;vf&RlcDD=8^0aev5dpIMO&XV9Yk_%3&6lm-N4=axnBWt6*lTp*CO$R0`BITTiC-t~E# z^6uYDm&E%0tbS2jwv6~-r(VT`vL7~}TPhs*PuU_ys9X2?Nu6{^&5eHAW^)mM`h|K! zVcDD*5j2lP*OoCr_U*o6WSBDvTNd=1$b*S*<05z+&d&=l(1MdG6Gh(RN~V#37_4EJ zTg-oeFhNsz-wF6d{h=4yY**2(QMDS8%2ZcOCeGRbC|?~hF9xE#+oOMs7HTC)_s*)2 zor2_B?2JP>wEh|ncE0RIfERh~xY{^@Y6pxlks4 z9M_NCt99PiHZC3(V^p8-ij(@F(P~}1+f^FqqA!HqYn=_;rpkabZwFWC=N2Cyj$2Av zn842=p}OC!m&1R@lRSmqmyZ^)l4^|FeLYA?OV&R>&}z6l&lHrJ`XHdh<#<@m{&+Rw zh1=6}eqN8^Dv20PfHwiI*_!vKl;8yo(NF;f=yGMvyYBpseW2WX0~wyM|C;?vsL?Ju zCm$HOOBO$L)h=C|Cq?{uR!p6PC|O@Y?9=C+8x$h+>jhGY$l1Rw9-Tt(!JcgT{*vVm z^ZzXo;MR$Cr&u+~q?BZo5s%Lp{TcAe_;RkDBx|~l(ODJ9F_CVGzicogjzQiIG-AeRsg)M(;?;Hb2Cv>D6{Lvy^U@k91n+LSK=Hsf{kFSa(iR zbQ~Eh?z!z}OT$PZccUqkMQ2cBLKg1d8SUkh%V##3Xd<9M@T|&BJxzRL4%iVm7B5~_ zAKLlBxku%$Bh^pLS`yYPkkf%!5o04556 zw-O{?|CM0=!xpeLf1!OAYXXH8CA`Dg(AnipxB+*pxtM}1IB&1|wvH%iFeBFqfWmZJ z8&d(}z)k}0+Gf3W$Ywj&<^AuOs~Dx(0jTcgNPMr6{qMi4Ovb&J$5+kMJ(!sSFY|6< zPSF=QDD)>l6!*T9x)8x|%sBJ2<53q1??rQA zPiuvl#^oNdqW|f^2n3t{$+;#Rdw?-R1zUHS%QUS!Y+S_r`#FrH&%o?kn1R+e8x>P0 zOWr7}Mks4!>P6cDJ5}5y)W_^A+6VQMf~=(Fo3J`D6)VP$QJRS_V7dv2%t4rh$m`#E zZ^#N%-B1#3j8Y}YZcs!zl#bv?>U#wvdO9-7LM=Ga5pupKF-O008Z-E5RTpLk)V$ci zNy6l9p=QEg+$}1n(q|VNW~CYZFJY>~n7NeS&~LPCprRAIMzyzUFbiFw}tvO<@F&wd(~$scfMS#&&8^b3qoJFd)XA7MjZ^2oKim64#vf1P zV3b)MrN(H}UKtiT`U2*l?SRg{ErF*)0N=8Rjh2^R^%a*aXCEnMKNtZ01oaf!!zrB!V9R- zhbA%BNH|1wp(|QpgL?}!#4Y%o+CTEf1B`5x4PjXUQC^IIiBXCx^;p9d93@pO20d}V zPi>MLv7oz6B`@%8~)32Y)2 z)q=REAU+ft(QNnSgl58ecLH{Ji)JNaj(2{1_#C4G=#1-Am z7G#)MhT$>LyXNNhb^NglUCWW{)+H<%lBFUPy$pl9*`*|1TiCI4V!B_m78F^97|8Lo zvwX)tuK8qai1bI;k3it$oI=iZwScCe1YWbX-+Eto9$sw?k|f#jujB{>HRt$9aCsVh zmgN~Bs4L*rr5LxQ4$-b=wmqI$oyYMiW2#ZE(h+`(cO&@ImA52C}n_1p$=YJ-%kGuWo_mo z1I~&qZ*hMMN-sqw2?lReldcgA?DS*K;g6Y4VsWe_DP#zvb*H%E;igrNDAglNvS9Gm z^BnRvIbewHP-G>=dzUfR1Q5FtObrr~cvm(Osg%Xd*2T;ErFZ8mOVl&m3 z)!9PQU^n5Jjk``M6?P~Y-52oo>#k6pnq}H~eGHv4t6p>h`LVtFJF*pmv3E<$Zs}c| z=1Tj~*xnIt4^px_;WfbmJf@?FtgEv_c}YEw`O>dKC~Ijc2q3)RLp4~7eg|w<8x-2B zi5rCQ)iv+dI^s)Ub^diyts6uArLzmvvl8qj-wjz~&9k}CTCAlI)~wFeB&wm<_Vxzv z`HR3%zrEvH1t*7S3g%U@hTH|!o>q`8mr2DPF~1j}uTd)8Mua?GoZVLKaL#nxls29p z&ip&^Cl@glbTi!UdUh|IMK&e?;fa||*Lzu74Sicqqkny3CfQ0sm; zH51nfcW|fRfUb3`5qh3*XAZtFoCJ%+!sm8KIPUH{9yYie-{3HCVQA2Ppne4RL~31) zRkB?<(9`+!Y8YH5a=oRmy2m$^SI!^hoMABMv0q2xJf7ip zO@5TI)s&v?k>x8Ntk62jS}9rYX|epr45vDuGKRu1(ItvbeN~_{%0&TDcyHv zbN{+d(?XINkD_RW4En(ADa$*1h&4NcPln*O0g*$%WqbLSeDvsm?Xi4x4y) zsfN_T~sJI z&;<9;sI+JgcuQ0zt?m;J{qAN{+$=4`nDX@7YLKu(Ve@=Hp@(EZtn`9$-_n=;Jmd|b z7&3BY$}6eE5Jn+WTfCd^)X@d{Di3s(U|o3=qa-kH1@RGe6h?gB&Ak*v4>Gfr?c0dP z>cE!k@q(meFNYICJsIDzq~HfFjv z>uu~r4Dc|u=``DTTPF_@%Ujx7|5Tw+e%f04i$ zT^FH_cnk1a6*fAIDs{bY2`WM%A9~r_zZ0lxWTzC^!%IldSie)F`p)l4i_X%6oHVI6 z07i@?lsJ1HPAskT<=S*rB?SW*_E_>3CWD@}%*x;65qgaW+MMW%q zbxL#XQ)^*wiZ_^?EobKHxQA~T75`+zUYd-PlzK7HkG=2543f=W?yZ*eM!+HyG2DOl zraT8B)|2`v1aZ47u$ElIXtDQw2)btM9vULEmi;w^_G5if@OUG}`5tf7DJ(z|0fr_| zm3br=Jg53yVqmiuVw^welH2H#jqtXE(=2=-Ig*h-zSu)fo{^t8CGUp9g3Aco?k&VctGbytH2YM-!6qa>spVuIH ztt4{2+6y&E?HfRJMIP_7Gh+r(EIla^>$kQuh9hS|%ek?+_(!0$sl9yxHm_0+6Xkky zIZ+L{v*V-?q*C1nD<~qzVgB*OJKRXHcV4HC!Bz*6irF42=22e`s|xTd;5W~a|0rKvo@TwRvM^!S~+F<8_>UbxBeyPzY5Q$$yyA_ZO>!4n@A=>VaC)M1?KhHHJ>poS>r z?^2R{B_T0ZM#67arGc@67r_rp^Ni1P23zL|COT7HNWOv&o5!8V$4KjWNXfd%x;v2V zaw!WaY2LbPc71)u0^5ki_32-MDrICpTwEj^;|bvv4hU zWa_*0Yd9^30#pm7Ao@lms?~6Z9fX5}viFTNruPSkg_mwgXSs*0Tr`pZMu#_+r4(zL^3Y}yT zS3zzOG7RKJ6vOvrOXep~MK^K8>3SCzv+cc3S8IWkl;V}_kCKnYR~fMbXKBP;kmBfUQ^T(c7Bsz}gc!l`oKu!V&Un=g%I~J4a@B zPa+W?h`RMN0Y=5LWtwpFF^At^8Ka79yAtO?Dxp*Y-P$h4GzGD@of<1eNGa&w97|Nc z`-4sZPeO=#A~|XUb6uG~knsMMby3i>kJTk==L?-?ci;&{?^tr&2j5Y+5eiecjD=EI zGCpT+747)FEbUU6!4qeK0dpv)Fo7GT%0lPI>W77*PYP`zv&MF3`}|EDL*FF@r`iWe z&X@YZ+uZgP_cV72xSg4>zjL6c~4l=O`TtGuzBhWkGL8PVS6*d5V{6C=Sh`+2r zp*3(L@W)83Y#xMzLu*~=?Kd3W4Jd@|n5i|qXHwDJs$PoOaj9Jwg)4!u6<#895#fyGvdI%E4AIrLCiXYyLeHhdb z))A;r4!nGv5+E6yKfX5z}wGlfQ#z0_x#7;SJD{tJ<8h9)YR@&ALza=P#vI8 zwFTclF&T(z31jyV@>BBAy6as1w2q>VdPYocR2;j)pbo}KKzy%I`PEYnl&(!@7 zuP*EhEC78MOne*)I>@iUt0SueC}6%UEk0f?AI@*g-1<7;iK?l*KgHMpRNxO6*A@)W zZ_NJe?)U^wK6r2(9v8sd<>M_M@LO=krjRv{Z`kj1`1+8GOY_iB`VYmsURip2H&sl8-r+vzo&7zK_YYT0L&P#4qknHj1AHBi|E*o~9qYGP{Usj}|L?3XJ-`pw zDwfyVULe3z@2NK47$|d)U;Qti+^1ghf8e*2|AybH#3zPVUg-F~cL9I#dLwdDYDW1G z?)qH<2GaQFaeJWRfB!54eRsD<;yY2Hb^bn22+HhD;zQNfeYYm4&M2zh;5(U6m1i99 zK0dVS9IlqGYYOmByv(Tf-k<_NbHTsy@xFOxtoPl*nZ3#X90T>Yp7zEi2dA%R{c1A7 z(%Sk*Bqc?FbHejVVQXsx(7C?xW(IBjVj2NpQ4Qfjl>zJ>zXSBW$e#7)Qm~=<%RHr> z+YJHd75#^OBfY0P*axW_{}H_tANE6_gYPT-!g2suQ}!cbm%C>31l3pijcEh4&g@0t zE?emhhl`c=C2B)m+zp58Gku{8jAP2m=%(ok`K|Bre4aEkd(HZ-YPCM`<97wXI)j5q zgZTj-VA1*l+&`!GBMWPtS{Up{1bHU{vTghT9{A@T0Ni)H_7!q%OXh34gASPfadUO2 z{z|zI!=jzuZ^fE%@!#Em*Mdj?*qr!@d%pGgoDP9@`!`dC^Ov+A>&n{d!1`Sb;LGEy zbnrFhkL5YYhZ0Qo;RAT^>iGwFu$9~AAC5imf9P2k{A}-@pG;jw6p}XkQ>_*l@6LY^ zGZub)w|I!8P5TQskN@F zVf}^&ZHKY9Xg0{h`Q`9VU$M~NR^kjed64nkKlMnd**{k5VW4Lv?L76vSf>up)?%%$m$q&wC2Ru%OSF&irC{ zWMH!rbi*P#szy(zMdDTIQuQa~g;sHYy3>~ej*okycXEwq8nxg5LftgLdlLf5aYR=4 zztsxQ?wE`%a}mu&I_QL>jfSK&&59DH&2VCrUocwYs(@U|eEPPQI*AN&qa*v&eDCQ8 z7Ca{kqg7RMt0d@%rBT)9U@$9jACIgN{t53Xytuo?v+o&6*iE2o7|MP}4wBge<8CjB zMvOWwt37yt`Z83^HkQ1D(IYk?=hf+2&G532iKaG*UCJdMvus@(v}OesO=2LdO}-Va z6OusDELP}@8awP3o8=*gW&$Al;h$saNi?XDzc$~ZS*%9J^u(MiY0NiyIZ|}l%;s?` zVi@`{S$fCP#`b#%bUg+F*LY{+?rnf~fGlDv zX2<1LLyCG9{mAD;yfo8xJyMx}e>ws!I)axVXkU}WxS6yJPXuy~R1 zWnq9|tc_o-iMjD(>fk?u=$4D#n7?q~0OtIH`3My&X5fN*>VQ38-}h>3u4yg4LhR1z zH9B^tli4dD7S8$sV)3`tPvS6DNv8X0yY`JFbV&dcHHwHY$wN1Eu_@=o^=sX@3S3-2 zm1Latd~HllRieh?E9tud#jS^Zk27d*x13&z+qa9sLP}`M`WN;6#&1!SgbV-h63X#l zGPQ#%VF5dGyW&A`8&+noy-TRhP$2v*r6rTIgnwM8F}ae@WxN^e`FtI|X~(_*O%7dh zug`$@6E54ks#P5;KEg#z#%;xVni*IRQH@NE*`L8fJxtPKkozv2qs^j~T`s)E>pi#n zmE}QmXLxG!zv5MMhqI^E(|^kEHv4LJkvg_D+|Y9{JB*xMn{9WRyX{1UJ(0r*;dk?k zV!L0SogH-$Fyt)}g291-*qC$uwiA`1zAdMI%~0%cfjcG4RV35a@2Adtdd6k$w9@s* zpZ87(f`KkkN>k!@h7lHAFhU^&HoZRH4B%2MegrvcK{X;9)5XLJ4hyPn0(0f*>~S(7GQr??X3HHq!I9V`&$A0d|oO8|v$q1F zqSB*G{6y_ILoNSuZpoIa)B30}+F3865W5s13rUY(Y7aHHkfjnyhpu5V&={ThqYI3j z$#aa+eNLm6&%RD?RY}nD*jEcUBQ7?{7unpKJT_K0eWeTw3teBGyi?Ui*GBp%B$G;v z)sDrnX~ZYW`jk8vsyfvY0VkhLkm;nm3z2NVP7=vg-;aP{c|4u2nSp;l<9)G&oMah6 zJZnatRE-a=bal(M(}W^V1bQFKFXJC72lD8M%9Rv0pxfZ*AFzgU5iFEKN|R@QGo7mOMUz z%ny@dG{o{R5b9cTMwza47ED!E2F~^4jAI4f^VltTmx;O!b;kF5(0XHq>mOQK z&@Ta2XNF_E+iqg82AU(+_N(Ta3cmv`LK5iW1h@SmpKGP7j3=Z7ywGi6Pr4M}ndbOw z$s3cG9X<(su|1#X?puf1E(*-hnjw~h(YXSnBM4I)jtPAetYJ1l6`MEpWdDvHOiSmG zQiK6U+f_^7W6vW0{knTPGvhvNPC*8wF3idlyN=qz&$Q5%Oc>s%$4l2a??EGXa*Z4# zIWv9nfGaCh=%X!KTfQ(J)oHCrN8@&XMANl^E1eY;J$K%fBIu$Pbj9lERE`GObCPq0 z;4Q@a9dGab(Ido-e0Zt7menTb`Uzumhgm+p(`fa+cku9aRq zxfYZ*>qaA;CY$umpDS?%`>NEdTM67~?nDXmS}B3joc@wLVuVDFwPq_9Pg9L<;1*W1 z{wW?l|L9q{!n11O2T37O+Uz4>*XqHj@PEdhqDPGBewTskcg2~Je+|^Y8Kc=&$*2N~ z7RknV4`9*ry_WoZN`WR(okd{797@AC2ljuhCDnD|@y6Uo&^EJiaxn)nb!)Q^y%vyD zUXyh)03t%G;!}rcVnYaFlVR(vX=O0f!a$|s$w{pr&l&7uMhkbb{36Ngp!EG@5k(u) zE;kt`Y$NF;rTVKU2DtjH3_2#~n~uX`*)1tmY_<2*02u$c zOQjjE^=_{5QQsjI(imLLvB%J|ua^}8how%bM3XB`*g6WP=80_l7h~rToNE`T+1R#i z+qP}z#I|kooY=N)+xcSKw$uL|+(B3Mpoe?B2fONh*Ls!=b)-r&?uTB>gH>z&GLx}# z{FiVj7jqY|vVnKQCq?mvxZc}$*wgdDX{2-3D*a-3G$?A}I?WWJ)$On!L1Oi{sO%zmx#8m}B;C6<6WCOlGvDGn2uO>lO!GgkTxUT&1;U~UGINkMm= zaOvzoL?`D)Sz*BSN3mAg@uIZ$E8fYLD6=@c*Dr;5m57CJ*A!lz!j|^kn2w;z8*}PD z00i{MY2^r?RnDEX%ys}~a(OW>IXb?gc!9HXrZHzKy!n9_vY8s4>F{oDYT7!c_^7Cc zai}bEmr=1jLCxX?JC0ET0znP`?y3Hk9XtrpjwH^i5WIuUD&p~-8wCB6#lflV=8CRi zTQUaEDXbR>b#x(r6Ox&M&qbqL_r8@NBO?pJ%3n0w6HcvXBwkMKbO_4CK6vjk*C4=+ z_SoFqHR?cdzzvM+wRU&Qx(Hc8J)$J42{DLdQdORu{y8~}#Xu_h!TKY`qnA@yx7-)3 z``AW40}m>}|8)fl$=!Uj9IYav%7THQax`j8`&nx_s0P5`QYly~>gZx?m1iB!+^=I` zwY4@7nbp<|q0BZq2pY3ut&2G{>M>HJ7=l{CflXz+V#W+m;rpxb3M z)`(feqbXw?Hh&@Z$?Ez6k&uoe1ASousRG})AiMSuX7+b6Z}=4pOjc*=`!0L?o(0}p zaE-rN4Q<-;P1$UuTFW`!uc?b|kRXN@MNdAJQU2*;Nkrr#yyhz>Cv(v zM$BkpiWPUUlA{N$d)z&{L>#1!AQzeWBzS<6Fv7mEdFnxa-me8GkSvk+UsX?My9Lso zPzC-!)akziL;&;Lk*_!+x&?kxH-BjH_?4N0!Td6CQ;fV;iAG}Y!I~PbYd$ZAe;L)$ zo>wd0SDgOMaul5W>@AMW+f2**wz^@wE|Awp?o&dBNRov#gaoWcc(B=6KK&8z~ zTKj@A0elAhK2+zJUfP%>nFR^tPttXQNIv318VgS|4Aw^@h%luY)h>>r3<8a-T$7j? zs59hl9p!eqZn%F&gx|Mf|ALO}!&xu0!j!?J0}buyc6a=h?v@F9rzfcUE6~VPl131- zrw>Ao*OkXcVAk9N&F|Lmwar1v`ox8&su~CR?_e&nN`@H&DB)y&uhR;dt;g`#%(M`y z_xZ6)#8ErnPp>vYPhu`6_e#Ow^`Z}%f-6-ydC}ngD!DtCQT&9mp^Sb&GWwL|F*7%c zl8z=)S)6=86=gs?Q)4_x$^(iySW=3ErJ6FU7uz@eAEkeDk)^X5E8~pme@kgd$=(J< zg}FWc|Ly!~P|MDye4Dxbvg_Cu1sKWsLx8p~bc2%BroV=Qwh%BEFVA6-X45SnqP#uY z6b?-5J)=$=D6)EuA);{IbEl)^5}Q3Q947-0k^($FVRvD*I>L6K&9;qaB+rr|q39iH z2p4ipk!=0VBCY$_#otzJU#(6XGBtI^G#_F(Mz;WhMTWaR!>&GtL5 z)bu_&oiwAtpbz{zQtF0LW%xAP_`CJji)5ANknuZ?3gVp{zrU=p9eQ^lZ1+wFIRPQ- zoqCr(5#Zyu&~+{kJ7 z`RA}?L&q5TINHba%#>UA-Zrp$#&mwmE|g?>jJ@C~q{%%gJ|{3j@4aDjW*9 zr&msPO78J4E_t0bfnxyQHVHA%Nhsdo_iQzcpXCX!K1456@YAz|^c6k`c`odnFUiGn zf&~L`8eHn5#YO-XUuII_PjPUcIsG}qSz}4TpJg<+J%rbEGm5l{o1{{f*gPMcBn4h% zr@_JnJ=;~V!EPx8e@-vTvGK>1yNqgp{Z`|B{y#?zr~TdulRTGYT=|NaCw;`(Hu5!y z_k#w?txQ)3FsKmeb%NWDpXp~lO>7O+7>hd~OBN=NxiX!#z!B zfuQ;d7PGE$rgRj#Q=cTZ{r#E)l7IfaOqs)A=e^!bkW0NE{){drdTDKFi#gT`feGnQ z?%y|4q|_qM(sm9%vm_;+m=rosxGOi=m`b;;@RuLjydhpTTg}<2 z4S6B@=&-to3h{$27(DB(Cg&fVGq5Q>13b6NvI{b5Zd-!<|D{tO0uG+Vr!M-4=e$?v zy?D2Nw}b3gg67e~>d?-cO_DOh{7Nvk^pDxJ@5PFke-bv*z_W7kTa-ahE)kU?)_ozs?x)xIDtURxuxO1IL!C z<>9uL!|07fOtVPa#(N^60iTQKYl!Qs;i?g2Ud|;~%fKat?+y0}8;d)j z4J{he)&Z@oT4i3?Rl^A(zRIxES0Z$$rz=C-S=IAZk9T!-Ux;wUFcX+k4Rv+FXu2wr z5I5L4AYtUxs^c!6ce3`i3e>F)vR7V<8AJI+79b6~9!kq?!SL#C;+{y+c8qn7(z61P z;nkE;(+x>8+Qikhqyq@-!?|z_OqpouetwFsNy#5@PT zBC0FvlvK%L{V+0_3qh8Q;{0yXf$S|-vS3UiVV|rLr67i#S!gUGw?-WSpW>)nyc7?B5 zl){+eg@6q5HnY^)dCO3PaH7@R=V}_%q;$tWAdN8;GFo1pj^i+kXzDC@Qq&Z=x0XW$ z-NBTQ=>~cgWCo5Ygf9H`bWF;v{Dbt>{PU$EyJInl>$8C4H1z|Gc)zs*|3KpIXOGaq z#tgeEy3oXmq3X{}_%RD|*Vd`SVbg(RrbUHR?N^C3jfI}u_o7jGPl9;H&I0ZN52|H) z$aiD&swoB*4*qeFN8|`z3ng4gKabEMZWZ?x_h%Y@6ZsvfNsb|!o{EJ5Id8)%?gZRy zBu&2hL*$8~6o58E9@4uK-RDfV7Yp$ciPKAG{DDnb;(W_8-bc*b$H!`7GF!3b%8bxE zFyQ3ft$di$;1rGIx^G`yFkq%DM9pV8b2<2`(na5I;Yh85I4VW-%H$DS<2vl@J))`4 zlyJvCx?gb10gb^LUR zbc^CAKU;Tpj7^7Ru_@dJhz>q|lOyF+RS7dknL%6K6F5{w4^Z8>&t47`cU|SFO6T}4 zrxo07teUs*9n@u$H1sB=B}fq{Br8};NQkMjL5^Y;^xl-@;mdNdeTa+6M2i%sjV8on z8ir*Uc(=8)zzw#dxf9pgr(Vls^JwoMIMik|m;!MW5az9IU1;Snej?TQ#&F-Y%ciG? z1BjI)*G(9?lczyoj)wITZTbPjGUY$=bUL66``TDq`4Q!@U+8FT1qLCFx?tH*B5@S+>iFw?M20%qa+|BgI#l7mi*jHm!5B)V{Ihvgl@%I_VktN}-7ur|$ z|2or;LW5vY>vGfUJbuqyWvZm$^c&o?-P1YUl3b&`eTizXk+GOGH@h?){dl)j+51z5 zAp2#;!|T2Zozcc$aJ7Ni{a>bYbGc}5#*UH5f9T((^Ux%wclvzj-ma!(03 zkD&Iy#w0^OZb_;6&^6_<;r*JR&vfx9oJb&|Df{Eus`y0e$p4*R7x!05^7!p+_*UN{ ze@@+DsZnILfmCDZSOD9b7klCBbl!(ay7=3(i>$1!})zIC925{3kG`D+X zipn|Sp88%VCP%5pC&bR2a@9)XK{tvVEVB))t8tWwLXTKAw>cY{SKF1p$;py8Wq;fu zq`J{4xbO}e{kDqU)H{CjpQ&$S5CsdqhU)H!M5Er$ZedU+0<^XkeKk0Q@x2!qSU#my z19zT^Y~!Vo+G44O)!F{`=`Z~nXpVsx&h9QCG&PHDxRMekD>tsGq;r1&G3!Jx#iJ89 z4XW3(EzFo0X2X{m^WUin-?9mlAnU=P6^QN5qWeZXJ<&Ujv93hrkKuCcuz|C`8u;?I zBq92&{)>Rx(c6>>9~LMNbaZfr6Oqg|>E0;M>OYe9{|;QDlr$@CS;R5qDk>&8*1PCj z$s}AaGw?mx`JDHD0Kxa3z7NEH_p+!43UT4`IL^UeJi@pgL#k6K%R^Ry=x)u;Kujz# z3f0yNm04}q4KzGW{yZnD{b=ovJ5~ADJV|LkcnTN?^;HMwHvXSHLK zjuhpx!)kHbW=6tp8-#-@|LRJh01C?H=6xzB16**O17eWRtDUEDp)Y5eW)y4ly7qDh zfwT6!adGsP8axOZr}nz2v@!?koPM+`g-;T9gZ%j#zLl~4 zSLaAf^X<%C2D?_2ak)%mOUF{?HO2DYRkz6Q>zP$Q7G<~MkhArEZ zcQPSAIWSD4{a3QlVIpDm~_O&Nv{=faLv1V=9wzEnQVrruYEFU?F zQ-1!oVCOpq-9eG!s&BsoEGdmHoK9DL`3_jT+i5pSieyN))NKBM?`_CmxGiHjt5*6k zR%NZQ4d@~gpilEz#I{cF)@8527(5=po#bg52igR*_ikreUn?!k%H>V6=>7aNtZz2QpL%IX3rBF_-g|JatdG;0lwEAaYP{DU6R5YJ{UkQj#*`V+4$H)G zj>Q7wm@<9?We*6F!7^zf3Cny()z0qj*or8heCUhOTE7)t8?YoIUpkU#Hn84mO<@__$5Jv7Wc^WEjZs11~hCbePz;rJ8%w zy0OVSVZ=PFd%}q{_l_QDB&}p5Jq>1C7Oio%QI$;+bGhIK505O2E1kpFpCy}a1c+?U zMX8bw13RMF)nJ%eYwtlIrq`Wl_T?I`GYL^5|6q`lT1eGbr=!q>0X+0w8{8s48&w$_@2u$yh_is*aFKF5Gm#3xhxFexN#MqHzG z<-84#5hZ@`MD~}}^@m;Z?46vY?Y5Sl%pyaeoPa{K1dOXWo_QOLPyw$3{?C2yif~Qs z@}9idY_}3eeAQz#xitKL{chd{#ibeL4HdV@R?>Y4C-(>GtDGVo-gcZeeODi1kd&;r zQ$|z=AKsNrbw1G>nT5-Q6!F&EO6_6aGhfWVlB|N((2FhoQvTpEPH8h+qR4dJeimdx zfuDHBQ&fT_$N%z|wS4&TohNx||Lt8hdiv^Wsf8 z^7!CunH-vV#I5mMf2WU&6&ZQ6*{E}X)bWS%+wo-&ruhan`QUzBNf|jw=Qu4y4pwY> zF`$k*)LljAH6!a8$yn7)lYFn&8cUT^)Zh`_QVyf;*0?g`jo`HNth5e?d|kPk8$Mj| zshDP-oO)dN+{&cJfM1D}sS^;<;jWbohL1v0k9HW0!GHIvK^8rj;%hh+E!HC(g4=Nn zkTUI9^%CO~-4x~yJ5@uiw`ej&LYjmbE~?AayJ& z!_~IwSxAm%g+%t6G)m!>qcOOXk(A&Weay2ptR;VHpt~COrdGcUkRU2P_u*?!7B=CJ zG0{JYIQd|n)SV1({rKX!rS-x|E0|i4JE$LjDG>+T62a0QxU2RmgU{E@O)0U2QjFL$4LITfV%_)EV5>7C$#j>yNop<6?wn$5gn@(YMJ#6C0^vVfVGfVtN+I~=U%{T{0(wX$ z3F{z7TOG9Ij^&YsK;q$GYKa|10^Kn%$w7)6;U1Mg$EQ@JNwI_zJ5uHC8(7WYE11no zTM{=}U{N|E+yMs1@wVDb+H_s?*l$$zDVdB;xXGy*;g6Oq^7N}{w7~86Y2aI6zX?p1 zllfHg@ph|xu85W{Sng`WaBou%czXex;IZ5}xI2(&sw>89iYQ646^m8Lwr>XfYnd8X zApyT$c0t5rC5)bR|C-*z{mi>&9%y`w`;!m!`$`HdZ;z(WT$@S0p!e4laXXGRuOjZZOfJJpNap=VKnN4xMN?>1ax09HsXm$oT_R0;-8fV2)MT_yQ04iy(;{ z?t~pY8NrhMCR#(L{m&&DCG)N4MD#$Fc|+=e6+dyzVLl{;$K9iBNb8@+ZMd;E;BVhW z*QGfQW63zPSdPyF&K^#TlN28D)eyo37w5(S&y=^AoAi#$xg#t#AESs{fx9>*Fd)6B{SZHraKlfAj za)}c+XskT3Dzm{%{iHg%k5ZG#ahQ!5N$e%qSeY-HECRll`8|ca>l0R|qoV8y8Wc1+ z#uh!Cx;{EV5IMmU24!!lnT{g~agoAz4Z*>tqZG{lY$#Diad$~_0-bfB z=nt}@m3QIif|`xem$6nM3~(UF0DWET^N4!?@M^tBjb>-$%|<<%ZBEAE{F45yVC(uj z0eMSEdaSJGUs%+#XID(J4BhmS4{_6TT2#0*J{5eF|Ry;-Q%|^^mr7t}0gk5;#z7ijY6$}_x{reBImYsn#sp07UrMkL0vmPbip zcnN0n)BQ56-;`~TVv^qTe8x=4qHDtyZ*)%d3>S;#a&Z9Xyw(-0#&4MxLA zH!qnJd?ByE;ZrpA;_a)^oVGm#$|n9S;2v}*g14*e=MF9alr7HwHrhM)-y8n`61J7R zHh1edr1mISUoW_IFRt^NMP4>&1)p-7=$VqQRm2V%6dpidB`;NgUDVj!po3u;|CMvEau3B zG=~yx7X$_d?tbvief|#<&Ascv1^JTMVSdwblH2jxFI81BL~RG(1U5C4f0NCpw^azF z&5n<648+Mvh{?&xNI_ckUlbPkD*^kjGU3$)M3AufFVT2su<(sz7T!4itZIlb@FYPz z5P<+tLOxYOA{8g+f2|1JBSEqBDk>p_Bj^T*awp(_#!B_L6h(B?>tpDDY)%mbPXq~p z=`^Z=2#}G9&BhjH?lt4Q+gL3vLb%OAKd6;OS*0xXkRO%x)JGyvdK!OYm z4D^U7IT_NcYRG6I`(Y34A?G2wg7o)9iX8KMOJM?-gg!CBM8x;TS~Zf`RMEK@={f$I)z-hh66n&am) zF_!NN_CtWQw7!!MZVX|ULx#0`2BsYS6!god_t~`?c?AA<5p7JIh(K2$Kt8(bzyT$z z&rT4($6r%$nn-)6a4ryQ{o;Y3L>xwMgmQwMiIiA-Hy6MNcb|a!U8uhikVfzg&X8Kc zcEn24yX*Vr^)tIz{_nL&4uQ;^zAv6YTYdgN-=;8xj5b(H3#Xjlb)VogXX8>;HVwZu zZ-#lXaMQSZW7JeYdq?Pd5ORVITEN~?`3c|yA3i*xL86*x7Pc1ABrQn zzi>cPf>Q%9Z?}3EOm9oUz;t~PTcNsvn=n5yr@vlHzcBB=S$BroKMtZlec%&5Ts%L= z=I>L#zfK|9MDYT@NPD7}`v`_W>{!0=`MqNhq{xXqS*LZw;VqzOAj_!;rev6wt48$B=|0Lk1>a zm!HpZ3nkk0=Q{L5bR@7qJ$+$;^gG#bQs6*BL|@FBft>(&W}tEKu#O>7AV)7?Al!7? zlRDh61aPH~|43*uFrdkwn9z1$q4!-WBp|i#xIMX+T`4Rewcj`~AfdQ`Bql_l@@Gbr zps^r-QqSyT2e5>u56t_=A!OfyfF`d!SRk-p*nRJyfZb6C5Jn)&$EDAf(%b1sA%oHIK!!=?FpQBaZ1V|s! z8tRwiw__*n*Tg%gpk6{2>G*ZR&0T-hn?9;<+JC%k?VjKUY}(Nc_$Mh)?$9A3ltbAm zA<_Tb%hIXQ;4 zA>oFTKCS%zNC$s>>r2U{dhotcaUd^X&_<9~{fREj3KJXll4fzib!MdTF5&S9&6aW% zb%&ydTR0DZ_PC9l3vWM}?Suih>wDP(uBtmUZbuomXlQqkM`5kJuY*;D#Y%}gy&DGo ze1ENRi?;32ag7s&2a*lsPud+O_b2tZwQpp>8x0CsVM#50$i^RGOm|B9f+MOsk7 zs~ho?b1vLF7tm5x|NT!z+)=%f29>R#hN*3}IhkUK;6i(Ms=!*GFp51RmqtUzS2SFy zv{0WgyzH<#^;6DB$hpB6HJB4Wcw2rG{VefX1NpxU&%v87H|0I=_{VF4C<{~7^%-4Kt^$>Wb z2@P_XJNT3w=bx-7#QP+jqIead+r14&cxCaL_9bP_mP1kLd!1QJmI5Yy_2yYNCQ3;h zOF$-duSq9cw^y`OnxdyJ3005uhg;D}q>6GHhvJ^iLxcC9>Z&U4urG+>M?aKLIHeAJ!=q!oN7Vd8O_*R@Kl*4sZ(8<2 z9?z&Q(L~(KUFfnED~98gS-kemjbo(KQcFCC2i=9w3Xf!DDwuzS zO5|vc1z7h+-K4L=msPK z^sMVjn|PBZo&7{FRcAM>8lxV@JBUvU>8G$VMVBe%U;(Q2K7;92{jwj?c01Dxx*Z&I zppT@OMM!~Rr1r@+YMOl}vPaR=s>drH)aq@?Z`ta{`1eaqW#m#^Wo91gM}X6$g7gj> zvukMSO*ETja+|z&7EGrj*~V&YM0Z?phwaE6-X#qUqbCdzOO3;1_&mQ=mM06fjV3<^ z9}`g|7Hfl*VR!^aL-N|Pqw8R1M>hoxyA;&`380JISl5o0YDf2Xs*dtcW}*o~wIm5d zI#p{wz9T9;J9`nE(LT!sn-m2J@#Q}d{0%Y$GnI#7!q?JaljylO>3+lSvC!F+KU-zA z%w<_z6|GQoh6c@EyRDRjlfE2tKIqUKM}6@8x4}l%^_#kD0p6!xcUZ1&hv<(rGk@)Z zp+-2c0wjRnX4z3+i+%`Jr@HB-Cg3g`*EidK z>d7gSd{1c@%q`LpgUIOzI_XUg#fhIu_BT&PPRaCrLxdOzaK~WF+o`NfJW8Zgx(Vt0a8_b)NBh|XJz6Lt*u%*$<3PI?4)<5l9KOvalln%N|B2pLZ8rbn(#>hA zAJlHC^?RCb?GfR<523&WJt#G;iFoP1P(Ndd_^Pi}u~LXd-@WFOl&o^=2T(D5ZN_BJ zbX@&Cg*wIOxZpd!a)~=xGRW zqbL>?GgK9~!9wv|xLjF;-pv-rBismEOlss*RGheIn6aARF!t&&ZFPrqk!Rai!n#TB zEvt^D?CvS0So71bjHU1bPWC4$@8yz4@G-f<{z$K@*cnhbdwQ442g(^SxwiH9P}Y0u z`CPGSqo6tc@s0AKb-CyUmu4Ws$g>?yxeP4SQs3!EtvX$gLWSYz zeW3}bP4}^{xBfH(7c#f-`E|dcU}3$CZkDR)atjjaUwqIQt3r`E%C7>-OTm7W3{pw- zLNVI12f8b}8I5V?)z-#L6RXXmV(z=Gxm)L-`$B@^+?ganmPZEH@BKoNeqcVT2 zLBcoV5BVoT)2Z}li$UbPduZJ%-Q(m}zGH;L){aC$z25HP2uGIXeNlKMW#cXDjzX|w zSC>@SiLKxgxvIEHw(P?tKzBv*&VR7vs-fPatWS=h5?vkGwp1=pCF2ZLCtR#Hzam*= z{Z=;e`6_3H`OcBHxE{9DV=&RgAj457P>TTr^27whorS?5czP30&J;Nm#4IoQ^!}m6 zVR{vq8`Tml{TnNK-j?c1IRDgwSLT7EL{4{W}D&PZrQ+@Om4D|B9GvcQNj~~ z;8Q@#g*^o&nX58cot1CfHxsQdLg$F49&=jb-K|2?ShKL&EY6P+6j{8m78GvH?vHPo zLt!?Hte+}Rd(rQ7yyxj*PR>z5RE$V>_E4~RcO=@`A*LTO$p*$&yS}L7~y(o=xUq!KxzUo$2_8r0P<2 z1Z>U--5+YUhP=d{4CrWX{>VxXs&XwS>^c70WwHA482@Pl_HdRO7N!BQ~;}1UhvTN!APqHSs0tR zXC%2RjCEooV^R9z@i2BrGBdWerFs@bS|W4TB5XZ*S+Fx^_%%ZC3}lm#mURiagMA@V zDxM)1Ne?gX1E6YB=OXu`uxSX2jODyY+D#i5f>uV(x1hv)1j`6nd-P*CNr6UcKVI!Ldy|9!sRSNfky3#V& zzk!#$5d=0|dH$Ipug7tzh5RcuozOTlv6K`M&K&L}!(A+>AOg69?`|jwK_65%Km14T z@>?)S#cpm2zXcY+2^ULRGzjrbE~#IgJJEPBm2>NFkXySafvAxCX4o9#$FgJ`k`bg^ z22`|i6$n>YcYn2_uX*KWFlB1ZQ*-#C|6#KTIvEWKR~&M_oZLOOO;(VsYY%#rOKfxe z*EE*tjX0_{`ZMqZ92L&`Gjy_1V_!sXLez=T@a^J zSD$d~Y(E{V$wAChU?2^Lza&lFUq~o#`Fz(cgl3FnoqNDDW?>CcNKLi^!AVK3#lFe6 zGx5IYpGlIMo~&^(;(&behruwCS4Rg06>|%7CI3f3AGm|oWkZ|4=+Oe zy5sCery}Cn;0kQv{ivI^79b%S5{!q|Q{Yk^X(%4A15ul0rgO1soxP3dCd0rB2iR?} zt1oq@*gDl;U;#Xhl?8tV&1 z`9(|`M%BA0o*?i$$=94WBx zq5gSLQ9(UM;{;&7((2UM@K!9ewxNoFfOKFGoHg)ZEECZa^1D=-9>%Vbeel;|+WupC zdP3Wf0{El{Zsh42RoET{c~+X}U3>gn6*EOaJ5j(RV z@h`j*{$w2Fw>j`IYL>Rx0z6KXKF*lKcil1&xM`eEaYb>Sm91(k2^5R^TINv52l|dN zOs(e>c7-~^oT{&?9}98O(gv{(p#v&Vu81!xj^OsZQDOBdjNB!18Pk1RR&}sB+!~#3 z)4tO&$vuF%*sS{WGa`_L%1|3k2@Hk;PQ#k!p}|jZBK97+nCMoC@Y1s}A+ADnv&%6< zD5lxPFw1Y99z7DL*&8DAb*V`E6nY37d-G)3TRb15H$3ibYG^Wk8SuI9w zc}6$Yb=1n07-)r`gO<)h-E#htOh-6rz69T|UX<$mMRtkE@Z{2@KKk=Q`F_SnUc512 ztA+c)hiN0us-<-%BeS-;#>DSEB#}`_=(g7asi03u^oA)x4X9&a)cO0ci^jsW|Hou3 z#K(WQk~M~}4G%J@yeLk-(|5B0=iOm^4y=Ilo2EBf+jFe;Z9T0alTk3R&xZp}i{|fL zkzg0nCmX)4VI5(BW7C#p1?LZd+XRL5_~z{)&eqR>+PYc-*DwdyA_jZ30Of#lf&mUZ z#djI;-km2_4bGT6!iunH)fLKxxfJ3b`OGw4LkqRCE3M$w?nSbozufgtB-2YF#`TEAk3i2h{|?B&lM*j{SSlu4WFGtu%@G}FA?w;z>$ zRxPXL`F1JRr{v$O1xf7*gj*-t+!w+4Lk~6mk!1bX?7uQL-QkHB7D_MsLjr!=kL~5# zD8m2zx&r8iP+IZ ztA5LGAUZsyR}&$GCVF-Ykju<}A@SiPF8cy>-QGbWTXY&U=!*_g@AE)ti)&=EC6981 zuI*;`F8S|G;ns2KvIlu^>OS>G7h2kO5wX4s{C#=+uGfdWZq9*Zjk}GPqW0z8`>SyH z)`RfV5;ENiZ0uE+2w+T7n*2jB8L@{@jV97E5GG&511uD`!*MA zsP*@A7iVmw$eB&mH- zzxK9yw!F5BwC)ms0()PJus&@{T#p>kv$dN&@dJ{nEu=WG?FjDZYtF!P+F1{{<~*H^KZJ;R;n2TlWK{{0U&bY*`gTTmp~+Q zDt3Bfh15)7N=En;(8iVm$j|zsOQt8ww2g-mV84LnTTfj>*cpjuK&LD;v!|0V&5_pS zV4~~Hw76e;2Bg=V=CRlM+)DS5?cfLN`!8XCKbCeI=@APtccOC3#*RZVYO@nLjL4L0<9shvFLGx$uW$yqoNu#{BT^Q)xfG(vdU!R$kXvoAp zkW1yIGsudIJ8GNr_uAxRRJ`{+CYdv+pPSfik|);>b$l5@lm>)Pp!j}b?&Gw_02Z>} z4{^BSyS~rGCXMbr2)lCQC=k%Hdoh^GSnr7ZSv)b(Xxzzene3A=($kch>B_g+m-lPL z5sFBq2ptuo37-iGfKop{jDL?VySZOigZ~qkq&e+4WlG2V(s6o(@F#r@7i4?se)$QK za&@9qA=ndsc?;Q469CJ??76dOEM23OEmy4WK+n1YqvSqeL5zSvF=()1v#J7H;@L5W zd0s+&=;K|E@E!+_Pi*1dSmlhYxA&4Z<9vhb@!lQ*VV8(u_o04+TI0qkI-Te|qr$Y- zsj`r_3+K|;3~eVea{HK+&t0JqE7ZqkpMiL;=?k#GkZ~krjv5QrnZaSu(05zP&Cj87W<^ z(+9=D^alE{>Yg{p<$R2jFSE<;LbW|Vz=0CIWYan&h1o+)f!5jyWVgO9BfL^MW7u_* z6Y$~&gq65S2v9X|DF}CfDKGDa8-xI1v&%@~27NYPG78cFX6^zM8))z&Ss1B+I4i~z zes_AidcKzt6pR8iW?pr$6h+bwcG*F8;9d8`U-rGtN=-{74f=H~F?loPx$jaKZguFd zN6$8lEceuMR`=lYG-pwIC&f8llI}Ugu4CMx=7JJPGbNBC?KygR+KdH7etNRX~a3wDIcQdP!m( zm-|y+Lj82w@K-Xo=R`r>k;e6wgW}%k%d`RbNANZdS>pUe#L$NF+@4HNIpJrejx_xl zNmsiI#lCWd>+P2{bfk2K%`4m0Ddb0Hrd2#4&@6KC zXZhV0;c0JSGk(mHSxzhZ&RTX}IJD9ik!mX$ES*N(&}9%mXY z+KfUgBWL!%1Xa<^#>Oq8s0|`2la_*ljxM+OKkjc)PXg;6zs^$kt!0Q&Of@JxlvJ?uTAo9Yr(L>V zX+#sbhe0@edMXKPK%1E>Z?ns!Y2?KPT%ph85R|`nTM{Eqx6VFmWbw(3CFl$i9tr7O z`t#q3=tO_O2HnQPi61b?rH&43p%-PoTFF9(f{FkrF<0K-?`_e>t;IjE`D}m8$ddn9 zBA!Q7u*X#Ue@~sdQX-D8B_yI)w9qE}XNdnZ9)quM~PaBYUDgeaDavd(&3H z$abE0eqe9;g$;F{Fp@wOL4D5N9+y@tSMyiJM-hM*t6^@29Pjd90M^S)kTr~2q|U8g z0t?xR2yaqQNBB?&@ULt|CleL7h~!|qllDK3jwnq{WOj`ATJHIFe!WD}GD-t}3Yk5x zM2Ms-iXI2q4TT@T6A)!n;gjUnga~qdVI4(&3NS#)AJ63(hCpaEWC38hhClOa4k)u) z^wA25}^xEnpe^pjyYF*b#4hKRr{&djPU0W!CH&%a5$(fw$>5? zP+m~3sK=ez8D>)$Rk8!Jfk2D@kc&C^X8aO;7$I~Kx3A^|1f?Ns`12OK*B5Rg<9d_v z;H&z`co+Xme@J47Y#8lK6LVBz>T~h*ke*$YHWMD(6nHL49qyrSN~>RL7)7$)OOwpt zElutf|~uBi&H=U294J&o_eV(CJ*M4cAI?|>|;N=l=5A)4v&#d{50fU7Y;INL=D zSz%O;;nC&S_AR86#hy_-4X#-;ORX&s14n3ZPs8f;+B#`0r9J)v#TFhOsni(b>8bzj zu`u=oOU`?1I>?d-@S<)Xt44asEfVL^qCaPE#H~}e)F~;I=-mv>m|_%7JQXSz6lpG_ z+Nn&CG<~qnih)f}>dL`_tSX{3OH~pDv)TB8)F@g!{}>9=`}n4Qlle>)em{e4lE(hn z(c9icS3wOcL0Y{Wj3W1x3rvh$vj9hF_Yi?r+%<-plJ)(!SZlu4m%GR%fy70@YJg+ zF!_@YqsODXoGxUKkZeWZ^^w6I(Tg{TK=YT1#b6(|T?FH6SSm>yo62!1>%U+*0prZF zXo)uI0bR4>23E=2^7Q%9nuuWgWjT$sI#?G8Dmh#XGvNF#sSTD#io zm~af^TV?FjLvH!HB)pU9CMLWYGf}PN@_d%<-^p02r~KcVvsIchl`qkfBAJI?D&o(= zBUi(T*8~VEf?oSa4jEEMv$mR1cJJ6OZK`5Vu@%XylDi&wxl@J`f+)(kZvT<9BdnN@ ziUR1s!q&lS>7G2hC63oGpL3u*Emdm7OS-bDH@2A6I8l<71`;Vq6L9A6yV z&kz1Gapa3-9*E%SvoDB2TA$jvgD{Xxz2fzR`OZ$qO8RVSDUvsKCPHuRqy$rkF~9_s zy27|ennIe|J9F-w`M~MH9g~?#j@gPIRBEB5e3{CmvHsvfR;Lw+bt1M36wz~{2Ra{M zAKKxFxptL!Gq(~~KnBOS4cchvQhC-*o?26(0-35(h+9wejAY{`O~^-ed2S|vTQ9;n zF6gE810L}(f{GGV>Kcx7tU?9t$5g;9Yt1PqVw%(47+n$6Y(}LD%`Y;P^uRpV zl>=aN85DVi-0YJV=I0ci-!PbtF%pl}gnKTxMheQnusI(Q z!KIx4-44JqXkc-e_maCvvqyiKQ7HO(bhT?Ew~Do`_r0}v4zjn#j*^1=XrEz|T-Y2= zsfNrXL4hf3!+;iVULvf^FGbHB1b%jYl4uVmkL#lMB}Hk^^<_e0^(*Fxe+-XqdyKup zA?T8Jgt{C;4PPaMu=*3V>q?#}zv||M-GP*lH*K*7BiXZ)0hhbHWxxD`;Rpuxp%>DS zv@g-dS&FwbkpuV3vs&5P{_A}FBd;Hq+Z+6U7(0hxQG#yG9{b#5+qP}nwr$(G$F^!U4!50)O6W74hEv-Y!%d$LzoQfB&YC-V4P4QG9?b>AW2^^dRo$SBVtCL!?bCy z8QZF>Ih)bXlcDFdhKu}$AD7jZZfaohik9b*f9Chd!cZK!&51+(1L{(lTe(!^d=j4SOo#;WMdw99b(y-nwiYL>fYN&QkvlNZaGb!N{~HPwdBe^_;_@ZLe9A!H}H?a0vXNOrN3NgPi`1?c9gl@8DMsZjmwaEVN><9#S*~u!fnp9&jV1SHJr=sZ#w(-S#d2zWqp=10+WK2#l3^kDC2l zKE5giK@!x3HtK;_jliK7RW4H9hpyh`OwL8oa`zkRMH)dfyLpzgDUn4i6`p=uaL8y? z*ygCq%=zG_A*Jw8Vw%VfncY3ix0irrly{@Qw0X`W0e?Ot9xgO>}FC74&ef~)x|S9op33vGL2`7qY{eN-|7=aFzrrcF)Wjq>(2zAbQo zGo_?v%}4C-!vLzX-kUb`sptp~Z^>O_#p}1c5H=3eva|br`DYi&LyIAC9SDY%Z++(( zTWgkpS9A4-mLkxW0%7i|3huN6JKU_Nn^9;SK8`1JEyu{C6uc4&l*Seor@+jy_1y?m zn_Tuq>>hs3lsoLt-ZXc`XX!?tMU8y`Y&3HCyTehisDQy&>k0dIIL^qo$?Y;!i%xa5 zkjPS8?<2pJj!K4QnunW3!6fxCOh1`7kV70)PpG?7t;++7>Z9N*%qKUeL1`=>_v+$v z0LsOB;wk(QF6*mD!bFQglZdCc?JBNo!eD1qeB)5N*4|RqxiBf43oIw@l-AL8%1h=~ zb^eTqQc9=37>#oaB>cllaJsgXef4v-rfsbqMSW7EL;1m^%1VpN;!J!bC+r9A#+hV$ z-#qPZlH*I6h|iWTfD|;WU(-mF#cZT|LOAc^qouD>gIDL$o~y93Xph_+Sql@0vzM2; zIs2gzf8M=Q&+_Dl2a8xUp_EIoS9g-=ux<0GZ`sX@ze%G zuP!0#)Q{IDXNdCN>t-0^G~l6+!h)f~wdLfr?9A+~$s5lQ!$l~%+JvEyPiZfDG$xp? z%!<9@PRhPhl}vmk0UUjH&1#p1Q%O_JOc}xMyeVp)k2){BFl^0+6&Va%10h4Z80{pF zay^mDM|)K3#Tt!$U7LMI(IEI?gA0iIb&ZdNyZnNBd-yKGD^Zky6?aj0pRf+WE~ctA zDZR!c23awxC3E#)mgY3^AuGyP*uj+Ah0m%k4bQJYER#z#6nc{%rLA_ql-ve#C2{~~ zikQN7P96kRc=qg*f@@IHkdG|x<6iEObHP+_TGHts$ncgc(imSn<#|$B`e3K>kCgB_ z6G)=Dr@B#(MZbP0QbgxD_LSmPt0aj$hj@k_VBEUIxDpgjA-Dtp$;Wt0&8My!_uNF{ zWyfzH68$JhsbJT>uKux^==R^~_6HC#`4=rhrd~u4f9hh1&tbD7k#L{6g}e7rNCsz^ z*5l4cP$P$!q7{Af9N@I?;o~Getic+MT82ZK8UeV~8rVvPp}dH^g}{%^fo(U_=NjDZ z<>)2tU4BaZ(3;Xj_j%`HTTY%sv{$25yjBO(-1^MZ$l^B`D~ohc5lUnJf%#2LABL9g zFY$Tx3?)CPCFVT3+^(Ya`&}0TXXqJGE}-oFsUm~}n2$6_i-hjp7;8TDzR6 z+aqeL*qg;~X{=Q4G}iMVGX86kEc^Eb&N)%x(F#oN# zzVh5GQf`q{h7Pymy{e^cHyup(rwgAqNFK66&2%~Q?MysnTlq?s?IK0)5Z$tQ2(=n# zEq2TkakYG!_Ngav2(#FSG~g`9gEt~SuK8%HeIEO7@~u$Isi=M@obC+N0N{aKjFy-_ zmiIBp9UVrDHUG{gHE;e<(j+YC@r#H_Z_AeO@K0QYBf629{L@+@niSZwQ0rLheUB{a z)*DLwCwU9{a#Uv4+@FSOZ9VLKq|EFzMD3?wm0onEQIKxC#Pc?jb)0-hs8B=oC;npo4W@f6Nn&M3FL=XlSLsgTP=8Lv zLP$EVA{y05EQg*xqn5ZQjUwPh1GNZ%#cH=AizoT6;hdUbV+$;?&Up@df4azJk} zNB{U37pDW3<<=LpTJ{-H6Lf5T zpqB+S_P@p?@JdvFIPA7zXid`QYD2q=mKZ*7@os@1E{;GSuE>%o?^?PJLY!ltJ`gkh z?yZ>) z9+9P$FH_>KrY}B$hrYlOIZI66su}0tTBY!@LG4l-%aryQ(M{UpCW9?OHqhSk_)1l0=N z$B2TuzOM|iVc}7a*xD&0im7h&2tEmFk$7plGT@sTDKh`Q{}Dk(*zdDawOV8bM|T;? z3gn6kJ^yK}j~<1rSFtrmiI}g9d6Ggg4j%)O{s>JaKhvf-^Rbz$zEL*nRE%XiPicyh zRWH@aioJ$ZyXsLgGM|MstevW#Ep8) z;~>fbU(%;-6q5>K``+VfRRYhj-un9av;st0%GH{#SS2GpPEz&O*f$X}kY=X=NxWT% zGuGN|l?m0Y-h|g7lDiFoajN@rlBXZ>#qoe`gdfsygQNBktbKYQFkK>{w%yE$f&5szc_#5TEGVh zQ1QwCNw=+mTY78F0ULnL2dVM(&0>Eg`>9P2udeV1hL2B9Mj#v94ubouNz~T>ZQ;$Z z{885+VcbBqfPAB2<$>3OelcQ@5&`p!fE_+2i(wlbU4i@i{k?*4hJavPI^Y}wGz0hj zKR*C$L{R)YfYCqDG>++jxb|)j0RcDyev3Z2KGX;yp4{li27pX&0Rlbw^Hu<;!g0{^ zXiEpquWqLP00_|dAqcq2)_uCiQ9&4jGxm~w+2Vr4C9Z(@-OH4ERGCTL%W@4;=0{{|+58u~6 zI13Bl2n`Jk64C~!`uS)NKrw6;6yX2SQ`mF^yEqhDv=tJTvkNv;r$eoF64beJ9hK;WPN zZrUrFPwX2Y#NM}|X5hA;6-M7MWC+J@7{E=~BanLkURtlMiaLBB=!eM%G$df#)UU_@ z0Bw}tTTh*_>HvnxWA3mQy^aFSooz*LNSFP1AT^AZN z+7sTG7ao*LJ=-sdkkuJp)#pFaYu~3U_K*MT^ZvghVp%_aK!DUoyt^N1?SCh?+kXIW zJDon!9=2AW-(Y_)b*Z)Y?ml0aZm-*Wn|}9p=|uj8Sy%^fFo8D0!@8^B%z6L&DcXuQ z@==Gm#(zHmg&Vc+ITt#;IXyI;tlz*mZ=4`;WYXd1SX8UFWQ;*eht~7MU*VNsb-&AY zke{^CP|6gO+s-ENixA3Env>mU#XmkHn~;w{?(d9eCLg@pz?IO{bhBdG>oKjk$%=6@ zBBo41QiiftwX*Qi%c^+@7F56ej7)JV^vIpz{2f;+sY3ZA|6R5}j3aJV)-c3yjj4A zjXf`Dzq$Cb(8SwtiNPDO$LBoeEU8s-G>9OfVC_<0!coHv&%b{uhVKO0>DEG#Clu%iv5Z{@Gi=TU1E84a?aII6RM^->5ijp{d!W~zOGWZO++fk zE;RXZ)zY4XpoS&Q*8dpO|Tsh*g{z~m) zPwOmr?c}5%(4Uvj8k~iaUtb=3gKZoWG1XTzM zlIU^^GZx1FE${VsB;z^ga(wVB=2A(2z1w&^w72`Y51Nf`z9xjJLoX7bnapYZZO_q# zym);0dy4$05bzigi}qoCK+BEBvBINh|N2aHz-LHs9*?q-c$!XmDx(-QGdcXB50+@~ z?=&bWgW9cp%_Pkfvk$BUdL0-vv#t&&Hk2IiQ~$~I0E`FTkE}u4x>AGq6ahzbj$M;{ zw{cVxS4{Pd6q4Hsk01<&x4Gdtywa50*eyt1~w19_=WWq&B zG~3R+;Lig%*x_ueL|!5cl=VGXfou{yW|OL-Al^@Jyjax>|0V&e)O2T$0}Q`(^| z!Kf=l#8<$a;juR_E0D1Zwk?XGC}fEUdAWE|^WZs5c62k0GBQsFkaxa>SFIkJ4j{1P z#Qu6+(KECBp&j$TWs}C!YH&_K#SD5y8Y~d)(NLSX==(n<+yv_eWELiJl90B+iSC;=QG{Thm)@DWKwo!y#)9khCcO2^nn>yXE7n6@`axp zYZQu=s&}&hC+obLSRi&ycz>wH-X{3s?D#mA=qQ&say$fbNPBK8#oOc{bii~a*b>NV zM^)Q@0W|||O@fpM!D{{ZQ$$H0PRc%XC{@hP;R}ViiX;yOYV`g}lkAh#FAztW;^(t{ z9!{o`+TOWQdHz-Eu)s4M=;>A(R0J|jB2gA9jB)14X^Xwp<=LNkhG46vxZFCRG-CUH zVQ|&MMwXh3k?#rgS3Cr3bHpo9Clrm3;Gz_cPcF@+)$kTSbA-brT(5CM-uR{UC%+t< zIgO79%uu4yVnBfZtp44DKErD5ur%CY7-L)aR;OEKt*qc&#!@XaP~#-~2)0KIF^{Nr z>jvdsv1ba~LGiPuiszXpu5{*}Rt`kyk-j2N8HcHb{$ePcZx} zGEGxLR*UcME5=u(fckQ!&|FT#TNZ#Hq2M8z>lHH)O~}wIdtdwU4>#Yw!qf%5vM8NC zDA^Ia)@iKA`GWYk2+0U6pRJx*=>{tPVh!x$gSO>JL%X({e9D4=(DzS8Ka6bFW zRk@+bmEPq9mVTW z$FZW{UeZc~=4!g%13#zTc+`V9&%#?(Bc563iOc09CRa@zj!drmr>TrIctMewhxt-qV-2C2zK^bsi$b$P< zC!(;Us-K!Z%{+)xJr~+>+#npc4YzxlDjt{c#lr zhL>hLp@V}-aD|h-m;nRV#v7~4e9YwOZ6WHq`e(&hi~}dC+Kd83yp;u+{tjF}XJk|P z=p^}4_~NO+8*bB$_LJ|y@`1vxQjs#8?@c>+fmi6M-#Prru+zbult9Yjt}u8fS@UyD zOq^Q99?ZYw^aC8Oza5ADalXBsO(@s!&m#3}NMtV?6PB~md~qZJ;;{{4{%_6RlSq?* zm0_YtReDWD2|f=x${P2p$Z&YeUOY#G=jJxuG@O*OcZM>)N~ReV<4A+2p}LuD4Cbm{0)o1A|klD5YtfL0oJ?3AHjlZ8aNhCo*l@0xeaTX-*%h zJ^wX|j44%tDi=_3>&vYCN%+74_AFi8iKGkvm(^PCPw*-hjL_QcYI0v5ym!372etl$D3gV!+fMo<0Z9Nb`$lULY=jm z4>m|>@_Fotk{G9Iv(=1EcU;&8;b*hTtVh(7?p`jjtLxO51cxpt76Ry^lD0`Hsm7&{ zZ@Pv@?y~X#0em9kM-8spRnn%YCRA`FS7pmrFZiKv85K0zB$@%ES2bpzhSu!Po#$?0 z^>4{+1j!p!m)3IHu1oW&r2 z{KGB{NUrT<7;;D)+xKEqMY2iSlS>lQ3Ra;tGE|oS#hvkz))CPkTulmgjYy+u3sAs7$wF+bn z#g!QyY&za`&T%I#JvhLjW$xGFa*`bER73Bb>2^vdO5et_2Mz8Jj*gGgNj-Dk#eb`$ zX443?YWqFm+8jDruDuR;gNv!(rBya)@|5(XZeX;WOTL(PrJd%On8tCdnjfX8gSPuC zxx{CEs+MXi1XS9-V*rBY%w0gfOnRe zJ*4fzzSM>)xxhu#*k)EhCnltG(RLG+0It(h!hHg&iuj_J+QxYrBf1~TW(U%>CiPNE z9x>xc$x1I8MsB~1c_EdoF;BNWTxCnWu~j1ZFc5;yJ`F-E1lyeI%#&72qNoi*R~#^V zAV~pMbW2Ey>%*&Eq~)>l%i}pP>OEYXl8ff2IjPX3y|127un}*|m|uBCl>rQ1CMn7H zbFC=W5`CRd)X8%z-(9R15!z0o&(t;Z?==HXh;VZ z%R`%V0aeJtoq#f5F~^O*`EVVE(wNGdm9hwG_*!j{cK^s+=dH@cPFQ!Rxj$2eH_wdJ ziuR0J)(xx_3?K0GCE7=8$)d2jMM?nz98F8*PQ{TP)_Cnc8J;&=I1A?@8Ak*&5Prt_ zcJ5Qc`U4>alMzvS?o(J^F1qvQgU(#IX8>mHq$kTpn$eZY*1a}3Zo)d_<%x9v^KE4? zpE-5fn6JmfNB29=GO{NWX@sGVjI6SQo-qrifj19B&*)+1jo&QbMjF=JHSKRj?o!-s z8$zw^-2ri}4OaItjnx*}>Qy>|=J$5Xulc;^_+OSex4k~oPc@f#rjRqfop>7lduBR( z<2Sc+UblzSijuH3yw&Oz8&Fr})UGN^i%3ls%z8t@vPRS>D;F9ltOj=oMtyi(#*!ez z5@)qGz$^LN!4)$-7a%h0-~MK0vxL!DWzJ6zU2ld*rn;o^UPh&3T&=RwRr7kV*mp`Z zd&GdC;rK*Mzmi4`k^QlWn6lPS0APMb${?}^(ZXI{7lo|1F)H9y&IXh1J&Q`aCSPQiIictOC=KgC5mxq)4}q!9uGLn zIA^jgu_J8XDIlw%(^v1qiKG*YI4?&e$zklR|h zxUZ}PX18&*9^wPZjZVvYa$1p}uLG@vz;5?o{*Dg?*`G1r9k;>2+WQvmn1KVq(ZF() zjG7hkWR$YQep!NPt!nAC-LZvp;}0|tJJV~3zb|VJb$161&q%M~O@X^{=scxsK7xu@ z<$oi@-jVpu(uq)9%lhi5JQ#f+!pxe68)#-+QYaE7ZAmgUCs>iT7Sq^QgVABJz>L4x zN1|)$ltb7_4(8OP%9JS}0%^Mq3{y0Ka*31%b**x@)sx#A)#O_HO#TqM5^v$$fU!^| zeM3KZqI4~bI}`7i7LwkAj!Is`i5*LqJx&={DFo~vo!M0v!#u%;H*Q-)`7yR@G2yO% zklp9%PzX!&O^2I>Hrc3^F%*NWy7574CEQ?|HjmZ9C;W%*Yws1B?GGwdNX1T~$J` zGLEVxY5P84vC?35zAnI8CL8SSocut!(V8H% z6?0UVm`yp-1!f;(71`npQAaen%2Z`(nzX}LqUjif!OsoRgLx zrfZ!3xwA$-z=R?3@jh=>-w zO4r%uNG%mg&1X>db7zu0c+=-hB>4fm;>1ApamR&X#u}A7A~j#iMqBIl zh~4}FJ_y1=UTP{pl?Ar{d^sz>MwTb4S70_NSXp}Eo^|53xp=d1too9sZ<6cp7IX{F z$H08vfTs-Ia(as!$7Hsxntz3R*`uee1bnFrjfU^V>eyot$^7t1LKmvING7wMS!M%GumOvz*m<6*l!^*Net>KjsAvoE^xFEW{ z*M$j{c+m5<6Jqw*%wq1XHNYSWqqIv2T6KBQ8f5^>V^avipI2Az^IZKn)MYuhmHz|2X&aqz*xoJ(aNJSnxHy|y^qn$I zJYK31YWaz5q(4avljC$i*c~vt%QZq$<0pU&yItHJuZ(o*MH>GVkgt8$^gf z(<8t!CNzgGii{TFG>6VKQ7(bQ5XLNH2%Jh+hy+N*jvF!Z+hX|}DsCk`v2Ghcjwszl zC>Ct#35>jHfuS1lC^6d!rBo;eVx-t>d#M1zqscYjAQL#a8FxO^Na_D&l7248G9Ib| zi@CQ6t#8Te*|M)PutT!CEhnBJQ!UX%Hp&HXlN+sA5IcASDP_vYiI&Wx*-G-ohxs_q zktL~1NPXx(10I%Q1z$V(5_~9)Fpg22?j(@ZdihP&->%25w`Av4GKxxu~ z7o#A@d_jwk!a#9=KuocmpSz3jgs$}+1&k#0hk^=1YkZkJ>*`c>^-jZi z6GI#q@JC@G`F64VG-Sp5omK?I_?bOH-F)e=E^I!q$^bm9ix9xd(KF1=25=F&`WI2V zB`Yy@W7K2vHOMUC7L-n*aiM+qN8T2s(51{7?MTE&Gt+0_FZ|rmj0I{o(-xrB!_%0H zPJ`v1gEZ64F_SwEMyVA0$LA73%bq(FVrd034Q|txFG;JdHdt*%lG+tV~OxWBtKooGcY&chgwW_2MfkREQB69;2Mqq_Buo-4ywr#CaNxGFGF z&8?u%8LtqVLEd%?qse6C2iPgs($kt39>}_O_hqePxFj@G1-`HK?wOifu9j)*XZ72^ zF7{Y%G}5RteNGqA%nmJr9t|{tvGnIn_v5QIEMt|zG^$Cf$hsjga1lq%J= zp+r8&6muGrdx7PnCxgWMqmZ$19K$g2ipv^U&r2MuP`;iAQ*8c`2MWaK;+BO5KiVWg z^A-zHc)LeEJp~3&BgF0)MDXb;bqQf<5=54PYpe^+QB7en=1R9wNMVnfzMMpS1oyv@ z02a2Nq7D#B28sTBMX0wH139 z8zwP^FPSmT+N?eT3Q72DdmjzEn-w-{)ED*;`Oc%lVHNhN6TLl6o$i(`v7LSs*$Oo< ziiJ~YUx?Qu#!d|ji_1CBXt<`qa1TgHRMs)wc+6h>l_G%SWAYiyFAtbX_*1gKw=UyG zJ%BeWIpjtE#M1Zn)}CtYd|A_B>O8o@Ss&i`Ee!uyPi3Q2LIx3zmn1WB>cm%Q(AGgm^SEMX zo3LJG^Z$WYM4s4_I>3*-yyc0Au5jbgG1oZwGYgJD5Wg^EyA0>4d$zPl_b^|00uimA zF@->WKf|bwwh1<1lwr15_33lhZvv;b&8N6@nK4Khvb*5jCB9gai+k^z23)p0yq})F zD(e)HW(W?RSNupF^OP^@MRTR90gLMHDfh7dWCT{Ed#&2B$dL=Pq+L+%StWj9aF0og zs#~GWkc4v#oos32ui~V8`wOx?@0$vyj(`M&BM4VIk3x%n7oKHPNV3+_OdryefSi}Ah@`VFnhnH# zLn}61*D>}u-}hLRWY zcFTxt5+oHPhfE2;aI9SEU*$?{oEn`j_k?!UL0A&?vaDGAMp&4M3Fhr~Qczz@D50Ot z7%D4?V&)f7tny@gnBqyHDvrVn&tkTPs>9>8m=2Fc*Lg~0-(WlQ?);mR)2$iDGdvzV z4$9Ubyl;GGLR&}UBpJ<)geHEFwusGHGwnx{5(L@JJX|WRz#~1DTfPh_1>`1KJP_yR=3Jnh#ARURcYUw4Ef`R%SPVVZ63 zRV{=i#jvLQV84L58^g03J0%_5Xx9&E02G~G)jWJN2kKK#+MV9q)z2{;x!ivC`{I6Z zU9Mr)0jK^G&&>9y?>)k~hx5Mth0^&Jrs|*BGqzo3K|Tzv;Rud|gAe#w`>_4@j7&DS zMUjP7b=NsnW?qLnj#xphRfO7X-00nJ%m7JZge4yr)O$yI{L=L1<82wur2Vco7tOT0*8lH9YR zd9))stc9Cj*RDw5lENHpKnog0A^V6gu=36-Cxu6^>N>`T1DWE&T!k~tf1QdBud?R! zVinVqy%Oa0N{|5I!uY*AlKXX6zV!$eB(9Mydz%<)db^&I-%rFH%yZTDg$#rHraRHU z*))|cB5UccY6xJTYtKQt0tRNM8@{P*f>NCx-;8LV-y==D%&Dk-H=}CrBNK@4jG2bo za_Eg>#Vd}|rIthS$-78q+7-6E&dCxnH0sS!q>^u{tGzEKmI>N9IfAV|K`htRU)11_ zrW3Q%fnVN{`fiJnBW7mp)zLZc6-%`Y*;(L}mbgt#7SDIR!hpmKLrzCg%2pmakxn(i z+?hwT=J3j{rA#!V8p_)C$y@8xYLnSbnrWOIWWmapR_ZLiu0VOwy!BPo9o^%|HCwnN@TT^h;%A)#L|Uq`xkh6hio|I%Fs9t{U7qV{K4NEe1~E zd2IR(k!<9PVz6$-bFbSw*uIDpGx7qf5h~cDt8(QD15`QNSDzdw&P-t?7#3Fhwdgyj z9un3c9gq+B^Y9(}-OraJ4dO8h$AZg@wv;5m`&b5wmW8OI+JZ7C73D|Cm*Q;#;>LL2 z(36{K8nL$ac%xWd3dYUcX-dBd*0Z(xeZJ(lSGWk$Y5ad z|K2d}wPW;XvRg)Gy_4qme@>^>usImH4-w+u^M#!}l$Me@TCQxk&93S)9>#=r_Xr)o zLeMy#t~R=};EJ8uLmbnf(=uQ2POfjMaB&c8R~p~(n2FoifvmLt z0+*a2o&8UskcFQ9KLLfzEdP(?mX(=}?LPs9Y|N}2|CcAkhoTd;uy!_a#HSOrHgGl( zHZig@Hi6>hg>rIsG%>J&a^Hw{2UkwqTwo;xBJ6epTCf6I@Vf1OMFav71R{2h(=C1_ zg;@9(?OpiqdO96}%oc$V5M}s!OHMc4r$4`%yR8kX4>x8x={$|6>Dg6v43`)W<6MN; z_>>Urk#IwBNdV-QSQ$b70QmCZco4`#Mn+751^RKl4&1Cr;b@4F{!(AyLi5muv4TdD zFXYS$Sb*T>?t=jM1OVc~K*W@QAV44i{P(>P5Ok3K3Egl@2zk?RbNqb-;@qqv!C}2< zOY5K^JEyl|0Ay*<0ElR4$KTg~;@raA1^Qp?{abh@Ko4Se5Uk;V(FyzHAtrwD2-u6Y z<;me70Ri>&^nAVCi1`L4HA17%_Mk=D0buaO!&Bjg5bi3C{bRbhj%pX)?$-cBY5!@! zg9rv7kF9P&0rLRSkzoRe=4p5A#lq15y1Dw~b)fSu0Rw&stiB2EfPOf!0mAW~YVG|z zexX1@eqCF@gAi_TU+n$Jw=niVLIeOhC^f&x`H<=QaBtqC%;AIx=YC#zU*yxefiv#7 zT)g}vyLkI>b@uh13Ho*A;gA#XAVa<@6!z56pBShHaZnI$Zs6IWB3`NZNXpPa{_?u{ z1@vZIf{DEi0RQT82<*qm_DG5UHXpDJ$|@g@IFW#FkHe&WWKG-WJdw+JJefQs(m{0;)zm>BeFM)4>6#k|!et^B&)!v+{c_=60K7*g%l3&|;|MH0V zmIm5cJ(AzavONDgV2}sMNZ_|25q|)92x1_36cjL!U#6HGo~K&wpFuU8n_z(OUkNlH zS-;Y&$5b%(Urcz~K0c#!{JDt`0QO%RXR17U_`klu-?_&>o5#PU4|)neYWcrzqLe%I zYj#|xcmuyit!`lhU%!TV$(PZBasjMBJkWQ)B^iQ0e`{EW5H7BNc2!-HcswzW_GipR z9s_XifbqWi^Tkoh@P-uNB7r);6sGaE-d4N%^X$WMfgrzK2J^c8`+ghp!(gsLdqm>Q zEq+)9=9r#leUkU}gAZ#BkdRRU*StK|KM(W0qIDsGfdtOVorB-LQJMJt@nZP%bpXyV zg8*eA=(_ZxQ4oP3^19)B{5C>bzC#5D0068NQrjvMqTw-CzU=PZ{m*X zYTZUlG2s105v`}=hIF1E7+1=^SFUraD#V>!(W;8f$$=|4NoWZv^`r%8Go z)8qHsp>IyFbyg(olqXO`oAEc4n?I=Cp4uiYl(3oqkOMJDN8nAno{}jfq-2UG!{@mr z`|KiiJt4nl$$;Qi)eL92&Z`4%x54=5Cv9Wff1-K(@V{^$qWG{8SO+fZPEsk9gZm2npajqZ-eBdI1r?oso219 z5HE)^{H!0HXCzBHwYG>6(74Ik*z=;*B~53wd-Ue;0256s)BIvRu^ef7W;`Nh_TL%Ro*!v~YQ$lH3%Bq?ctV z!HPgg$x5g}zp&9iHs5YVi$H?MDJ3m_;;K9>->Xgaa1!C{-sxPTPJXvPv2;&E@+~($ zB$N62^x3;zFX3>1-OnKm1Y6E#zdO7X?~r=FzJ z+g#-+>5b_8az^9wl#uEDL>*dPTXa`?tmJx;4Tc9*NaBfuu7nr%$=>+pt-RTnZ32O&cTGc7N;ixa+ zS;322;?Yed_V5w#@tdI=6KkQEZ};5#^?Q8mG#tOF02G#&3M(7 z+l8&+yMk1NER`-|rU_}wwwQ`BF2sSk<4SPU0L@hzOS^ zN>*Pu5L?OAP(nk~y5kSqRwl^Ks08fYYHJPXp4-KffN!WF4&FMx=5t)mw@I!cD@(pv)#jpU-Os?{nm>o_@PjxBGxv|&Z!+z#DT|7irSF3Jdk-@G6K$J-5TH|cK01!c{50GPL-VV&{QfVM7!zmyXDun`*TLF;+PfdHV; z$UvtwiP77tbH&)h%8ej*G)dXr=BrJ7I1g?k0u9ap{ls>-@b-(|#}N5PTjMNS%)aRE)+&p_IC}Sja6e>axCr$Y+Icl)UxB z@VX+Q`&E+*qCiiu23BruvFb~U(&P5Jg%*L@j2iZ=U`_~1UAy1Op1Ycc!Ve8Mqt-PW z{{W&z(kC}fwVjV5&3Kg_x?*t0Ojj$!r( zzu1#?PPy0FLYkbVfg8{F@{b#jKQuO0_vg+T!!EDYQ8B4J8=HD$CR_)hAS-d4HoSFuYbdStx0--?)S(cJoRX|Idc zM^q93*UBMqDMC0FTr1(Dw;?nRBZqX69xvdb5@}&YA+XGhe`rN=yuNZA@A(c2ZtlI_C?W3!Sxb z?_$UMV}L!}FHoWn{eGV*LcCp)i&1<0b{F3>RQ7^c__2Xhq^`85qIgfb_2?}w4MmHC zjn1!Jtua$KeSq=I{o2nvSkn^M6T?URZ?E^a?0W8Z}IZc>ou_V^xn8y%uz3d ziP5o+*iY#d;=4ZcHFlqEMLO!2JquFtH&OzGPtOuvH8ycuAWyV(vajL*1`0Pzu57GW z|It+;-lW#}=VGTg%ji7%@ICQ1v#({2kER)Qxa!r&harLwPhiSqCnq8t!@dOSB!PYp zQ*Kf|m~J}I?ckJ>SUI`}C?m_Q&HV#$BJh8N%PURss4j+{U37fNgi7`Kvh~ZQeP#|% zagSpLwWcISeJt#%GX9wa3DrPd1kn*Im1M@nfqd&p2;nI>o3d<=47zE8czA zHr8+P;?7HGd#?Et9*V+f>MeZZcv>3__Se!Q_mUN^iQSa@?HD^FC0z@4=4TzLGLuTJ zX>Fq(6|ePRE?)Jz!x&-C4ihvqJQFbZK&{VXa3^(@ou+6#zLbU|A8S0wcg9LKz@idn z9)paA1!XOsuWA?7Sc^Xn1I71WU>sZEci61YhKU@3ueZaZBm6i5!$BX0*Rtea_a%@#S4%h;?GpeU!A^`V+#Z^qcH!cBNm zgL0DxjJ!vK0m^o`J0oXDBNm@KdH%rX=obq*AKm_KQb>Q9rLK%F9ztjL@2wC>(|iUK zojzmR?bYDVlTm z8iIG^TWt!8=t(cK;vjc>3l7Y@Tbrj|I01Zsl8CUS?zj;8QE8H#$P6m8vz|IF{j*DJ z?~!*n_04r`?2obLsLP0H3uxEOt>*_Ci_yA2XLnqaFWW88zV~=gcfV{TvG7<`S54%V z|E4&1T{4|JXu=yngEo4`ISlI48d?%uY*G(;cYJU}x7t|iw|D?)YWFk_PKbz+U3R>R za&v!}WeynAu8mX@Hw%9^=$97yXyYNoP?p8=NE%2$csT$e=6}Q?rmBT@_you|Uq&l|`*yqdM6sT#PC?VWL-QwdeLJaioanh-(XfHdS`;aRgZorZ9UU1=KRYH1R z31ifA9yz_5)-GSIiVCW%({w?Y-JY1pbBBw$I)8kwa_I+mtq&itPZ6r36p~$eXf}g($pCW z^$4~oH2NsUDrjdL#741w=4~Q45rCT;_hdmGWM89&_{&r)W2>I6O$~^iwG7tB6-GHP zZj3CD6HYtcs`!DRl`ATo%AWxp z8OGpCBxeGFbsu${UGil`92a8j)MBFXhJOL$VEafvfRL}>-6J>kG4d=a{LJHfQ#c^E z(61IoKW-mBKBA_~BugW)dcW3zrV3kdJ2f(P)AjaRLGYb@`A_wfT~q`rVnDS*XBul{ zr8X>hyN-0!h^!8Rg|zi)c%OIDt5&1CnOr5UTUKtvullmFd*oCyxFT#t3k1>s)3m_lCLr>ZQitm~K%CUQle}@k?wY7kihhwnM<#C^CqS zI50gdrm(6>2siMUg9Jb@F5Pe}|1Z-B(2ju4Vwi1MR{`!S^ecq!y(0vg3^X=GPEM)3v z!{E@&B!Pv?vC+FcMkK{b$VdnB4uPPmfQakg1_YLl0!DqClI_h2$qEnp z(Me2p!WG-FPMuzT`I zR-AJy{rw|O(l9W1ja=e3S8hO9emmPka9f=vL3qJr{8H#qJckK$i~G{6EU9h~#OkUJ ztz*p{RrhECWdr2dxN41u@E#VKGxaN7_B`qy9@!sQ+EW~oYzfMh6S8xaVi@o9eU&z{ z1?*k8>yD{6FKO>EyYG@Q6|^5|{)z@y5&k@qrSnFQ*B;v2U)6YhZ7Ob7aCJC!km#K$ zy1ErToUh$#vn7VkmHn#IvwOK&*RYe0oW+|Xhh->GgJNf7^}+6Z6Lu=iF&ONlx`#=`ERm03=!a5sP>-j zlS>0CMl%H+*t%p(cmliX_rOK1NR_1Uk9c(>LcNR0=S0E2c2LkLf)jc$+tqn(IBsSk zfI0o<2P$P5y&s8`Kb@#{?crRH&AilK{B2*A+A(11p&8!;(xPZT(dT9|rJUbZ&H%`~ zJhh$9;T!gsmYERBO2DO_;}gs7gZX&1sTdpN7_JYjn|kT&*zUnziI3S+aVt6mZK+kR zh?Ju+G1sBm0^C@8UC6{J-rFqP&{%eVA&i)dR5vEfFStndVsCfidHlsFP+C*Hzpg{( zHVWTRkV(aeg||p-8YILVdVcM98E@LqGjfN_8EyH(hVPFum7iiQahBGq?2mbf;A3(ylBo4L>ubDMU|aJ*TH1N`7aDptQXINSC$!H^AT6YYpZbel z>Lp{!p=(*{khoFr^Y=3@6aorA+14|PRrpSjx7jt6&FS9T1oLDa)b@2}AIxi;)k61^ z7hKCj_R8*?`#Z(h$M4%{4Vwx8s?@p1N|tqLhW|Yt<47+_OE2;k(_8*$+x{YD@_8Wy z$G1H;kPT<#eB0jlyO=(Ylmixik~8H_)ef;bfT8i7#1(ftzB0o7odY>_rXJn0c+}H! z8@kXJ`<_TjW(+L!`JWjgSmR{GEP|dC;UfRDVrKgrtVK<`R!`q?$2d-O$Ne@~I2`4> zX4r?$HSP}M@EO8h&MPImY_W}#E|!if>QN)L(%>D=PwxRSSBAsF=W$gM+Eo|!dhGp8 zh12|qFKsBzGJ?MEtuQ$g1S>seC}Y>lOGGrmMVzH--=N-$gY&5f3rw?KC7VtRBeC>L z!ttU4QNzGqP#_?+tGsZ}ayk?zRPEEW$#gV2gno{2x!;i=8qOLh1_2KI*5YJDh}DO2 z3Wy5lZtrC*yK)Lf^3WT5OFArk2R6Kk6*V9#GK=mH2)ET;zLG)6lADBGhqehQO6Lt{ zc5s>OcGJB?(ADGe=>o(?$1gt4*Z#}Wnw=<2$EV6>*T~33qw(CQVgz9crpapYgokl z?B3jOkwBD9b^^m}&Owim#owOBWXDiCdh5~SQb|cqcjp;0qKU$J<7xf;|0+jYGe^f) zx56H5d63tvXLknT5nM|=Mhq8xf2)4(mF@j5_%_)18*>t>ghx?zC?O|O1V zsF1~raE#?}e*r{;^vb+B_2o9Z!JHJ~l5Az~9H3BRCq0z^KK8u7iR>zN0J33g4yC|# z+WIDDRhx_&fai{UMLur+{2GC=IC>&^WvbAz=3ps)vPyWS?*14wpFnl?DAEuy7VV1; znc%1{9Iun?V+$=b_6JN;Z503(GDMpk#emo{!BsK*nT0 zSmnVK(z>f}a3>5|SwN6q^%BYrp6N}z+X=*-q-;mb z7kA8MN?y|ex12R(XXbp)G!93gaB8ep=m1F{e2R(e|~Vad_H+8GmCz?jjw>)??wLn8xb&o|-+${&>zz7M%v=^g9Q z6+rqFSYGH$RQuv;W++)l?{TKu&b(-5Hir;v_$ii}D(YhkFx8f%E1l$VjdHko)AR{P z4BInqBgV3$HV@h=40W8zZ1p=u@(wWU+^snbU;l+quwh&Mj)zNl{;)i1TSbv?9jknK znX`5IP1V!sD2_b*W++xaY3qqP73=bf)qLeJB!m#TOEIx4@{DXi@JH6!JSk^Jv&0v~ z0)oxae+p?C|3gU2$jrq2Ke6mj6)777$Ny%w|A&2HXJusn@0j-gPido`z~r#k=D?s0 z{da8q`*yB=u3rI6KR59GP**Z`q5O8KJ6HXi+5x>B$CA^ZU%hAD8I&d}>t|P+-E1ht z6Be+9B`zcU5?q?5-~dq*9lx}I4o(;*zx4Fz`1EvC{{TUp11q?XSZsemCj4$jZw)?L=(HeY3KVTyM zrD?!P5I=#5>GduoW2+tb{l_tx-&zoO-@xFY$agCj{}A{IBqKuu7&$tJCXjVs6shPE z5DOPF;_T|aub|w$COg;qJtISVdpiT>Dm%j(M+VH?EXW4`Gyq6B4|(<_DOf- z?wc_W;-ixT2q2pB9rE7#tn1I;;>(4Vp0={Og0<@5XEpW91m6;$UqT`0-|A*MuAY(Q zTWWrMb0W^l_Shn%k+l&l>kF4NkzY|5bRVSFZtuAxqdn*<7Z(Gkf7N$I{An%y4y)8k zM#Sps3XpYnHsaUBPoq2}^RMU4z|-weOJmDplk-PBBk%?mmiJ0nMKn)RIL){~7&+08 zZB{*`0lajMS&Y8^$;rv75ojPzAipb8z#m_vGS3x|Zd1)J@cT6u!px4I=SF;sKvGi!DaM5}^`V==H;B9-<4}WVe8L3IY$F}O&l>+w zNHBh{wKKlHr~YJ<3)4$$`?0UvVv|L05>k?qAy?ZZ->MXp7C&1C26_7bHy>2$cBx;9b?3cgesA070w6EjY7s6w?O?!V-%+oo zCh#W6H+U)EGlO5Pf4_S%zNsd@&(prQqlmSus=t>MzyI!hi`g21G(5eug1OmfXZ3UC z=|OCPu76urfPF0QQ36Q|(6WB3QtexTZh;|bI-Y;C{yrdva)8L749t?I{>q8=4QKgU zW3r5+kz1XHdRU$U*3;8B`QqceaY$G5w?!bk5&x68 z7!Lu=$!50zBPvQt+50rlVTR;yym=AI??*+miBk3!$ ztsi#67m>=X=Nq3sa1Cz=UQgjWun*PzLqN}E>gTzdN8KS1QPFP^s~xQm0Ub?w~ge1bbp= z8q@3J+t}z^%E0iRtg-hfVEh3Oal?8i7jg&M|9i0SHFc{Wtm}`jR-bX&r~NN7D{u#} zFCm1Kv~MV&7S```91z=nJY+i^fNujRn(3~-?XRZKP%t&{Hwe(ysJRq z?W{i6ci;BD51A!V8NfFrNXd61?8}moLyH~g*Zx@lBS+`z=io2w!NZv#|HhYh3|Gc_ z#}CuQKC+!R1W1{gOT2yPlG_&uP}hZf*#tdR*}VN;^6z6{MtghE)`iz=HefHVPl)f& z0}%f!e&nEHcP``}y}unhE!!-Fpf<3b`@l>nof_rLusTvPm$b3ub`s2s>Hg<2u7nao zJZ4nuzZY5u{tnUUj7~k;<R?BY9fZJCnc2+}1CbEMdgN+hyzw-?sGnT(IM*%gu>8-BT*Cpvs@={{L98~Laj*Jdcf-?m$25NHscIr869i{#iZ zVdGL%F%6DBH=(0ll{L1yGNE%I+YU9@J3;c2{j&z0fnQVm0G#57MYSQYz(>>2yBKXG?9@mFt$QIm`IRy;c4y~)QE z`eh*!G`GFd#7tJx)HkZZPt5e_h?F9^c0p;Ow3iJX+86Ig+_Yi`6QuNHx4E?))Undr zi^|nYgDwA@qsp=cEVeGVdyc$=^6U56PGB*Rlw0HI=ApJ;y1b;$`JyFU$p=dKbgE(WGfj!KaOoMHwiwBC8Bks;8Q(oFG}4D zIm1!0v^N9h$p($AgT0(^a5k zn{Iwq3E34tdfG64U1wd2nHvaUpTV`X7?1b?m z!09SWJwc{dmW(I+EW_YM9;#-_!&{ws2t}8ZqqNbi&*6|sXw5* z;8n#`z*K)?>>27qLIC0rZ__t%s&FShc zoYAJg=t8IQee&$5IXN38xDQ+1DLs!po#Ubmy;Qa{4$WA&PirDl)t!#k=$6V>YWM0h ztq;iELQ-@N4SpUX-oG|bMoMeW$81+O%lvcB^@%E3jq1BGwwS9U9Q}$$$C}`cr!w=G z&t$RG={NOEFp+a=cMjLN%ieF3AjNTxfI1yI)UoEEKT@{^g139|TU(<;Xk4B95L z5!>a{*1^NYCiN1l_VM^>{_eq{t^8xUAb`yrNqQtbv$jbJFMsS4=??@n9MRaX3Ej9J zqo8~6ym;^B*n#ZLyPCRm+7{IQ^%9ri4`=z}-qs<>Q=@-{Nanm;dk=@|28 zwn;QcOp!V$lzBHg3dL(=QXk-!6r3u}OUJr)XCXKuYQSNNGz$H_I=BBU3%+5n#9wDep#r?q3A+d$gzjXm2`B1KTmZFRDd&L^+1D-xgVlLE~>8HPvwfDXlJ zrw9iD5Qaqr*^8M zw>z%Q;&%*s?N;I()74El_OJmLLx)t29HP1R6WfMJ>DDTavM8}ARh*N=;%oF@l29xV z8=sjs5fLrVoZbqm*E`r+BVm}nOVd=ug3cRGnS40Y8H@r#3UI z!1WitvCQ~&S&OeL`EP#(vgF8eJ;S}yoD^2R%*a9c#U^afZ~MjWN7^Qa{QAiMHkKsn+V4HMWg^QY%$y8;txICR2e+>X%n|An?kt&C9u_pp%~ zSf~-f!;}QK9v`5HcS`Q{fIs-W_$2KYAN-Uio?nflHQu-~J0-vMRwilijU!9h#SMS1 z)GbD`UQ9df0XLwA#fB?vgv}2Y+8D4zN1|bJsu@i8m8$k_-tnm;e3FHG@oD+B*PjMQ zyTI#MRiLgPoV*}MC^wASrbk%2p}`wcF=`5USW zEg-(wXq!77lL?`yoYQG6a4;2r4uCoW(^d$>+A{)sk{2D~9Zc;uGt+#fOk>*{vgKJT zA>h68MfDxj9vIN%rzes>Xt89c0Ny1_&2YtGd-z(h^a6Wusxs3%=Jd= zwjn6*PDRxYPREYh_%OVq@4FzzWq zuq4moeMe7>)^ER!c~k+7Q;x1+gVF`VxD##tf_upm3XM86;*GL>I>D~B+d2ZP2e^}* zE-5`~lukNv&X{b_S60x{&BwzJYDZkQ54r9!t)?wZa7T&^v3-#{@fM9R6rp9ES_u2s z&AQJvkp%qfIapN#>l-BPdtpL-lFO5*Iuvyn{u`yHU)ChH0@EvQXYUiwXJ~ggjW8s*@cPkJ>!AL(lH%n$~ZCS=-xR z^~Sl@Edvu+p7nw}NgcYLYLDJ8Xe+^AWVox=g3@~PvWhRnCdt`W!o@~0Fp)Z`_!_Gh z3NU*Qgu@bc;->iH>{~vnE)5Qw;aRj^A&*Ro&c*{M}%VZsrw!JkMy9Pel`W0)}H zufbywMeY~86Wr~BG|1$ZnRp=B!Ec8l=(}tL67R@;0zj?DZ(a_)?Y6@e=u6M5OQi&= z>g*A{jQF_6y0?t<7H4nKz>5u+c?lRJCdZMeMPYp=9UA@d-`;_Y0L?@!^R8@&Zk=lf zrn<1^&>obwrEfO?w{4M~EN9aEbAm6G;n~cjq;iE;hu*O|X13`R8G7#-@u|q!ZfBhw zi7|8*sANHTz3W+Z7oxCzRf00#lT(@Ai())JlY-;4`eAd`MfAB@uS~W!m0jxWydJ_Q z#(E%eg}qFt73kT0ANv&t*Mm4Xb|1C=BKoa8z2ADb+^;vq z-rY+SIO4hZ)K|Sf6JG`QV<->kW=6H;c89=et86>f4?>1vMc9(VXRdY(8szlaJ$mIh zHo?KlCz05&dqiioNqUYaS2b0X34MHBRS1nYFcp6aExz61YV%|PZU#4s6w3#)OeBe= zSFaZi&adFw7+0mNr~bynUcDvKrd*+^DKe_N1F#AH1iuLo{F(cL-QSZ>wm})Vk1=6N z92rl?9({YV5cYAj+&;NaVu!!9XovC#YTlIUHdF-NtpoN2VUOW3FOBXiT}Qp}Cn~9| zP8pM@k{(b_<+;b{9ZKyQdu3wAqjWhh8XwZ$4Ly?qAxVEUq5X)0BCS>PMwdkZR~%*F zb~R!O=z5ah6N9B)eJp?dyWJ1Nr2YJ8pdixz7l%JZA z{&?}};fogYo_HlCy_^_IR0Y8zLWzbbH=m9enwmp@f(J2}oP8BKFG)Q3@aik)>0bTU zmh?XFeGreaeS0jgBZv}8g>~E39~iBL`I1PNHu*5BJnP9jYl4$IE%RO7d&t=pDPlor!czyGMkOOK;Gqx%ov+KKz_QeZ%8hTrfgr0hdAav zQfZz|!|sUzG8kC<01KxKkru>tvryTI0)*!*_PtngM+r z))fAdnHFBW-wMzcbMSPBR`x<9kJV6g&t#l0R!bY@Pbf+V@$Cb{$8_)t+t&Xqru4Gx8l8LQsER$ z3Os>d#`ZJkpS#z_*Hbbhv$i7qR{Fh{SXPz7cQ*ew>8VCZ>ZH6J&Kfjs6W(RO8KC=t z`>Lkf)u}tb$myqH(jDudDRvXr0*Bs3*AA_+FWm1Qw$cd%$Z~c~s1tbWMO##16O#(| z*AA(E?3bTWWRt`}fPwu>k?Y4eIGx5aLt&Ba-Tk6u#DAPMmM|(w#2CBRtaJD7wN7l7 zPs&!)HKdrXVqC;5VlCjjoRs3Wt#OOu$}oaH??B#!0y9y)Ih@yvju{*0juTrwn>Syg z3Y2lGw<&vgchniVI;FxpK#~mGMfk_(JY^QKB+z)(7!z*bW_kZ1owMTt&z%sRuasAY z^T@V2#X&(*mNepaa-F^B{_4=7YOpQEBX0+;JHgrOv%gnV+lIDydtUW>vvp1T6;Wkd z*T&KsfY<6N^e+D%`63Q{dR-#yKuypnN?VY0_~4B9aP%Te#N`5^|86ytuB7R7rt7z* z)!M^x)$2~owRfx^iVP&7MhjVZTppfx`3yjW!lM#-efCyq~55NfM!S!UkZ9 zH}}$`FK-0EI(I3cf_l_>y4OE0>jB{J5zY(mXO(74-bSIue{PG^ElkyC8=1FN3HG{KyXenMx|>=*8HUwh8?$4pb?AHk_4WxXnK zvz4?EiWjL;v-Vt6ay}LamG=CzbP(($Y8AWF#oL8Sy$ht2Ll1;>=3xl``(911k(j#h2%wP&ejI}{P;Z>|HPD4XLo)m7j*iL!6 z(|b^B_v=u^62U_hHb?6*?Xu7wIck`T!RB+E3#CAMpLe0B&%iN9aht? z+X$K-PuevundZUWLc})_jzvR?Gu)!J{jTiEuA^h#HeHWAcjx>5F*EHz=WOsmVu~@i&)a z5P%A6r)Q)jqO)1~;R7DL@tmwe`JPgY>MkFJf02om!&pv104{SP>ehjP56CJIqHQ_g zYN=3TuPYqO@>-4DrK$g}L;z2zA))MJxvdTj8N4RVhGO_voHk7RWPgczA<21r8Paj@t^HySTc<8K320C-`USZv5_#jF?%OD<; zFW*BhQ$)Wt6n5^lh5x29OwA#>Y=`O1E;ND{AD3U_&}_Cq_egQ}J^FWgzy(GG@=m&iSD90sK1OfLjR#sgX&(QFd) zIf9^+b6135dqbw>sG|S~LCLpvMDw$>h$B%rgpO`kT4= ztr!>se&B^NB8qm1oG$+kUO%V1La~1LG>Y!GE^iFWWlDs@%wp5|G7N3P6MoYNzzjtz zP7ehPWTIfs!P%g!xWxt;x{kU%xug0TYQu%fip&(6(m{+9#`Gy&zNZ#*Kl3?jQv97p zA!Xy(`kfXObMp4o^#>M+AfRS{WX~s#F~qsKJN?(dEHR^wSu3KPr5$R>4=2Ym7Yi+3dIG?2v;^z3Eh|=ZZPW<}~ ztG)y>U$nhcMWi+RHMMyxrq`0=Gr%(iig;=@I$Tg6ty2;)F}x^uRNc=yQ2S}5YO%t z%AAs*+#*j;Fp*s*kS|lKL2J-tr(e|lZE{iQQxozN1M4Bew9QjBogC1SlEqcf`t>Ztrzl=mcR{)=@rc8L1Iwa~1@xK~4sD>9 zL#w>6%7BO!?oLLE;b0JW{dbwxFAJ;wiWZ)IZ8g9L1)3)H-qU4I2^lUQy0n)sSlq)H zcMYHQuOPMS3i7RZoM@4-Ju}j}(wuAN$c&K$2RmrqJKGapTelfsgakkHCxK28trSiy}MEc4xNUYTn86PQ&?6jcw2H@SbfY+skP!rkk@bn43nWp|SKuuyy^aO&=wJCTLx`5>+yBUcyDJPrn~gnIv4<=9hOsD6q>PL`=TJ%w zC!rbeY6!*dYG*C&vxL}3Ct_|4=aRIb6R3qHY3T5n{qeB`4WbCc6V%qtwM~WEml?oP z?QQg>0}(4Ub9WaCQf@OE7hq3K?2peNd!K1Fbi=d71-ACik^v%f!18W!3V1~r%tmlX zcieooTpF`?)5<`HW}FYfe)-U4@5s7al0DZ-n)YgKJ$smf4*tvuY$7{O!!o&#?rqI) z(>mP;r`txxZOm#w3x5Vc6EKp z=L7LM(qALbm)>p4u5p%6rO95vn>fw-Yle8FB2opY$+^Th}M#0P!JA*EpoRvMu@2TSfri%0tu0b zq+Ra(E%U4m$1*7-D&7t#`EY`i-73`M7l^Tofk}M8#$}p1@k*7?(lYgl?3My)qA~l% z%!^!wjvjD^>E-rO^R;p%{VBJlcpYDZl!3*celIqtb%v__U~?#_ZSnZ+sE}(Kht`Bm zE4Nt+ilk~e(MiA&p!v0V#KnK=(O2s1$lZf(Kg_88qXxIaK8ltw@x>o>H(FAuWIyrB ziy$gZfZ>nNnTg94Ia{w*QngfKIskqZ+tHKx(d}FkIwVx0AowVKxtej_cqb|HGbx3< zx>3V%&-B2tWL3R|bY5+YwhBSAkvVQOj|awK7!)*$>#*`Bs*8y6{Y#wn-t&%+UgwTr z+T5Dkgf|SmVWy2e%l@2oEOG^P6tnFn(oRLO)q(^Tvjrdgq|51pO-qL&q&hMc)?lB`FQQh~CxlMX>Wc z^Dn}ZU_sE3HM&|oqfzZPa1_((iI=_vn(VXcazgAAkn;jT6%HH}!?>W5=PWbg1Mo!V z$~+ZGfA_i2X_+w%F1@hWnEskZPsB^GYoj^jq?cNY>^0TAym0tDJ2qV^AaasqwA7Z5esi<)d*_cD zytl+@S*;d2A2?3m>hu=*M<=kajLftHi5*;jPR>xJt2i7cLFM{1eOT`<8=F61O){v7 z{Qxss4HpvwoI@Q=UvZ&pJ&J90WrDiNIf1(aV!o@(fc(TKJ=cF94C=oxkp%@F~DPiD4#O(p4L*Zq7<+#Q)KjOdg`EJpFf4xEcy zJ_3eh=VeB`YnN;l9TQ7XhiLrJb0DlQ9f97l4wtZ|z)~n!t?u*kZv%QzGF0qLtdc)6 z4bVXyTDhKg#G*VJSD{F^R+C8gOFl9Ke}rYeRpZZMtQ-sSNp6n51CRW{$WGJ}?UurE zgaLMkN5T!mQ&oi@4`+q*fgTZu{klgbSpp)P5w5y>S9(;X_g3BR)4Z$j^oUwlC)CVW z$$iWsc>@A%Ne3t4xZ0~=i2qN$!#-syF{=TICYYqAn5vo^Oj#3p|Bgp982JORE6On$z`@QY~j7%iC3SEH{pGm zoDf!w!jjl1Hsnv~Vec2IyTzg(KUnv1bWVkf#ipmZMko5oSv8(Ag|$sX5~v{2P#~QF zrz6f3CK?xO+aD*5*K-JEVGg$(q3bjEv>N7Ig@p9gv!2v2xC*^RSia8|jXpdKL~N3# ze}dMR;>mALVMd`X5-zTMNK9egkqLj!c$sf#j!Fk$sG9|W>Mzxj<9$R{-fCrZoQ#>K zq>3P(qo#}>W7u8WAC!q?Y)>R!S~a<|KQS6l>7IsJosWfE0$8cq;gX|9ND<&QNf=md zaZ3|U(rdlW*v+&B7N$dtOvTos80EVdMCm|pI~gbTD(Q+Np7FJ%Rag7G_?$)oivYHo z)s86Ot6g>E1;v5^Y}x=@N-fx4g$on0cxt6e)}cJ6RBJtvCi*lD64Dux!z(x)c82>{ z&tByeuIZ!N^`XhFApC}K@!U&HTL)^MA#Eliz@}|R_1Zc*I3BAt*p&MIzIbIxAL@)4 zS6rjg?|yW+qK;y~9-^D@2cmaMbn-j}$F0=VAe;PQCR62W!e3Q~oEX z+u2FR_GNQpY^aN^)6Vk?S1&NFTd@T64clIpC`@LCE#5OPxE`H_yrY=~$r3yDrx~N7 zy`*sI7+X+=AoXKOQD2#AK5RycvX2Cdlaqssg#!16E1j*t*LQUyh{)+VHi`qj_dT9y5(wRZ)6n9VXBMJCQ~n&=d^V)#gqF> z5`Qj+&pB1O(&cw2QFC-^Sd}j(y2J*5`fS{Ib|y1938eJI(g^}jQd0$s=CZ^9G#cK! z-%fn8LgWPm4cFYC?cv?8x`QUgOqM$qRX-tjXvt@9$`wDaw?H)(6k%+aAn{OT8w3!J7Fn>Zoa+^5rcELB(j_*A*wiK}GlZ7K>F)wu^q@H~d-)>H-Qz4B^Q zAYR88xPpWP42i=__ zfp1pR}XhjOW1w(P;V7tl3}Q zHs)+7@3kk5Dq@mZJ@v!UH_?OUZkmxhAJ+Mnt$orMav-?Wk8sW7Rw_fXaDAoPO*-dZ zDIl_-Wh_k0OiyAMd&FQ^!0=P`f&GN&;Brg~37JLmE|LLmjndx9&LjgdQA|6n^;6WW z+v}PB>bf0w*Dh-H=mg}60Xd4Bmm_7ruGi?%%|$tvV8Gs=vU+v%o^uw~I*e@#{Q1-$ z5%5Hh>rn#&aPX1Nj)xCs=Og~=BmS&JdJ}(Ijc(EC#xS^gzo*PgGQiV0&leQK7qcb! zEuJxo;%Q!5?>+(snN*(g+cbZv$7!P9gaI8?QfIuY+_X^WNU6%*N;DgYx6v;+85Q@g znXglf3$NL2ZE@Zuf-;vTy83bsWsOf22wO}v#_nZUG9J_#8|Ln(t3nM80PIM^iI7Bsv;yY>8&>KvhEE z0%wEJNcA_dpvzphn;rSg_;Qb38eP>KQxI^h#I{Jcue1*=^2dtJuv;d7jIkzIkh%7U z!di9Zm9=hLUVbyEooP!@HreK!1pUW;7%h@*Ynsbvi3emIHz^{Ly$ZuLdaDjlZeM-T zfl|?W)Z~fS7y%y<YBp8MIDnbfk-?o#!GE)HYhj^WEGJb8_!xgdxKbb24a7?& zKS03VO=d+die@^&DXVVzCIfh_4UNLmj%J10E2fWkPLDr0d_Kq>41>Lr2+r8ViZSJ;h+qP}nwr$(oZQHi3*YDn(^UjYi zc_&$aW-_ZXYh@)h*QilrsOniJEKDjvdWP%V!fCP*Ab&UYZk-~vhbfKX=u|a2n)Mh) zAyMn8Bp^M0T%XU$iQlRoadlD9wd;vWm~Oh#BKBr4HaoX|x6TflY*j`uy?1M(qDU=a zB_I*`asf^xU)6%7Vm;b#F^=lULo|$3VG%QOLoyXPJ7hakrEw|lU%5syVcTzF&`-)0 z4^%uW8>F}4!ailan`__MN^#-s%w%T7O|-^G7sAS(({W+*GJbb(8SgsE_c6uEp#pRZEmUF66#U(S9-)n_M(E4}1zU+YH^n?#-&q^~ey> zY7aBr@y0Pnv$a_`(0)|I99^7?bTW{S1uXaLTNoN3@NSp$$X@!Y-PSWcQ@*;fHX(EK zoPPtdUdEn5d$Jb7yOqX_qur7$707UR%rl7Qay{+7wB z=(NIBBXo(YMU*}fG3U?x*YFp*#+l@=XDP6KAU=i zZr{rbYsMI3B>7k89mI+H->g3M?nPDc3HPdL(5TVKKs9cLxii#BoCdWXxT>dn{&eG{ zdy%BU5G?wBS<$$)L+=J1CuOV67{9$`d1Kn>#uYstqrO7buMEPxjf*6UR?=fdS}$yH z)dhmLbRXaJI)otEsM=0V3WF=s+)#1Osdv3u22@+#OQ%6XUfQy`Vv7fC-;x>)KGl8-F#-BB2dO7cBN{V`kir)=`jzA?sY1DC#yPd<|#zcyr#q;T7T}GSc}|? z%r~Y07y-#u*booJI(ZXcwC4P=0O%DbQ+(dyGWLr!-*;SIi-n7*iz1AFBG26*) z*j8vfVW*zFACy<^}F)ISF_^n%<9P&ehKRD`b=4Z$@iz~aL zyU0x1I1B8i0nJ0tEsO(^+x;JHSWd>o-V4qUO7?l-6jL@LVUHQzKW{elNaC(7Wzq{U zgB)?}?P$C!RQ2sL59G0*t)5X;nX`t&;3TnwjtnX%Qp(!#;V6wjZ3lx|`3GAU@A23t zBQn56!k+xV0BE9u^3&NjE-c%s&hr=SKRz;!4I*96tms6^2T`%SP_C;PFq7f15g#7h z^#QEFZFfv?JP+2l;&Hc|m$Kuxy|9u~dd-jlsP_f$nGx0?1_Gv()Q>vtOoGYA`c?Y` z%rbB%b9b-QFYR6Yx2|rJT2ZmmR`luw$QD8Gj+{Kp#Lhjesv_YMNi5OyXHH5sm<@CR znVyal9?Z%N#1HjB>^wGI_fOEuc=#U+M#2Tf1WWUGTCDgQ;-BrS&2MkV&ho#0eUn03 z*Jb4awz|Oxudqk*mgOGiovbm>pN5Fg7%1wLPgxz@qR8y;Yj4HIk$3qO^X0)8#M>l{ zjA}mYiN_hftl8x7V)(#Zg-{s%QOGBQ1`nmwS+oPF3n?d7^6n{piLo*5P>el8Y2x1Asp6OQ)kCnnTnTjH%;%I-te1fh(U zA`Jq>v>VF)HAS}jEBX}mL|viMl3RT=W)hbnPRW$RC2A2xz zMu{Go87APk8d#Ku*{jJHLXB4^xKI!`N@hm5rX!6ZRJf)tN(AhWDgUOnWM(>z-|~`R zPSLs^M!K^vMVn{D$;vxKeKbFiJxKnL;-m56_=pkF2wfgM%`9(H)?Y61c!_&takt-$ zq6_{Z!jUE@i;-1x^z7eauKx&gfH_5yso~*vJr4D(FNm7b?E*ojRdo+>b&jK~!PNMM z0T=sDx)lAO*SeXAA~e!NIa65X5X>h&wsvP@#OLeiubpd=+;v(}?l^YY4HK529K=Jo z=t=p^j)#GZ1W*#5^L{lqh@P*E0$^j7S}Ak*C;E1qMGOp}s_K{}a(l&17j+U@oNzd& zCeAI#+nL}1IaAfg~+=m~VWp8i!pp4FM6!k>rqiz=hpgxbU%3eBoDZtP5qYOBKlpW0N1CyW*GBfn|~PD_Cb%C9@$i90(DC*Os0)g}Fvz z{piv0&{0>YV^up|*$o)xtsKgrw;6Q$U~o2vnu04x`XKIK#r(m9O&oq?s~kL?u!5aA zBJ9x~y^P~F8ZpRLO)1L?SR|QbVfn5(Udy={`k?+)tgajo+DU_f>b&S&XRr5elnq@i z2N(vaV$6I1HvU%h)m^CgfOkrdT?Lai<(%cYsB~eoM>&B*4(NHr=_jMb_5oF=`e&Zw zoy)*P3nN7Q&ta=*=8 zOlh9r(FW!=(!^W!PV1iutUL~t+SNY(*{->ji5axu6GAM~ScQyoLE$W1`89jO@1AVI zrr>im&7R}oDPzf%I$4wSF2OLOnUrX0hTKQU&7E#6zrk0EQ|9eiWZBSsLcR}6NtbeY zaQ_BDFIj|?zb&~AuYxcf0s@^I9<86>%+;Kf2Q!;i@z!5dWBkeW8q-g3k-^|!dn0bI zpl`^0X_2=jGY(q`*!gV^US%)o$Gu?Vd&1iWB!lsiL$jHH>|;T)2)(zHwMBeKN6x86 z_!~2SLuIKPrN<#E9s5?DX>$-o?&xwC5N5~z^uT=L%SP(P-YhW%Mt!%Svd(B+h zMB7_&5vh92PVnq%&!7*J3#J#2zn7n!A{49m{a%h1MP0Wq_$kMAtZvA0G#WEU?X!>mp^Vfd|(}%rKukzHE>5TgXigJ`OIkvxn zkfTJP61|2!>=!c&i?l;ougCD6Re|8{?LY2V!lKq+5q?@0)>3?#K&~kluTCh2>@9Dl z`j$7~DVQK|7_7qtPSvtW+LLHh(nE1z2O~17CBwhPk$4c4UNhCv#M&;^W@Vg2TeEbE zD!ZGut2dglXix<|eDKLC*@D{Dp9VLT638&84)ib6$%F{<6CGv4^OBZzxoW6?AC^=^o(cz z260>W++a|F%^23z5^o%>P;LM?9W&*?y7Ob4cejsA?=mIbeDls+AC2^uI5$fG{;?4U z#wHXo=juH!vZ5oM&cmEHWEWUbAfcAC*k*fyLq@Q4j?2$feaqqqr#T|k5Q(W@r;edw z-JrDC;yv^|GOiY7hU`?jtaWDK@m5L**`J28{&gZVH(q{15q(X`WJc`U1??1el`=JO zHpVFl%1rE6xqT#ZAnG_wJT90QG()MMYF2MR#}!+yeT{mFQ0cfmWkV_Z2WNVl zRkXpX|Hep$E#fM+&Ym#UeyG5m{7@&Xo_ZyJhXS}VB+p0A5or_fiU1KieQ$^9?)iEo z#qCoji@NN~FY(Q%n143x_`5FX#=mEg8LYt-pAxd2waegMITR8dN>SR?HhqBd!2hc2 zok$`Ff_zkq0TExZ-}%qQyUKGg!)H;7FpR_&woalC_KI5k`+Ms&wKjA&XG{1A);B!D zHfyAi<#BRx4NSXF-(mA;|52O;x-uKEb8iOSvt}YZVVA(U8epK%b9VW83%FvT3%U1H zz~hDgA2>s4Z0NN)eFQYHb+o>DH|&SE+gTyFl{&Z_Q%#$cvM?~x_*Z>q3}NXB7okxj z+MkTWk!hGXnl)b=W*XXM?=qq`2Myc`@cUhE;OrWM6bN-?wZ3Gc_v|2&iz|{z3fw`s zb9Fh?azuvxt54UHS9O9{ zcayadUY1bP5NJ>&3Li1wT^`Zv^&LLP`io!V68;$^A}ha30U*8PqXu7Zo6l3q|ELxmf=8<^lTyY(Bv zDsAiD-q`^7mGpJ!Bq6O56{S(^WHGh8wV4|jCT)PjR+0INEW3Yp-I4llW;oOX2WKXGST>znnwj76QvixE&q@O_CT(<9=e> z8+p3gMlyheJa_&rh1|xELd3UMFC^EGDh1DohWyBPcNU)lYe>h0qMuPFxMy4|$bft} z6WVx$82JcxGK-p{UZ?zc9x_{K_$jY;Xf24vuF}g#K?x#OELkiD3hcVlz^caTIi#*` zFH-iv+>vG1GhbEvR53mKnre)yW!hEy$*n!s@!h|>lzriP?AJN1P=0vXH)0lc8+nuKvw^gJ z?8+Q%gNoWPX-c04|2o99>jsw`S=emP9Cm~(Q2@+|i{f4@JS^OHkZ%JD2yU(pR#9>C zkiD6#Jg@Drh2h(CAfakW%gh=hRH3=z9~9S?zK*?HuH(u3LXe&oe))UXt&}P)$>$@e8KNQ|`OvQHf(m(Of6Ez!4JCgj2EQNi9idSDue)uuLiIA%{a# zNp$`V&+t|bZx7Naf@Zik|Jxo#rDtkmy=d*CO$HIi!g-iaM^S7=#QcNFNV@6$65N`a zSh0bicn~h>?Qr*9@U@%W`;oKJ>bcgZB_B-qH8D}2z2MF;D49ESG$Z*FG9dZ zDg`I}y}fP{`Pdza6dLB)Jqq@(EtK2r4-yFY$Lqwjo6ggbLSS#nGaa35H2%e#@%~Mb z{)r2&hg587L>o|5N>b2`l}@FSPLjNRG1dW#*pjIX}QL;*T$aZjYZ;G1OdLV6hy?}H-h z?M**(XQGOe+0xu3Yoa_W<0;0BM3e1?$amy4fO6<5qTVP%5!}hQ95uw4L0DwN;HPQ@ z&?cq@tsTjV7SmxHw&KCW8~n%G>2@e!b3kw zuaiM?d;ClC$DRT+Lt&!nWmC3kUJ6ex_;xik0h0Au{TK8%5>b!GnFk|Y&@CHf%TqaQ zkcVC8D8u*zbnQZ@W;`;M^7=&}RSjj+MwS&j|>rg(tOjoUx(fOEvfwpb`3B<4v}`bPh)#TB5S? zRvB0(<;3Ck%F2D`H)52!n&i#8LoiB>b~VQm>=BE+BC;KF%~NO*R~B$~i=?N7l6yA$ z&#`@=w1OS^79>0&Z`;W?v|ULfi&0~Ee_qd9!Kp~6*b&9f3=w=@bc^b_#0SUHo*h~CHvxxMC#pmWr*lc} zx1!*TqnbEfo-w5I0VcQCNiH@$N47BnfI|s>lB&SX-> z_qZQpU`$Z4E&h||4x21P$48+oO_6e_X(hpzx2vE(NgV$HW)%6HyA|Hc{o02V?g|&) zdp!LW%);+%9rZcw#9AEi&%;X1ycHcC78DwJ7PyH|&Do2ABzZEP~qCdY|5{ge#X zcrR18ja<2%5q~&Aykx3(u#%ohwz8R>dHZbCNBs$$4*Mg?;yuwzqmfqjjdco``-<*n zD5#7-ICD)PrG-ap*np=d%y&^i>1yQ{yrk1Tg!cENe$L( zzNxu{-H0UCcxvK@m2+R`4uzr$ZMGSARsqF6xGG5dU*JW1SZs~}`} z&MkKIaPz>%?B^=XzEZB_rxl+dFgw#J^kqCakoAd-D8_;UXns{<6oiBAF60Z1mOuBl zaw|$*eN2&YtJ8AHWP=ySq6&kGw)|>vpdUy{&&3Q%c7N~awzgqg?2jK}nm%%2)i0|l z2QC}N1r8e6Oa~j(=v3}z6N8JSC^-^_bY9=ZVE+xPU~8rwj|5Fg$9EJow>nJ&T(Eav z0NK9qwR{ES>(jxE$guBhJx-xsQ|+ry1g^5wtvFYyx$|aod2i^7a+JarGdE9qPmw17 zY14(+Qp?qOqdXxC1KcFW*(wko%acB%V#F7d=yJphajGX}94qEcX6W6{Is%|w^}{=I z8me-XaK0j5i})H*9h|6Z|25-_ijH`>!`F`9KA2?RmA#h??8RcU>1D@3#=z&k0m#1- zK=TB5?x#LKLCQ`S%c1>vUFZ7@8QhCnr};V1wTu=Z>*LlJ8)_W!In2a{Guk&Xj)I-l z1>r>-LVtnEmA-~P4A`v;qQP(L=_ZyVRm0ia7)`s3!%!&Xg)2kGRsg#0pynXTb0t;W zI&u4BaSOSQ{@Yk86YJcAGNYA4n(?a9-Z>{9?m3Pkk73Q#RkSImfS8DoO6p123OKWkb&ZOk zablo{W&cJ#;4XJ08c}N#1EEMiWMt}Cy=G1^qS<&^#c(>PAG2x) z?&!hC7|4V41I;)GeIFn*n04z*3G?TpylRC<%+OJy6vy$0fPqO8{7^UyGm^Cy$H0d# z@Bn1R2PFpgQ`2kI>qESPwoxS=C52jb=TEY3r$G>f6-wj>K;0#P5zFht{RW{H1KS-0 z)I@UZivU4viJoeOHS)XZ29$Fs!`tiev;_-*(401d2|1M%Kr(~xN9yBugRf&`PAvqJ zHwIt^>@*J>Q4Fi&4udh+ZCQZdV7m~+2SXI-0mc^U+h>jJdryRa1{s^dW1jo50g7Dp z`NzdinHMBPsEiT^a0>3nk4`sa;>L*yxv5aVI&253Pe4ov+1D@Zrw{)pNY{!0*$Dtb zZv0zMZLBt82LTHTRF41#IWDyTgaC@ym*mQFQIM~x6r z_QSl#xfcRjP*Tua^6VvK&yq7rb6Xa3_)P`;nPo zH_UI{rN%Dtdz@|e<4MvB4_18lMJ+`rtq~vpE^(h$jD#5MoByjfyyxh?_s)~)-;eN} zFPYJ4?5J%yNz3DpVDxkFZPN`Hz42E${TEHxEj;|U^*Mk|Y!zr&@Td1rHA{ZM9>Xk& z+rDoTLIJt}KmIaifk9VGSUpY44mJaq5I}V}vYBHkKm$4Py)K7a4L#58c3z*?rh@ZeZuAfVuo&2(y40;3;k;v6$8u@sRz2?;(3C>$Xw%(?-+UmOv9`L7(3DjL{sm{H3V zw6Y=~{HgRDs)9-o!V|cXARq)BBjy3{fG5IBq}E=j*N2NQ`62+wF}Og#m9Sqg-|Ex@ zS{W4nEiVBSBtFD%ocan(IpDts`i?{e41p+xOZbci$L@Yl2mc4}b^%bNAK#sG-xg=+ zHF5kFY^{KObKCcJxdW&lAAWFoO=QSS$TL-lSG~Q~lU~l{FuIG-^Go0pv2w5u(6?_j zoSb`VNjNwY5~qXZMifcq<1g9KjaTKZ{UQH*);08$nB2)3wdXzfw4oSjQ@Wz1b%Mb) z9j}Fow)a16f4`^0bZ@f+EG~D4aNw;~->ZSBGQ2#EYA|ohpoW$ro8+6C6$>*hd z^UZe5Hj7{1rWI9s|8@Z9%1jYine!_qC_lG% z_-}~hkIv!hG{s6x-p_L;a%3L5GgW?xqnsjQtdS&k6xvf4J6^y<4QH)TKZfi>eZ*f; z0Mb&VczV*|jx>_x&vuOmncrJ?YZX_V?BFC?nx=N~yU^iW5yMuFNr; z<|}e98$HCbVBv7Nzp?vLC}rSTv#3IA+^9yFN_-xxf1gJnMJtH@18XpF#IUtUPZ!^X zuXZhrlkAr4eN`acymoRK?m9t7WAPj!XIo35-4L(Qx$&*yp<E+~K0`+E_vI`ujoR9% zwRp$!!BhH!1H+>hL7qbdb79bMh?> zdNUuPH_7?5amGAy>cc;u$q5l4b4#C>m#!UkY76+9#H8H?6&E*~;`sLHy{VPcnd!Jh z_3qq<_EncY4!xriaChB_kK10X5XSRgk8XS8_{~ltn?^_bWDBz>9scL=>PlRR2=2$+ zqZw5#VCS26p`0V8Q^^>l85`rO73OG%xqa?TK@fMTuqwzl)Fg*M;plCiPcx477xFYs zQ!zK@+h{f={UC86pQ=gb3FyJajk_In3@P#LlMT#EQUaIIB;lW;! zu;UOo>)n!CVDMa2uYH%j_HkHmwer{tsO6d)M2Gl&lIL5bo17&Ex3!GC(Mjot>=8wm zWuoeZVYBA)8{z9&B`{jX1|bP4mNK`AvfEw}rA5?2kPf<~piO$hjiO%DrPQm7F;}h5 zVs~yarfZSDEnXI>_s81jV9unF7gkNk)#(swoi2EKCc7~vSqT!2Bb7n3sRoMB{8ppO zbiX&oQox=+-`M%81F8eC?V3oge9zGOkiIa=Y2NcWOPd@2JdAArIp^E~kE-My|NSDC z&1bKjRXyZ=yga~8%$4aHYpRRXaO7VA1-#_DKReZtcPt)dc)zy2-tJj3nCY3wmz_2$ zN=OGDPI6-)3Rtr?%NL8ytu);~#P5^MQiJlP=n5#fOf&aV?u6e$H;#NktElQ|o@6}Dm@^BQ^9L@u^i<*KRKt6Nd#7;VopoX_)XiPyLXwy+D1 zQED?@IoIZ=S{il({#wDV7l8P^9S!Cl2eWXq_wfK4Oa^& zewZcBp_rd`@%Gk~NQ9ds!Rj*1qKfsi$>boyRmaO?%lp@}LaFd+V5qxY6|YmNhZPe# z99$Ac7%WSZol$k&+6CRzY-{wv)h=$C2I&Ur>8kLAOh`J^+-W>KjLg9vu3$-%4CQcT zK*B5%E&E|~4bC$i7;Pgo161g|#OtMYlRO#%;lqnxz3_UmJSgUc;bAfwiB{QhD=UPi{o*!V>vBfPQ^poLz9umQ#C!C21N=b1GF1l0i6O8TH@s7Mj(ayWf1H#33W3~(J*?H1-xcYhi{c`0D zF)qiIa(Yia9do+&XTojP6(BfSzi|I*kVLjO^cT&1Ldia_i#-I5ok zm9iZ=3zBEZIRCw_6N$W}dIOF0LqhI+VhPVgeQdg6AS0D&hwa(;j+zoizH*cH0x#X( zod~6nZy0ZmE>otYD1bjp(bJ??b*3S64PzTB!-ggNEQ)aAz5ZAxib+a^g4MbjhH)8Z z_8u>@%hAQb5v|BkKd&1SUZ+0OO`&m z<@a-6YOK?HA(c()N z;8g2_?#S-8)p+2W&l&GCM>IDFK|HstmdEIkn!Jk=Iqr?g@+s4n8kPryDP#{@?gpuG z;cxF+^d`%`dWrTZAXW|xvh^x*R~R#1g{1M}t6o)0Wx00jB*GY^NRni`7 zPdY#A%`t5edoSb54;1#uK{vI*^PU9EBHRe%2Ls{EPH3-a#jufq?x(*?KDIh`ZyH?S zWE%V>aZN43f|TFdp6HUL*u#gygw0iCjD$;5+8YahT}pHNckP}2s$N;HXHXlg$XYn8 znO%M7DT+T6wMj>p)*^t8>a2;9;8yDYIl-}Av4_hyL{d>li2{XjWwgk)-`Zt)+L#z} z*%iCJdq&a$V~hJ(7;C~Jc?+>~EY&Xn zgA1z5X3~sn#J%d@^flypT8{Y@aG7WMBqfRP)#Uy$d4P}eKd4+w}u&+)}FT?AdsZla;_OK^Lfo> zS|OY(Ux0tZQTwC-dZ0VVI0PJcH>s{V7inuSeSfv}u;1otdYPMF(3reUL{Q8S7TA!1 z?7=?>quoYUY*D}FsCV1%VO(|B7|KGZ4feXK(rW$=DCYW!o41zE?$`66>f`*QoC4L7 z((SKct}Ijz4LtVeeq;<;jABY{d|JvlHYmhy)Nm=0Xu7Sp>l^WBe^VDv*Xz`$ zuGDjT{Jz1~Xhv!a+4YwI*N=9LPQx8SnLHVtcX$5H0DPSGFHnanc469bSuT~g8rIQ= z-(y%C(#QClm=*yJR^~}x9LOXRqQ^N0N$V+hzwGE^>;Pw9y8K^1e6xvlKtl%kRl|-6 zsZZWC;gfsG+Iml@?Ul8=>GMT>c(W^P4eCNoM-d9`2^eN~idNmF%W+O_~ z1eju;BRNGer7HF-bDwRaawEI=N3>E&VTCTiLm=&YWpxJ6*4PE(=QT!Ul~#w6@{Oab zC+Bs-;!qOg$hz4r;uwPIe?CuGpb_V;bMOB$tj}E>FvJh+MFOgSK#5C89sUyl!TO&7 z2qs4M|AHKT)tl+r{yX}g00hHd28RDN0HFx2gk*|J8&L*mxg;Ef@~R2SFKz+`6DL;e z4*_zNFULRokNzPvikQisk(eJgJZ2$UiC)vs7WcN$wOKh@*QBu4VBaFV8Ojx)ag<3(FR!}~Gu(uHT zIgOoTWHCF7mp)n>O=wk&f!P^Vm>%jrwr%18He8gR8GKF&SQ=cPS8-Y{vaml}goF+N zh6>bSTMi+A5JV6Y^CIMC0ERV(WN{fg1^OEZZV*`?F(NOWBYT`Vz9a%WETNqYD9dLg zNDUYR3_m}uw-Qt=0LrL&F@ymY{puC0{Twp2;aF?GkI{=bC(HZqz zi4d~&pApD_IWv2X>;gr<1O-8Olm?+8jK6e#3q_D&mCi7Du^09d(PR$*j%Vx;@id_9 zJNd=DIQG{S@ zeB-TvOcW%CKOh(!T5y1)CH<4XB7TJ|fFXbhgzRxJisTx%jGVS1|MatWcqK9T3I8p? z0hgVz-8V;=qL0s>Kc?C}L?8%X?x)q&xu>LvX7Te!As9h0FhETL33k&EIy#k@Wi=?> zOEd+jF!tHD>G>$oMJ#Ko?dtzUkfrBPB_4D{1By*m(hZN04?tV}k#=(&^F3&%nD{Ar zzd#8tDqk|`9i`vp{cHzjWd%x5$~bne4H$f*ikqW2kH3@efd2Kp-8uPxB1QKCyoW-w6#KUKocsOMOJ);eB5-g7Qgd9K8@Hv&%I=*rzz=lU2Bq9dULCZK4kj1id zyvSXX^tFb6Y5h_R&NyfSd)gVfLRa40o7DS|OZ%6%G$loaaX!v)0{Xp#nR_X>qN-)4 z;2E>_>C5{iX*tK6L}K{{7J5=Fb;gj&{0!k8IP!-8-$jlV<2V=GSE@-*C3v z^Ym)-iX${wD|aDI<`Gu*pk()SIP~^?3#?|*d2>BClD`%QR$f`X!q+qXtYEW%1A0eH zW?EO83`csa1TX8K+Rxu#_Cwvw6(O<>_s8XGmCds8NIJMvPF#Grw)HdL;+m=cXP@B? zutiG)gG$8#EZ7O3cRnO-2P4~{1ELey9wwX4<*7~hdOORdjzH_`dn&4FPsW5$nHgS- z@pdgw`2^gK-uFD624uBLSkD#Uykxh|#V}jFQx|ogzL|2_JUY!$KpL3ad|1x+jJ4rJ)1u`@iyD-MGgX52gx1s_R|F9Esvn*E^ zCuTmBuykJ5*RCZOaP3`;=FrACZ>oLiY;|EhPnDKfFY|YD3EngZr3-f~R`owPzVPf& zqI7c0%SQfdp@d1_UMPsc!js8zV>$!lUNSE+>R-~-PKBG~=Cq%{xLYk}+Zz2gY}vn- zHaS55>AR$SagDbs z3B9~XHd`IxQv?4s1o}djWkxd(vrE(7Y`?kZdYjnVeK5eI&(l!9yN~`8w0u&BQ!G)Q z!o=Ia+@(LdTx57_Z1_G7#ojjc6-WKMNUx9WU~_{GUUMilORLKKFiSSS#4;_jCr#|4 zLlW}!baQ0)#Ufj*`^E9L^-8mon#_82@v&;Gx{Xe-rn0H>x!0#R?49X&K=!w{_sM^g zT(Wr_?%jT_B*qGXK@G+`H0J#S1ivL&X(2Q4Bl}(f+~g43y8Y!@ms`StiGwXji(zcA zwB5=oR*C3(`Ke#rAg$NI#UC|xqpp!H|KKf{J@c!DLO#c(l=Mj(xj6}ErY5Vq)2Sk` zsDb_&&}Dxor@LO_KWz65?EhoC=XZ88vvnZH=eII(FgCKq=cmONv~_SW)~BFTGIz2v zCdZeS6KBF#b1{hMWI zrSIs7?_%uWXl`qR&qT|_Np0M?smrb zbg~8(icZ#0bTas?zx|Ymxr3tGb8zjy2Y*9SURSmwWMgy&A5VXr9pG)G-w z_^Q~|yJh>$Rq1)mk$@BoO)PaK3Jy`X4NYMSg1H0p`m9VJ#6iAaQ zLj~|QkCHBbsy>AQ7Dh`#CK-A(E-eyhs2t8=!oCr6QsE)c5bWr}pv3KgC_sESXr_1yeSlarsc`_eE+Yx52|J*2EVYsq6KZI{7L7gi zBvf>^7-3k-6re>5pelwkoxxrGM?Qc_q(EqH3Xn*Iwgey)ILuLm7yo!a9x2bJA|QyQ zk-jJe_Sq1|*I6n(`F$;bF{q#dJ{dv*nU^XM+M=K()JeKxu+Dsh;wJpMWIkDrNcdZP zZZUHbbZihI2(bwWu$FluWbK=`2VCASR$BBO%;Bf&YU86gwUm4jHCrqG!P?<{SPDvGaD;Z4)_X}vAKmQ((Mct&9k{oni zuE~G9+a<3AIuKuhdEOO(|9oT0__3aL>Jx`i2OMxk#rLMg#tpNZ4w)bvTtlKm{uY-G zON77-H2q0j0-1-%>6idPDKC3S%(uaBL@d)Jm;Ufh5mgvGrWMB|;uVtrK^)~LvbiYP z1QLB(dWT5EPa-nI4w$T0XfyR813kKW&lh`zM;- zO$_w+6C1#vo-W!e2jj6>y0GW^KJiZzSle`!6=RM*@+pZP4<=l*Sh2^5k4XMB@n?&@ zMzg{KQ4lY7e{Apj>u7zg+TKU)*fC91k4~a{>9z9Or$Cy_2$>Kxq=T&}cbOm_PV2fg z+0sLO;fgwoa=T4`DjxPqFAD(ZDjGUoI{-a6CT&H(F8-o*UQY5^BVN)R;Q4&PW-$p+ zm32)QIaoPyV&~NMCf#K_sHu=0bm`WajN0q02(tv3WZUTaRKLicchRO5h2!o}iL#v8 zz!eO#c0T7721O+6WqWKLhj?tAMGt_rukm}+5A?zO+#c`mlyM7q0bdiF#>P}O_$v4r z__uFykH|*WvRDe`&1PoNl+%A0(r?EF9b}?lk)$ig$uFw^{n@>4z+^EMBA1!$(ul1`-iA0-72)&}%G2(O5xV>lJoG zhIrwbu$epQ7(@9k_`@!P!90LnLy2;@ge~t<)ceyQ%&)U>&Q}qll<_qZi)27jlow?; zE}1V5h02$~0BfgY@C>6fl4~HDqDyK3<^JIPl#0}4#$#Gb8G0b@?i!t@hB!sY!G3-* zBjtcGy_%YtsS}jf?VSqe>1D-EgTs)VFwo~~D#@+vdU0w8f->EuJ`X}&CId|hiBTn} zyz0^CA(eoD!@<@?1KI1XKpKKkTBO$&&%^Gs>Dxu``N)wMLeG~TY!^6L5Rg_fkdMtMZ?R}5K9G^5ADKaM8yT* zjp8!^7t6~7DKuUi+|fCMhX8HM^*lfor=uyyl+ZEH zYDJxDT->SB{@e)WwU4hOCO&J9jMwY-*;;ua%%W;CSm2%fNRAnwJvG;~+#Gq$p8Q zAvVb8lxE_TLRou<+#Ubdn8F_YaTpmX zWQQ^59)8MAma(L}`pJ9|^Ju9KS8t4cp>#y=-f5bUH6Rn=9+i3W0hkBZ(lt`N7c=Uj zL1I*kOKV`x?xeJOuo}-r2QMo}0#G*OZ7C?d$;h+w6H3%)oD~JzGaFa~!53^T;}M#j zEO6uHRffk$8K$NtQ?n>whBaA%+XsMbh2PiGiG_N#1f+`0+5X{AU0-_JnlUqB`!p37 zDDQzL*9n@}5bpyWR(NNtjV{!G^y9NGO#0K(N*$ggy*G+ zg{=c^_~%2T5;O2#WZ7C0Z5pU)c~S64=O%>z7#QECZuXoD{s zqg#e}P1As@hi&yEoZmjdEIcmGKLFi{8l3;6B0yMNSj?8^cE87;m*-}R`s)PzT`;He zB*>|Es{40umUg!%D}8g${xq{b={arAdU9?7P&QGKk&vyjRD!+L?E+``nq;eWy zI4jMu$glKkju`%MEv?n<;RpTfknloTGjV#926`mgo*oYUch_d1KlC+B=sq-jWv3=5 zfbBIMI$WkJHQyg($H%R=*B5}Wmij%D6veyp>jl~SE-!(%7VXM{p%x`A z_Z}-OMx3YP@D0S`MDHH5>#*S82SqMYlGAgkGAPl5g-VbnTyZu<_l=@(XhO2oj|IggHi#}yfsHZ{jRa_T~VAkmi9 z){5R!?Vh{4sA=nyvu1yI)Fdi7|2~!-DbpDzx)Vwe?z`SOMat@?rh@($D|EG2hX%?s zt>qA>2^`RkM zyNICFG%kJ5yp+@mg=hl>QzL^|F8$!ls#Fk@OFvv8+EBq*!4N1Il%HRsU}|U%G$IJ3 zBP}1u19i$lJQsy%8&gv=S7Sp zYF;|9Q*LNzVn*y3z3R2?{V|~;2Od^m-+eFk`J2+H=NryhN7mLw>a%;UX5SYy{hqJ`LJ#46--SsdktY_=_zAHxm0{>e_RBqiXvo`f=dafz&<-$wp&#!L! z^`Ck62Fd6RqPI85zTIQJk$d+>_UeuNzb}3(5@;`wa4!%!{z0zqF^gaFqG_h-x6S-+ zUd?-Rv}}v8?Xk!?hbv;GB}7Pzb83s16Oe1s8GONePNReqk7ApE zvxh?2)uMmzw~0rlL~AOE2zGI}wkSCXF6;Ry zuD9BOXD>LG3`7rS81IigtuDXKzb7eIH~(LTzAbZ=u_5&})(;G?eg5!j z=jrd?4y@hxJo4J@erAgfXNw+Xi>_pgM&7M8me;-&-Kk&w?OFQ{ZvRR5GUpt`NNx_N zKdR+DG<)|z?%iWXxyLEG>*Ke^+HP8HHg9=Fo2SijZjD3d`)_T%-K=EgTGI7G9>H*^{Jd%ks5mS9bl5iOyY9`sw_(TOqGkugx_Le7$}D z_N&jL{?EM{7X2&y@2<5=F3l`SzckmgT:-Wall -Wextra -Werror -Wno-error=deprecated-declarations>) # -Wpedantic -Wunused -Wnon-virtual-dtor -Woverloaded-virtual -Wcast-qual -Wformat=2>) #-Wsign-conversion -Wshadow -Wconversion - target_compile_options(${TEST_EXE} PRIVATE $<$:-Wall -Wextra>) - - target_compile_options( - ${TEST_EXE} - PRIVATE - $<$: - -Xcompiler=-Werror,-Wall,-Wextra,-Wcast-align,-Wcast-qual,-Wno-deprecated-declarations,-Wno-double-promotion,-Wduplicated-branches,-Wduplicated-cond,-Wformat-truncation,-Wformat=2,-Wlogical-op,-Wmisleading-indentation,-Wno-missing-include-dirs,-Wnon-virtual-dtor,-Wno-missing-declarations,-Wnon-virtual-dtor,-Wnull-dereference,-Woverloaded-virtual,-Wpointer-arith,-Wno-redundant-decls,-Wno-shadow,-Wno-switch-enum,-Wno-unknown-pragmas,-Wtrampolines,-Wuninitialized,-Wunused,-Wunused-but-set-variable,-Wunused-result,-Wno-zero-as-null-pointer-constant - --expt-relaxed-constexpr --extended-lambda --Werror=cross-execution-space-call -Xcudafe=--display_error_number -Xcudafe=--diag_error=incompatible_assignment_operands -Xcudafe=--diag_error=returning_ptr_to_local_variable -Xcudafe=--diag_error=subscript_out_of_range -Xcudafe=--diag_error=used_before_set -Xcudafe=--diag_error=undefined_preproc_id -Xcudafe=--diag_error=implicit_func_decl -Xcudafe=--diag_error=implicit_return_from_non_void_function -Xcudafe=--diag_error=missing_type_specifier - > - $<$,$>,$>>: - -Wno-unknown-pragmas - # -Wextra-semi (gcc 8, not in gcc 7) - -Wpedantic - # -Wabi=13 -Wabi-tag (maybe important when linking with very old libraries) - -Waddress - # -Waddress-of-packed-member (gcc 11, not in gcc 8) - # -Waggregate-return (disallow return classes or structs, seems a C-compatibility warning) - -Waggressive-loop-optimizations - -Wzero-as-null-pointer-constant - > - $<$,$,$>: - -Wno-unknown-pragmas - -Wcast-align - -Wcomplex-component-init - -Wconstexpr-not-const - -Wconsumed - -Wconversion -Wconversion-null - -Wmove - -Wzero-as-null-pointer-constant -Wzero-length-array - -Wno-c++98-compat-unnamed-type-template-args - -Wno-ignored-qualifiers - -Wno-range-loop-analysis # (this check is overzealous in clang 9) - > - $<$: # also IntelLLVM, XL (ibm), XLClang (ibm) - -Werror - -Wall - -Wextra - -diag-disable=remark - -wd161 - -Wabi - -Warray-bounds - -Wcast-qual - -Wchar-subscripts - -Wunused -Wunused-but-set-variable -Wunused-function -Wunused-parameter -Wunused-variable - -Wwrite-strings - > - $<$,$>: - -Werror - -Wall - -Wcast-qual - -Wformat=2 - -Wshadow - -Wunused-parameter - > - $<$: - /WX - /W4 - > - ) - - target_include_directories(${TEST_EXE} PUBLIC "../../../..") - - # TODO(correaa) simplify include directories - target_include_directories(${TEST_EXE} PUBLIC "../include") - target_include_directories(${TEST_EXE} PUBLIC "../include/mpi3") - target_include_directories(${TEST_EXE} PUBLIC "../include/mpi3/dummy") - - #target_link_libraries(${TEST_EXE} PRIVATE mpi3) - - list(FIND NEED_BOOST_SERIALIZATION_SRCS ${TEST_FILE} NEED_BOOST_SERIALIZATION) - if (NOT (${NEED_BOOST_SERIALIZATION} EQUAL -1)) - target_compile_options(${TEST_EXE} PUBLIC -D_MAKE_BOOST_SERIALIZATION_HEADER_ONLY) -# target_link_libraries(${TEST_EXE} PRIVATE boost_serialization) - endif() - - target_link_libraries(${TEST_EXE} PRIVATE pthread) - target_link_libraries(${TEST_EXE} PRIVATE multi) - - set(NPROC 3) - if ( - (TEST_FILE STREQUAL "communicator_divide.cpp" ) - OR (TEST_FILE STREQUAL "communicator_operator.cpp" ) - OR (TEST_FILE STREQUAL "communicator_scatter.cpp" ) - OR (TEST_FILE STREQUAL "cartesian.cpp" ) - OR (TEST_FILE STREQUAL "simple_send_receive.cpp" ) - ) - set(NPROC 6) - endif() - - if ( - (TEST_FILE STREQUAL "uniform_abort.cpp" ) - OR (TEST_FILE STREQUAL "communicator_grip_handle.cpp") - ) - set(NPROC 4) - endif() - - if ( - (TEST_FILE STREQUAL "cartesian.cpp" ) - ) - set(NPROC 6) - endif() - - if ( - (TEST_FILE STREQUAL "reduce_maxloc.cpp" ) - ) - set(NPROC 4) - endif() - - if(DEFINED ENV{VALGRIND_EXE}) - add_test(NAME ${TEST_EXE} COMMAND ${MPIEXEC} $ENV{MPI_OVERSUBSCRIBE} -n ${NPROC} valgrind --leak-check=full --track-origins=yes --show-leak-kinds=all --error-exitcode=1 ./${TEST_EXE}) - else() - add_test(NAME ${TEST_EXE} COMMAND ${MPIEXEC} $ENV{MPI_OVERSUBSCRIBE} -n ${NPROC} ./${TEST_EXE}) - endif() - set_tests_properties(${TEST_EXE} PROPERTIES TIMEOUT 30) -endforeach() - -set_tests_properties(communicator_abort.cpp.x PROPERTIES WILL_FAIL true) -set_tests_properties(uniform_abort.cpp.x PROPERTIES WILL_FAIL true) - -# Compile-time check for multiply defined symbols -add_library(library_check library_check.cpp) -target_compile_features(library_check PUBLIC cxx_std_17) - -# TODO(correaa) simplify include directories -target_include_directories(library_check PUBLIC "../include") -target_include_directories(library_check PUBLIC "../include/mpi3") -target_include_directories(library_check PUBLIC "../include/mpi3/dummy") - -add_executable(library_main.x library_main.cpp) -target_compile_features(library_main.x PRIVATE cxx_std_17) -target_link_libraries(library_main.x library_check) diff --git a/external_codes/mpi_wrapper/mpi3/test/all_reduce.cpp b/external_codes/mpi_wrapper/mpi3/test/all_reduce.cpp deleted file mode 100644 index f46ad383f51..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/all_reduce.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - - assert( world.size() > 1); - - std::vector local(120); - iota(begin(local), end(local), 0); - - std::vector global(local.size()); - - auto last = world.all_reduce_n(local.begin(), local.size(), global.begin()); - assert(last == global.end()); - - assert( - std::inner_product( - global.begin(), global.end(), local.begin(), - true, std::logical_and{}, - [sz = static_cast(world.size())](auto& e1, auto& e2) { return e1 == e2 * sz; } - ) - ); - // for(std::size_t i = 0; i != sz; ++i) { - // assert(global[i] == local[i] * world.size()); - // } - - auto const sum_of_ranks = (world += world.rank()); - assert( sum_of_ranks == world.size()*(world.size()-1)/2 ); - - auto rank = world.rank(); - auto sum_rank = 0; - world.all_reduce_n(&rank, 1, &sum_rank); -// world.all_reduce_n(&rank, 1, &sum_rank, std::plus<>{}); -// world.all_reduce_n(&rank, 1, &sum_rank, mpi3::plus<>{}); -// sum_rank = (world += rank); - assert(sum_rank == world.size()*(world.size()-1)/2); - - auto max_rank = -1; - world.all_reduce_n(&rank, 1, &max_rank, mpi3::max<>{}); - assert( max_rank == world.size() - 1 ); - - auto min_rank = -1; - world.all_reduce_n(&rank, 1, &min_rank, mpi3::min<>{}); - assert( min_rank == 0 ); - - - { - std::vector local(20, 1); - if(world.rank() == 2){local[1] = 0;} - - std::vector global(local.size()); - world.all_reduce_n(local.begin(), local.size(), global.begin()); - - assert(global[0] == world.size()); - assert(global[1] == world.size() - 1); - } - { - std::vector local(20, 1); - if(world.rank() == 2){local[1] = 9;} - - std::vector global(local.size()); - world.all_reduce_n(local.begin(), local.size(), global.begin(), std::logical_and<>{}); - - assert(global[0] != 0); - assert(global[1] == 1); - } - { - int b = 1; - if(world.rank() == 2){b = 0;} - int all = (world += b); - assert( all == world.size() - 1 ); - } - { - int const b = world.rank() == 2 ?0:1; - int const all = (world &= b); - assert( all == false ); - } - { - bool const b = not(world.rank() == 1); - bool const all_of = (world &= b); - assert( all_of == false ); - } - { - assert(world.size() != 1); - - bool const b = (world.rank() == 1); - bool any_of = false; - world.all_reduce_n(&b, 1, &any_of, std::logical_or<>{}); - assert(any_of); - - bool all_of = true; - world.all_reduce_n(&b, 1, &all_of, std::logical_and<>{}); - assert(not all_of); - } - { - assert(world.size() != 1); - - bool const b = (world.rank() == 1); - bool const any_of = (world |= b); - assert(any_of); - - bool const all_of = (world &= b); - assert(not all_of); - } - - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/async_interaction.cpp b/external_codes/mpi_wrapper/mpi3/test/async_interaction.cpp deleted file mode 100644 index 7f2c46b0516..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/async_interaction.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#include -#include -#include - -#include - -namespace mpi3 = boost::mpi3; - -auto async_send(mpi3::communicator& comm, int val, int target) { // NOLINT(bugprone-easily-swappable-parameters) - return std::async([=, &comm] () { // was: [=] () mutable { in original Joseph's S. posting - auto value = val + 1; - comm[target] << value; // same as comm.send(&value, &value + 1, target); - }); -} - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - if(world.rank() == 0) { - auto fut = async_send(world, 41, 1); - fut.wait(); - } - - if(world.rank() == 1) { - int value{}; - world[0] >> value; // same as world.receive(&value, &value + 1, 0); - std::cout<< "value = "<< value << std::endl; - assert( value == 42 ); - } - return 0; -} catch(...) {return 1;} - diff --git a/external_codes/mpi_wrapper/mpi3/test/broadcast.cpp b/external_codes/mpi_wrapper/mpi3/test/broadcast.cpp deleted file mode 100644 index 374bfd51dd0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/broadcast.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2023 Alfredo A. Correa - -#include -#include - -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - - { - auto v = std::vector(10, world.root()? 3.14 : 99.9); - - auto it = world.broadcast_n(v.begin(), 10); - - assert( it == v.end() ); - assert( std::find_if(v.cbegin(), v.cend(), [](auto& e) {return e != 3.14;}) == v.cend() ); - } - { - namespace multi = boost::multi; - auto arr = multi::array(10, world.root()? 3.14 : 99.9); - - auto it = world.broadcast_n(arr.begin(), 10); - - assert( it == arr.end() ); - assert( std::find_if(arr.cbegin(), arr.cend(), [](auto& e) {return e != 3.14;}) == arr.cend() ); - } - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/cartesian.cpp b/external_codes/mpi_wrapper/mpi3/test/cartesian.cpp deleted file mode 100644 index eabb7addf90..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/cartesian.cpp +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2021-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/cartesian_communicator.hpp" - -namespace mpi3 = boost::mpi3; - -void division_tests1() { - -{ - auto div = mpi3::cartesian_communicator<2>::division(6); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 3 ); - assert( div[1] == 2 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 3 ); - assert( div[1] == 2 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {mpi3::fill}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 3 ); - assert( div[1] == 2 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {mpi3::fill, mpi3::fill}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 3 ); - assert( div[1] == 2 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {2}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 2 ); - assert( div[1] == 3 ); -} -} - -void division_tests2() { -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {2, mpi3::fill}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 2 ); - assert( div[1] == 3 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {mpi3::fill, 3}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 2 ); - assert( div[1] == 3 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(6, {mpi3::_, 3}); - assert( div[0]*div[1] == 6 ); - assert( div[0] == 2 ); - assert( div[1] == 3 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(7); - assert( div[0]*div[1] == 7 ); - assert( div[0] == 7 ); - assert( div[1] == 1 ); -} -{ - auto div = mpi3::cartesian_communicator<2>::division(7, {mpi3::fill, mpi3::fill}); - assert( div[0]*div[1] == 7 ); - assert( div[0] == 7 ); - assert( div[1] == 1 ); -} - -try { // this is an error in MPICH and openMPI - auto const div = mpi3::cartesian_communicator<2>::division(7, {2, mpi3::fill}); - assert( div[0]*div[1] == 4 ); - assert( div[0] == 2 ); - assert( div[1] == 2 ); -} catch(std::runtime_error&) {} - -try { // this is an error in MPICH - auto const div = mpi3::cartesian_communicator<2>::division(6, {2, 2}); - assert(div[0] * div[1] == 4); - assert(div[0] == 2); - assert(div[1] == 2); -} catch(std::runtime_error&) { -} - -} - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { // NOLINT(readability-function-cognitive-complexity) test function - assert( world.size() == 6 ); - - division_tests1(); - division_tests2(); - -{ - mpi3::cartesian_communicator<2> cart_comm(world, {3, 2}); - assert( cart_comm.dimensions()[0] == 3 ); - assert( cart_comm.dimensions()[1] == 2 ); - - auto row = cart_comm.axis(0); - auto col = cart_comm.axis(1); - assert( row.size() == 3 ); - assert( col.size() == 2 ); -} -{ - mpi3::cartesian_communicator<2> cart_comm(world, {mpi3::fill, 2}); - assert( cart_comm.dimensions()[0] == 3 ); - assert( cart_comm.dimensions()[1] == 2 ); - - auto row = cart_comm.axis(0); - auto col = cart_comm.axis(1); - assert( row.size() == 3 ); - assert( col.size() == 2 ); -} -{ - mpi3::cartesian_communicator<2> cart_comm(world, {3, mpi3::fill}); - assert( cart_comm.dimensions()[0] == 3 ); - assert( cart_comm.dimensions()[1] == 2 ); - - auto row = cart_comm.axis(0); - auto col = cart_comm.axis(1); - assert( row.size() == 3 ); - assert( col.size() == 2 ); -} - -{ - mpi3::cartesian_communicator<2> cart_comm(world, {3, 2}); - assert( cart_comm.dimensions()[0] == 3 ); - assert( cart_comm.dimensions()[1] == 2 ); - - auto row = cart_comm.axis(0); - auto col = cart_comm.axis(1); - assert( row.size() == 3 ); - assert( col.size() == 2 ); - - { - auto comm_sub0 = cart_comm.axis(0); - assert( comm_sub0.shape()[0] == 3 ); - assert( comm_sub0.size() == 3 ); - } - { - auto comm_sub1 = cart_comm.axis(1); - assert( comm_sub1.shape()[0] == 2 ); - assert( comm_sub1.size() == 2 ); - } - { - auto plane0 = cart_comm.hyperplane(0); - static_assert( decltype(plane0)::dimensionality == 1 , "!" ); - assert( plane0.size() == 2 ); - } - { - auto plane1 = cart_comm.hyperplane(1); - static_assert( decltype(plane1)::dimensionality == 1 , "!" ); - #if not defined(MPICH_VERSION) - { // cartesian communicators of dimension 0 do not work in MPICH - assert( plane1.size() == 3 ); - auto point = plane1.hyperplane(0); - assert( point.num_elements() == 1 ); - } - #endif - } - { - auto comm_sub0 = cart_comm.axis<0>(); - assert( comm_sub0.shape()[0] == 3 ); - assert( comm_sub0.size() == 3 ); - } - { - auto plane = cart_comm.plane<0, 1>(); - assert( plane.shape() == cart_comm.shape() ); - } -} -{ - mpi3::cartesian_communicator<3> cart_comm(world, {3, 2 ,1}); - assert( cart_comm.dimensions()[0] == 3 ); - assert( cart_comm.dimensions()[1] == 2 ); - assert( cart_comm.dimensions()[2] == 1 ); - - { - mpi3::cartesian_communicator<2> plane = cart_comm.plane<0, 1>(); - assert( plane.shape()[0] == cart_comm.shape()[0] ); - assert( plane.shape()[1] == cart_comm.shape()[1] ); - } - { - auto plane = cart_comm.plane<0, 1>(); - assert( plane.shape()[0] == cart_comm.shape()[0] ); - assert( plane.shape()[1] == cart_comm.shape()[1] ); - } - { - mpi3::cartesian_communicator<2> plane = cart_comm.plane<0, 2>(); - assert( plane.shape()[0] == cart_comm.shape()[0] ); - assert( plane.shape()[1] == cart_comm.shape()[2] ); - } - { - auto plane_comm = cart_comm.plane(0, 1); - assert( plane_comm.shape()[0] == cart_comm.shape()[0] ); - assert( plane_comm.shape()[1] == cart_comm.shape()[1] ); - } - { - auto plane_comm = cart_comm.plane(); - assert( plane_comm.shape()[0] == cart_comm.shape()[0] ); - assert( plane_comm.shape()[1] == cart_comm.shape()[1] ); - } -} -{ - mpi3::cartesian_communicator<2> cart_comm(world, {3, 2}); - - assert( cart_comm.rank() == cart_comm.rank(cart_comm.coordinates()) ); - assert( cart_comm.coordinates() == cart_comm.coordinates(cart_comm.rank()) ); - - assert( cart_comm(2, 1).rank() == cart_comm.rank({2, 1}) ); - assert( std::apply(cart_comm, cart_comm.coordinates()).rank() == cart_comm.rank() ); - - assert( cart_comm.rank(cart_comm.coordinates(4)) == 4 ); - - std::cout<< cart_comm.rank({5, 3}) < comm_1D(world, {world.size()}); - assert( comm_1D.periods()[0] == true ); - assert( comm_1D.rank({-1}) == comm_1D.size() - 1 ); - - auto next_rank = comm_1D.rank({comm_1D.coordinates()[0] + 1}); - auto prev_rank = comm_1D.rank({comm_1D.coordinates()[0] - 1}); - - assert(comm_1D.rank() != next_rank ); - assert(comm_1D.rank() != prev_rank ); -} -{ - auto const periodic = mpi3::cartesian_communicator<1>{world}; // implivit , {}, {true}}; - { - auto const [source, dest] = periodic.shift<0>(+1); - assert( source == ((periodic.rank() - 1 + periodic.size()) % periodic.size()) ); - assert( dest == ((periodic.rank() + 1) % periodic.size()) ); - } - { - auto const [source, dest] = periodic.shift<0>( 0); - assert( source == periodic.rank() ); - assert( dest == periodic.rank() ); - } - { - auto const [source, dest] = periodic.shift<0>(-1); - assert( source == ((periodic.rank() + 1) % periodic.size()) ); - assert( dest == ((periodic.rank() - 1 + periodic.size()) % world.size()) ); - } -} -{ - auto const nonperiodic = mpi3::cartesian_communicator<1>{world, {}, {false}}; - { - auto const [source, dest] = nonperiodic.shift<0>(+1); - - if( nonperiodic.rank() > 0 ) {assert( source == nonperiodic.rank() - 1);} - if( nonperiodic.rank() == 0 ) {assert( source == MPI_PROC_NULL );} - - if( nonperiodic.rank() < nonperiodic.size() - 1) {assert( dest == nonperiodic.rank() + 1);} - if( nonperiodic.rank() == nonperiodic.size() - 1) {assert( dest == MPI_PROC_NULL );} - } - { - auto const [source, dest] = nonperiodic.shift<0>( 0); - assert( source == nonperiodic.rank() ); - assert( dest == nonperiodic.rank() ); - } - { - auto const [source, dest] = nonperiodic.shift<0>(-1); - if( nonperiodic.rank() < nonperiodic.size() - 1) {assert( source == nonperiodic.rank() + 1);} - if( nonperiodic.rank() == nonperiodic.size() - 1) {assert( source == MPI_PROC_NULL );} - - if( nonperiodic.rank() > 0 ) {assert( dest == nonperiodic.rank() - 1);} - if( nonperiodic.rank() == 0 ) {assert( dest == MPI_PROC_NULL );} - } -} - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_abort.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_abort.cpp deleted file mode 100644 index 663d35ae62b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_abort.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "../main.hpp" -#include "../communicator.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int { - world.abort(911); - - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_all_gather.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_all_gather.cpp deleted file mode 100644 index 47752e78ef7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_all_gather.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../communicator.hpp" -#include "../detail/iterator.hpp" -#include "../main.hpp" - -namespace mpi3 = boost::mpi3; - -void part1(mpi3::communicator& world) { - using T = std::tuple; - std::vector v_local(10, T{world.rank(), world.rank()}); - std::vector v(v_local.size() * static_cast(world.size())); - auto end = world.all_gather_n(v_local.begin(), v_local.size(), v.begin()); - assert(end == v.end()); - assert((v[ 0] == T{0.0, 0.0})); - assert((v[10] == T{1.0, 1.0})); - assert((v[20] == T{2.0, 2.0})); -} - -void part2(mpi3::communicator& world) { - using T = std::tuple; - std::vector v_local(10, T{world.rank(), world.rank()}); - std::vector v(v_local.size() * static_cast(world.size())); - auto d_last = world.all_gather(begin(v_local), end(v_local), begin(v)); - assert(d_last == end(v)); - assert((v[0] == T{0.0, 0.0})); - assert((v[10] == T{1.0, 1.0})); - assert((v[20] == T{2.0, 2.0})); -} - -void part3(mpi3::communicator& world) { - using T = std::pair; - std::vector v_local(10, T{world.rank(), world.rank()}); - std::vector v(v_local.size() * static_cast(world.size())); - auto end = world.all_gather_n(v_local.begin(), v_local.size(), v.begin()); - assert(end == v.end()); - assert((v[0] == T{0.0, 0})); - assert((v[10] == T{1.0, 1})); - assert((v[20] == T{2.0, 2})); -} - -void part4(mpi3::communicator& world) { - using T = std::pair; - std::vector v_local(10, T{world.rank(), world.rank()}); - std::vector v(v_local.size() * static_cast(world.size())); - auto d_last = world.all_gather(begin(v_local), end(v_local), begin(v)); - assert(d_last == end(v)); - assert((v[0] == T{0.0, 0})); - assert((v[10] == T{1.0, 1})); - assert((v[20] == T{2.0, 2})); -} - -void part5(mpi3::communicator& world) { - using T = std::pair; - std::vector v_local(static_cast(world.rank()) + 10, T{world.rank(), world.rank()}); - std::vector v(v_local.size() * static_cast(world.size())); - auto d_last = world.all_gather(begin(v_local), begin(v_local) + 4, begin(v)); - assert(std::distance(begin(v), d_last) == 4L * world.size()); - // assert(e == end(v)); - assert((v[0] == T{0.0, 0})); - assert((v[4] == T{1.0, 1})); - // assert(( v[10] == T{1.0, 1} )); - // assert(( v[20] == T{2.0, 2} )); -} - -void part6(mpi3::communicator& world) { - auto cs = world.all_gather_as>(world.rank()); - assert(cs[0] == 0); - assert(cs[1] == 1); - assert(cs[2] == 2); -} - -void part7(mpi3::communicator& world) { - using T = double; - std::vector v_local(static_cast(world.rank() + 1), world.rank()); - std::vector v(1 + 2 + 3); - auto end = world.all_gatherv_n(v_local.begin(), v_local.size(), v.begin()); - assert(end == v.end()); - assert((v[0] == 0.0)); - assert((v[1] == 1.0)); - assert((v[2] == 1.0)); - assert((v[3] == 2.0)); - assert((v[4] == 2.0)); - assert((v[5] == 2.0)); -} - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - part1(world); - part2(world); - part3(world); - part4(world); - part5(world); - part6(world); - part7(world); - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv.cpp deleted file mode 100644 index 7af3855d822..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/detail/iterator.hpp" - -#include // for std::iota - -namespace mpi3 = boost::mpi3; - -template void f(int); - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - assert( world.size() > 2 ); -{ - using T = std::tuple; - std::vector v_local(10, T{world.rank(), world.rank()}); - std::vector v( v_local.size() * static_cast(world.size()) ); - auto end = world.all_gather_n(v_local.begin(), v_local.size(), v.begin()); - assert(end == v.end()); - assert(( v[ 0] == T{0.0, 0.0} )); - assert(( v[ 9] == T{0.0, 0.0} )); - assert(( v[10] == T{1.0, 1.0} )); - assert(( v[19] == T{1.0, 1.0} )); - assert(( v[20] == T{2.0, 2.0} )); - assert(( v[29] == T{2.0, 2.0} )); -} -{ - using T = std::tuple; - std::vector v_local(static_cast(world.rank() + 5), T{world.rank(), world.rank()}); - std::vector v(1000, T{-99.0, -99.0}); - auto d_last = world.all_gatherv_n(begin(v_local), v_local.size(), begin(v)); - - // int predict_size = 0; - // for(auto i = 0; i != world.size(); ++i) {predict_size += i + 5;} - int predict_size = world.size()*(world.size() - 1)/2 + 5*world.size(); - - assert( std::distance(begin(v), d_last) == predict_size ); - - assert(( v[ 0] == T{0.0, 0.0} )); - assert(( v[ 4] == T{0.0, 0.0} )); - assert(( v[ 5] == T{1.0, 1.0} )); - assert(( v[10] == T{1.0, 1.0} )); - assert(( v[11] == T{2.0, 2.0} )); - assert(( v[17] == T{2.0, 2.0} )); -} -{ - using T = std::tuple; - std::vector v_local(static_cast(world.rank() + 5), T{world.rank(), world.rank()}); - std::vector v(1000, T{-99.0, -99.0}); - auto d_last = world.all_gatherv(begin(v_local), end(v_local), begin(v)); - assert( d_last < end(v) ); -} -{ -// initialize data - using T = double; - assert( world.size() == 3 ); - std::vector v_loc; - switch(world.rank()) { - case 0: v_loc = {0.0, 0.0, 0.0} ; break; - case 1: v_loc = {1.0, 1.0, 1.0, 1.0} ; break; - case 2: v_loc = {2.0, 2.0, 2.0, 2.0, 2.0}; break; - } -// gather communication - std::vector v; -// v.reserve(v_local.size()*world.size()); // to avoid _some_ reallocations - world.all_gatherv(begin(v_loc), end(v_loc), std::back_inserter(v)); -// v.shrink_to_fit(); // to save _some_ memory - -// check communication - assert((v==std::vector{0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0})); -} - return 0; -} catch(...) {return 0;} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv_output_iterator.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv_output_iterator.cpp deleted file mode 100644 index fa801240610..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_all_gatherv_output_iterator.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// © Alfredo Correa 2018-2021 - -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - using T = double; - assert( world.size() > 2 ); - -// initialize local data /////////////////////////////////////////////////////// - std::vector v_loc; - switch(world.rank()){ - case 0: v_loc = {0., 0., 0.} ; break; - case 1: v_loc = {1., 1., 1., 1.} ; break; - case 2: v_loc = {2., 2., 2., 2., 2.}; break; - } - -// gather communication //////////////////////////////////////////////////////// - std::vector v; -// v.reserve(v_local.size()*world.size()); // optional! avoid a few allocations - world.all_gatherv(begin(v_loc), end(v_loc), std::back_inserter(v)); -// v.shrink_to_fit(); // optional! save a few memory - -// check communication ///////////////////////////////////////////////////////// - assert((v==std::vector{0., 0., 0., 1., 1., 1., 1., 2., 2., 2., 2., 2.})); - return 0; -}catch(...) {return 1;} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_all_to_all.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_all_to_all.cpp deleted file mode 100644 index 2087caeadf2..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_all_to_all.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2019-2022 Alfredo A. Correa - -#include -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world) { - assert(world.size() == 4); - - std::vector send_buff = { - world.rank() * 10 + 0, - world.rank() * 10 + 1, - world.rank() * 10 + 2, - world.rank() * 10 + 3, - }; - - assert((int)send_buff.size() == world.size()); - - std::vector recv_buff(world.size(), 99.); - { - assert((int)recv_buff.size() == world.size()); - world.all_to_all(send_buff.begin(), recv_buff.begin()); - - if(world.rank() == 0) - assert(recv_buff[0] == 00 and recv_buff[1] == 10 and recv_buff[2] == 20 and recv_buff[3] == 30); - - if(world.rank() == 1) - assert(recv_buff[0] == 01 and recv_buff[1] == 11 and recv_buff[2] == 21 and recv_buff[3] == 31); - } - std::vector recv_buff2(world.size(), 99.); - for(std::size_t i = 0; i != send_buff.size(); ++i) - world[i] << send_buff[i] >> recv_buff2[i]; - - assert(recv_buff == recv_buff2); - - { - auto const& csend_buff = send_buff; - auto const r = world & csend_buff; - assert(recv_buff == r); - } - { - auto const& csend_buff = send_buff; - auto const r = world(csend_buff); - assert(recv_buff == r); - } - { - world& send_buff; - assert(recv_buff == send_buff); - } - - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_attributes.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_attributes.cpp deleted file mode 100644 index 07b69df66dc..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_attributes.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++17 -Wall -Wextra $0 -o $0x.x && time mpirun -n 8 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/environment.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - assert(world.get_attribute_as(mpi3::tag_ub) >= 32767); - - int h = world.get_attribute_as(mpi3::host); - if((h < 0 || h >= world.size()) && h != mpi3::process_null) assert(0); - - int io = world.get_attribute_as(mpi3::io); - if((io < 0 || io >= world.size()) && io != mpi3::any_source && io != mpi3::process_null) assert(0); - - bool wtime_is_global = world.get_attribute_as(mpi3::wtime_is_global); - assert(!wtime_is_global); - -// int* appnum = world.get_attribute_as(mpi3::application_number); - -// int us = world.get_attribute_as(mpi3::universe_size); -// (void)us; -// cout << us << std::endl; -// assert(us >= world.size()); - -// int luc = world.get_attribute_as(mpi3::last_used_code); -// (void)luc; - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_barrier.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_barrier.cpp deleted file mode 100644 index c06f21d9296..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_barrier.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2018-2021 Alfredo A. Correa -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -auto mpi3::main(int/*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - cout <<"Before barrier, I am "<< world.rank() <<" of "<< world.size() << std::endl; - world.barrier(); - cout <<"After barrier, I am "<< world.rank() <<" of "<< world.size() << std::endl; - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_cctor.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_cctor.cpp deleted file mode 100644 index ce482ee79cb..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_cctor.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" - -#include -#include -#include - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - - static_assert(sizeof(MPI_Comm) == sizeof(mpi3::communicator) ); - -{ - mpi3::communicator& w2 = mpi3::grip_communicator(world.handle()); - assert( w2 == world ); - assert( &w2 == &world ); - - assert( mpi3::grip_communicator(MPI_COMM_WORLD) == world ); - assert( &mpi3::grip_communicator(MPI_COMM_WORLD) != &world ); -} - -// assert( reinterpret_cast(MPI_COMM_WORLD) == world ); - -{ - mpi3::communicator mty; - assert( mty.empty() ); -// assert( mty.size() == 0 ); - mpi3::communicator mty2 = mty; - assert( mty2.empty() ); -// assert( mty2.size() == 0 ); -} -{ - std::vector comms; - comms.emplace_back(world); - comms.emplace_back(world); - comms.emplace_back(world); - - std::vector comms2; - comms2.reserve(3); -// for(auto& e:comms) {comms2.emplace_back(e);} // ok, but old style -// std::copy(comms.begin(), comms.end(), std::back_inserter(comms2)); // doesn't work because it calls cctor - std::transform(comms.begin(), comms.end(), std::back_inserter(comms2), [](auto&& e) {return e;}); // calls dup ctor -} -{ - std::size_t const NTHREADS = 10; - std::vector> fs; - for(int i=0; i != NTHREADS; ++i) { // NOLINT(altera-unroll-loops) -#if 0 // this is problematic because copy (mpi_comm_dup) is not thread safe - fs.emplace_back(std::async([&world](){ - auto comm = world; // hangs here - return 5; - })); -#endif - fs.emplace_back(std::async([thread_comm=world](){ - // auto comm2 = thread_comm; // works, just to test - return 5; - })); -#if 0 // more correct - fs.emplace_back(std::async([world](){ - auto comm2 = world; // works, just to test - return 5; - })); -#endif - std::cout << "created thread" << std::endl; - } - for(std::size_t i = 0; i != NTHREADS; ++i) { // NOLINT(altera-unroll-loops) - auto five = fs[i].get(); - assert( five == 5 ); - std::cout << "joined thread" << std::endl; - } -} - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_create.hpp b/external_codes/mpi_wrapper/mpi3/test/communicator_create.hpp deleted file mode 100644 index 9e54369725a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_create.hpp +++ /dev/null @@ -1,20 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x `#-lboost_serialization` && time mpirun -np 15 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::communicator world2 = world; - mpi3::group world2g = world2.group(); - mpi3::communicator world3 = world2.create(world2g); - - assert(world3.rank() == world.rank()); - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_custom_attributes.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_custom_attributes.cpp deleted file mode 100644 index 6c5cca0eee0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_custom_attributes.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++17 -Wall -Wextra `#-Wfatal-errors` $0 -o $0x.x && time mpirun -n 1 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/environment.hpp" -#include "../../mpi3/communicator.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -mpi3::environment e; -mpi3::communicator::keyval k; -mpi3::communicator::keyval> k_map; - -int main(){ - - auto world = e.get_world_instance(); - - world.set_attribute(k, 10); - world.get_attribute(k) = 20; - int const& v = world.get_attribute(k); - assert(v == 20); - - auto w2 = world; - assert(w2.get_attribute(k) == 20); - - world.delete_attribute(k); - assert(not world.has_attribute(k)); - assert(w2.has_attribute(k)); - -// assert(not world.has_attribute(k_map)); -// world.attribute(k_map); -// assert(world.has_attribute(k_map)); -// world.get_attribute(k_map)["leader"] = std::string{"red"}; -// assert(mpi3::any_cast(world.get_attribute(k_map)["leader"]) == "red"); -// world.attribute(mpi3::environment::named_attributes_key()); - world.attribute("lop"); -// world.attribute("color") = std::string("red"); -// std::string s = mpi3::any_cast(world.attribute("color")); -// assert( s == "red" ); - -// assert(mpi3::any_cast(world.named_attributes()["leader"])); - - std::map m = {{"node", 555}}; - m["color"] = 11; -// int n = - std::any_cast(m["color"]) = 99; - int n = std::any_cast(m["color"]); - assert( n == 99 ); - - assert( std::any_cast(m["node"]) == 555 ); - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_divide.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_divide.cpp deleted file mode 100644 index 71f322e5dc4..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_divide.cpp +++ /dev/null @@ -1,20 +0,0 @@ -// © Alfredo Correa 2018-2021 - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world)-> int try { - assert( world.size() == 6 ); - - mpi3::communicator fifth = world/5; - - cout << "I am rank " << world.rank() << " in " << world.name() << ", "; - - if(fifth){cout<<"I am also "<< fifth.rank() <<" in "<< fifth.name() <<'\n';} - else {cout<<"I am not in "<< fifth.name() <<'\n';} - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_error.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_error.cpp deleted file mode 100644 index c7a945cc0db..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_error.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wextra `#-Wfatal-errors` $0 -o $0x.x && time mpirun -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright Alfredo A. Correa 2018. -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - std::error_code ec = mpi3::error::invalid_buffer_pointer; - assert( ec == mpi3::error::invalid_buffer_pointer); - assert( ec != std::io_errc::stream ); - - try{ - world.broadcast_n((int*)nullptr, 0, -1); - }catch(std::system_error const& e){ - if(world.root()) - cout - <<"code --> "<< e.code() <<'\n' - <<"message --> "<< e.code().message() <<'\n' - <<"what --> "<< e.what() <<'\n' - ; - } - cout <<"finish\n"; - - return 0; - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_exception.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_exception.cpp deleted file mode 100644 index 33db03e372e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_exception.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -O3 -Wall -Wextra $0 -o $0x.x -D_MAKE_BOOST_SERIALIZATION_HEADER_ONLY `#-lboost_serialization` && time mpirun -n 3 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright Alfredo A. Correa 2018. - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - assert( world.size() > 1 ); - std::vector v(3); - switch(world.rank()){ - case 0: v = {3, 4, 5}; try{world.send_n(begin(v), -1, 1);}catch(...){world.send_n(begin(v), 3, 1);} break; // -1 is a bad argument - case 1: world.receive(begin(v), 0) ; break; - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_gather.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_gather.cpp deleted file mode 100644 index ede25ac19f3..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_gather.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx.mpich -g $0 -o $0x -lboost_serialization&&mpirun.mpich -n 3 valgrind --error-exitcode=1345 $0x&&rm $0x;exit -#endif -// © Alfredo A. Correa 2018-2021 - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/ostream.hpp" -#include "../../mpi3/process.hpp" - -#include - -#include - -namespace mpi3 = boost::mpi3; - -using std::cout; -using std::list; -using std::vector; - -void part1(mpi3::communicator& world) { - vector> small(10, {0.0, world.rank()}); - vector> large(world.root() ? small.size() * static_cast(world.size()) : 0, std::pair(0.0, -10)); - - auto it = world.gather_n(small.begin(), small.size(), large.begin(), 0); - assert(it == large.end()); - - if(world.rank() == 0) { - assert(it != large.begin()); - assert((large[9] == std::pair(0.0, 0))); - assert((large[11] == std::pair(0.0, 1))); - } else { - assert(it == large.begin()); - } -} - -void part2(mpi3::communicator& world) { - vector> small(10, {0., world.rank()}); - vector> large(world.root() ? small.size() * static_cast(world.size()) : 0, std::pair(0., -1)); - - auto it = world.gather_n(small.begin(), small.size(), large.begin()); - assert(it == large.end()); - - if(world.root()) { - assert(it != large.begin()); - assert((large[9] == std::pair(0., 0))); - assert((large[11] == std::pair(0., 1))); - } else { - assert(it == large.begin()); - } -} - -void part3(mpi3::communicator& world) { - list small(10, world.rank()); - vector large(world.root()?small.size()*static_cast(world.size()):0, -1.0); - - world.gather(small.begin(), small.end(), large.begin(), 0); - if(world.root()) { - cout << "large: "; - for(auto& e : large) {cout<< e <<" ";} // NOLINT(altera-unroll-loops) - cout << '\n'; - } - if(world.root()) { - assert(large[ 1] == 0); - assert(large[11] == 1); - assert(large[21] == 2); - } -} - -void part4(mpi3::communicator& world) { - auto val = std::string{"5.1 opa"}; - - using T = decltype(val); - - vector small(10, val); - vector large(world.root() ? small.size() * static_cast(world.size()) : 0); - - world.gather(small.begin(), small.end(), large.begin(), 0); - - if(world.rank() == 0) { - assert(all_of(large.begin(), large.end(), [val](auto& e) { return val == e; })); - } -} - -void part5(mpi3::communicator& world) { - auto Lval = std::to_string(world.rank() + 1000); - auto vals0 = (world[0] |= Lval); - if(world.rank() == 0) { - assert(vals0.size() - static_cast(world.size()) == 0); - assert(vals0[2] == "1002"); - } else { - assert(vals0.empty()); - } -} - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - - mpi3::vector v; - - assert( world.size() > 2); - - part1(world); - part2(world); - part3(world); - part4(world); - part5(world); - - return 0; - -}catch(...){ - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_gather2.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_gather2.cpp deleted file mode 100644 index 5ac11ed26b2..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_gather2.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wextra -Wfatal-errors $0 -o $0x.x && time mpirun -n 3 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/detail/iterator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - using dd = std::tuple; - - std::vector
v_local(10, dd{world.rank(), world.rank() + 1}); - std::vector
v_local(10, dd{world.rank(), world.rank() + 1}); - std::vector
v(world.root()?v_local.size()*static_cast(world.size()):0); - auto last = world.gather(begin(v_local), end(v_local), begin(v)); - if(world.root()){ - assert(last == end(v)); - assert(( v[ 0] == dd{0.,1.} )); - assert(( v[10] == dd{1.,2.} )); - assert(( v[20] == dd{2.,3.} )); - }else{ - assert( last == begin(v) ); - } - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/group.cpp b/external_codes/mpi_wrapper/mpi3/test/group.cpp deleted file mode 100644 index 70529ccce6c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/group.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/*-*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4-*-*/ -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/environment.hpp" -#include "../../mpi3/group.hpp" - -#include "../../mpi3/main.hpp" - -#include - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - mpi3::group wg{world}; - mpi3::communicator w2{wg}; - - assert(w2.rank() == world.rank()); - assert(w2.size() == world.size()); - - mpi3::communicator half = world / 2; - mpi3::group hg{half}; - - mpi3::communicator h2{hg}; - assert(half.rank() == h2.rank()); - - static_assert(std::is_same{}, "!"); - - [[maybe_unused]] mpi3::group const& wgc = wg; - static_assert(std::is_same{}, "!"); - - // static_assert( std::is_same{}, "!" ); - - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/group_operations.cpp b/external_codes/mpi_wrapper/mpi3/test/group_operations.cpp deleted file mode 100644 index d7b02a4a6f8..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/group_operations.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 8s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/environment.hpp" -#include "../../mpi3/group.hpp" -#include "../../mpi3/communicator.hpp" - -using std::cout; -namespace mpi3 = boost::mpi3; - -int main(int argc, char* argv[]){ - mpi3::environment env(argc, argv); - - mpi3::communicator world = env.world(); - mpi3::communicator comm = env.world(); - - mpi3::group basegroup(world); - int grp_rank = basegroup.rank(); - assert(basegroup.rank() == comm.rank()); - - /* Form a new communicator with inverted ranking */ - mpi3::communicator newcomm = comm.split(0, comm.size() - comm.rank()); - mpi3::group g1(newcomm); - std::vector ranks_out = g1.translate_ranks(basegroup); - for(int i = 0; i != ranks_out.size(); ++i) assert(ranks_out[i] == comm.size() - 1 - i); - - assert( basegroup.compare(g1) == mpi3::similar ); - mpi3::communicator dupcomm = comm; - mpi3::group g2(dupcomm); - assert( basegroup.compare(g2) == mpi3::identical ); - - mpi3::communicator splitcomm = comm.split(comm.rank() < comm.size()/2, comm.rank()); - mpi3::group g3(splitcomm); - assert( compare(basegroup, splitcomm) == mpi3::unequal ); - - mpi3::group g3a = basegroup.include({comm.rank(), (comm.rank() + 1)%comm.size()}); - mpi3::group g3b = basegroup.include({comm.rank(), (comm.rank() + comm.size() -1)%comm.size()}); - assert( compare(g3a, g3b) == mpi3::unequal ); - - - std::vector ranks(basegroup.size()); - std::iota(ranks.begin(), ranks.end(), 0); - mpi3::group g4 = basegroup.exclude_n(ranks.begin(), 1); - mpi3::group g5 = basegroup.exclude_n(ranks.begin() + 1, ranks.size() - 1); - mpi3::group g6 = set_union(g5, g4); // order matters - assert( basegroup == g6 ); - - mpi3::group g7 = set_union(basegroup, g4); - assert( basegroup == g7 ); - - mpi3::group g8 = basegroup.range_exclude({{1, basegroup.size()-1, 1}}); - assert( g5 == g8 ); - - mpi3::group g9 = set_intersection(basegroup, g4); - assert( g9 == g4 ); - - mpi3::group g10 = basegroup.range_exclude({{0, basegroup.size()-1, 1}}); -// assert( g10 == mpi3::group::empty() ); // g10 == mpi3::group() -// assert( g10.size() == 0 ); - assert( g10.empty() ); -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/ibarrier.cpp b/external_codes/mpi_wrapper/mpi3/test/ibarrier.cpp deleted file mode 100644 index 20109500cf9..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/ibarrier.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2022 Alfredo A. Correa -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - auto const my_rank = world.rank(); - mpi3::request r = world.ibarrier(); - - using namespace std::literals::chrono_literals; - std::this_thread::sleep_for(2s); - - std::cout<<"mpi process "<< my_rank <<" call ibarrier."<< std::endl; - r.wait(); - assert( r.completed() ); - std::cout<<"mpi process "<< my_rank <<" the barrier is complete."<< std::endl; - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/keyval.cpp b/external_codes/mpi_wrapper/mpi3/test/keyval.cpp deleted file mode 100644 index 43f6171141b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/keyval.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 8s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/environment.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace boost{ -namespace mpi3{ - struct keyval{ - template - static int default_MPI_Comm_copy_attr_function( - MPI_Comm oldcomm, int comm_keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag) - { - // std::memcpy(attribute_val_in, attribute_val_out, (std::size_t)(*extra_state)); - attribute_val_out = new T(*reinterpret_cast(attribute_val_in)); - *flag = 1; - return MPI_SUCCESS; - } - template - static int default_MPI_Comm_delete_attr_function(MPI_Comm old, int comm_keyval, void *attribute_val, void *extra_state){ - delete reinterpret_cast(attribute_val); - return MPI_SUCCESS; - } - std::vector key_; - std::vector attrval_; - keyval(int size) : key_(size), attrval_(size){} - int size(){return key_.size();} - ~keyval(){ - for(int i = 0; i != size(); ++i){ - MPI_Comm_free_keyval(&key_[i]); - } - } - template - void set(int idx){ - MPI_Comm_create_keyval( - default_MPI_Comm_copy_attr_function,// MPI_NULL_COPY_FN, - default_MPI_Comm_delete_attr_function,// MPI_NULL_DELETE_FN, - &key_[idx], (void*)0 - ); - } - template - void set(){ - set(0); - set(1); - } - template - void set(){ - set(0); - set(1); - set(2); - } - }; - - -template -void communicator::set_attribute(keyval const& kv, int idx, T const& val){ - int status = MPI_Comm_set_attr(impl_, kv.key_[idx], new T(val)); - if(status != MPI_SUCCESS) throw std::runtime_error("cannot set attribute"); -} -void communicator::delete_attribute(keyval const& kv, int idx){ - int status = MPI_Comm_delete_attr(impl_, kv.key_[idx]); -} - -template -void communicator::get_attribute(keyval const& kv, int idx, T& val){ - int flag = 0; - T* p = nullptr; - int status = MPI_Comm_get_attr(impl_, kv.key_[idx], &p, &flag); - assert(flag); - val = *p; -} - -bool communicator::has_attribute(keyval const& kv, int idx){ - int flag = 0; - void* p = nullptr; - int status = MPI_Comm_get_attr(impl_, kv.key_[idx], &p, &flag); - return flag; -} - -template -T const& communicator::get_attribute_as(keyval const& kv, int idx){ - int flag = 0; - T* p = nullptr; - int status = MPI_Comm_get_attr(impl_, kv.key_[idx], &p, &flag); - assert(flag); - return *p; -} - -}} - - -namespace mpi3 = boost::mpi3; -using std::cout; - -int main(int argc, char* argv[]){ - - mpi3::environment env(argc, argv); - mpi3::communicator comm = env.world(); - - mpi3::keyval kv(3); - kv.set(); - - comm.set_attribute(kv, 0, std::string("hola")); - comm.set_attribute(kv, 1, 5.1); - comm.set_attribute(kv, 2, 8); - - assert( comm.has_attribute(kv, 0) ); - std::string ret; - comm.get_attribute(kv, 0, ret); - assert(ret == "hola"); - assert( comm.get_attribute_as(kv, 0) == "hola" ); - - mpi3::communicator comm2 = comm; - assert( comm2.has_attribute(kv, 0) ); - comm2.set_attribute(kv, 0, std::string("chau")); - assert( comm2.get_attribute_as(kv, 0) == "chau" ); - - assert( comm.get_attribute_as(kv, 0) == "hola" ); - - comm.delete_attribute(kv, 0); - comm.delete_attribute(kv, 1); - comm.delete_attribute(kv, 2); - - assert( not comm.has_attribute(kv, 0) ); - - cout << "TAG_UB = " << comm.attribute_as(MPI_TAG_UB) << std::endl; - cout << "HOST = " << comm.attribute_as(MPI_HOST) << std::endl; - cout << "IO = " << comm.attribute_as(MPI_IO) << std::endl; - cout << "WTIME_IS_GLOBAL = " << comm.attribute_as(MPI_WTIME_IS_GLOBAL) << std::endl; - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/legacy_fftw3.cpp b/external_codes/mpi_wrapper/mpi3/test/legacy_fftw3.cpp deleted file mode 100644 index feb3b697341..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/legacy_fftw3.cpp +++ /dev/null @@ -1,65 +0,0 @@ -//#ifdef COMPILATION_INSTRUCTIONS -//(echo '#include"'$0'"'>$0.cpp)&&mpic++ $0 -o $0x -lfftw3_mpi -lfftw3 -lboost_unit_test_framework&&mpirun -n 4 $0x&&rm $0x $0.cpp;exit -//#endif -// Copyright 2018-2021 Alfredo A. Correa - -#include - -#include -#include "../main.hpp" - -const ptrdiff_t N0 = 16, N1 = 16; - -std::complex myf_x(ptrdiff_t i, ptrdiff_t j){ - return {cos(2*M_PI*i/N1), sin(2*M_PI*i/N1)}; -} - -std::complex myf_y(ptrdiff_t i, ptrdiff_t j){ - return {cos(2*M_PI*j/N1), sin(2*M_PI*j/N1)}; -} - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int argc, char **argv, mpi3::communicator world) { - fftw_mpi_init(); - - /* get local data size and allocate */ - ptrdiff_t local_n0, local_0_start; - ptrdiff_t alloc_local = fftw_mpi_local_size_2d(N0, N1, &world, &local_n0, &local_0_start); - assert( alloc_local >= local_n0*N1 and alloc_local%4==0 ); // seems to be multiple of 4 (number of complexs) - auto data = reinterpret_cast*>(fftw_alloc_complex( alloc_local )); - - /* create plan for in-place forward DFT */ - fftw_plan plan = fftw_mpi_plan_dft_2d( - N0, N1, reinterpret_cast(data), reinterpret_cast(data), - &world, // pass an old fashion comm handle to legacy libraries - FFTW_FORWARD, FFTW_ESTIMATE - ); - - /* initialize data to some function my_function(x,y) */ - for(ptrdiff_t i = 0; i != local_n0; ++i) - for(ptrdiff_t j = 0; j != N1; ++j) - data[i*N1 + j] = myf_x(local_0_start + i, j); - - /* compute transforms, in-place, as many times as desired */ - fftw_execute(plan); - { - int i = 0, j = 0; - if(local_0_start <= i and i < local_0_start + local_n0) - assert( abs( data[(i - local_0_start)*N1 + j] ) < 1e-12 ); - } - { - int i = 1, j = 0; - if(local_0_start <= i and i < local_0_start + local_n0) - assert( abs( data[(i - local_0_start)*N1 + j] - 256.) < 1e-12 ); - } - { - int i = 0, j = 15; - if(local_0_start <= i and i < local_0_start + local_n0) - assert( abs( data[(i - local_0_start)*N1 + j] ) < 1e-12 ); - } - fftw_destroy_plan(plan); - - fftw_mpi_cleanup(); - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/library_check.cpp b/external_codes/mpi_wrapper/mpi3/test/library_check.cpp deleted file mode 100644 index 8e2fba4f616..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/library_check.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Test for separate compilation / library usage. - -#include "../communicator.hpp" -#include "../environment.hpp" - -namespace mpi3 = boost::mpi3; - -void do_broadcast(mpi3::communicator &c) { // cppcheck-suppress unusedFunction - int a = 2; - c.broadcast_value(a); -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/library_main.cpp b/external_codes/mpi_wrapper/mpi3/test/library_main.cpp deleted file mode 100644 index c9f9ee4e744..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/library_main.cpp +++ /dev/null @@ -1,35 +0,0 @@ -//#if COMPILATION_INSTRUCTIONS -//mpic++ library_check.cpp library_main.cpp -//#mpic++ -c library_check.cpp -//#ar rcs liblibrary_check.a library_check.o -//#mpic++ library_main.cpp -o library_main.x -L. -llibrary_check -//mpirun -n 4 ./library_main.x;exit -//#endif -// Compile-time test that all functions are appropriately declared 'inline' and -// will not give multiple definition errors - -// For a simple check on multi-file compilation, use -// mpic++ library_check.cpp library_main.cpp - -// To check when one file is in a library, use -// mpic++ -c library_check.cpp -// ar rcs liblibrary_check.a library_check.o -// mpic++ library_main.cpp -L. -llibrary_check - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; - -// defined in library_check.cpp -void do_broadcast(mpi3::communicator& c); - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - int b = 1; - world.broadcast_value(b); - do_broadcast(world); - - return 0; -}catch(...){ - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/minimal.cpp b/external_codes/mpi_wrapper/mpi3/test/minimal.cpp deleted file mode 100644 index 4683a61e628..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/minimal.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -$CXX `mpic++ -showme:compile|sed 's/-pthread/ /g'` $0 -o $0x `mpic++ -showme:link|sed 's/-pthread/ /g'`&& -#(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --suppressions=$0.openmpi.supp $0x)&&rm $0x;exit -mpic++ -O3 -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --gen-suppressions=all $0x 2>&1|grep -v '==' > $0.openmpi.supp )&& -mpic++ -g -O3 -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --gen-suppressions=all $0x 2>&1|grep -v '==' > $0-g.openmpi.supp )&& -mpic++ -g -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --gen-suppressions=all $0x 2>&1|grep -v '==' > $0-O0.openmpi.supp )&& -cat $0-g.openmpi.supp $0-O0.openmpi.supp >> $0.openmpi.supp&&rm $0x;exit -#endif -// © Alfredo A. Correa 2018-2020 -#include "../../mpi3/environment.hpp" - -namespace mpi3 = boost::mpi3; - -int main(int argc, char* argv[]){ -// mpi3::environment env(argc, argv); -// MPI_Init(&argc, &argv); -// MPI_Finalize(); - - MPI_Init(&argc, &argv); - MPI_Comm dup_comm_world; - MPI_Comm_dup( MPI_COMM_WORLD, &dup_comm_world ); - MPI_Comm_free(&dup_comm_world); - MPI_Finalize(); - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/multi_array.cpp b/external_codes/mpi_wrapper/mpi3/test/multi_array.cpp deleted file mode 100644 index 07481789b50..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/multi_array.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wextra `#-Wfatal-errors` -I$HOME/prj/alf $0 -o $0x.x && mpirun -n 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/shm/allocator.hpp" -#include "../../mpi3/main.hpp" - -#include "boost/multi/array.hpp" -#include - -namespace mpi3 = boost::mpi3; -namespace multi = boost::multi; -using std::cout; - -namespace boost{namespace mpi3{namespace shm{ -namespace multi{ - template - using array = boost::multi::array>; -} -}}} - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::shared_communicator node = world.split_shared(); - - multi::array> A({16, 16}, node); - assert(A[2][2] == 0); - node.barrier(); - if(node.root()) A[2][2] = 3.14; - node.barrier(); - assert(A[2][2] == 3.14); - - shm::multi::array B = A; -// multi::array> B = A; - assert(A[2][2] == 3.14); - assert(B[2][2] == 3.14); - - shm::multi::array C({1, 1}, node); - C.reextent({30,30}); - assert(C.size() == 30); -// C = std::move(A); -// assert(C[2][2] == 3.14); -// assert(A.size()==0); - node.barrier(); - - { - std::vector> w; - std::vector> v(2, shm::multi::array(shm::allocator{node})); - v.reserve(3); - v.emplace_back(shm::multi::array({10,10}, node)); - v.emplace_back(node); - v.emplace_back(shm::multi::array({20,20}, node)); - v.emplace_back(shm::multi::array({15,15}, node)); - - - if(world.rank() == 0) v[4][2][3] = 5.; - world.barrier(); - if(world.rank() == 1) assert(v[4][2][3] == 5.); - world.barrier(); - v[2].reextent({30, 30}); - assert( v[2].size() == 30 ); - v.pop_back(); - assert( v.back().size() == 20 ); - } - shm::multi::array AA({0,0}, node); - assert(AA.size() == 0); - { - boost::multi::array, 1, shm::allocator>> v(std::array{{3}}, shm::allocator>{node}); - v[0].reextent({10, 10}); - v[1].reextent({20, 20}); - v[2].reextent({30, 30}); - if(world.rank() == 1) v[1][2][3] = 5.; - world.barrier(); - if(world.rank() == 1) assert(v[1][2][3] == 5.); - } - { - using inner_ma = boost::multi::array>; - // using dd = inner_ma::allocator_type; - using outter_vec = std::vector>; - using scoped_alloc = std::scoped_allocator_adaptor, shm::allocator>; - shm::allocator aaaa{node}; - scoped_alloc sa(std::allocator{}, aaaa ); - outter_vec v(sa); - // std::vector v; - v.emplace_back(shm::allocator{node}); - // v.emplace_back(std::array{{10, 10}}); -// v.resize(10, {node}); - // boost::multi::array, 1, std::scoped_allocator_adaptor>>, shm::allocator> > v(sa); - cout << "fdfsd " << std::uses_allocator>{} << std::endl; - cout << "fdffddf " << std::is_constructible&>{} << std::endl; - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/package.cpp b/external_codes/mpi_wrapper/mpi3/test/package.cpp deleted file mode 100644 index d71fe13a7a2..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/package.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x -lboost_serialization && time mpirun -n 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -//#include "../../mpi3/process.hpp" - -#include "mpi.h" -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char *[], mpi3::communicator world){ - - std::vector buffer(110); - if (world.rank() == 0){ - mpi3::package p(world); - int i = 123; - p.pack_n(&i, 1); - double d = 5.; - p.pack_n(&d, 1); - cout << "about to send" << std::endl; - cout << "size " << p.buffer_.size() << std::endl; - p.send(1, 12345); - cout << "already sent" << std::endl; - } - - if (world.rank() == 1){ - mpi3::package p(world); - cout << "about to receive" << std::endl; - p.receive(0, 12345); - cout << "already received" << std::endl; - int i = -1; - p.unpack_n(&i, 1); - assert( i == 123 ); - double d = -1; - p.unpack_n(&d, 1); - assert( d == 5. ); - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/parallel_dot_product.dat b/external_codes/mpi_wrapper/mpi3/test/parallel_dot_product.dat deleted file mode 100644 index d85eac99f6a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/parallel_dot_product.dat +++ /dev/null @@ -1,12 +0,0 @@ -# n time -1 12.6676 9.5459 -2 6.81593 9.26805 -3 4.79953 11.2034 -4 6.62449 9.36368 -5 3.5278 9.29483 -6 4.73039 9.38357 -7 4.23245 11.156 -8 12.5752 9.7146 -9 11.3129 13.952 -10 11.2814 9.29381 -11 10.099 10.1061 diff --git a/external_codes/mpi_wrapper/mpi3/test/process.cpp b/external_codes/mpi_wrapper/mpi3/test/process.cpp deleted file mode 100644 index 4b493d1d0b8..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/process.cpp +++ /dev/null @@ -1,159 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include -#include -#include - -#include - -// nontrivial nonpod class -struct B { // NOLINT(readability-identifier-naming) - std::string name_ = "unnamed"; // NOLINT(misc-non-private-member-variables-in-classes) - std::size_t n_ = 0; // NOLINT(misc-non-private-member-variables-in-classes) - double* data = nullptr; // NOLINT(misc-non-private-member-variables-in-classes) - - B() = default; - explicit B(std::size_t n) : n_{n}, data{new double[n]} { std::fill_n(data, n, 0.0); } - B(B const& other) : name_{other.name_}, n_{other.n_}, data{new double[other.n_]} {} - B(B&&) = delete; - - B& operator=(B&&) = default; - B& operator=(B const& other) { - if(this == &other) { - return *this; - } - name_ = other.name_; - n_ = other.n_; - delete[] data; - data = new double[other.n_]; // NOLINT(cppcoreguidelines-owning-memory) - std::copy_n(other.data, other.n_, data); - return *this; - } - - ~B() { delete[] data; } -}; - -// nonintrusive serialization -template -void save(Archive& ar, B const& b, unsigned int const /*version*/) { - ar << b.name_ << b.n_ << boost::serialization::make_array(b.data, b.n_); -} -template -void load(Archive& ar, B& b, unsigned int const /*version*/) { - ar >> b.name_ >> b.n_; - delete[] b.data; // NOLINT(cppcoreguidelines-owning-memory) - b.data = new double[b.n_]; // NOLINT(cppcoreguidelines-owning-memory) - ar >> boost::serialization::make_array(b.data, b.n_); -} -BOOST_SERIALIZATION_SPLIT_FREE(B) // cppcheck-suppress unknownMacro - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - - assert(world.size() > 1); - - switch(world.rank()) { - case 0: { - int a = 5; - world[1] << a; - break; - } - case 1: { - int a = -1; - world[0] >> a; // specific source (any tag) - assert(a == 5); - break; - } - } - - switch (world.rank()) { - case 0: { - int a = 7; - world[1] << a; - break; - } - case 1: { - int a = -1; - world >> a; // any source (any tag) - assert(a == 7); - break; - } - } - - int b = world.rank(); - world[1] & b; // broadcast (from rank 1) - assert(b == 1); - - // if(world.root()) { - // B b1(4); - // b1.data[2] = 4.5; - // world[1] << b1; - // } else { - // B b2; - // world[0] >> b2; - // assert(b2.data[2] == 4.5); - // } - - { - switch(world.rank()) { - case 0: { - int a = 7; - world[1] << a; - break; - } - case 1: { - int a = -1; - world >> a; // any source (any tag) - assert(a == 7); - break; - } - } - - switch(world.rank()) { - case 0: { - world[1] << true; - break; - } - case 1: { - bool bo = false; - world >> bo; - assert(bo == true); - break; - } - } - - { - std::vector v = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}; - - switch(world.rank()) { - case 0: { - world[1] << v; - break; - } - case 1: { - std::vector w; - world >> w; - assert(v == w); - } - } - } - - { - std::vector v = {false, true, false}; - switch(world.rank()) { - case 0: { - world[1] << v; - break; - } - case 1: { - std::vector v; - world >> v; - assert((v == std::vector{false, true, false})); - } - } - } - } - return 0; -} catch(...) { return 1; } diff --git a/external_codes/mpi_wrapper/mpi3/test/process_vector.cpp b/external_codes/mpi_wrapper/mpi3/test/process_vector.cpp deleted file mode 100644 index 13166c941cc..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/process_vector.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 $0 -o $0.x -lboost_serialization -lboost_container&&time mpirun -np 2 $0.x&&rm -f $0.x;exit -#endif - -#include "../main.hpp" -#include "../process.hpp" - -#include - -struct long_long{ - long long value; - long_long& operator=(long long v){value = v; return *this;} -}; - -template -void serialize(Archive& ar, long_long& l, unsigned = 0){ - ar & l.value; -} - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - - assert( world.size() == 2 ); - long long size = 100000000; - switch(world.rank()){ - case 0: - { - std::vector v(size); std::iota(v.begin(), v.end(), 0.); - // assert(std::accumulate(v.begin(), v.end(), 0.) == size*(size-1)/2 ); - world[1] << v; - } - break; - case 1: - { - std::vector w; world[0] >> w; - assert( w.size() == size ); - assert( w[45].value = 45 ); - // assert(std::accumulate(w.begin(), w.end(), 0.) == size*(size-1)/2 ); - } - break; - } - return 0; - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/reduce_maxloc.cpp b/external_codes/mpi_wrapper/mpi3/test/reduce_maxloc.cpp deleted file mode 100644 index 8b0a422b5c5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/reduce_maxloc.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "../../mpi3/main.hpp" -#include "../../mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - - int const data = [&] {switch(world.rank()) { - case 0: return 12; - case 1: return 34; - case 2: return 56; - case 3: return 78; // default: world.abort(EXIT_FAILURE); - } return 0;}(); - - { - auto reduction = world.max_loc(data); - - assert( reduction.value == 78 ); - assert( reduction.location == 3 ); - } -#if __cpp_structured_bindings >= 201606 - { - auto [value, location] = world.max_loc(data); - assert( value == 78 ); - assert( location == 3 ); - } - { - auto&& [value, procs] = world.max_location(data); - assert( value == 78 ); - assert( procs.rank() == 3 ); - } -#endif - { - int const* max_ptr = world.max_element(data); - if(world.rank() == 3) { - assert( max_ptr and max_ptr == &data ); - } else { - assert( not max_ptr ); - } - } - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/request_test_some.cpp b/external_codes/mpi_wrapper/mpi3/test/request_test_some.cpp deleted file mode 100644 index 2ffdf1cc5c0..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/request_test_some.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/environment.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/request.hpp" - -#include // sleep_for - -namespace mpi3 = boost::mpi3; -using std::cout; -int main(){ - mpi3::environment env; - auto& world = env.world(); - assert( world.size() == 4 ); - - if(world.rank() == 0){ - int rem = 3; - std::vector rs(3); - for(int i = 1; i != world.size(); ++i){ - std::vector buffer(100); - rs[i - 1] = world.ireceive(buffer.begin() + i, buffer.begin() + i + 1, i); - } - while(rem > 0){ - std::vector completed = mpi3::completed_some(rs); - if(completed.size() > 0){ - cout << "finished " << completed.size() << "\n"; - rem -= completed.size(); - }else std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }else{ - std::vector buffer(100); - world.send(buffer.begin(), buffer.begin() + 1, 0); - } -} - - diff --git a/external_codes/mpi_wrapper/mpi3/test/request_wait.cpp b/external_codes/mpi_wrapper/mpi3/test/request_wait.cpp deleted file mode 100644 index 99972b42b83..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/request_wait.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - assert( world.size() == 4); - - std::vector buffer(10); - std::vector buffer2(10); std::iota(buffer2.begin(), buffer2.end(), 0); - - int right = (world.rank() + 1)% world.size(); - int left = world.rank() - 1; if(left < 0) left = world.size() - 1; - - mpi3::request r = world.ireceive(buffer.begin(), buffer.end(), left, 123); - world.send(buffer2.begin(), buffer2.end(), right, 123); -// r.wait(); - mpi3::wait(r); - assert( buffer == buffer2 ); - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/request_wait_all.cpp b/external_codes/mpi_wrapper/mpi3/test/request_wait_all.cpp deleted file mode 100644 index bb2f454616e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/request_wait_all.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - assert( world.size() == 4); - - std::vector buffer(400); - if(world.root()){ - for(int i = 0; i != world.size(); ++i) buffer[i] = i/100; - std::vector rs(3); - for(int i = 0; i != world.size() - 1; ++i){ - rs[i] = world.isend(&buffer[i*100], &buffer[i*100] + 100, i + 1, 123); - } - wait_all(rs.begin(), rs.end()); - }else world.receive(buffer.begin(), buffer.begin() + 100, 0, 123); - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/request_wait_any.cpp b/external_codes/mpi_wrapper/mpi3/test/request_wait_any.cpp deleted file mode 100644 index dbc49f1dc56..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/request_wait_any.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - assert( world.size() == 4); - - std::vector buffer(400); - if(world.root()){ - for(int i = 0; i != world.size(); ++i) buffer[i] = i/100; - std::vector rs(3); - for(int i = 0; i != world.size() - 1; ++i){ - rs[i] = world.isend(&buffer[i*100], &buffer[i*100] + 100, i + 1, 123); - } - for(int i = 0; i != world.size() - 1; ++i){ - wait_any(rs.begin(), rs.end()); - } - }else world.receive(buffer.begin(), buffer.begin() + 100, 0, 123); - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/request_wait_some.cpp b/external_codes/mpi_wrapper/mpi3/test/request_wait_some.cpp deleted file mode 100644 index 048a9ec27bf..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/request_wait_some.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - assert( world.size() == 4); - - std::vector buffer(400); - if(world.root()){ - for(int i = 0; i != world.size(); ++i) buffer[i] = i/100; - std::vector rs(3); - for(int i = 0; i != world.size() - 1; ++i){ - rs[i] = world.isend(&buffer[i*100], &buffer[i*100] + 100, i + 1, 123); - } - int remaining = world.size() - 1; - while(remaining > 0){ - int count = wait_some(rs.begin(), rs.end()).size(); - if(count > 0){ - cout << "sends completed " << count << '\n'; - remaining =- count; - } - } - }else world.receive(buffer.begin(), buffer.begin() + 100, 0, 123); - -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/ring.cpp b/external_codes/mpi_wrapper/mpi3/test/ring.cpp deleted file mode 100644 index e1c92351d54..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/ring.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// © Alfredo A. Correa 2021 -#include "../../mpi3/main.hpp" -#include "../../mpi3/process.hpp" - -#include -#include // for iota -#include - -namespace bmpi3 = boost::mpi3; - -int bmpi3::main(int /*argc*/, char ** /*argv*/, bmpi3::communicator world) try { - auto const size = world.size(); assert(size != 0); - - mpi3::process&& next = world[(world.rank() + size + 1) % size]; - mpi3::process&& prior = world[(world.rank() + size - 1) % size]; - - int token; // NOLINT(cppcoreguidelines-init-variables) - if(not world.is_root()) {prior >> token;} - else {token = -1; } - - next << token; - - if( world.is_root()) {prior >> token;} - - assert(token == -1); - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/send_complex.cpp b/external_codes/mpi_wrapper/mpi3/test/send_complex.cpp deleted file mode 100644 index 25290e62680..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/send_complex.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2023 Alfredo A. Correa - -#include -#include - -#include -#include -#include - -namespace mpi3 = boost::mpi3; - -template -struct my_complex { - T _re; // NOLINT(misc-non-private-member-variables-in-classes) aggregate - T _im; // NOLINT(misc-non-private-member-variables-in-classes) aggregate - T real() const {return _re;} - T imag() const {return _im;} - bool operator==(my_complex const& other) const {return _re == other._re and _im == other._im;} - bool operator!=(my_complex const& other) const {return _re != other._re or _im != other._im;} - -}; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - assert(world.size() > 1); - - using complex = my_complex; - - switch(world.rank()) { - case 0: { - complex const c{1.0, 2.0}; - world.send_n(&c, 1, 1); - break; - }; - case 1: { - complex c; - world.receive_n(&c, 1); - assert((c == complex{1.0, 2.0})); - break; - }; - } - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/serialization_intrusive.cpp b/external_codes/mpi_wrapper/mpi3/test/serialization_intrusive.cpp deleted file mode 100644 index 732f62458a6..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/serialization_intrusive.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -std=c++14 -O3 -Wfatal-errors -D_MAKE_BOOST_SERIALIZATION_HEADER_ONLY `#-lboost_serialization` $0 -o $0x.x && time mpirun -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif - -// use this to avoid linking to -lboost_serialization -//#define _MAKE_BOOST_SERIALIZATION_HEADER_ONLY - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -struct A{ - std::string name_ = "unnamed"; - int n_ = 0; - double* data_ = nullptr; - - A() = default; - A(int n) : n_(n), data_{new double[n]}{} - A(A const& other) : - name_{other.name_}, - n_{other.n_}, - data_{new double[n_]} - { - std::copy_n(other.data_, n_, data_); - } - A& operator=(A const& other){ - A cpy(other); - return swap(cpy); - } - A& swap(A& other){ - std::swap(name_, other.name_); - std::swap(n_, other.n_); - std::swap(data_, other.data_); - return *this; - } - ~A(){delete[] data_;} - - // begin intrusive serialization - BOOST_SERIALIZATION_SPLIT_MEMBER() - template - void save(Archive & ar, const unsigned int) const{ - ar - << name_ - << n_ - << boost::serialization::make_array(data_, n_) - ; - } - template - void load(Archive & ar, const unsigned int){ - int n; - ar - >> name_ - >> n; - ; - if(n != n_){ - delete[] data_; - data_ = new double[n_]; - } - ar >> boost::serialization::make_array(data_, n_); - } - // end intrusive serialization -}; - -int mpi3::main(int, char*[], mpi3::communicator world){ - assert( world.size() == 2); - - if(world.root()){ - std::vector v(5, A(3)); - v[2].data_[2] = 3.14; - world.send(v.begin(), v.end(), 1, 123); - }else{ - std::vector v(5); - world.receive(v.begin(), v.end(), 0, 123); - assert(v[2].data_[2] == 3.14); - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/shared_communicator.cpp b/external_codes/mpi_wrapper/mpi3/test/shared_communicator.cpp deleted file mode 100644 index ee6c69a51dd..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shared_communicator.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ $0 -o $0x&&mpirun -n 4 $0x $@&&rm $0x;exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/shared_communicator.hpp" -#include "../../mpi3/ostream.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -void check_isa_communicator(mpi3::communicator const&){} -void check_isa_shared_comm (mpi3::shared_communicator const&){} - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::ostream wout(world); - - auto node = world.split_shared(); - wout << "I am rank " << node.rank() << " in comm " << node.name() << std::endl; - wout << "----" << std::endl; - - auto core = world.split_shared(mpi3::communicator_type::core); - wout << "I am rank " << core.rank() << " in comm " << core.name() << std::endl; - wout << "----" << std::endl; - - auto hw = world.split_shared(mpi3::communicator_type::hw_thread); - wout << "I am rank " << hw.rank() << " in comm " << hw.name() << std::endl; - wout << "----" << std::endl; - - auto l1 = world.split_shared(mpi3::communicator_type::l1_cache); - wout << "I am rank " << l1.rank() << " in comm " << l1.name() << std::endl; - wout << "----" << std::endl; - - auto l2 = world.split_shared(mpi3::communicator_type::l2_cache); - wout << "I am rank " << l2.rank() << " in comm " << l2.name() << std::endl; - wout << "----" << std::endl; - - auto l3 = world.split_shared(mpi3::communicator_type::l3_cache); - wout << "I am rank " << l3.rank() << " in comm " << l3.name() << std::endl; - wout << "----" << std::endl; - - auto socket = world.split_shared(mpi3::communicator_type::socket); - wout << "I am rank " << socket.rank() << " in comm " << socket.name() << std::endl; - wout << "----" << std::endl; - - auto numa = world.split_shared(mpi3::communicator_type::numa); - wout << "I am rank " << numa.rank() << " in comm " << numa.name() << std::endl; - wout << "----" << std::endl; - - auto board = world.split_shared(mpi3::communicator_type::board); - wout << "I am rank " << board.rank() << " in comm " << board.name() << std::endl; - wout << "----" << std::endl; - - auto host = world.split_shared(mpi3::communicator_type::host); - wout << "I am rank " << host.rank() << " in comm " << host.name() << std::endl; - wout << "----" << std::endl; - - auto cu = world.split_shared(mpi3::communicator_type::cu); - wout << "I am rank " << cu.rank() << " in comm " << cu.name() << std::endl; - wout << "----" << std::endl; - - return 0; -#if 0 - mpi3::shared_communicator node2 = node.split(node.rank() % 2, node.rank()); - - wout << "I am rank " << node.rank() << " in comm " << node.name() << " and rank " << node2.rank() << " in " << node2.name() << std::endl; - - check_isa_shared_comm(node); - check_isa_communicator(node); - - mpi3::communicator virtual_node{node}; - assert( &virtual_node != &node ); - - mpi3::communicator virtual_node2; - virtual_node2 = node2; - assert( &virtual_node2 != &node2 ); - - mpi3::communicator virtual_node3 = world.split_shared(); - assert( &virtual_node3 != &world ); - - assert( sizeof(mpi3::communicator) == sizeof(mpi3::shared_communicator) ); -#endif - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/shared_mutex.cpp b/external_codes/mpi_wrapper/mpi3/test/shared_mutex.cpp deleted file mode 100644 index d08645e6d48..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shared_mutex.cpp +++ /dev/null @@ -1,11 +0,0 @@ -// © Alfredo Correa 2018-2020 - -#include "../../mpi3/main.hpp" -#include "../../mpi3/environment.hpp" -#include "../../mpi3/shm/mutex.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator/*world*/) -> int { - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/shared_window.cpp b/external_codes/mpi_wrapper/mpi3/test/shared_window.cpp deleted file mode 100644 index 4270e28d54e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shared_window.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#if COMPILATION_INSTRUCTIONS// -*- indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4 -*- -mpicxx -O3 -std=c++14 $0 -o $0x.x && time mpirun -n 3 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/shared_window.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char *argv[], mpi3::communicator world){ - - auto node = world.split_shared(); - int n = 100; - auto sw = node.make_shared_window(node.root()?n:0); - assert( sw.size(0) == n ); - assert( sw.size(1) == 0 ); - -// mpi3::shared_window sw(node, node.rank()==0?n:0); - - -// int* arr = (int*)sw.base(0); -#if 0 -// std::cout<<"size "<< sw.size(0) < - -using std::cout; -using std::endl; - -namespace mpi3 = boost::mpi3; -namespace shm = mpi3::shm; - -int main(int, char*[]){ - mpi3::environment env; - auto world = env.world(); - mpi3::shared_communicator node = world.split_shared(); - - std::vector> v(100, node); - - cout << "v = " << v[32] << std::endl; - return 0; - - cout<<" rank: " < A1(node); - shm::allocator A2(node); - shm::allocator A3(node); - shm::allocator A4(node); - shm::allocator A5(node); - shm::allocator A6(node); - shm::allocator A7(node); - shm::allocator A8(node); - shm::allocator A9(node); - - auto data1 = A1.allocate(80); - auto data2 = A2.allocate(80); - auto data3 = A3.allocate(80); - auto data4 = A4.allocate(80); - auto data5 = A5.allocate(80); - auto data6 = A6.allocate(80); - auto data7 = A7.allocate(80); - auto data8 = A8.allocate(80); - auto data9 = A9.allocate(80); - -// assert(data9 != nullptr); - - using ptr = decltype(data1); - std::pointer_traits::element_type dd = 5.6; - std::pointer_traits::pointer pp = data1; -// double* dppp = std::pointer_traits::to_address(data1); - - assert(dd == 5.6); - - if(node.root()) data9[3] = 3.4; - node.barrier(); - assert(data9[3] == 3.4); - node.barrier(); - A1.deallocate(data1, 80); - A2.deallocate(data2, 80); - A3.deallocate(data3, 80); - A4.deallocate(data4, 80); - A5.deallocate(data5, 80); - A6.deallocate(data6, 80); - A7.deallocate(data7, 80); - A8.deallocate(data8, 80); - A9.deallocate(data9, 80); - - - -} diff --git a/external_codes/mpi_wrapper/mpi3/test/shared_window3.c b/external_codes/mpi_wrapper/mpi3/test/shared_window3.c deleted file mode 100644 index c490acfffef..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shared_window3.c +++ /dev/null @@ -1,62 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicc -Wfatal-errors $0 -o $0x.x && mpirun `#srun -p pdebug` -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include -#include -#include - -int main(int argc, char* argv[]){ - MPI_Init(&argc, &argv); - MPI_Comm node; - int ss = MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &node); - assert(ss == MPI_SUCCESS); - - int rank = -1; - MPI_Comm_rank(node, &rank); - - - #define N 65536 - - MPI_Win win[N]; - void* base_ptr[N]; - int s[N]; - MPI_Aint size[N]; - int a[N]; - char* ptr[N]; - - for(int i = 0; i != N; ++i){ - base_ptr[i] = 0; - printf("before alloc shared # %i\n", i); fflush(stdout); - int sam = MPI_Alloc_mem((rank==0?99:0)*sizeof(char), MPI_INFO_NULL, &base_ptr[i]); - assert(sam == MPI_SUCCESS); -// s[i] = MPI_Win_create(base_ptr[i], (rank==0?99:0)*sizeof(char), 1, MPI_INFO_NULL, node, &win[i]); - s[i] = MPI_Win_allocate_shared((rank==0?99:0)*sizeof(char), 1, MPI_INFO_NULL, node, &base_ptr[i], &win[i]); - assert(s[i] == MPI_SUCCESS); - if(rank == 0) assert(base_ptr[i] != 0); - size[i] = -1; - a[i] = -1; - ptr[i] = 0; -// MPI_Win_get_attr(&win, MPI_WIN_BASE, &ptr[i], 0); - MPI_Win_shared_query(win[i], 0, &size[i], &a[i], &ptr[i]); - assert(&ptr[i] != 0); - printf("alloc # %i ok\n", i); fflush(stdout); - } - - MPI_Barrier(node); - printf("barrier at %i\n", rank); fflush(stdout); - - for(int i = 0; i != N; ++i){ - if(rank == 0) ptr[i][3] = 'z'; - MPI_Barrier(node); - assert(ptr[i][3] == 'z'); - } - - for(int i = 0; i != N; ++i){ - if(win[i] != MPI_WIN_NULL) MPI_Win_free(&win[i]); - } - - MPI_Finalize(); - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/shared_window3.cpp b/external_codes/mpi_wrapper/mpi3/test/shared_window3.cpp deleted file mode 100644 index b933ac151ea..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shared_window3.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++17 -I${HOME}/prj/ -Wfatal-errors $0 -o $0x.x && time mpirun -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/environment.hpp" -#include "alf/boost/mpi3/shared_window.hpp" - -#include - -using std::cout; -using std::endl; - -namespace mpi3 = boost::mpi3; - -int main(int argc, char* argv[]){ - MPI_Init(&argc, &argv); - MPI_Comm node; - int s = MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &node); - int rank = -1; - MPI_Comm_rank(node, &rank); - assert(s == MPI_SUCCESS); - -#define ALLOC(N) \ - MPI_Win win##N; \ - void* base_ptr##N = nullptr; \ - int s##N = MPI_Win_allocate_shared((rank==0?80:0)*sizeof(char), 1, MPI_INFO_NULL, node, &base_ptr##N, &win##N); \ - MPI_Aint size##N = -1; \ - int i##N = -1; \ - char* ptr##N = nullptr; \ - MPI_Win_shared_query(win##N, 0, &size##N, &i##N, &ptr##N); \ - -ALLOC(1); -ALLOC(2); -ALLOC(3); -ALLOC(4); -ALLOC(5); -ALLOC(6); -ALLOC(7); -ALLOC(8); -ALLOC(9); - - MPI_Barrier(node); - -#define CHECK(N) \ - if(rank == 0) ptr##N[3] = 'z'; \ - MPI_Barrier(node); \ - assert(ptr##N[3] == 'z'); -// mpi3::environment env(argc, argv); -// auto node = env.world(); -// mpi3::shared_communicator node = world.split_shared(); -// cout<<" rank: " < A1(node); - - auto data1 = A1.allocate(80); - if(node.root()) data1[3] = 'z'; - node.barrier(); - assert(data1[3] == 'z'); - node.barrier(); - A1.deallocate(data1, 80); -#endif -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/shm_allocator.cpp b/external_codes/mpi_wrapper/mpi3/test/shm_allocator.cpp deleted file mode 100644 index 974e19f4c12..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shm_allocator.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wextra -Wpedantic `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright 2019 Alfredo A. Correa -#include "../../../mpi3/main.hpp" -#include "../../../mpi3/shm/allocator.hpp" - -namespace mpi3 = boost::mpi3; - -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::shared_communicator node = world.split_shared(); - - mpi3::shm::allocator A1(&node); - mpi3::shm::pointer data1 = A1.allocate(80); - - using ptr = decltype(data1); - std::pointer_traits::pointer pp = data1; - double* dp = std::addressof(*data1); - double* dp2 = mpi3::shm::pointer_traits::to_address(data1); - - if(node.root()) data1[3] = 3.4; - data1.w_->fence(); - node.barrier(); - assert( *dp == *data1 ); - assert( *dp2 == *data1 ); - assert(data1); - assert(!!data1); - assert(not (data1 < data1)); - assert(data1[3] == 3.4); - - A1.deallocate(data1, 80); - - return 0; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/shm_mapped_region.cpp b/external_codes/mpi_wrapper/mpi3/test/shm_mapped_region.cpp deleted file mode 100644 index 5434c59b7c7..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shm_mapped_region.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 `#-Wfatal-errors` $0 -o $0x.x && mpirun -np 2 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/shm/memory.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - mpi3::shm::shared_memory_object shm(world); - shm.truncate(1000); // collective - mpi3::shm::mapped_region region(shm); - if(world.rank() == 0){ - // mpi3::shm::mapped_region region(shm); - std::memset(region.get_address(), 1, region.get_size()); - } - world.barrier(); - if(world.rank() == 1){ - // mpi3::shm::mapped_region region(shm); - char* mem = static_cast(region.get_address()); - for(std::size_t i = 0; i != region.get_size(); ++i){ - if(*mem++ != 1) assert(0); - } - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/shm_multi_array.cpp b/external_codes/mpi_wrapper/mpi3/test/shm_multi_array.cpp deleted file mode 100644 index f1e7c265e09..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shm_multi_array.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ $0 -o $0x -lboost_serialization&&mpirun -n 4 $0x&&rm $0x;exit -#endif -// © Alfredo A. Correa 2018-2020 - -#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/process.hpp" -#include "../../mpi3/shm/allocator.hpp" - -#include -#include //sleep_for -#include //lock_guard - -#include "../../../boost/multi/array.hpp" - -namespace mpi3 = boost::mpi3; -namespace multi = boost::multi; - -using std::cout; - -namespace boost{ -namespace mpi3{ -namespace shm{ - -template -using array = multi::array>; - -}}} - -template using shm_vector = multi::array>; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - mpi3::shared_communicator node = world.split_shared(); -{ - multi::array> V(100, 99., &node); - assert( V[13] == 99. ); - - multi::array> W(100, 88., &node); - assert( W[13] == 88. ); - W = V; - assert( V[13] == 99. ); -} -{ - shm_vector V(200, 99., &node); - assert( V.size() == size(V) ); - assert( size(V) == 200 ); - - cout << V[10] << std::endl; - - double x = 10.0 * V[10]; - node.barrier(); - assert( x == 10.*99. ); -} -{ - auto const V = [&]{ - shm_vector V(1000, 0., &node); - if(node.root()) std::iota(V.begin(), V.end(), 10.); - node.barrier(); - return V; - }(); - std::cout - << "accumulation result in rank "<< node.rank() - <<' '<< std::accumulate(V.begin(), V.begin() + node.rank(), 0.) << std::endl - ; - node.barrier(); -} -{ - multi::array> A({5, 5}, 1.); // implicitly uses self communicator - multi::array> B({5, 5}, 2.); - - assert( A[1][2] == 1 ); - assert( B[2][1] == 2 ); - - B = A; - - assert( B[2][1] == 1 ); - assert( B[3][1] == 1 ); -} -{ - mpi3::shm::array A({5, 5}, 1., &node); - mpi3::shm::array B({5, 5}, 2., &node); - - assert( A[1][2] == 1 ); - assert( B[1][2] == 2 ); - - B() = A(); - - assert( A[1][2] == 1 ); - assert( B[1][2] == 1 ); -} - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/shm_vector.cpp b/external_codes/mpi_wrapper/mpi3/test/shm_vector.cpp deleted file mode 100644 index 643e2b5b6fe..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/shm_vector.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++17 `#-Wall` -Wfatal-errors -I$HOME/prj $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/process.hpp" -#include "alf/boost/mpi3/shm/vector.hpp" -#include "alf/boost/mpi3/mutex.hpp" - -#include -#include //sleep_for -#include //lock_guard - -#include - -int rand(int lower, int upper){ - static std::mt19937 rng(std::random_device{}()); - static std::uniform_int_distribution uni(lower, upper); - return uni(rng); -} -int rand(int upper = RAND_MAX){return rand(0, upper);} - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator world){ - -// mpi3::shm::vector::allocator_type alloc(world); - mpi3::shm::vector v(10, world); - assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); - - mpi3::mutex m(world); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); - { - std::lock_guard lock(m); - // m.lock(); - for(int i = 0; i != 10; ++i){ - v[i] = world.rank(); - std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); // slow down the writing - } - // m.unlock(); - } - - world.barrier(); - - if(world.rank() == 0){ - for(int i = 0; i != 10; ++i) - cout << v[i] << " "; - cout << std::endl; - } - // check that only one process had exclusive access - assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); - -// if(world.rank() == 0){ - v.resize(2); -// } - world.barrier(); - assert( v.size() == 2 ); - return 0; - { - boost::container::flat_map, std::allocator>> fm; - fm[4.1] = 6.; - assert( fm[4.1] == 6. ); - } - { - using shm_flat_map = boost::container::flat_map, mpi3::shm::allocator>>; - shm_flat_map fm(world); - -// if(world.rank() == 0) fm[4.1] = 6.; - world.barrier(); - -// assert( fm[4.1] == 6. ); - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/simple_broadcast.cpp b/external_codes/mpi_wrapper/mpi3/test/simple_broadcast.cpp deleted file mode 100644 index d9a4de6322d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/simple_broadcast.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/process.hpp" - -namespace bmpi3 = boost::mpi3; - -auto bmpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - { - using T = int; - std::vector v(10); - if(world.is_root()) {iota(begin(v), end(v), 0);} - - world.broadcast(begin(v), end(v)); - assert( v[9] == T(9) ); - } - { - using T = double; - std::vector v(10); - if(world.is_root()) { - iota(begin(v), end(v), 0); - } - world.broadcast(begin(v), end(v)); - - assert( v[9] == T(9) ); - } - - { - using T = double; - std::vector v; - if(world.is_root()) {v.resize(10); iota(begin(v), end(v), 0);} - - auto size = v.size(); - world.broadcast_n(&size, 1); - - v.resize(size); - assert( v.size() == 10UL ); - - world.broadcast_n(v.data(), v.size()); - - assert( v[9] == T(9) ); - } - { - using T = double; - std::vector v; - if(world.is_root()) {v.resize(10); iota(begin(v), end(v), 0);} - - world[0] & v; - - assert( v.size() == 10UL ); - assert( v[9] == T(9) ); - } - - return 0; -} catch(...) {return 1;} - - diff --git a/external_codes/mpi_wrapper/mpi3/test/simple_send_receive.cpp b/external_codes/mpi_wrapper/mpi3/test/simple_send_receive.cpp deleted file mode 100644 index 0e26bb134ff..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/simple_send_receive.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" - -#include -#include // for std::iota -#include - -namespace bmpi3 = boost::mpi3; - -int bmpi3::main(int /*argc*/, char ** /*argv*/, bmpi3::communicator world) try { - if(world.size()%2 == 1) { - if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< xsend(10); iota(begin(xsend), end(xsend), 0.); - std::vector xrecv(xsend.size(), -1.); - - auto last = world.send_receive( - cbegin(xsend), cend(xsend), (world.rank()/2)*2 + (world.rank()+1)%2, - begin(xrecv) - ); - - assert( last == end(xrecv) ); - assert( xrecv[5] == 5. ); - } - { - std::vector xsend(20); iota(begin(xsend), end(xsend), 100.); - std::vector xrecv(xsend.size(), -1.); - - world.send(cbegin(xsend), cend(xsend), (world.rank()/2)*2 + (world.rank()+1)%2); - world.receive(begin(xrecv), end(xrecv)); - - assert( xrecv[5] == 105.0 ); - } - { - std::vector xsend(20); iota(begin(xsend), end(xsend), 100.); - std::vector xrecv(xsend.size(), -1.0); - - world.send(cbegin(xsend), cend(xsend), (world.rank()/2)*2 + (world.rank()+1)%2); - world.receive(begin(xrecv), end(xrecv)); - - assert( xrecv[5] == 105.0 ); - } - - if(world.is_root()) {std::cerr<<"successfully completed"< -#include - -// #include - -namespace mpi3 = boost::mpi3; - -struct spinor { - std::complex up; // NOLINT(misc-non-private-member-variables-in-classes) - std::complex dn; // NOLINT(misc-non-private-member-variables-in-classes) - - bool operator==(spinor const& other) const { return up == other.up and dn == other.dn; } - bool operator!=(spinor const& other) const { return up != other.up or dn != other.dn; } -}; - -mpi3::environment mpienv; // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) - -template<> struct mpi3::datatype : mpi3::struct_< - std::complex, - std::complex -> {}; - -auto main(int /*argc*/, char** /*argv*/) -> int try { - - mpi3::communicator world = mpienv.world(); - - using namespace std::complex_literals; // i - - switch(world.rank()) { - case 0: { - std::vector v(5); - v[2] = spinor{3.14 + 6.28i, 4.0 + 5.0i}; - world.send_n(begin(v), 5, 1); - break; - }; - case 1: { - std::vector v(5); - world.receive_n(begin(v), 5, 0); - assert(( v[2] == spinor{3.14 + 6.28i, 4.0 + 5.0i} )); - break; - }; - } - - static_assert(boost::mpi3::has_datatype{}); -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/status.cpp b/external_codes/mpi_wrapper/mpi3/test/status.cpp deleted file mode 100644 index c9930ad1b6e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/status.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2019-2022 Alfredo A. Correa - -#include -#include - -#include - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - std::vector bufsizes = {1, 100, 10000}; - mpi3::communicator& comm = world; - - int const dest = comm.size() - 1; - for(std::vector::size_type cs = 0; cs != bufsizes.size(); ++cs) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm - if(comm.rank() == 0) { - auto const n = bufsizes[cs]; - comm.send_n(&n, 1, dest); - std::vector buf(n); - mpi3::request req = comm.isend(buf.begin(), buf.end(), dest, static_cast(cs + n + 1UL)); - // req.cancel(); - // mpi3::status s = - // req.wait(); - // if(not s.cancelled()) cout << "failed to cancel request\n"; - // else - // n = 0; - } else if(comm.rank() == dest) { - std::size_t n; // NOLINT(cppcoreguidelines-init-variables) delayed init - comm.receive_n(&n, 1, 0); - if(n > 0) { - std::vector btemp(n); - comm.receive_n(btemp.data(), n, 0); - } - } - comm.barrier(); - } - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/test.cpp b/external_codes/mpi_wrapper/mpi3/test/test.cpp deleted file mode 100644 index 331ca7c52b6..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/test.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -c++ -std=c++14 -O3 -Wall -Wextra -Wfatal-errors $0 -o $0x.x -lstdc++fs && $0x.x $@ ; rm -f $0x.x; exit -#endif - - -#include -#include -#include -#include - -namespace fs = std::experimental::filesystem; -using namespace std::string_literals; -using std::cout; - -int main(int argc, char* argv[]){ - std::vector some; - if(argc == 1 or argv[1] == "all"s){ - const std::regex communicator_related( "communicator_.*\\.cpp" ); - for(fs::path const& p: fs::directory_iterator("./")){ - if(p == "./test.cpp") continue; - std::smatch what; - std::string tmp =p.filename().string(); - if(not regex_match(tmp, what, communicator_related)) continue; - some.push_back(p); - } - }else if(argc==2 and argv[1] == "some"s){ - some = { - "./communicator_split.cpp", - "./communicator_send.cpp", - "./communicator_reduce.cpp" - }; - } - for(fs::path const& p: some){ - cout << p << '\n'; - std::system(("sh " + p.string()).c_str()); - } -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/test_some.cpp b/external_codes/mpi_wrapper/mpi3/test/test_some.cpp deleted file mode 100644 index 7f07ad89470..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/test_some.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/environment.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/request.hpp" - -#include // sleep_for - -namespace mpi3 = boost::mpi3; -using std::cout; - -int main(){ - mpi3::environment env; - auto& world = env.world(); - assert( world.size() == 4 ); - - if(world.rank() == 0){ - int rem = 3; - std::vector rs(3); - for(int i = 1; i != world.size(); ++i){ - std::vector buffer(100); - rs[i - 1] = world.ireceive(buffer.begin() + i, buffer.begin() + i + 1, i); - } - while(rem > 0){ - std::vector completed = mpi3::completed_some(rs); - if(completed.size() > 0){ - cout << "finished " << completed.size() << "\n"; - rem -= completed.size(); - }else std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }else{ - std::vector buffer(100); - world.send(buffer.begin(), buffer.begin() + 1, 0); - } -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/type_commit.cpp b/external_codes/mpi_wrapper/mpi3/test/type_commit.cpp deleted file mode 100644 index 9e0052a24b6..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/type_commit.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// #define BOOST_MPI3_DISALLOW_AUTOMATIC_POD_COMMUNICATION - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/type.hpp" - -#include "../../mpi3/detail/tuple_offset.hpp" - -#include - -namespace mpi3 = boost::mpi3; - -//template -//auto gen_type(T const&); - -//template<> auto gen_type(int const&) {return type{MPI_INT };} -//template<> auto gen_type(char const&) {return type{MPI_CHAR};} - -//template -//auto gen_type_aux>(std::tuple const& t, std::index_sequence) { -// std::vector blocklen = { (il.size(), 1); -// std::vector disp; -// return type::struct_({get_type(std::get(t), ...}); -//} - -//template -//auto gen_type>(std::tuple const& t) { -// return gen_type_aux(t, std::index_sequence_for{}); -//} - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator /*world*/) try { - - { - using Tuple = std::tuple; - Tuple tup; - auto offset4 = mpi3::detail::element_offset<4, Tuple>(); - assert( reinterpret_cast(&tup) + offset4 == reinterpret_cast(&std::get<4>(tup)) ); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert,cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-bounds-pointer-arithmetic) for some compiler this is not a constexpr - } - - mpi3::type t = mpi3::int_[100]; // mpi3::type::int_.contiguous(100); - - -#if 0 - t.commit_as>(); - t.commit_as(); -// std::array buffer; - int buffer[100]; - - if(world.rank() == 0) world.send_n(&buffer, 1, 1, 123); - else if(world.rank() == 1) world.receive_n(&buffer, 1, 0, 123); -#endif - - return 0; -} catch(...) {return 1;} - diff --git a/external_codes/mpi_wrapper/mpi3/test/type_size.cpp b/external_codes/mpi_wrapper/mpi3/test/type_size.cpp deleted file mode 100644 index 34dc5e7cb44..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/type_size.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#define BOOST_MPI3_DISALLOW_AUTOMATIC_POD_COMMUNICATION - -#include "../main.hpp" -#include "../communicator.hpp" - -namespace mpi3 = boost::mpi3; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - { - mpi3::type t = mpi3::int_; - assert( t.size() == sizeof(int) ); - assert( t.extent() == sizeof(int) ); - assert( t.lower_bound() == 0 ); - assert( t.upper_bound() == t.extent() - t.lower_bound() ); - } - { - mpi3::type t = mpi3::int_[3]; - assert( t.size() == sizeof(int)*3 ); - assert( t.extent() == sizeof(int)*3 ); - assert( t.lower_bound() == 0 ); - assert( t.upper_bound() == t.extent() - t.lower_bound() ); - } - { - mpi3::type t = mpi3::make_type()[3]; - assert( t.size() == sizeof(int)*3 ); - assert( t.extent() == sizeof(int)*3 ); - assert( t.lower_bound() == 0 ); - assert( t.upper_bound() == t.extent() - t.lower_bound() ); - } - { - mpi3::type t = mpi3::make_type()[3]; - assert( t.size() == sizeof(double)*3 ); - assert( t.extent() == sizeof(double)*3 ); - } - { - mpi3::type t = mpi3::make_type>()[3]; - assert( t.size() == sizeof(std::complex)*3 ); - assert( t.extent() == sizeof(std::complex)*3 ); - } - { - mpi3::type t = mpi3::float_int; - assert( t.size() == sizeof(std::pair) ); - struct foo_t { - float a; - int b; - }; - [[maybe_unused]] // nvcc false warning - foo_t foo{}; - - foo.a = 1.2F; - foo.b = 5; - - assert( t.size() == sizeof(float) + sizeof(int) ); - assert( t.extent() == sizeof(foo) ); - assert( t.lower_bound() == 0 ); - assert( t.upper_bound() == t.extent() - t.lower_bound() ); - } - { - using T = std::complex; - T buffer[100]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) test legacy type - - switch(world.rank()) { - break; case 0: { - buffer[10] = 42.; - MPI_Send(buffer, 1, &mpi3::make_type()[100], 1, 0 , world.handle()); - } - break; case 1: { - MPI_Status status; - MPI_Recv(buffer, 1, &mpi3::make_type()[100], 0, MPI_ANY_TAG, world.handle(), &status); - assert( buffer[10] == 42. ); - } - } - } - - return 0; -} catch(...) {return 1;} - diff --git a/external_codes/mpi_wrapper/mpi3/test/uniform_abort.cpp b/external_codes/mpi_wrapper/mpi3/test/uniform_abort.cpp deleted file mode 100644 index 674e4ce2743..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/uniform_abort.cpp +++ /dev/null @@ -1,231 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#include -#include - -#include -#include // std::runtime_error -#include - -namespace mpi3 = boost::mpi3; - -// failures - -void uniform_fail(mpi3::communicator const& comm) { // cppcheck-suppress [unusedFunction,unmatchedSuppression] - using namespace std::chrono_literals; - std::this_thread::sleep_for(comm.rank() * 1s); - - std::cout << "uniform_fail in n = " << comm.rank() << " is about to fail" << std::endl; - throw std::logic_error{"global but unsynchronized error"}; -} - -void nonuniform_fail(mpi3::communicator const& comm) { // cppcheck-suppress [unusedFunction,unmatchedSuppression] - using namespace std::chrono_literals; - std::this_thread::sleep_for(comm.rank() * 1s); - - if(comm.rank() > 1) { - std::cout << "nonuniform_fail in n = " << comm.rank() << " is about to fail" << std::endl; - throw std::logic_error{"nonglobal error"}; - } -} - -// handlers - -void unconditional_abort(mpi3::communicator const& comm) { // cppcheck-suppress unusedFunction - std::cout << "not essential message: aborting from rank " << comm.rank() << std::endl; - comm.abort(); -} - -void barriered_abort(mpi3::communicator& comm) { // cppcheck-suppress unusedFunction - comm.barrier(); - std::cout << "not essential message: aborting from rank " << comm.rank() << std::endl; - comm.abort(); -} - -// template -// void abort_after(mpi3::communicator& comm, Duration d) { -// auto const t0 = mpi3::wall_time(); -// while((mpi3::wall_time() - t0) < d) { // NOLINT(altera-unroll-loops) spin loop -// } -// std::cout << "not essential message: aborting from rank " << comm.rank() << " after others join" << std::endl; -// comm.abort(); -// } - -// template -// void timedout_abort(mpi3::communicator& comm, Duration d) { -// auto rbarrier = comm.ibarrier(); -// auto const t0 = mpi3::wall_time(); -// while(not rbarrier.completed() and (mpi3::wall_time() - t0) < d) { // NOLINT(altera-unroll-loops,altera-id-dependent-backward-branch) spin loop -// } - -// if(not rbarrier.completed()) { -// std::cout << "non essential message: aborting from rank " << comm.rank() << " after timeout" << std::endl; -// } else { -// std::cout << "not essential message: aborting from rank " << comm.rank() << " after others join" << std::endl; -// } - -// comm.abort(); -// } - -// template -// void timedout_throw(mpi3::communicator& comm, Duration d) { -// auto rbarrier = comm.ibarrier(); -// auto const t0 = mpi3::wall_time(); -// while(not rbarrier.completed() and (mpi3::wall_time() - t0) < d) { // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) spin loop -// } - -// if(rbarrier.completed()) { -// std::cout << "non essential message: throwing from rank " << comm.rank() << " before timeout" << std::endl; -// throw; // cppcheck-suppress [rethrowNoCurrentException,unmatchedSuppression] ; experimental line -// } -// std::terminate(); -// } - -template -[[noreturn]] void mpi3_timed_terminate(Duration d, mpi3::communicator& comm = mpi3::environment::get_world_instance()) { - std::cout << "terminate called" << std::endl; - auto rbarrier = comm.ibarrier(); - auto const t0 = mpi3::wall_time(); - while(not rbarrier.completed() and (mpi3::wall_time() - t0) < d) { // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) spin loop - using namespace std::chrono_literals; - std::this_thread::sleep_for(1s); - } - - if(rbarrier.completed()) { - if(comm.root()) { - std::cout << "not essential message: terminate from rank " << comm.rank() << " after others joined before timeout of " << std::chrono::duration_cast(d).count() << " seconds" << std::endl; - comm.abort(911); - } - } else { - std::cout << "non essential message: terminate from rank " << comm.rank() << " after timeout of " << std::chrono::duration_cast(d).count() << " seconds, not all processes failed within that time." << std::endl; - } - - comm.abort(911); - // never call std::terminate from here - std::abort(); // necessary to avoid error for returning in a [[noreturn]] function -} - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int { // NOLINT(bugprone-exception-escape) part of the test is that there is no `try` here - assert(world.size() == 4); - -// unconditional abort -#if 0 - // (-) prints only one message, (+) program terminates immediately - try { - uniform_fail(world); - } catch(std::logic_error&) { - unconditional_abort(world); - } -#endif - -#if 0 - // (-) prints only one message, (+) program terminates immediately - try { - nonuniform_fail(world); // non-uniform error - } catch(std::logic_error& e) { - unconditional_abort(world); - } -#endif - -// barriered abort -#if 0 - // (+) prints all available messages, (+) program terminates immediately - try { - uniform_fail(world); - } catch(std::logic_error& e) { - barriered_abort(world); - } -#endif - -#if 0 - // (+) prints all available messages, (-) it DEADLOCKS (here or later) - try { - nonuniform_fail(world); - } catch(std::logic_error& e) { - barriered_abort(world); - } -#endif - -// abort after hard sleep -#if 0 - // (+) prints all available messages, (~) program terminates after hard timeout - try { - uniform_fail(world); // non-uniform error - } catch(std::logic_error&) { - using namespace std::chrono_literals; - abort_after(world, 20s); - } -#endif - -#if 0 - // (+) prints all available messages, (~) program terminates after hard timeout - try { - nonuniform_fail(world); // non-uniform error - } catch(std::logic_error&) { - using namespace std::chrono_literals; - abort_after(world, 20s); - } -#endif - -// timedout_abort -#if 0 - // (+) prints all available messages, (+) program terminates very quickly - try { - uniform_fail(world); - } catch(std::logic_error&) { - using namespace std::chrono_literals; - timedout_abort(world, 20s); - } -#endif - -#if 0 - // (+) prints all available messages, (~) program terminates after timeout - try { - nonuniform_fail(world); - } catch(std::logic_error&) { - using namespace std::chrono_literals; - timedout_abort(world, 20s); - } -#endif - -// timedout_terminate -#if 1 - // (+) prints all available messages, (+) program terminates very quickly - std::set_terminate([] { - using namespace std::chrono_literals; - mpi3_timed_terminate(20s); - }); - - try { - uniform_fail(world); - } catch(std::logic_error&) { - throw; - } -#endif - -#if 0 - // (+) prints all available messages, (~) program terminates after timeout - std::set_terminate([]{ - using namespace std::chrono_literals; - mpi3_timed_terminate(20s); - }); - - try { - nonuniform_fail(world); - } catch(std::logic_error&) { - std::terminate(); - } -#endif - - // I am putting a collective here to produce an deadlock if some abort strategy leaks processes - { - int n = 1; - int total = 0; - world.all_reduce_n(&n, 1, &total); - assert(total == world.size()); - } - - return 0; -} -// catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/wall_time.cpp b/external_codes/mpi_wrapper/mpi3/test/wall_time.cpp deleted file mode 100644 index 0067cd6397c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/wall_time.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x && time mpirun -np 4s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], communicator& world){ - using namespace std::chrono_literals; - - auto t1 = mpi3::wall_time(); - std::this_thread::sleep_for(2s); - auto t2 = mpi3::wall_time(); - cout - << "started at " << t1 << " seconds.\n" - << "ended at " << t2 << " seconds.\n" - << "duration " << t2 - t1 << " seconds.\n" - << "resolution " << mpi3::wall_tick() << " seconds.\n" - ; - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/window_accumulate.cpp b/external_codes/mpi_wrapper/mpi3/test/window_accumulate.cpp deleted file mode 100644 index 9138590daf4..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/window_accumulate.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 2 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright Alfredo A. Correa 2018. -#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include "../../mpi3/main.hpp" -#include "../../mpi3/window.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -#define NROWS 100 -#define NCOLS 100 - -int mpi3::main(int, char*[], mpi3::communicator world){ - - double A[NROWS][NCOLS]; - - if(world.root()){ - for(int i = 0; i != NROWS; ++i) - for(int j = 0; j != NCOLS; ++j) - A[i][j] = i*NCOLS + j; - - mpi3::window<> w = world.make_window(); - w.fence(); // note the two fences here - w.accumulate_n( (double*)A, NROWS*NCOLS, 1 ); - w.fence(); - }else{ - for(int i = 0; i != NROWS; ++i) - for(int j = 0; j != NCOLS; ++j) - A[i][j] = i*NCOLS + j; - mpi3::window<> w = world.make_window( (double*)A, NROWS*NCOLS ); - w.fence(); // note the two fences here - w.fence(); - for(int i = 0; i != NROWS; ++i){ - for(int j = 0; j != NCOLS; ++j){ - if(world.rank() == 1) assert( A[i][j] == (i*NCOLS + j)*2 ); - else assert( A[i][j] == i*NCOLS + j ); - } - } - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/window_blocking_put.cpp b/external_codes/mpi_wrapper/mpi3/test/window_blocking_put.cpp deleted file mode 100644 index a04833126c8..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/window_blocking_put.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 2 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/main.hpp" -#include "alf/boost/mpi3/window.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ - - std::vector buf(10); - for(int i = 0; i != 10; ++i) buf[i] = world.rank()*10.0 + i; - { - auto w = world.make_window(buf.data(), buf.size()); - - if(world.rank() == 0){ - cout << "before the put from proc " << world.rank() << std::endl; - for(int i = 0; i != 10; ++i) cout << buf[i] << " "; - cout << std::endl; - } - else if(world.rank() == 1){ - w.blocking_put_n(buf.begin(), 10, 0); - } - // window destructor (calling Win_free_implicitly syncrhonizes) - // alternatively use w.fence(); - } - if(world.rank() == 0){ - cout << "after the put from proc " << world.rank() << std::endl; - for(int i = 0; i != 10; ++i){ - cout << buf[i] << " "; - assert(buf[i] == 10. + i); - } - cout << std::endl; - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/window_group.cpp b/external_codes/mpi_wrapper/mpi3/test/window_group.cpp deleted file mode 100644 index e72e0a003dc..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/window_group.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright Alfredo A. Correa 2018. -#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 -#include "../../mpi3/main.hpp" -#include "../../mpi3/window.hpp" -#include "../../mpi3/group.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int argc, char*[], mpi3::communicator world){ - int buf[10]; - mpi3::window win{buf, 10, world}; - mpi3::group wing(win); - mpi3::group g(world); - assert( g == wing ); - assert( g.rank() == wing.rank() ); - assert( g.rank() == world.rank() ); - assert( mpi3::group(world) == mpi3::group(win) ); - assert( mpi3::group(world).rank() == mpi3::group(win).rank() ); - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/window_put_get.cpp b/external_codes/mpi_wrapper/mpi3/test/window_put_get.cpp deleted file mode 100644 index 519a7b1deae..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/window_put_get.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/window.hpp" - -#include -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - mpi3::communicator comm = (world < 2); -#if 0 - if(comm){ - std::vector inbuf(100); - std::vector outbuf(100); - - std::iota(outbuf.begin(), outbuf.end(), 0.0); - mpi3::window win{ - comm, comm.rank()==1?inbuf.data():nullptr, - comm.rank()==1?inbuf.size():0 - }; - win.fence(); - if(world.rank() == 0) win.put_n(outbuf.data(), outbuf.size(), 1); - win.fence(); - if(world.rank() == 1) assert( inbuf[7] == 7.0 ); - } -#endif - return 0; -}catch(...){ - return 1; -} - - From fdaf7d107a0953142a4d681c1d58a4c9fa169b81 Mon Sep 17 00:00:00 2001 From: Alfredo Correa Date: Mon, 23 Oct 2023 12:37:32 -0700 Subject: [PATCH 088/168] Squashed 'external_codes/mpi_wrapper/mpi3/' content from commit 7f82048b1 git-subtree-dir: external_codes/mpi_wrapper/mpi3 git-subtree-split: 7f82048b1963e4b38b1f61b5d6402c03f509e6c1 --- .clang-format | 233 + .clang-tidy | 32 + .cppcheck-suppressions | 5 + .deepsource.toml | 9 + .gitignore | 5 + .gitlab-ci.yml | 276 + .gitmodules | 0 CMakeLists.txt | 40 + LICENSE | 23 + README.md | 972 +++ examples/01-hello.cpp | 21 + examples/02-send.cpp | 47 + examples/03-isend.cpp | 34 + examples/04-array.cpp | 68 + examples/05-array_mult.cpp | 109 + examples/average.cpp | 52 + examples/broadcast.cpp | 131 + examples/calculate_pi.cpp | 49 + examples/groups.cpp | 23 + examples/hello.cpp | 21 + examples/hello_world.cpp | 23 + examples/managed_shared_memory.cpp | 30 + examples/managed_shared_memory_0.cpp | 57 + examples/pi_reduce.cpp | 56 + examples/ping_pong.cpp | 30 + examples/random_walk.cpp | 82 + examples/rank.cpp | 35 + examples/reduce_average.cpp | 54 + examples/ring.cpp | 28 + examples/send_receive.cpp | 26 + examples/shared_window.cpp | 50 + examples/shared_window_2.cpp | 47 + examples/shared_window_3.cpp | 50 + examples/shm_vector.cpp | 86 + examples/split.cpp | 28 + examples/test3.cpp | 137 + examples/test4.cpp | 155 + examples/test5.cpp | 125 + fake/examples/MPI_Barrier.c | 15 + fake/examples/MPI_Comm_call_errhandler.c | 40 + fake/examples/MPI_Comm_compare.c | 36 + fake/examples/MPI_Comm_dup.c | 25 + fake/examples/MPI_Comm_free.c | 90 + fake/examples/MPI_Comm_rank.c | 14 + fake/examples/MPI_Comm_set_errhandler.c | 45 + fake/examples/MPI_Gather.c | 58 + fake/examples/MPI_Gatherv.c | 39 + fake/examples/MPI_Recv.c | 39 + fake/examples/MPI_Reduce.c | 43 + fake/examples/MPI_Send.c | 39 + fake/examples/MPI_Sendrecv_replace.c | 24 + fake/examples/MPI_Type_contiguos.c | 29 + fake/examples/MPI_Type_dup.c | 31 + fake/examples/MPI_Type_get_extent.c | 62 + fake/examples/MPI_Type_size.c | 62 + fake/examples/MPI_Type_vector.c | 45 + fake/examples/MPI_Wtick.c | 14 + fake/examples/MPI_Wtime.c | 17 + fake/examples/Makefile | 25 + fake/examples/abort.cpp | 10 + fake/examples/testme | 3 + fake/load | 2 + fake/mpi.h | 2763 +++++++ fake/mpi_qmcpack_compile.h | 741 ++ fake/mpic++ | 2 + fake/mpicc | 2 + fake/mpirun | 2 + fake/test/CMakeLists.txt | 33 + fake/test/test_c_mpi_init.c | 11 + fake/test/test_mpi_init.cpp | 11 + fake/test/test_wrapper_mpi_init.cpp | 11 + include/mpi3/FILE.hpp | 145 + include/mpi3/address.hpp | 53 + include/mpi3/allocator.hpp | 137 + include/mpi3/buffer.hpp | 111 + include/mpi3/cartesian_communicator.hpp | 387 + include/mpi3/communication_mode.hpp | 69 + include/mpi3/communicator.hpp | 3329 +++++++++ include/mpi3/communicator/operatorators.hpp | 35 + include/mpi3/communicator/operators.hpp | 36 + include/mpi3/communicator_fwd.hpp | 24 + include/mpi3/communicator_iterator.hpp | 55 + include/mpi3/config/NODISCARD.hpp | 27 + include/mpi3/core.hpp | 36 + include/mpi3/detail/basic_communicator.hpp | 343 + include/mpi3/detail/buffer.hpp | 42 + include/mpi3/detail/call.hpp | 76 + include/mpi3/detail/datatype.hpp | 234 + include/mpi3/detail/equality.hpp | 46 + include/mpi3/detail/iterator.hpp | 170 + include/mpi3/detail/iterator_traits.hpp | 115 + include/mpi3/detail/just.hpp | 159 + include/mpi3/detail/package.hpp | 74 + include/mpi3/detail/tuple_offset.hpp | 51 + include/mpi3/detail/value_traits.hpp | 83 + include/mpi3/dummy/.dummy | 0 include/mpi3/dynamic_window.hpp | 63 + include/mpi3/environment.hpp | 315 + include/mpi3/error.hpp | 102 + include/mpi3/error_handler.hpp | 90 + include/mpi3/exception.hpp | 21 + include/mpi3/future.hpp | 49 + include/mpi3/generalized_request.hpp | 162 + include/mpi3/graph_communicator.hpp | 140 + include/mpi3/group.hpp | 108 + include/mpi3/handle.hpp | 140 + include/mpi3/info.hpp | 96 + include/mpi3/main.hpp | 45 + include/mpi3/main_environment.hpp | 31 + include/mpi3/match.hpp | 27 + include/mpi3/message.hpp | 58 + include/mpi3/mutex.hpp | 379 + include/mpi3/nccl/communicator.hpp | 367 + include/mpi3/nccl/detail/basic_datatype.hpp | 48 + include/mpi3/nccl/detail/basic_reduction.hpp | 37 + include/mpi3/nccl/test/CMakeLists.txt | 145 + include/mpi3/nccl/test/nccl_constructor.cu | 63 + include/mpi3/nccl/universal_communicator.hpp | 57 + include/mpi3/operation.hpp | 222 + include/mpi3/ostream.hpp | 198 + include/mpi3/package_archive.hpp | 410 ++ include/mpi3/pointer.hpp | 205 + include/mpi3/port.hpp | 48 + include/mpi3/process.hpp | 119 + include/mpi3/request.hpp | 184 + include/mpi3/rma/counter.hpp | 38 + include/mpi3/rma/memory.hpp | 266 + include/mpi3/rma/mutex.hpp | 40 + .../serialization_hack/archive_exception.cpp | 160 + .../mpi3/serialization_hack/basic_archive.cpp | 87 + .../serialization_hack/basic_iarchive.cpp | 602 ++ .../serialization_hack/basic_iserializer.cpp | 35 + .../serialization_hack/basic_oarchive.cpp | 474 ++ .../serialization_hack/basic_oserializer.cpp | 34 + .../serialization_hack/extended_type_info.cpp | 194 + .../extended_type_info_typeid.cpp | 168 + include/mpi3/serialization_hack/singleton.cpp | 38 + include/mpi3/shared_communicator.hpp | 139 + include/mpi3/shared_main.hpp | 39 + include/mpi3/shared_mutex.hpp | 124 + include/mpi3/shared_window.hpp | 110 + include/mpi3/shm/allocator.hpp | 135 + include/mpi3/shm/managed_shared_memory.hpp | 187 + include/mpi3/shm/memory.hpp | 162 + include/mpi3/shm/mutex.hpp | 69 + include/mpi3/shm/pool.hpp | 47 + include/mpi3/shm/ptr.hpp | 261 + include/mpi3/shm/vector.hpp | 195 + include/mpi3/status.hpp | 71 + include/mpi3/timed_terminate.hpp | 33 + include/mpi3/type.hpp | 404 + include/mpi3/types.hpp | 13 + include/mpi3/vector.hpp | 123 + include/mpi3/version.hpp | 108 + include/mpi3/wall_clock.hpp | 68 + include/mpi3/window.hpp | 301 + pre-push | 17 + pres/.gitkeep | 0 pres/QMCPack_Alfredo_MPI3Wrapper_slides.pdf | Bin 0 -> 138556 bytes test/CMakeLists.txt | 288 + test/all_reduce.cpp | 104 + test/async_interaction.cpp | 33 + test/broadcast.cpp | 32 + test/cartesian.cpp | 280 + test/communicator_abort.cpp | 10 + test/communicator_all_gather.cpp | 99 + test/communicator_all_gatherv.cpp | 76 + ...mmunicator_all_gatherv_output_iterator.cpp | 29 + test/communicator_all_to_all.cpp | 55 + test/communicator_attributes.cpp | 37 + test/communicator_barrier.cpp | 13 + test/communicator_cctor.cpp | 76 + test/communicator_create.hpp | 20 + test/communicator_custom_attributes.cpp | 57 + test/communicator_divide.cpp | 20 + test/communicator_error.cpp | 32 + test/communicator_exception.cpp | 21 + test/communicator_gather.cpp | 116 + test/communicator_gather2.cpp | 29 + test/communicator_grip_handle.cpp | 41 + test/communicator_ibroadcast.cpp | 36 + test/communicator_igather.cpp | 42 + test/communicator_iprobe.cpp | 28 + test/communicator_ireceive.cpp | 36 + test/communicator_issend.cpp | 30 + test/communicator_list.cpp | 37 + test/communicator_main.cpp | 18 + test/communicator_main.cpp.openmpi.supp | 6558 +++++++++++++++++ test/communicator_mutable.cpp | 161 + test/communicator_operator.cpp | 45 + test/communicator_ostream.cpp | 54 + test/communicator_pack.cpp | 65 + test/communicator_passby.cpp | 35 + test/communicator_reduce.cpp | 66 + test/communicator_reduce_in_place.cpp | 50 + test/communicator_scatter.cpp | 100 + test/communicator_scatterv.cpp | 47 + test/communicator_send.cpp | 57 + test/communicator_send_async.cpp | 55 + test/communicator_send_class.cpp | 198 + test/communicator_send_class_nonintrusive.cpp | 92 + test/communicator_send_receive.cpp | 102 + test/communicator_set_error_handler.cpp | 23 + test/communicator_split.cpp | 33 + test/compare.cpp | 25 + test/datatype.cpp | 28 + test/datatype_struct_vector3.cpp | 107 + test/deino_all_to_all.cpp | 38 + test/deino_broadcast.cpp | 59 + test/deino_op_create.cpp | 60 + test/empty_main.cpp | 10 + test/enum.cpp | 118 + test/environment_self.cpp | 43 + test/environment_thread.cpp | 20 + test/gather2.cpp | 23 + test/group.cpp | 37 + test/group_operations.cpp | 63 + test/ibarrier.cpp | 21 + test/keyval.cpp | 133 + test/legacy_fftw3.cpp | 65 + test/library_check.cpp | 12 + test/library_main.cpp | 35 + test/minimal.cpp | 26 + test/multi_array.cpp | 94 + test/package.cpp | 45 + test/parallel_dot_product.dat | 12 + test/process.cpp | 159 + test/process_vector.cpp | 43 + test/reduce_maxloc.cpp | 44 + test/request_test_some.cpp | 38 + test/request_wait.cpp | 27 + test/request_wait_all.cpp | 25 + test/request_wait_any.cpp | 27 + test/request_wait_some.cpp | 32 + test/ring.cpp | 27 + test/send_complex.cpp | 45 + test/serialization_intrusive.cpp | 80 + test/shared_communicator.cpp | 88 + test/shared_mutex.cpp | 11 + test/shared_window.cpp | 43 + test/shared_window2.cpp | 74 + test/shared_window3.c | 62 + test/shared_window3.cpp | 73 + test/shm_allocator.cpp | 37 + test/shm_mapped_region.cpp | 32 + test/shm_multi_array.cpp | 97 + test/shm_vector.cpp | 77 + test/simple_broadcast.cpp | 59 + test/simple_send_receive.cpp | 51 + test/spinor.cpp | 50 + test/status.cpp | 41 + test/test.cpp | 38 + test/test_some.cpp | 38 + test/type_commit.cpp | 55 + test/type_size.cpp | 80 + test/uniform_abort.cpp | 231 + test/variant.cpp | 179 + test/wall_time.cpp | 26 + test/window_accumulate.cpp | 47 + test/window_blocking_put.cpp | 42 + test/window_group.cpp | 27 + test/window_put_get.cpp | 33 + 262 files changed, 34788 insertions(+) create mode 100644 .clang-format create mode 100644 .clang-tidy create mode 100644 .cppcheck-suppressions create mode 100644 .deepsource.toml create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 .gitmodules create mode 100644 CMakeLists.txt create mode 100644 LICENSE create mode 100644 README.md create mode 100644 examples/01-hello.cpp create mode 100644 examples/02-send.cpp create mode 100644 examples/03-isend.cpp create mode 100644 examples/04-array.cpp create mode 100644 examples/05-array_mult.cpp create mode 100644 examples/average.cpp create mode 100644 examples/broadcast.cpp create mode 100644 examples/calculate_pi.cpp create mode 100644 examples/groups.cpp create mode 100644 examples/hello.cpp create mode 100644 examples/hello_world.cpp create mode 100644 examples/managed_shared_memory.cpp create mode 100644 examples/managed_shared_memory_0.cpp create mode 100644 examples/pi_reduce.cpp create mode 100644 examples/ping_pong.cpp create mode 100644 examples/random_walk.cpp create mode 100644 examples/rank.cpp create mode 100644 examples/reduce_average.cpp create mode 100644 examples/ring.cpp create mode 100644 examples/send_receive.cpp create mode 100644 examples/shared_window.cpp create mode 100644 examples/shared_window_2.cpp create mode 100644 examples/shared_window_3.cpp create mode 100644 examples/shm_vector.cpp create mode 100644 examples/split.cpp create mode 100644 examples/test3.cpp create mode 100644 examples/test4.cpp create mode 100644 examples/test5.cpp create mode 100644 fake/examples/MPI_Barrier.c create mode 100644 fake/examples/MPI_Comm_call_errhandler.c create mode 100644 fake/examples/MPI_Comm_compare.c create mode 100644 fake/examples/MPI_Comm_dup.c create mode 100644 fake/examples/MPI_Comm_free.c create mode 100644 fake/examples/MPI_Comm_rank.c create mode 100644 fake/examples/MPI_Comm_set_errhandler.c create mode 100644 fake/examples/MPI_Gather.c create mode 100644 fake/examples/MPI_Gatherv.c create mode 100644 fake/examples/MPI_Recv.c create mode 100644 fake/examples/MPI_Reduce.c create mode 100644 fake/examples/MPI_Send.c create mode 100644 fake/examples/MPI_Sendrecv_replace.c create mode 100644 fake/examples/MPI_Type_contiguos.c create mode 100644 fake/examples/MPI_Type_dup.c create mode 100644 fake/examples/MPI_Type_get_extent.c create mode 100644 fake/examples/MPI_Type_size.c create mode 100644 fake/examples/MPI_Type_vector.c create mode 100644 fake/examples/MPI_Wtick.c create mode 100644 fake/examples/MPI_Wtime.c create mode 100644 fake/examples/Makefile create mode 100644 fake/examples/abort.cpp create mode 100755 fake/examples/testme create mode 100755 fake/load create mode 100644 fake/mpi.h create mode 100644 fake/mpi_qmcpack_compile.h create mode 100755 fake/mpic++ create mode 100755 fake/mpicc create mode 100755 fake/mpirun create mode 100644 fake/test/CMakeLists.txt create mode 100644 fake/test/test_c_mpi_init.c create mode 100644 fake/test/test_mpi_init.cpp create mode 100644 fake/test/test_wrapper_mpi_init.cpp create mode 100644 include/mpi3/FILE.hpp create mode 100644 include/mpi3/address.hpp create mode 100644 include/mpi3/allocator.hpp create mode 100644 include/mpi3/buffer.hpp create mode 100644 include/mpi3/cartesian_communicator.hpp create mode 100644 include/mpi3/communication_mode.hpp create mode 100644 include/mpi3/communicator.hpp create mode 100644 include/mpi3/communicator/operatorators.hpp create mode 100644 include/mpi3/communicator/operators.hpp create mode 100644 include/mpi3/communicator_fwd.hpp create mode 100644 include/mpi3/communicator_iterator.hpp create mode 100644 include/mpi3/config/NODISCARD.hpp create mode 100644 include/mpi3/core.hpp create mode 100644 include/mpi3/detail/basic_communicator.hpp create mode 100644 include/mpi3/detail/buffer.hpp create mode 100644 include/mpi3/detail/call.hpp create mode 100644 include/mpi3/detail/datatype.hpp create mode 100644 include/mpi3/detail/equality.hpp create mode 100644 include/mpi3/detail/iterator.hpp create mode 100644 include/mpi3/detail/iterator_traits.hpp create mode 100644 include/mpi3/detail/just.hpp create mode 100644 include/mpi3/detail/package.hpp create mode 100644 include/mpi3/detail/tuple_offset.hpp create mode 100644 include/mpi3/detail/value_traits.hpp create mode 100644 include/mpi3/dummy/.dummy create mode 100644 include/mpi3/dynamic_window.hpp create mode 100644 include/mpi3/environment.hpp create mode 100644 include/mpi3/error.hpp create mode 100644 include/mpi3/error_handler.hpp create mode 100644 include/mpi3/exception.hpp create mode 100644 include/mpi3/future.hpp create mode 100644 include/mpi3/generalized_request.hpp create mode 100644 include/mpi3/graph_communicator.hpp create mode 100644 include/mpi3/group.hpp create mode 100644 include/mpi3/handle.hpp create mode 100644 include/mpi3/info.hpp create mode 100644 include/mpi3/main.hpp create mode 100644 include/mpi3/main_environment.hpp create mode 100644 include/mpi3/match.hpp create mode 100644 include/mpi3/message.hpp create mode 100644 include/mpi3/mutex.hpp create mode 100644 include/mpi3/nccl/communicator.hpp create mode 100644 include/mpi3/nccl/detail/basic_datatype.hpp create mode 100644 include/mpi3/nccl/detail/basic_reduction.hpp create mode 100644 include/mpi3/nccl/test/CMakeLists.txt create mode 100644 include/mpi3/nccl/test/nccl_constructor.cu create mode 100644 include/mpi3/nccl/universal_communicator.hpp create mode 100644 include/mpi3/operation.hpp create mode 100644 include/mpi3/ostream.hpp create mode 100644 include/mpi3/package_archive.hpp create mode 100644 include/mpi3/pointer.hpp create mode 100644 include/mpi3/port.hpp create mode 100644 include/mpi3/process.hpp create mode 100644 include/mpi3/request.hpp create mode 100644 include/mpi3/rma/counter.hpp create mode 100644 include/mpi3/rma/memory.hpp create mode 100644 include/mpi3/rma/mutex.hpp create mode 100644 include/mpi3/serialization_hack/archive_exception.cpp create mode 100644 include/mpi3/serialization_hack/basic_archive.cpp create mode 100644 include/mpi3/serialization_hack/basic_iarchive.cpp create mode 100644 include/mpi3/serialization_hack/basic_iserializer.cpp create mode 100644 include/mpi3/serialization_hack/basic_oarchive.cpp create mode 100644 include/mpi3/serialization_hack/basic_oserializer.cpp create mode 100644 include/mpi3/serialization_hack/extended_type_info.cpp create mode 100644 include/mpi3/serialization_hack/extended_type_info_typeid.cpp create mode 100644 include/mpi3/serialization_hack/singleton.cpp create mode 100644 include/mpi3/shared_communicator.hpp create mode 100644 include/mpi3/shared_main.hpp create mode 100644 include/mpi3/shared_mutex.hpp create mode 100644 include/mpi3/shared_window.hpp create mode 100644 include/mpi3/shm/allocator.hpp create mode 100644 include/mpi3/shm/managed_shared_memory.hpp create mode 100644 include/mpi3/shm/memory.hpp create mode 100644 include/mpi3/shm/mutex.hpp create mode 100644 include/mpi3/shm/pool.hpp create mode 100644 include/mpi3/shm/ptr.hpp create mode 100644 include/mpi3/shm/vector.hpp create mode 100644 include/mpi3/status.hpp create mode 100644 include/mpi3/timed_terminate.hpp create mode 100644 include/mpi3/type.hpp create mode 100644 include/mpi3/types.hpp create mode 100644 include/mpi3/vector.hpp create mode 100644 include/mpi3/version.hpp create mode 100644 include/mpi3/wall_clock.hpp create mode 100644 include/mpi3/window.hpp create mode 100755 pre-push create mode 100644 pres/.gitkeep create mode 100644 pres/QMCPack_Alfredo_MPI3Wrapper_slides.pdf create mode 100644 test/CMakeLists.txt create mode 100644 test/all_reduce.cpp create mode 100644 test/async_interaction.cpp create mode 100644 test/broadcast.cpp create mode 100644 test/cartesian.cpp create mode 100644 test/communicator_abort.cpp create mode 100644 test/communicator_all_gather.cpp create mode 100644 test/communicator_all_gatherv.cpp create mode 100644 test/communicator_all_gatherv_output_iterator.cpp create mode 100644 test/communicator_all_to_all.cpp create mode 100644 test/communicator_attributes.cpp create mode 100644 test/communicator_barrier.cpp create mode 100644 test/communicator_cctor.cpp create mode 100644 test/communicator_create.hpp create mode 100644 test/communicator_custom_attributes.cpp create mode 100644 test/communicator_divide.cpp create mode 100644 test/communicator_error.cpp create mode 100644 test/communicator_exception.cpp create mode 100644 test/communicator_gather.cpp create mode 100644 test/communicator_gather2.cpp create mode 100644 test/communicator_grip_handle.cpp create mode 100644 test/communicator_ibroadcast.cpp create mode 100644 test/communicator_igather.cpp create mode 100644 test/communicator_iprobe.cpp create mode 100644 test/communicator_ireceive.cpp create mode 100644 test/communicator_issend.cpp create mode 100644 test/communicator_list.cpp create mode 100644 test/communicator_main.cpp create mode 100644 test/communicator_main.cpp.openmpi.supp create mode 100644 test/communicator_mutable.cpp create mode 100644 test/communicator_operator.cpp create mode 100644 test/communicator_ostream.cpp create mode 100644 test/communicator_pack.cpp create mode 100644 test/communicator_passby.cpp create mode 100644 test/communicator_reduce.cpp create mode 100644 test/communicator_reduce_in_place.cpp create mode 100644 test/communicator_scatter.cpp create mode 100644 test/communicator_scatterv.cpp create mode 100644 test/communicator_send.cpp create mode 100644 test/communicator_send_async.cpp create mode 100644 test/communicator_send_class.cpp create mode 100644 test/communicator_send_class_nonintrusive.cpp create mode 100644 test/communicator_send_receive.cpp create mode 100644 test/communicator_set_error_handler.cpp create mode 100644 test/communicator_split.cpp create mode 100644 test/compare.cpp create mode 100644 test/datatype.cpp create mode 100644 test/datatype_struct_vector3.cpp create mode 100644 test/deino_all_to_all.cpp create mode 100644 test/deino_broadcast.cpp create mode 100644 test/deino_op_create.cpp create mode 100644 test/empty_main.cpp create mode 100644 test/enum.cpp create mode 100644 test/environment_self.cpp create mode 100644 test/environment_thread.cpp create mode 100644 test/gather2.cpp create mode 100644 test/group.cpp create mode 100644 test/group_operations.cpp create mode 100644 test/ibarrier.cpp create mode 100644 test/keyval.cpp create mode 100644 test/legacy_fftw3.cpp create mode 100644 test/library_check.cpp create mode 100644 test/library_main.cpp create mode 100644 test/minimal.cpp create mode 100644 test/multi_array.cpp create mode 100644 test/package.cpp create mode 100644 test/parallel_dot_product.dat create mode 100644 test/process.cpp create mode 100644 test/process_vector.cpp create mode 100644 test/reduce_maxloc.cpp create mode 100644 test/request_test_some.cpp create mode 100644 test/request_wait.cpp create mode 100644 test/request_wait_all.cpp create mode 100644 test/request_wait_any.cpp create mode 100644 test/request_wait_some.cpp create mode 100644 test/ring.cpp create mode 100644 test/send_complex.cpp create mode 100644 test/serialization_intrusive.cpp create mode 100644 test/shared_communicator.cpp create mode 100644 test/shared_mutex.cpp create mode 100644 test/shared_window.cpp create mode 100644 test/shared_window2.cpp create mode 100644 test/shared_window3.c create mode 100644 test/shared_window3.cpp create mode 100644 test/shm_allocator.cpp create mode 100644 test/shm_mapped_region.cpp create mode 100644 test/shm_multi_array.cpp create mode 100644 test/shm_vector.cpp create mode 100644 test/simple_broadcast.cpp create mode 100644 test/simple_send_receive.cpp create mode 100644 test/spinor.cpp create mode 100644 test/status.cpp create mode 100644 test/test.cpp create mode 100644 test/test_some.cpp create mode 100644 test/type_commit.cpp create mode 100644 test/type_size.cpp create mode 100644 test/uniform_abort.cpp create mode 100644 test/variant.cpp create mode 100644 test/wall_time.cpp create mode 100644 test/window_accumulate.cpp create mode 100644 test/window_blocking_put.cpp create mode 100644 test/window_group.cpp create mode 100644 test/window_put_get.cpp diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000..02fed447340 --- /dev/null +++ b/.clang-format @@ -0,0 +1,233 @@ +--- +Language: Cpp +# BasedOnStyle: Google +#AccessModifierOffset: -1 +AlignAfterOpenBracket: BlockIndent # Align +AlignArrayOfStructures: Right +#AlignConsecutiveMacros: None +AlignConsecutiveAssignments: Consecutive # None +#AlignConsecutiveBitFields: None +AlignConsecutiveDeclarations: Consecutive +#AlignEscapedNewlines: Left +AlignOperands: AlignAfterOperator +AlignTrailingComments: false +#AllowAllArgumentsOnNextLine: true +#AllowAllParametersOfDeclarationOnNextLine: true +#AllowShortEnumsOnASingleLine: true +#AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: true # false +#AllowShortFunctionsOnASingleLine: All +#AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +#AllowShortLoopsOnASingleLine: true +#AlwaysBreakAfterDefinitionReturnType: None +#AlwaysBreakAfterReturnType: None +#AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: No # Yes +#AttributeMacros: +# - __capability +#BinPackArguments: true +#BinPackParameters: true +BraceWrapping: +# AfterCaseLabel: false +# AfterClass: false +# AfterControlStatement: Never +# AfterEnum: false +# AfterFunction: false +# AfterNamespace: false +# AfterObjCDeclaration: false +# AfterStruct: false +# AfterUnion: false +# AfterExternBlock: false +# BeforeCatch: false +# BeforeElse: false +# BeforeLambdaBody: false +# BeforeWhile: false +# IndentBraces: false + SplitEmptyFunction: false # true +# SplitEmptyRecord: true +# SplitEmptyNamespace: true +#BreakBeforeBinaryOperators: None +#BreakBeforeConceptDeclarations: true +#BreakBeforeBraces: Attach +#BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeComma +#BreakBeforeTernaryOperators: true +#BreakConstructorInitializersBeforeComma: false +#BreakConstructorInitializers: BeforeColon +#BreakAfterJavaFieldAnnotations: false +#BreakStringLiterals: true +ColumnLimit: 0 +#CommentPragmas: '^ IWYU pragma:' +QualifierAlignment: Right # Leave +#CompactNamespaces: false +ConstructorInitializerIndentWidth: 0 +ContinuationIndentWidth: 99 +#Cpp11BracedListStyle: true +#DeriveLineEnding: true +#DerivePointerAlignment: true +#DisableFormat: false +#EmptyLineAfterAccessModifier: Never +#EmptyLineBeforeAccessModifier: LogicalBlock +#ExperimentalAutoDetectBinPacking: false +#PackConstructorInitializers: NextLine +#BasedOnStyle: '' +#ConstructorInitializerAllOnOneLineOrOnePerLine: false +#AllowAllConstructorInitializersOnNextLine: true +FixNamespaceComments: true +#ForEachMacros: +# - foreach +# - Q_FOREACH +# - BOOST_FOREACH +#IfMacros: +# - KJ_IF_MAYBE +#IncludeBlocks: Regroup +#IncludeCategories: +# - Regex: '^' +# Priority: 2 +# SortPriority: 0 +# CaseSensitive: false +# - Regex: '^<.*\.h>' +# Priority: 1 +# SortPriority: 0 +# CaseSensitive: false +# - Regex: '^<.*' +# Priority: 2 +# SortPriority: 0 +# CaseSensitive: false +# - Regex: '.*' +# Priority: 3 +# SortPriority: 0 +# CaseSensitive: false +#IncludeIsMainRegex: '([-_](test|unittest))?$' +#IncludeIsMainSourceRegex: '' +#IndentAccessModifiers: true # false +AccessModifierOffset: -98 # 2 +#IndentCaseLabels: true +#IndentCaseBlocks: false +#IndentGotoLabels: true +#IndentPPDirectives: None +#IndentExternBlock: AfterExternBlock +#IndentRequires: false +IndentWidth: 99 +#IndentWrappedFunctionNames: false +#InsertTrailingCommas: None +#InsertBraces: true # clang format 15 +#JavaScriptQuotes: Leave +#JavaScriptWrapImports: true +#KeepEmptyLinesAtTheStartOfBlocks: false +#LambdaBodyIndentation: Signature +#MacroBlockBegin: '' +#MacroBlockEnd: '' +#MaxEmptyLinesToKeep: 1 +#NamespaceIndentation: None +#ObjCBinPackProtocolList: Never +#ObjCBlockIndentWidth: 2 +#ObjCBreakBeforeNestedBlockParam: true +#ObjCSpaceAfterProperty: false +#ObjCSpaceBeforeProtocolList: true +#PenaltyBreakAssignment: 2 +#PenaltyBreakBeforeFirstCallParameter: 1 +#PenaltyBreakComment: 300 +#PenaltyBreakFirstLessLess: 120 +#PenaltyBreakOpenParenthesis: 0 +#PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +#PenaltyExcessCharacter: 1000000 +#PenaltyReturnTypeOnItsOwnLine: 200 +#PenaltyIndentedWhitespace: 0 +PointerAlignment: Left +#PPIndentWidth: -1 +#RawStringFormats: +# - Language: Cpp +# Delimiters: +# - cc +# - CC +# - cpp +# - Cpp +# - CPP +# - 'c++' +# - 'C++' +# CanonicalDelimiter: '' +# BasedOnStyle: google +# - Language: TextProto +# Delimiters: +# - pb +# - PB +# - proto +# - PROTO +# EnclosingFunctions: +# - EqualsProto +# - EquivToProto +# - PARSE_PARTIAL_TEXT_PROTO +# - PARSE_TEST_PROTO +# - PARSE_TEXT_PROTO +# - ParseTextOrDie +# - ParseTextProtoOrDie +# - ParseTestProto +# - ParsePartialTestProto +# CanonicalDelimiter: pb +# BasedOnStyle: google +#ReferenceAlignment: Pointer +#ReflowComments: true +#RemoveBracesLLVM: false +#SeparateDefinitionBlocks: Leave +#ShortNamespaceLines: 1 +#SortIncludes: CaseSensitive +#SortJavaStaticImport: Before +#SortUsingDeclarations: true +#SpaceAfterCStyleCast: false +#SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: false # true +#SpaceBeforeAssignmentOperators: true +#SpaceBeforeCaseColon: false +#SpaceBeforeCpp11BracedList: false +#SpaceBeforeCtorInitializerColon: true +#SpaceBeforeInheritanceColon: true +SpaceBeforeParens: Custom # ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: false # true +# AfterForeachMacros: true +# AfterFunctionDefinitionName: false +# AfterFunctionDeclarationName: false +# AfterIfMacros: true +# AfterOverloadedOperator: false +# BeforeNonEmptyParentheses: false +#SpaceAroundPointerQualifiers: Default +#SpaceBeforeRangeBasedForLoopColon: true +#SpaceInEmptyBlock: false +#SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +#SpacesInAngles: Never +#SpacesInConditionalStatement: false +#SpacesInContainerLiterals: true +#SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 +# Maximum: -1 +#SpacesInParentheses: false +#SpacesInSquareBrackets: false +#SpaceBeforeSquareBrackets: false +#BitFieldColonSpacing: Both +Standard: c++17 +#StatementAttributeLikeMacros: +# - Q_EMIT +#StatementMacros: +# - Q_UNUSED +# - QT_REQUIRE_VERSION +TabWidth: 99 +#UseCRLF: false +UseTab: ForContinuationAndIndentation # Never +WhitespaceSensitiveMacros: + - BOOST_REQUIRE + - BOOST_TEST + - BOOST_TEST_REQUIRE + - assert + - MPI_ +#WhitespaceSensitiveMacros: +# - STRINGIZE +# - PP_STRINGIZE +# - BOOST_PP_STRINGIZE +# - NS_SWIFT_NAME +# - CF_SWIFT_NAME +... diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000000..bb16502d09f --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,32 @@ +# -*-indent-tabs-mode:nil;c-basic-offset:2;tab-width:2;autowrap:nil;-*- +--- +Checks: '*, + -altera-struct-pack-align, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -fuchsia-default-arguments-declarations, + -fuchsia-default-arguments-calls, + -fuchsia-overloaded-operator, + -fuchsia-trailing-return, + -google-runtime-references, + -hicpp-no-array-decay, + -llvmlibc-callee-namespace, + -llvm-header-guard, + -llvmlibc-implementation-in-namespace, + -llvmlibc-restrict-system-libc-headers, + -modernize-use-trailing-return-type, + -modernize-concat-nested-namespaces, + -modernize-use-nodiscard, + -readability-identifier-length, + -readability-magic-numbers +' +CheckOptions: + - { key: readability-identifier-naming.NamespaceCase, value: lower_case } + - { key: readability-identifier-naming.ClassCase, value: lower_case } + - { key: readability-identifier-naming.PrivateMemberSufix, value: _ } + - { key: readability-identifier-naming.StructCase, value: lower_case } + - { key: readability-identifier-naming.FunctionCase, value: lower_case } + - { key: readability-identifier-naming.TemplateParameterCase, value: CamelCase } +WarningsAsErrors: '*' +HeaderFilterRegex: '.' +FormatStyle: file diff --git a/.cppcheck-suppressions b/.cppcheck-suppressions new file mode 100644 index 00000000000..0f9cf7386cd --- /dev/null +++ b/.cppcheck-suppressions @@ -0,0 +1,5 @@ +#syntaxError +#missingInclude +missingIncludeSystem +#unmatchedSuppression +#preprocessorErrorDirective diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000000..0ea2ab62c10 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,9 @@ +version = 1 + +[[analyzers]] +name = "shell" +enabled = true + +[[analyzers]] +name = "test-coverage" +enabled = true \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..95c242d2c8f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +build* +.build* +.vscode + + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..ee7e33b98c4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,276 @@ +# -*-indent-tabs-mode:nil;c-basic-offset:2;tab-width:4;-*- +# Copyright 2020-2023 Alfredo A. Correa + +image: debian:testing + +variables: + GIT_SUBMODULE_STRATEGY: recursive + +openmpi: + stage: build + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make g++ git libboost-serialization-dev + - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 + - cd test + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="--oversubscribe" + - cmake .. -DCMAKE_BUILD_TYPE=Debug + - cmake --build . --parallel 2 || make VERBOSE=1 + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + +exampi: + allow_failure: true + stage: build + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates cmake g++ git libboost-serialization-dev make python3 ssh strace # ca-certificates libopenmpi-dev openmpi-bin cmake make g++ git libboost-serialization-dev + - git clone https://correaa:${EXATOKEN}@github.com/tonyskjellum/ExaMPI.git + - cd ExaMPI + - mkdir build && cd build + - cmake .. --install-prefix=$HOME/exa + - make -j 4 + - make install + - export PATH=$HOME/exa/bin:$PATH + - export PATH=$HOME/exa/runtime:$PATH + - export LD_LIBRARY_PATH=$HOME/exa/lib:$LD_LIBRARY_PATH + - export MPI_PATH=$HOME/exa/bin + - export MPI_LIB=$HOME/exa/lib + - export MPI_INC=$HOME/exa/include + - export MPI_HOME=$HOME/exa + - which mpicxx + - which mpirun + - strace mpirun -n 4 tests/integration_tests/allreduce + - strace mpirun -n 4 tests/integration_tests/alltoall + - ctest --output-on-failure + - cd ../.. + - mkdir build && cd build + - which mpicxx + - mpicxx --version + - cmake .. -DCMAKE_BUILD_TYPE=Debug -DUSE_EXAMPI=1 -DMPI_HOME=$HOME/exa + - make -j 2 || make VERBOSE=1 + - ls + - ctest --output-on-failure + +icpc-intelmpi: + stage: build + image: intel/oneapi-hpckit:latest + script: + - apt-get update && apt-get install --no-install-recommends -y --quiet bash ca-certificates cmake curl g++ git make libboost-test-dev libboost-serialization-dev + - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 + - cd test + - mkdir build && cd build + - icpc --version + - CXX=icpc CXXFLAGS="-diag-disable=593,2196,1786,1478" cmake .. -DCMAKE_BUILD_TYPE=Debug #https://community.intel.com/t5/Intel-C-Compiler/Inline-and-no-inline-warning/td-p/1216764 + - cmake --build . --parallel 2 || cmake --build . --verbose + - ctest --output-on-failure + needs: ["openmpi"] + +icpx-intelmpi: + stage: build + image: intel/oneapi-hpckit:latest + allow_failure: true + script: + - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl g++ git make libboost-test-dev libboost-serialization-dev + - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 + - cd test + - mkdir build && cd build + - icpx --version + - CXX=icpx CXXFLAGS="-O1" cmake .. -DCMAKE_BUILD_TYPE=Debug + - cmake --build . --parallel 2 || make VERBOSE=1 + - ctest --output-on-failure + needs: ["openmpi"] + +openmpi-clang: + stage: build + image: debian:testing + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang g++ git libstdc++-12-dev libboost-serialization-dev + - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 + - cd test + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="--oversubscribe" + - clang++ --version + - mpirun --version + - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug + - cmake --build . --parallel 2 || make VERBOSE=1 + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + needs: ["openmpi"] + +"openmpi-clang20": + stage: build + image: debian:stable + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin cmake make clang git libstdc++-12-dev libboost-serialization-dev + - cd test + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="--oversubscribe" + - clang++ --version + - mpirun --version + - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 + - cmake --build . --parallel 2 || make VERBOSE=1 + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + needs: ["openmpi-clang"] + +openmpi-clang-tidy: + stage: build + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates libopenmpi-dev openmpi-bin clang libstdc++-12-dev clang-tidy cmake git make libboost-serialization-dev + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="--oversubscribe" + - clang++ --version + - clang-tidy --version + - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_CLANG_TIDY="clang-tidy" + - make --jobs=2 || make VERBOSE=1 + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + needs: ["openmpi-clang"] + +openmpi-cppcheck: + stage: build + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends libopenmpi-dev openmpi-bin g++ libstdc++-12-dev ca-certificates cmake cppcheck git make libboost-serialization-dev + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="--oversubscribe" + - g++ --version + - cppcheck --version + - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_CPPCHECK="cppcheck;--force;--enable=all;--inline-suppr;--language=c++;--suppress=missingIncludeSystem;--suppress=syntaxError;--suppress=unmatchedSuppression;--std=c++17;--error-exitcode=666;-UEXCLUDE_CPPCHECK" + - make --jobs=2 || make VERBOSE=1 + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + needs: ["openmpi"] + +mpich-debug: + stage: build + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends libopenmpi-dev openmpi-bin g++ libstdc++-12-dev ca-certificates cmake cppcheck git make libboost-serialization-dev + - cd test + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="" + - export OMPI_ALLOW_RUN_AS_ROOT=1 + - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + - cmake .. -DCMAKE_BUILD_TYPE=Debug + - cmake --build . --parallel 2 || cmake --build . --verbose + - ctest --output-on-failure + +mpich-valgrind: + stage: build + allow_failure: true + script: + - apt-get update -qq && apt-get install -qq -y --no-install-recommends ca-certificates cmake git libboost-test-dev libboost-serialization-dev libmpich-dev make mpich valgrind + - mpirun --version + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="" + - export VALGRIND_EXE="valgrind --leak-check=full --track-origins=yes --show-leak-kinds=all --suppressions=.valgrind_suppressions --gen-suppressions=all --error-exitcode=1 " + - cmake .. -DCMAKE_BUILD_TYPE=Debug + - make --jobs=2 || make VERBOSE=1 + - ulimit -n # reports current value + - ulimit -n 1024 # workaround neededed by valgrind in docker running in Fedora 37 + - ctest --output-on-failure + needs: ["mpich-debug"] + +qmcpack-openmpi: + stage: test + image: debian:testing + script: + - apt-get -qq update && apt-get -qq install --no-install-recommends -y libblas-dev liblapack-dev libfftw3-dev libboost-serialization-dev libopenmpi-dev gfortran g++ cmake make git ca-certificates numdiff python3 python3-numpy python3-h5py python3-mpi4py python3-scipy libxml2-dev libhdf5-dev + - git clone https://github.com/QMCPACK/qmcpack.git + - cd qmcpack + - git config --global user.email "alfredo.correa@gmail.com" && git config --global user.name "Alfredo Correa" + - git rm -r external_codes/mpi_wrapper/mpi3 && git commit -m "remove mpi3 subtree" + - git subtree add --squash -P external_codes/mpi_wrapper/mpi3 https://gitlab.com/correaa/boost-mpi3.git $CI_COMMIT_BRANCH + - cd build + - cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DBUILD_PPCONVERT=1 -DQMC_MIXED_PRECISION=1 -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS="--allow-run-as-root;--bind-to;none" .. #-DCMAKE_CXX_FLAGS="-Werror" + - make --jobs=2 || make VERBOSE=1 # afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R afqmc --output-on-failure + needs: ["openmpi"] + +qmcpack-cuda-runner: + allow_failure: false + image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 + tags: + - nvidia-docker + stage: test + script: + - apt-get -qq update && apt-get -qq install --no-install-recommends -y libblas-dev liblapack-dev libfftw3-dev libboost-serialization-dev libopenmpi-dev gfortran g++ cmake make git ca-certificates numdiff python3 python3-numpy python3-h5py python3-mpi4py python3-scipy libxml2-dev libhdf5-dev + - cmake --version + - git clone --depth=1 https://github.com/QMCPACK/qmcpack.git + - cd qmcpack + - git config --global user.email "alfredo.correa@gmail.com" && git config --global user.name "Alfredo Correa" + - git rm -r external_codes/mpi3 && git commit -m "remove mpi3 subtree" + - git subtree add --squash -P external_codes/mpi3 $CI_REPOSITORY_URL $CI_COMMIT_BRANCH # e.g. https://gitlab.com/correaa/boost-multi.git + - cd ../qmcpack + - cd build + - CUDACXX=/usr/local/cuda/bin/nvcc cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DQMC_CXX_STANDARD=17 -DENABLE_CUDA=1 -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DQMC_GPU_ARCHS=sm_61 -DCMAKE_CUDA_HOST_COMPILER=g++ -DCMAKE_CXX_FLAGS="-Wno-deprecated -Wno-deprecated-declarations" .. + - make -j4 afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R afqmc --output-on-failure + needs: ["openmpi-cuda-11", "qmcpack-openmpi"] + +inq-openmpi: + stage: test + image: debian:testing + tags: + - cpu + script: + - apt-get update && apt-get install --no-install-recommends -y --quiet libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-serialization-dev libboost-iostreams-dev libopenmpi-dev libhdf5-dev gfortran g++ cmake pkg-config python3-dev make git ca-certificates wget + - cmake --version + - git clone https://gitlab.com/npneq/inq.git --recurse-submodules + - cd inq + - cd external_libs/mpi3 + - git checkout $CI_COMMIT_BRANCH # check that multi repo is mirrored correctly from this repo to the submodule repo (npneq) + - cd ../.. + - mkdir build && cd build + - cmake .. --install-prefix=$HOME -DCMAKE_BUILD_TYPE=Release + - make --jobs=2 || make VERBOSE=1 + - make install + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + needs: ["openmpi"] + +# - export DEBIAN_FRONTEND=noninteractive +# - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates git gnupg software-properties-common +# - apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/3bf863cc.pub +# - add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/ /" +# - add-apt-repository contrib +# - apt-get update +# - apt-get -y install cuda + +# TODO(correaa) -> openmpi-cuda-11 +openmpi-cuda-11: + stage: build + allow_failure: false + image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 + script: + - apt-get update && apt-get install --no-install-recommends -y cmake libboost-test-dev libboost-serialization-dev libopenmpi-dev make git + - /usr/local/cuda-11/bin/nvcc --version + - cd .. && ln -s boost-mpi3 mpi3 && cd mpi3 + - cd test + - mkdir build && cd build + - export MPI_OVERSUBSCRIBE="--oversubscribe" + - cmake .. -DCMAKE_CUDA_COMPILER=/usr/local/cuda-11/bin/nvcc -DCMAKE_BUILD_TYPE=Debug -DENABLE_CUDA=1 + - cmake --build . --parallel 2 || cmake --build . --verbose + - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure + needs: ["openmpi"] + +# - wget https://cmake.org/files/v3.21/cmake-3.21.3-linux-x86_64.sh --no-verbose # following https://askubuntu.com/a/865294/15943 +# - mkdir /opt/cmake +# - sh cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/opt/cmake +# - ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake + +inq-cuda-11-openmpi-compileonly: + stage: build + allow_failure: true + image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 + tags: + - nvidia-docker + script: + - apt-get update && apt-get install --no-install-recommends -y --quiet cmake git libblas-dev liblapack-dev libfftw3-dev libboost-filesystem-dev libboost-iostreams-dev libboost-serialization-dev libopenmpi-dev libhdf5-dev ca-certificatesgfortran g++ make pkg-config python3-dev wget + - cmake --version + - git clone https://gitlab.com/npneq/inq.git --recurse-submodules + - cd inq + - cd external_libs/mpi3 + - git checkout $CI_COMMIT_BRANCH + - cd ../.. + - mkdir build && cd build + - /usr/local/cuda-11/bin/nvcc -V + - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. --install-prefix=$HOME -DENABLE_CUDA=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=61 + - make silicon --jobs=2 + - make install + - ctest -R silicon + needs: + - openmpi-cuda-11 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..e69de29bb2d diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..f72eaee20b0 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,40 @@ +# -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- +cmake_minimum_required(VERSION 3.16) + +project( + bmpi3 + VERSION 0.79.0 + DESCRIPTION "B-MPI3 is a C++ library wrapper for version 3.1 of the MPI standard interface that simplifies the utilization and maintenance of MPI code." + HOMEPAGE_URL "https://gitlab.com/correaa/boost-mpi3" + LANGUAGES CXX +) + +find_package(MPI REQUIRED) # might need to `module load mpi` + +add_library(${PROJECT_NAME} INTERFACE) + +target_compile_features(${PROJECT_NAME} INTERFACE cxx_std_17) +target_include_directories(${PROJECT_NAME} INTERFACE $ $ $) +target_link_libraries(${PROJECT_NAME} INTERFACE MPI::MPI_CXX) + +# to use this project directly from CMake +# FetchContent_Declare( +# bmpi3 +# GIT_REPOSITORY git@gitlab.com:correaa/boost-mpi3.git # https://gitlab.com/correaa/boost-mpi3.git +# GIT_TAG master) +# FetchContent_MakeAvailable(bmpi3) +# add_executable(main main.cpp) +# target_link_libraries(main PUBLIC bmpi3) + +# this makes CM FetchContent friendly https://www.foonathan.net/2022/06/cmake-fetchcontent/ +if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + return() +endif() + +include(GNUInstallDirs) + +include(CTest) + +enable_testing() + +add_subdirectory(test) diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000000..36b7cd93cdf --- /dev/null +++ b/LICENSE @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000000..d3a602d8452 --- /dev/null +++ b/README.md @@ -0,0 +1,972 @@ + +[comment]: # (Comment) + +# B.MPI3 +*Alfredo A. Correa* + + +[//]: <> () + +B-MPI3 is a C++ library wrapper for version 3.1 of the MPI standard interface that simplifies the utilization and maintenance of MPI code. +B-MPI3 C++ aims to provide a more convenient, powerful and an interface less prone to errors than the standard C-based MPI interface. + +B-MPI3 simplifies the utilization of MPI without completely changing the communication model, allowing for a seamless transition from C-MPI. +B-MPI3 also provides allocators and facilities to manipulate MPI-mediated Remote Access and shared memory. + +For example, pointers are not utilized directly and it is replaced by an iterator-based interface and most data, in particular custom type objects are serialized automatically into messages by the library. +B-MPI3 interacts well with the C++ standard library, containers and custom data types (classes). + +B.MPI3 is written from [scratch](https://octo-repo-visualization.vercel.app/?repo=llnl%2Fb-mpi3) in C++17 and it has been tested with many MPI library implementations and compilers, OpenMPI +1.9, MPICH +3.2.1, MVAPICH or Spectrum MPI, using the following compilers gcc +5.4.1, clang +6.0, PGI 18.04. +(Any standard compliant MPI library can be used.) + +B.MPI3 is not an official Boost library, but is designed following the principles of Boost and the STL. +B.MPI3 is not a derivative of Boost.MPI and it is unrelated to the, [now deprecated](https://web.archive.org/web/20170421220544/http://blogs.cisco.com/performance/the-mpi-c-bindings-what-happened-and-why/), official MPI-C++ interface. +It adds features which were missing in Boost.MPI (which only covers MPI-1), with an iterator-based interface and MPI-3 features (RMA and Shared memory). + +B.MPI3 optionally depends on Boost +1.53 for automatic serialization. + +## Contents +[[_TOC_]] + +## Introduction + +MPI is a large library for run-time parallelism where several paradigms coexist. +It was is originally designed as standardized and portable message-passing system to work on a wide variety of parallel computing architectures. + +The last standard, MPI-3, uses a combination of techniques to achieve parallelism, Message Passing (MP), (Remote Memory Access (RMA) and Shared Memory (SM). +We try here to give a uniform interface and abstractions for these features by means of wrapper function calls and concepts brought familiar to C++ and the STL. + +## Motivation: The problem with the standard interface + +A typical C-call for MP looks like this, + +```cpp +int status_send = MPI_Send(&numbers, 10, MPI_INT, 1, 0, MPI_COMM_WORLD); +assert(status_send == MPI_SUCCESS); +... // concurrently with +int status_recv = MPI_Recv(&numbers, 10, MPI_INT, 0, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); +assert(status_recv == MPI_SUCCESS); +``` + +In principle this call can be made from a C++ program. +However there are obvious drawbacks from using this standard interface. + +Here we enumerate some of problems, + +* Function calls have many arguments (e.g. 6 or 7 arguments in average) +* Many mandatory arguments are redundant or could easily have a default natural value (e.g. message tags are not always necessary). +* Use of raw pointers and sizes, (e.g. `&number` and `1`) +* Data argument are type-erased into `void*`. +* Only primitive types (e.g. `MPI_INT`) can be passed. +* Consistency between pointer types and data-types is responsibility of the user. +* Only contiguous memory blocks can be used with this interface. +* Error codes are stored and had to be checked after each function call. +* Use of handles (such as `MPI_COMM_WORLD`), handles do not have a well defined semantics. + +A call of this type would be an improvement: + +```cpp +world.send(numbers.begin(), numbers.end(), 1); +... // concurrently with +world.receive(numbers.begin(), numbers.end(), 0); +``` + +For other examples, see here: [http://mpitutorial.com/tutorials/mpi-send-and-receive/](http://mpitutorial.com/tutorials/mpi-send-and-receive/) + +MPI used to ship with a C++-style interfaces. +It turns out that this interface was a very minimal change over the C version, and for good reasons it was dropped. + +The B.MPI3 library was designed to use simultaneously (interleaved) with the standard C interface of MPI. +In this way, changes to existing code can be made incrementally. + +## Usage + +The library is "header-only"; no separate compilation or configuration of the library is necessary. +It requires an MPI distribution (e.g. OpenMPI or MPICH2), a C++14 compiler and Boost libraries installed. +A typical compilation/run command looks like this: + +```bash +$ mpic++ communicator_send.cpp -o communicator_send.x -lboost_serialization +$ mpirun -n 8 ./communicator_send.x +``` + +In a system such as Red Hat or Fedora, the dependencies can by installed by `sudo dnf install gcc-c++ boost-devel openmpi-devel mpich-devel`. + +Alternatively, the library can be fetched on demand by the CMake project: + +```cmake +include(FetchContent) +FetchContent_Declare(bmpi3 GIT_REPOSITORY https://gitlab.com/correaa/boost-mpi3.git) # or git@gitlab.com:correaa/boost-mpi3.git +FetchContent_MakeAvailable(bmpi3) + +target_link_libraries(your_executable PRIVATE bmpi3) +``` + +Some systems require loading the MPI module before compiling and using MPI programs, `module load mpi` (or `mpich`). + +The library is tested frequently against `openmpi` and `mpich` implementations of MPI. + +## Testing + +The library has a basic `ctest` based testing system. + +```bash +# module load mpi/mpich # or mpi/openmpi , needed in systems like Fedora +cd mpi3/test +mkdir build && cd build +cmake .. +cmake --build .. +ctest +``` + +## Initialization + +Like MPI, B.MPI3 requires some global library initialization. +The library includes a convenience header `mpi3/main.hpp`, which provides a "main" function that does this initialization. +In this way, a parallel program looks very much like normal programs, except that the main function has a third argument with the default global communicator passed in. + +```cpp +#include "mpi3/version.hpp" +#include "mpi3/main.hpp" + +#include + +namespace mpi3 = boost::mpi3; + +int mpi3::main(int argc, char** argv, mpi3::communicator world) { + if(world.rank() == 0) {std::cout << mpi3::version() << '\n';} + return 0; +} +``` + +Here `world` is a communicator object that is a wrapper over MPI communicator handle. + +Changing the `main` program to this syntax in existing code can be too intrusive. +For this reason a more traditional initialization is also possible. +The alternative initialization is done by instantiating the `mpi3::environment` object (from with the global communicator `.world()` is extracted). + +```cpp +#include "mpi3/environment.hpp" +int main(int argc, char** argv){ + mpi3::environment env(argc, argv); + auto world = env.world(); // communicator is extracted from the environment + // ... code here + return 0; +} +``` + +## Communicators + +In the last example, `world` is a global communicator (not necessarily the same as `MPI_COMM_WORLD`, but a copy of it). +There is no global communicator variable `world` that can be accessed directly in a nested function. +The idea behind this is to avoid using the global communicators in nested functions of the program unless they are explicitly passed in the function call. +Communicators are usually passed by reference to nested functions. +Even in traditional MPI it is a mistake to assume that the `MPI_COMM_WORLD` is the only available communicator. + +`mpi3::communicator` represent communicators with value-semantics. +This means that `mpi3::communicator` can be copied or passed by reference. +A communicator and their copies are different entities that compare equal. +Communicators can be empty, in a state that is analogous to `MPI_COMM_NULL` but with proper value semantics. + +Like in MPI communicators can be duplicated (copied into a new instance) or split. +They can be also compared. + +```cpp +mpi3::communicator world2 = world; +assert( world2 == world ); +mpi3::communicator hemisphere = world/2; +mpi3::communicator interleaved = world%2; +``` + +This program for example splits the global communicator in two sub-communicators one of size 2 (including process 0 and 1) and one with size 6 (including 2, 3, ... 7); + +```cpp +#include "mpi3/main.hpp" +#include "mpi3/communicator.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator world){ + assert(world.size() == 8); // this program can only be run in 8 processes + mpi3::communicator comm = (world <= 1); + assert(!comm || (comm && comm.size() == 2)); + return 0; +} +``` + +Communicators give also index access to individual `mpi3::processes` ranging from `0` to `comm.size()`. +For example, `world[0]` referrers to process 0 or the global communicator. +An `mpi3::process` is simply a rank inside a communicator. +This concept doesn't exist explicit in the standard C interface, but it simplifies the syntax for message passing. + +Splitting communicators can be done more traditionally via the `communicator::split` member function. + +Communicators are used to pass messages and to create memory windows. +A special type of communicator is a shared-communicator `mpi3::shared_communicator`. + +## Message Passing + +This section describes the features related to the message passing (MP) functions in the MPI library. +In C-MPI information is passed via pointers to memory. +This is expected in a C-based interface and it is also very efficient. +In Boost.MPI, information is passed exclusively by value semantics. +Although there are optimizations that amortize the cost, we decided to generalize the pointer interface and leave the value-based message passing for a higher-level syntax. + +Here we replicate the design of STL to process information, that is, aggregated data is passed mainly via iterators. (Pointer is a type of iterator). + +For example in STL data is copied between ranges in this way. +```cpp +std::copy(origin.begin(), origin.end(), destination.begin()); +``` + +The caller of function copy doesn't need to worry about he type of the `origin` and `destination` containers, it can mix pointers and iterators and the function doesn't need more redundant information than the information passed. +The programmer is responsible for managing the memory and making sure that design is such that the algorithm can access the data referred by the passed iterators. + +Contiguous iterators (to built-in types) are particularity efficient because they can be mapped to pointers at compile time. This in turn is translated into a MPI primitive function call. +The interface for other type of iterators or contiguous iterators to non-build-in type are simulated, mainly via buffers and serialization. +The idea behind this is that generic message passing function calls can be made to work with arbitrary data types. + +The main interface for message passing in B.MPI3 are member functions of the communicator. +For example `communicator::send`, `::receive` and `::barrier`. +The functions `::rank` and `::size` allows each process to determine their unique identity inside the communicator. + +```cpp +int mpi3::main(int argc, char* argv[], mpi3::communicator world) { + assert(world.size() == 2); + if(world.rank() == 0) { + std::vector v = {1.,2.,3.}; + world.send(v.begin(), v.end(), 1); // send to rank 1 + } else if(world.rank() == 1) { + std::vector v(3); + world.receive(v.begin(), v.end(), 0); // receive from rank 1 + assert( v == std::vector{1.,2.,3.} ); + } + world.barrier(); // synchronize execution here + return 0; +} +``` + +Other important functions are `::gather`, `::broadcast` and `::accumulate`. +This syntax has a more or less obvious (but simplified) mapping to the standard C-MPI interface. +In Boost.MPI3 however all, these functions have reasonable defaults that make the function call shorted and less prone to errors and with the C-MPI interface. + +For more examples, look into `./mpi3/tests/`, `./mpi3/examples/` and `./mpi3/exercises/`. + +The interface described above is iterator based and is a direct generalization of the C-interface which works with pointers. +If the iterators are contiguous and the associated value types are primitive MPI types, the function is directly mapped to the C-MPI call. + +Alternatively, value-based interface can be used. +We will show the terse syntax, using the process objects. + +```cpp +int mpi3::main(int, char**, mpi3::communicator world) { + assert(world.size() == 2); + if(world.rank() == 0) { + double v = 5.; + world[1] << v; + } else if(world.rank() == 1) { + double v = -1.; + world[0] >> v; + assert(v == 5.); + } + return 0; +} +``` + +## Remote Memory Access + +Remote Memory (RM) is handled by `mpi3::window` objects. +`mpi3::window`s are created by `mpi3::communicator` via a collective (member) functions. +Since `mpi3::window`s represent memory, it cannot be copied (but can be moved). + +```cpp +mpi3::window w = world.make_window(begin, end); +``` + +Just like in the MPI interface, local access and remote access is synchronized by a `window::fence` call. +Read and write remote access is performed via put and get functions. + +```cpp +w.fence(); +w.put(begin, end, rank); +w.fence(); +``` + +This is minimal example using `put` and `get` functions. + +```cpp +#include "mpi3/main.hpp" +#include + +namespace mpi3 = boost::mpi3; using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator world) { + + std::vector darr(world.rank()?0:100); + mpi3::window w = world.make_window(darr.data(), darr.size()); + w.fence(); + if(world.rank() == 0) { + std::vector a = {5., 6.}; + w.put(a.begin(), a.end(), 0); + } + world.barrier(); + w.fence(); + std::vector b(2); + w.get(b.begin(), b.end(), 0); + w.fence(); + assert( b[0] == 5.); + world.barrier(); + + return 0; +} +``` + +In this example, memory from process 0 is shared across the communicator, and accessible through a common window. +Process 0 writes (`window::put`s) values in the memory (this can be done locally or remotely). +Later all processes read from this memory. +`put` and `get` functions take at least 3 arguments (and at most 4). +The first two is a range of iterators, while the third is the destination/source process rank (called "target_rank"). + +Relevant examples and test are located in For more examples, look into `./mpi3/tests/`, `./mpi3/examples/` and `./mpi3/exercises/`. + +`mpi3::window`s may carry type information (as `mpi3::window`) or not (`mpi3::window<>`) + +## Shared Memory + +Shared memory (SM) uses the underlying capability of the operating system to share memory from process within the same node. +Historically shared memory has an interface similar to that of remove access. +Only communicators that comprise a single node can be used to create a share memory window. +A special type of communicator can be created by splitting a given communicator. + +`mpi3::shared_communicator node = world.split_shared();` + +If the job is launched in single node, `node` will be equal (congruent) to `world`. +Otherwise the global communicator will be split into a number of (shared) communicators equal to the number of nodes. + +`mpi3::shared_communicator`s can create `mpi3::shared_window`s. +These are special type of memory windows. + +```cpp +#include "mpi3/main.hpp" + +namespace mpi3 = boost::mpi3; using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator world) { + + mpi3::shared_communicator node = world.split_shared(); + mpi3::shared_window win = node.make_shared_window(node.rank()==0?1:0); + + assert(win.base() != nullptr and win.size() == 1); + + win.lock_all(); + if(node.rank()==0) *win.base(0) = 42; + for (int j=1; j != node.size(); ++j){ + if(node.rank()==0) node.send_n((int*)nullptr, 0, j);//, 666); + else if(node.rank()==j) node.receive_n((int*)nullptr, 0, 0);//, 666); + } + win.sync(); + + int l = *win.base(0); + win.unlock_all(); + + int minmax[2] = {-l,l}; + node.all_reduce_n(&minmax[0], 2, mpi3::max<>{}); + assert( -minmax[0] == minmax[1] ); + cout << "proc " << node.rank() << " " << l << std::endl; + + return 0; +} +``` + +For more examples, look into `./mpi3/tests/`, `./mpi3/examples/` and `./mpi3/exercises/`. + +# Beyond MP: RMA and SHM + +MPI provides a very low level abstraction to inter-process communication. +Higher level of abstractions can be constructed on top of MPI and by using the wrapper the works is simplified considerably. + +## Mutex + +Mutexes can be implemented fairly simply on top of RMA. +Mutexes are used similarly than in threaded code, +it prevents certain blocks of code to be executed by more than one process (rank) at a time. + +```cpp +#include "mpi3/main.hpp" +#include "mpi3/mutex.hpp" + +#include + +namespace mpi3 = boost::mpi3; using std::cout; + +int mpi3::main(int, char**, mpi3::communicator world) { + + mpi3::mutex m(world); + { + m.lock(); + cout << "locked from " << world.rank() << '\n'; + cout << "never interleaved " << world.rank() << '\n'; + cout << "forever blocked " << world.rank() << '\n'; + cout << std::endl; + m.unlock(); + } + return 0; +} +``` + +(Recursive mutexes are not implemented yet) + +Mutexes themselves can be used to implement atomic operations on data. + +# Ongoing work + +We are implementing memory allocators for remote memory, atomic classes and asynchronous remote function calls. +Higher abstractions and use patterns will be implemented, specially those that fit into the patterns of the STL algorithms and containers. + +# Advanced Topics + +## Thread safety + +If you are not using threads at all, you can skip this section; +however here you can find some rationale behind design decisions taken by the library and learn how to use `mpi3::communicator` as a member of a class. + +Thread-safety with MPI is extremely complicated, as there are various aspects to it, from the data communicated, to the communicator itself, to operations order, to asynchronous messaging, to the runtime system. +This library doesn't try to hide this fact; in place, it leverages the tools available to C++ to deal with this complication. +As we will see, there are certain steps to make the code _compatible_ with threads to difference degrees. + +Absolute thread-safety is a very strong guarantee and it would come at a very steep performance cost. +Almost no general purpose library guarantees complete thread safety. +In opposition to thread-safety, we will discuss thread-compatibility, which is a more reasonable goal. +Thread-compatibility refers to the property of a system to be able to be thread-safe if extra steps are taken and that you have the option to take these steps only when needed. + +The first condition for thread compatibility is to have an MPI environment that supports threads. +If you have an MPI system provides only a `thread_support` at the level of `mpi3::thread::single` it means that there is probably no way to make MPI calls from different threads an expect correct results. +If your program expects to call MPI in concurrent sections, your only option would be to change to a system that supports MPI threading. + +In this small example, we assume that the program expects threading and MPI by completely rejecting the run if the any level different from `single` is not provided. +This is not at all terrible choice, _optionally_ supporting threading in a big program can be prohibitive from a design point of view. + +```cpp +int main() { + mpi3::environment env{mpi3::thread::multiple}; + switch( env.thread_support() ) { + case mpi3::thread::single : throw std::logic_error{"threads not supported"}; + case mpi3::thread::funneled : std::cout<<"funneled" < mpi3::single`, since the levels `multiple > serialized > funneled > single` are ordered. + +### From C to C++ + +The MPI-C standard interface is expressed in the C language (and Fortran). +The C-language doesn't have many ways to deal with threads except by thorough documentation. +This indicates that any level of thread assurance that we can express in a C++ interface cannot be derived by the C-interface syntax alone; +it has to be derived, at best, from the documentation and when documentation is lacking from common sense and common practice in existing MPI implementations. + +The modern C++ language has several tools to deal with thread safety: the C++11 memory model, the `const`, `mutable` and `thread_local` attributes and a few other standard types and functions, such as `std::mutex`, `std::call_once`, etc. + +### Data and threads + +Even if MPI operations are called outside concurrent sections it is still your responsibility to make sure that the *data* involved in communication is synchronized; this is always the case. +Clear ownership and scoping of *data* helps a lot towards thread safety. +Avoiding mutable shared data between threads also helps. +Perhaps as a last resort, data can be locked with mutex objects to be written or accessed one thread at time. + +### Communicator and threads + +The library doesn't control or owns the communicated data for the most part, therefore the main concern of the library regarding threading is within the communicator class itself. + +The C-MPI interface briefly mentions thread-safety, for example most MPI operations are accompanied by the following note (e.g. https://www.mpich.org/static/docs/latest/www3/MPI_Send.html): + +> **Thread and Interrupt Safety** +> +> This routine is thread-safe. This means that this routine may be safely used by multiple threads without the need for any user-provided thread locks. However, the routine is not interrupt safe. Typically, this is due to the use of memory allocation routines such as malloc or other non-MPICH runtime routines that are themselves not interrupt-safe. + +This doesn't mean that that _all_ calls can be safely done from different threads concurrently, only some of them, those that refer to completely different argument can be safe. + +In practice it is observable that for most MPI operations the "state" of the communicator can change in time. +Even if after the operation the communicator seems to be in the same state as before the call the operation itself changes, at least briefly, the state of the communicator object. +This internal state can be observed from another thread even through undefined behavior, even if transiently. +A plausible model to explain this behavior is that internal buffers are used by individual communicators during communication. + +In modern C++, this is enough to mark communicator operations non-`const` (i.e. an operation than can be applied only on a mutable instance of the communicator). + +(`MPI_Send` has "tags" to differentiate separate communications and may help with concurrent calls, but this is still not a enough since the tags are runtime variables, of which the library doesn't know the origin. +Besides, the use of tags are not a general solution since collective operation do not use tags at all. +It has been known for a while that the identity of the communicator in some sense serves as a tag for collective communications. +This is why it is so useful to be able to duplicate communicators to distinguish between collective communication messages.) + +This explains why most member functions of `mpi3::communicator` are non-`const`, and also why most of the time `mpi3::communicators` must either be passed either by non-`const` reference or by value (depending on the intended use, see below.) +Be aware that passing by `const`-reference `mpi3::communicator const&` is not very productive because no communication operation can be performed with this instance (not even duplication to obtain a new instance). +(This behavior is not unheard of in C++: standard "printing" streams generally need be _mutable_ to be useful (e.g. `std::cout` or `std::ofstream`), even though they don't seem to have a changing state.) + +This brings us to the important topic of communicator construction and assignment. + +More material: ["C++ and Beyond 2012: Herb Sutter - You don't know const and mutable"](https://web.archive.org/web/20170119232617/https://channel9.msdn.com/posts/C-and-Beyond-2012-Herb-Sutter-You-dont-know-blank-and-blank) and ["related"](https://web.archive.org/web/20160924183715/https://channel9.msdn.com/Shows/Going+Deep/C-and-Beyond-2012-Herb-Sutter-Concurrency-and-Parallelism). + +### Duplication of communicator + +In C, custom structures do not have special member functions that indicate copying. +In general this is provided by free functions operating in pointer or _handle_ types, and in general in their signature ignores `const`ness. + +In C-MPI, the main function to duplicate a communicator is `int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)`. +When translating from C to C++ we have to understand that `MPI_Comm` is a handle to a communicator, that is, it behaves like a pointer. +In a general library the source (first argument) is conceptually constant (unmodified) during a copy, so we could be tempted to mark it as `const` when translating to C++. + +```cpp +struct communicator { + ... + communicator bad_duplicate() const; // returns a new communicator, congruent to the current communicator, +}; +``` +Furthermore, we could be tempted to call it `copy` or even to make it part of the copy-constructor. + +But, alas, this is not the case according to the rules we delineated earlier. +We know that duplication is an operation that requires communication and it is observable (through concurrent threads) that the internal state of the _original_ communicator is changed *while* it is duplicated. +Therefore to be honest with the semantics of communicator duplication we are forced to implement this function as non-`const`. + +```cpp +struct communicator { + ... + communicator duplicate(); // returns a new communicator, congruent to the current communicator +}; +``` + +The consequence of this line of though is that a `const` communicator (i.e. non-mutable) cannot be duplicated. +That is, not only such communicator cannot do any communication operation but it cannot be duplicated itself. + +```cpp +mpi3::communicator new_comm{comm.duplicate()}; +``` + +This syntax also makes very explicit what the operation really does. + +### Pass-by-value or pass-by-reference + +As indicated earlier, a useful communicator is one that is mutable. +Therefore when passing a communicator to a function we have two main options, either pass by reference (non-const reference) or by value. + +```cpp +void f(mpi3::communicator& comm); +void g(mpi3::communicator comm); +``` + +These two cases have different meanings and different things can be done with the corresponding communicators. + +Case `f` implies that, first, we are reusing a communicator, even if all communication operations are internal to the function or second, that `f` can communicate messages with a communicator that is external to the function. + +Although reusing a communicator sound reasonable (since duplicating communicators can be an expensive operation), even if all communication is contained in `f` there is a risk that some communicator is mixed inadvertedly with communication external to `f`. +The logic of collective operation would be distributed in different functions in the code, which is possible but difficult or impossible to reason about. +If `f` is running in a multi-threaded environment, it could be dealing with a communicator that is being shared with other threads. + +Case `g` is different in the sense that it knows that it has exclusive access to the communicator, send and receive operations cannot be captured by other functions and collective operations only need to be fully contained inside `g`. + +For completeness we can also imagine a function declared as pass-by-const-reference. + +```cpp +void h(mpi3::communicator const& comm); +``` + +In the current system, this is not very useful since only a handful of operations, which do not include communication or duplication, can be done. +(An example is probing the `.size()` of the communicator.) +Take into account that, inside the `h` function it is also "too late" to produce a duplicate of the communicator. + +## Communicator as an implementation detail + +Note that so far we didn't indicate how to use `mpi3::communicator` member with threads, we are simply following the logic being transparent of what each MPI operation is likely to perform behind the scenes regarding the (transient) state of the communicator. + +In C++ it is very useful to include a communicator to each object that requires to perform communication to maintain its internal consistency. +Suppose we have a data that is distributed across many processes, and that we store a instance of the communicator containing these processes. +Such class could have operations that modify its state and others that do not. +The correct design is to mark the function in the latter category as `const`. + +```cpp +struct distributed_data { + void set() { ... } + void print() const { ... } + + private: + mpi3::communicator comm_; +}; +``` + +However such design doesn't work, because for `print` to do any actual communication (e.g. to communicate some data to the root process) would need to have access to a mutable communicator, the `const` mark prevents that. + +One option is to make `print` non-`const`, this is bad because we will lose any concept of mutability just because an implementation detail. +The other option to remove const by force, +```cpp + void print() const { const_cast(comm_).do_something()... } +``` +which would work but it is not very idiomatic. +Besides, this class would become now **hostile** to threads, because two simultaneous `print` calls (which are marked as `const`) on the same class could overlap, the messages could be mixed and weird behavior can appear under threads and we would need to look inside the implementation of `print`. +Ending up with hostile class is an basically a show stopped for threading and must be avoided. + +Note that making the communicator member a pointer `mpi3::communicator* comm_;` doesn't solve any problem, it just kick the can down the road. + +This leads to a more modern design which would use the keyword `mutable`. + +```cpp +struct distributed_data { + void set() { ... } + void print() const { ... } + + public: + mutable mpi3::communicator comm_; +}; +``` + +This will allow the use of the communicator from internals of `print() const` without the use of `const_cast`. +This doesn't save us from the problem of using the communicator concurrently but at least it is clear in the declaration of the class. +As a matter of fact this `mutable` attribute is exactly what marks the class a thread unsafe. +(A mutable member without a synchronization mechanism is a red flag in itself.) +If a single instance of the class is never used across threads *or* the program is single threaded there is nothing else that one needs to do. + +Note also that different instances of the class can also be used from different threads, since they don't share anything, nor internal data or their internal communicator. + +What if you want to make your class, that contains a communicator thread-safe, at least safe for calling concurrently non mutating (`const`) members? +For that you need to implement your own synchronization or locking mechanism. +There is no single recipe for that, you can use a single mutex to lock access for the communicator alone or both the communicator and data. + +```cpp +struct distributed_data { + void set() { ... } + void print() const { std::lock_guard guard{mtx_}; ... use comm_ ... } + + private: + mutable std::mutex mtx_; + mutable mpi3::communicator comm_; +}; +``` + +I don't recommend doing this specifically; the code above is just to illustrate the point. +I can not give a general recipe beyond this point, because there are many possible choices on how to make class thread safe (e.g. data-safe) or thread safe to some specific level (operation-safe). +Ideally concurrent data structure should be able to do some of the work without the synchronization bottleneck. +The whole point is that the library gives you this option, to trade-off safety and efficiency to the desired degree but no more. + +In fact a (bad) blanket way to make the library thread safe could be to wrap every communicator in class with a mutex and make all most communication operations `const`. +This would force, from a design perspective, an unacceptable operation cost. + +### Not a copy-constructor, but a duplicate-constructor + +So far we have shown the `duplicate` interface function as a mechanism for duplicating communicators (used as `auto new_comm{comm.duplicate()}`), which is nice because it makes the operation very explicit, but it also makes it difficult to integrate generically with other parts of C++. + +A reasonable copy constructor of the class containing a communicator would be: + +```cpp +struct distributed_data { + distributed_data(distributed_data const& other) : comm_{other.comm_.duplicate()} {} + + private: + ... + mutable mpi3::communicator comm_; +}; +``` +Note that this code is valid because `comm_` is a mutable member of `other`. +The worst part of forcing us to use the "non-standard" `duplicate` function is that we can no longer "default" the copy constructor. + +Copying in C++ is usually delegated to special member functions such as the copy-constructor or copy-assignment. +However these function take their source argument as `const` reference and as such it cannot call the `duplicate` member. +(And even if we could we would be lying to the compiler in the sense that we could make the system crash by copying concurrently a single (supposedly) `const` communicator that is shared in two threads.) + +However the language is general enough to allow a constructor by non-const reference. +The signature of this constructor is this one: + +```cpp +communicator::communicator(communicator & other) {...} // "duplicate" constructor? +communicator::communicator(communicator const& ) = delete; // no copy constructor +``` + +There is no standard name for this type of constructor, I choose to call it here "duplicate"-constructor, or mutable-copy-constructor. +This function does internally call `MPI_Comm_dup`, and like `duplicate()` it can only be called with a source that is mutable. +This makes the copy constructor of the containing class more standard, or even can be implemented as `= default;`. + +```cpp +struct distributed_data { + distributed_data(distributed_data const& other) : comm_{other.comm_} {} // or = default; + ... + private: + mutable mpi3::communicator comm_; +}; +``` + +**In summary**, +1) all important communication operations are non-`const` because according to the rules and practice of modern C++ the internal state of the communicator is affected by these operations, +2) ... including the `duplicate` operation; +3) `mutable` is a good marker to indicate the _possible_ need for custom (thread) synchronization mechanism; it also makes possible the use of communicator as member of a class. +4) the need may be critical or not (the user of the library decides), +5) mutable instances of communicators (i.e. non-`const` variables or mutable members) can be duplicated using standard C++ syntax, via "duplicate"-constructor or via `duplicate` member functions. +6) In general, it is likely to be a good idea to duplicate communicator for specific threads *before* creating them; otherwise duplication will happen "too late" with a shared (non-const) communicator. + +(Thanks Arthur O'Dwyer for the critical reading of this section.) + +## NCCL (GPU communication) + +If the underlying MPI distribution is GPU-aware, in principle you can pass GPU pointers to the communication routines. +This is generally faster than copying back and forth to CPU. + +Nvidia's NCCL conceptually implements a subset of MPI operations and it might be faster than GPU-aware MPI. +To obtain an NCCL communicator you pass an MPI communicator. + +```cpp + mpi3::nccl::communicator gpu_comm{mpi_comm}; +``` + +The original MPI communicator is assumed be working with non-overlapping devices (e.g. one process per GPU). +This can be achieved by `cudaSetDevice(world.rank() % num_devices);` generally at the start of the program or autommically by using certain ways to run the MPI program (e.g. `lrun` tries to attach each MPI process to a different GPU device). + +With some limitations, the NCCL communicator can be used to perform operations on GPU memory without the need to obtaining raw pointers. +By default it works with `thrust[::cuda]::device_ptr` or `thrust[::cuda]::universal_ptr`. +For example this produces a reduction in GPU across processes (even processes in different nodes): + +```cpp +// thust::device_vector> A(1000, gpu_comm.rank()); + thrust::device_vector> A(1000, gpu_comm.rank()); + + gpu_comm.all_reduce_n(A.data(), A.size(), A.data()); +``` + +Like B-MPI3 communicator the NCCL communicator is destroyed automatically when leaving the scope. + +The implementation is preliminary, the NCCL communicator is moveable but not copyable (or duplicable). +Congruent NCCL communicators can be constructed from the same (or congruent) B-MPI3 communicator (at the cost of a regular MPI broadcast). +There is not mechanism to create NCCL subcommunicators from other NCCL communicators, except using MPI subcommunicators as constructor arguments. + +# Conclusion + +The goal is to provide a type-safe, efficient, generic interface for MPI. +We achieve this by leveraging template code and classes that C++ provides. +Typical low-level use patterns become extremely simple, and that exposes higher-level patterns. + +# Mini tutorial + +This section describes the process of bringing a C++ program that uses the original MPI interface to one that uses B.MPI3. +Below it is a valid C++ MPI program using send and receive function. +Due to the legacy nature of MPI, C and C++ idioms are mixed. + +```cpp +#include + +#include +#include +#include + +int main(int argc, char **argv) { + MPI_Init(&argc, &argv); + MPI_Comm comm = MPI_COMM_WORLD; + + int count = 10; + + std::vector xsend(count); iota(begin(xsend), end(xsend), 0); + std::vector xrecv(count, -1); + + int rank = -1; + int nprocs = -1; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nprocs); + if(nprocs%2 == 1) { + if(rank == 0) {std::cerr<<"Must be called with an even number of processes"< +#include +#include + +namespace bmpi3 = boost::mpi3; + +int main(int argc, char **argv) try { + bmpi3::environment::initialize(argc, argv); + MPI_Comm comm = &bmpi3::environment::get_world_instance(); assert(comm == MPI_COMM_WORLD) +... + bmpi3::environment::finalize(); + return 0; +} +``` + +Notice that we are getting a reference to the global communicator using the `get_world_instance`, then, with the ampersand (`&`) operator, we obtain a `MPI_Comm` handle than can be used with the rest of the code untouched. + +Since `finalize` will need to be executed in any path, it is preferable to use an RAII object to represent the environment. +Just like in classic MPI, it is wrong to create more than one environment. + +Both, accessing the global communicator directly is in general considered problematic. +For this reason it makes more sense to ask for a duplicate of the global communicator. + +```cpp +int main(int argc, char **argv) { + bmpi3::environment env(argc, argv); + bmpi3::communicator world = env.world(); + MPI_Comm comm = &world; assert(comm != MPI_COMM_WORLD); +... + return 0; +} +``` + +This ensures that `finalize` is always called (by the destructor) and that we are not using the original global communicator, but a duplicate. + +Since this pattern is very common, a convenient "main" function is declared by the library as a replacement declared in the `mpi3/main.hpp` header. + +```cpp +#include "../../mpi3/main.hpp" + +#include +#include +#include + +namespace bmpi3 = boost::mpi3; + +int bmpi3::main(int, char **, bmpi3::communicator world) { + MPI_Comm comm = &world; assert(comm != MPI_COMM_WORLD); +... + return 0; +} +``` + +The next step is to replace the use of the MPI communicator handle by a proper `mpi3::communicator` object. +Since `world` is already a duplicate of the communicator we can directly use it. +The `size` and `rank` are methods of this object which naturally return their values. + +```cpp +... + int rank = world.rank(); + int nprocs = world.size(); +... +``` + +Similarly the calls to send and receive data can be transformed. +Notice that the all the irrelevant or redundant arguments (including the receive source) can be omitted. + +```cpp +... + world.send_n (xsend.data(), count, partner_rank); + world.receive_n(xrecv.data(), count); +... +``` + +(We use the `_n` suffix interface to emphasize that we are using element count (container size) as argument.) + +The condition `(rank == 0)` is so common that can be replaced by the `communicator`'s method `is_root()`: + +```cpp + if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< +#include +#include + +namespace bmpi3 = boost::mpi3; + +int bmpi3::main(int /*argc*/, char ** /*argv*/, bmpi3::communicator world) try { + int count = 10; + + std::vector xsend(count); iota(begin(xsend), end(xsend), 0); + std::vector xrecv(count, -1); + + if(world.size()%2 == 1) { + if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< +#include +#include + +namespace bmpi3 = boost::mpi3; + +int bmpi3::main(int /*argc*/, char ** /*argv*/, bmpi3::communicator world) { + if(world.size()%2 == 1) { + if(world.is_root()) {std::cerr<<"Must be called with an even number of processes"< xsend(10); iota(begin(xsend), end(xsend), 0); + std::vector xrecv(xsend.size(), -1); + + world.send_receive(cbegin(xsend), cend(xsend), (world.rank()/2)*2 + (world.rank()+1)%2, begin(xrecv)); + + assert(xrecv[5] == 5); + if(world.is_root()) {std::cerr<<"successfully completed"< + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + if(world.rank() == 0) cout << mpi3::version() << '\n'; + + cout << "hello from task " << world.rank() << " in host " << mpi3::processor_name() << std::endl; + + if(world.rank() == 0) cout << "numer of tasks " << world.size() << std::endl; + return 0; +} + diff --git a/examples/02-send.cpp b/examples/02-send.cpp new file mode 100644 index 00000000000..0673223604d --- /dev/null +++ b/examples/02-send.cpp @@ -0,0 +1,47 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/version.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], boost::mpi3::communicator& world){ + + if(world.size() % 2 != 0){ + if(world.root()) cout << "Quitting. Need an even number of tasks: numtasks = " << world.size() << "\n"; + }else{ + int message = -1; + if(world.root()) cout << "MASTER: number of mpi tasks is " << world.size() << "\n"; + if(world.rank() < world.size()/2){ + int partner = world.rank() + world.size()/2; + int rank = world.rank(); + world.send_value(rank, partner); + world.receive_value(message, partner); + }else if(world.rank() >= world.size()/2){ + int partner = world.rank() - world.size()/2; + world.receive_value(message, partner); + int rank = world.rank(); + world.send_value(rank, partner); + } + cout << "Task " << world.rank() << " is partner with " << message << "\n"; + } + + if(world.rank()==0){ + std::vector code = {1.,2.,3.,4.}; + world.send_n(code.begin(), 4, 1); + } + + if(world.rank()==1){ + std::vector code(4); + world.receive_n(code.begin(), 4, 0); + assert(( code == std::vector{1.,2.,3.,4.} )); + } + + return 0; +} + diff --git a/examples/03-isend.cpp b/examples/03-isend.cpp new file mode 100644 index 00000000000..10c861988c0 --- /dev/null +++ b/examples/03-isend.cpp @@ -0,0 +1,34 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++17 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/version.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + if(world.size() % 2 != 0){ + if(world.root()) cout << "Quitting. Need an even number of tasks: numtasks = " << world.size() << "\n"; + return 1; + } + + int message = -1; + cout << "Hello from task " << world.rank() << " on host " << mpi3::processor_name() << "\n"; + if(world.root()) cout << "MASTER: number of mpi tasks is " << world.size() << "\n"; + + int partner = world.rank() + +using std::cout; +using std::endl; + +double update(std::vector& data, int offset, int chunk_size){ + double sum = 0; + for(int i = offset; i != offset + chunk_size; ++i){ + data[i] += i*1.0; + sum += data[i]; + } + return sum; +} + +int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator const& world){ + if(world.size() % 4 != 0){ + if(world.master()) cout << "Quitting. Need an even number of tasks: numtasks = " << world.size() << std::endl; + return 1; + } + + int const array_size = 16000000; + int chunk_size = array_size/world.size(); + + if(world.master()){ + std::vector data(array_size); + { + double sum = 0; + for(int i = 0; i != data.size(); ++i){ + data[i] = i*1.0; + sum += data[i]; + } + cout << "serial sum " << sum << endl; + } + + int offset = chunk_size; + for(int dest = 1; dest != world.size(); ++dest){ + world.send_n(data.begin() + offset, chunk_size, dest); + offset += chunk_size; + } + + double partial_sum = 0; + for(int i = 0; i != chunk_size; ++i) + partial_sum += data[i]; + + double parallel_sum = world.reduce(partial_sum); + cout << "parallel sum is " << parallel_sum << endl; + }else{ + std::vector partial_data(chunk_size); + world.receive_n(partial_data.begin(), chunk_size, 0); + + double partial_sum = 0; + for(int i = 0; i != partial_data.size(); ++i) + partial_sum += partial_data[i]; + + world.reduce(partial_sum); + } + + +} + diff --git a/examples/05-array_mult.cpp b/examples/05-array_mult.cpp new file mode 100644 index 00000000000..806ee98f2c5 --- /dev/null +++ b/examples/05-array_mult.cpp @@ -0,0 +1,109 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++17 `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +// based on https://computing.llnl.gov/tutorials/mpi/samples/C/array_mm.c +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/version.hpp" +#include "alf/boost/mpi3/processor_name.hpp" +#include "alf/boost/multi/array.hpp" + +#include + +using std::cout; +using std::endl; + +int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator const& world){ + assert( world.size() > 1 ); + + if(world.master()){ + + std::string name = "master"; + std::list l = {1.,2.,3.,4.,5.,6.}; + + boost::multi::array a({{62, 15}}); + boost::multi::array b({{15,7}}); + + for(int i : a.extensions()[0]) + for(int j : a.extensions()[1]) + a[i][j] = i + j; + + for(int i : b.extensions()[0]) + for(int j : b.extensions()[1]) + b[i][j] = i*j; + + boost::multi::array c_serial({{62,7}}); + + for(int i : c_serial.extensions()[0]){ + for(int j : c_serial.extensions()[1]){ + c_serial[i][j] = 0; + for(int k : a.extensions()[1]){ + c_serial[i][j] += a[i][k] * b[k][j]; + } + } + } + + auto averow = a.extensions()[0].size()/(world.size()-1); + auto extra = a.extensions()[0].size()%(world.size()-1); + + auto offset = 0; + for(int dest = 1; dest != world.size(); ++dest){ + auto p = world[dest]; + p << name; + world.send_n(l.begin(), l.size(), dest); + int rows = (dest < extra + 1)?(averow + 1): averow; + p + << offset + << rows + ; + world.send_n(&a[offset][0], rows*a.extensions()[1].size(), dest); + world.send_n(&b[0][0], b.num_elements(), dest); + offset += rows; + } + + boost::multi::array c({{62,7}}); + for(int source = 1; source != world.size(); ++source){ + auto proc = world[source]; + int rows; + proc + >> offset + >> rows + ; + world.receive_n(&c[offset][0], rows*c.extensions()[1].size(), source); + } + for(int i : c.extensions()[0]) + for(int j : c.extensions()[1]) + assert( c[i][j] - c_serial[i][j] == 0 ); + }else{ + auto proc = world[0]; + std::string name; + proc >> name; + std::list l(6); + world.receive_n(l.begin(), l.size(), 0); + assert(( l == std::list{1.,2.,3.,4.,5.,6.} )); + int offset; + int rows; + proc >> offset >> rows; + boost::multi::array a_partial({{rows, 15}}); + world.receive_n(a_partial.data(), a_partial.num_elements(), 0); + boost::multi::array b({{15,7}}); + world.receive_n(b.data(), b.num_elements(), 0); + + boost::multi::array c_partial({{rows,7}}); + for(int i = 0; i != c_partial.extensions()[0].size(); ++i){ + for(int j = 0; j != c_partial.extensions()[1].size(); ++j){ + c_partial[i][j] = 0; + for(int k = 0; k != a_partial.extensions()[1].size(); ++k){ + c_partial[i][j] += a_partial[i][k] * b[k][j]; + } + } + } + proc + >> offset + >> rows + ; + world.send_n(c_partial.data(), c_partial.num_elements(), 0); + } + cout << "ok" << endl; +} + diff --git a/examples/average.cpp b/examples/average.cpp new file mode 100644 index 00000000000..e13ada314f5 --- /dev/null +++ b/examples/average.cpp @@ -0,0 +1,52 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +//#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/environment.hpp" +#include "alf/boost/mpi3/communicator.hpp" +#include "alf/boost/mpi3/process.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +std::random_device rd; +std::mt19937 gen(rd()); + +int main(int, char*[]){//, mpi3::communicator& world){ + + mpi3::environment env; + auto& world = env.world(); + + int elements_per_proc = 5000; + + std::vector v; + if(world.rank() == 0){ + std::uniform_real_distribution<> dis; + v.resize(elements_per_proc*world.size(), -1.); + std::generate(v.begin(), v.end(), [&](){return dis(gen);}); + cout << "serial average " << std::accumulate(v.begin(), v.end(), 0.)/v.size() << '\n'; + } + + std::vector partial_v(elements_per_proc, -1.); + world.scatter_from(partial_v.begin(), partial_v.end(), v.begin(), 0); + double partial_ave = std::accumulate(partial_v.begin(), partial_v.end(), 0.)/partial_v.size(); + { + std::vector partial_aves = world.gather_value(partial_ave, 0); // std::vector subaves = (world[0] += subave); + if(world.rank() == 0){ + assert(partial_aves.size() == (unsigned)world.size()); + cout << "parallel average: " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << '\n'; + }else{ + assert(partial_aves.empty()); + } + } + { + std::vector partial_aves = (world |= partial_ave); + assert(partial_aves.size() == (unsigned)world.size()); + cout << "parallel average on " << world.rank() << ": " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << std::endl; + } + return 0; +} + diff --git a/examples/broadcast.cpp b/examples/broadcast.cpp new file mode 100644 index 00000000000..8e5a39feb4e --- /dev/null +++ b/examples/broadcast.cpp @@ -0,0 +1,131 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/process.hpp" +#include "alf/boost/mpi3/detail/package_archive.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +void my_broadcast(mpi3::communicator& comm, std::vector& data, int root){ + if(comm.rank() == root){ + for(int i = 0; i != comm.size(); ++i){ + if(i != comm.rank()) comm[i] << data; + } + }else{ + comm[root] >> data; + } +} + +void my_bcast(void* data, int count, MPI_Datatype datatype, int root, + MPI_Comm communicator) { + int world_rank; + MPI_Comm_rank(communicator, &world_rank); + int world_size; + MPI_Comm_size(communicator, &world_size); + + if (world_rank == root) { + // If we are the root process, send our data to everyone + int i; + for (i = 0; i < world_size; i++) { + if (i != world_rank) { + MPI_Send(data, count, datatype, i, 0, communicator); + } + } + } else { + // If we are a receiver process, receive the data from the root + MPI_Recv(data, count, datatype, root, 0, communicator, MPI_STATUS_IGNORE); + } +} + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + { + std::vector data; + if(world.rank() == 0){data.resize(10); std::iota(data.begin(), data.end(), 0);} + my_broadcast(world, data, 0); + assert( data[5] == 5 ); + } + // ... less efficient but same effect as ... + { + std::vector data; + if(world.rank() == 0){data.resize(10); std::iota(data.begin(), data.end(), 0);} + world[0] & data; + assert( data[5] == 5 ); + } + int num_elements = 100000; + int num_trials = 1000; + { + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + + double total_my_bcast_time = 0.0; + double total_mpi_bcast_time = 0.0; + int i; + double* data = (double*)malloc(sizeof(double) * num_elements); + std::iota(data, data + num_elements, 0); + assert(data != NULL); + + for (i = 0; i < num_trials; i++) { + // Time my_bcast + // Synchronize before starting timing + MPI_Barrier(MPI_COMM_WORLD); + total_my_bcast_time -= MPI_Wtime(); + my_bcast(data, num_elements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + // Synchronize again before obtaining final time + MPI_Barrier(MPI_COMM_WORLD); + total_my_bcast_time += MPI_Wtime(); + + // Time MPI_Bcast + MPI_Barrier(MPI_COMM_WORLD); + total_mpi_bcast_time -= MPI_Wtime(); + MPI_Bcast(data, num_elements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); + total_mpi_bcast_time += MPI_Wtime(); + } + + // Print off timing information + if (world_rank == 0) { + printf("Data size = %d, Trials = %d\n", num_elements * (int)sizeof(double), + num_trials); + printf("Avg my_bcast time = %lf\n", total_my_bcast_time / num_trials); + printf("Avg MPI_Bcast time = %lf\n", total_mpi_bcast_time / num_trials); + } + free(data); + } + { + + std::vector data(num_elements); std::iota(data.begin(), data.end(), 0); + + double total_my_broadcast_time = 0.0; + double total_broadcast_time = 0.0; + for(int i = 0; i != num_trials; ++i){ + // if(world.rank() == 0) cout << i << std::endl; + world.barrier(); + total_my_broadcast_time -= mpi3::wall_time(); + my_broadcast(world, data, 0); + world.barrier(); + total_my_broadcast_time += mpi3::wall_time(); + world.barrier(); + total_broadcast_time -= mpi3::wall_time(); + // world.broadcast(data.begin(), data.end(), 0); + // MPI_Bcast(data.data(), num_elements, MPI_DOUBLE, 0, MPI_COMM_WORLD); + // world[0] & data; + int n = data.size(); world[0] & n; data.resize(n); + world.broadcast(data.begin(), data.end(), 0); + world.barrier(); + total_broadcast_time += mpi3::wall_time(); + } + if(world.rank() == 0){ + cout << "data size = " << num_elements * (int)sizeof(double) << ", trials = " << num_trials << '\n'; + cout << "avg my_broadcast time = " << total_my_broadcast_time / num_trials << '\n'; + cout << "avg broadcast time = " << total_broadcast_time / num_trials << '\n'; + } + } + + return 0; +} + diff --git a/examples/calculate_pi.cpp b/examples/calculate_pi.cpp new file mode 100644 index 00000000000..e72aab92537 --- /dev/null +++ b/examples/calculate_pi.cpp @@ -0,0 +1,49 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" +#include "alf/boost/mpi3/process.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +std::random_device rd; +std::mt19937 gen(rd()); +std::uniform_real_distribution<> dis; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + + int elements_per_proc = 5000; + + std::vector v; + if(world.rank() == 0){ + v.resize(elements_per_proc*world.size(), -1.); + std::generate(v.begin(), v.end(), [&](){return dis(gen);}); + cout << "serial average " << std::accumulate(v.begin(), v.end(), 0.)/v.size() << '\n'; + } + + std::vector partial_v(elements_per_proc, -1.); + world.scatter_from(partial_v.begin(), partial_v.end(), v.begin(), 0); + double partial_ave = std::accumulate(partial_v.begin(), partial_v.end(), 0.)/partial_v.size(); + { + std::vector partial_aves = world.gather_value(partial_ave, 0); // std::vector subaves = (world[0] += subave); + if(world.rank() == 0){ + assert(partial_aves.size() == (unsigned)world.size()); + cout << "parallel average: " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << '\n'; + }else{ + assert(partial_aves.empty()); + } + } + { + // std::vector partial_aves = world.all_gather_value(partial_ave); + std::vector partial_aves = (world |= partial_ave); + assert(partial_aves.size() == (unsigned)world.size()); + cout << "parallel average on " << world.rank() << ": " << std::accumulate(partial_aves.begin(), partial_aves.end(), 0.)/partial_aves.size() << std::endl; + } + return 0; +} + diff --git a/examples/groups.cpp b/examples/groups.cpp new file mode 100644 index 00000000000..7b5957a7a97 --- /dev/null +++ b/examples/groups.cpp @@ -0,0 +1,23 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 15 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + + mpi3::communicator prime = world.create(world.group().include( {2, 3, 5, 7, 11, 13} )); + + // If this rank isn't in the new communicator, it will be an invalid (null) communicator + // Using rank() or size() in a null communicator is erroneous + cout << "world " << world.rank() << "/" << world.size() << " ---> "; + if(prime) cout << "prime " << prime.rank() << "/" << prime.size() << std::endl; + else cout << "not in prime comm" << std::endl; + + return 0; +} + diff --git a/examples/hello.cpp b/examples/hello.cpp new file mode 100644 index 00000000000..44d02a5c420 --- /dev/null +++ b/examples/hello.cpp @@ -0,0 +1,21 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++17 -Wfatal-errors $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/version.hpp" +#include "alf/boost/mpi3/processor_name.hpp" + +#include + +using std::cout; +using std::endl; + +int boost::mpi3::main(int argc, char* argv[], boost::mpi3::communicator const& world){ + if(world.rank() == 0) cout << boost::mpi3::version() << '\n'; + + cout << "hello from task " << world.rank() << " in host " << boost::mpi3::processor_name() << endl; + + if(world.rank() == 0) cout << "numer of tasks " << world.size() << endl; +} + diff --git a/examples/hello_world.cpp b/examples/hello_world.cpp new file mode 100644 index 00000000000..58e4fd9ae62 --- /dev/null +++ b/examples/hello_world.cpp @@ -0,0 +1,23 @@ +#if COMPILATION_INSTRUCTIONS +#export PATH=/home/correaa/prj/alf/boost/mpi3/fake:$PATH +mpic++ $0 -o $0x&&mpirun -n 4 $0x $@&&rm $0x;exit +#endif + +#include "../../mpi3/main.hpp" +#include "../../mpi3/process.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator world){ + + cout + << "Hello world from processor '" << mpi3::processor_name() + << "' rank " << world.rank() + << " out of " << world.size() << " processors\n" + ; + + return 0; + +} + diff --git a/examples/managed_shared_memory.cpp b/examples/managed_shared_memory.cpp new file mode 100644 index 00000000000..2146a1abd6b --- /dev/null +++ b/examples/managed_shared_memory.cpp @@ -0,0 +1,30 @@ +#if COMPILATION_INSTRUCTIONS +time mpicxx -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x -lboost_system && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/shm/managed_shared_memory.hpp" // there is a bug in boost 1.64, this needs to be included first +#include "alf/boost/mpi3/main.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + + mpi3::communicator node = world.split_shared(0); + int N = 1000; + + mpi3::shm::managed_shared_memory msm(node, 100000); + std::atomic& i = *msm.construct>(0); + node.barrier(); + for(int j = 0; j != N; ++j) ++i; + node.barrier(); + int snapshot = i; + if(node.rank() == 0) assert(snapshot == node.size()*N); + msm.destroy>(i); + node.barrier(); + + return 0; +} + diff --git a/examples/managed_shared_memory_0.cpp b/examples/managed_shared_memory_0.cpp new file mode 100644 index 00000000000..46e73743d0e --- /dev/null +++ b/examples/managed_shared_memory_0.cpp @@ -0,0 +1,57 @@ +#if COMPILATION_INSTRUCTIONS +time mpicxx -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x -lboost_system && time mpirun -np 1 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/shm/managed_shared_memory.hpp" // there is a bug in boost 1.64, this needs to be included first +#include "alf/boost/mpi3/main.hpp" + +#include +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +struct list_node{ +// boost::interprocess::offset_ptr next; + list_node* next; + int value; +}; + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + + mpi3::communicator node = world.split_shared(0); + + mpi3::shm::managed_shared_memory segment(node, 65536); + list_node* prev = 0; + list_node* current; + list_node* first; + + list_node* ln = static_cast(segment.allocate(sizeof(list_node))); + assert(ln != nullptr); + ln -> value = 0; + ln -> next = 0; + list_node* ln2 = static_cast(segment.allocate(sizeof(list_node))); + assert(ln2 != nullptr); + ln2 -> value = 1; +// ln2 -> next = ln; + + for(int i = 0; i < 10; ++i, prev = current){ + current = new list_node; + // current = static_cast(segment.allocate(sizeof(list_node))); + current->value = i; + current->next = 0; + if(!prev) first = current; + else prev->next = current; + } + +#if 0 + for(current = first; current; ){ + prev = current; + current = current->next; + segment.deallocate(prev);//.get()); + } +#endif + + return 0; +} + diff --git a/examples/pi_reduce.cpp b/examples/pi_reduce.cpp new file mode 100644 index 00000000000..99431503cbc --- /dev/null +++ b/examples/pi_reduce.cpp @@ -0,0 +1,56 @@ +#if COMPILATION_INSTRUCTIONS +OMPI_CXX=$CXXX mpicxx $CXXFLAGS $0 -o $0x&&mpirun --oversubscribe -n 6 $0x&&rm $0x;exit +#endif + +#include "../examples/../process.hpp" +#include "../examples/../main_environment.hpp" + +#include + +namespace mpi3 = boost::mpi3; + +// this program is based on https://computing.llnl.gov/tutorials/mpi/samples/C/mpi_pi_reduce.c + + + +double pi_approx(int n_samples){ + + std::random_device rd; + static std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis(-1.0, +1.0); + + int score = 0; + for(int i = 0; i != n_samples; ++i){ + double x = dis(gen); + double y = dis(gen); + if(x*x + y*y <= 1.0) ++score; + } + return 4.*score/static_cast(n_samples); // pi +} + +int mpi3::main(int, char*[], mpi3::environment& menv){ + auto world = menv.world(); + + constexpr int n_samples = 50000000; + + for(int s = 0; s != world.size(); ++s){ + auto t0 = mpi3::wall_time(); + if(auto continent = (world <= s)){ + + double pi = (continent += pi_approx(n_samples/continent.size())/continent.size()); + + if(world.root()){ + std::cout + <<"After "<< n_samples <<" throws " + <<"(in "<< continent.size() <<" procs), " + <<"average value of pi = " << pi + <<" (vs. real value of pi = "<< M_PI <<")" + <<" (time "<< (mpi3::wall_time() - t0).count() <<" sec)" + <> count; + cout << "proc " << world.rank() << " received count " << count << " from proc " << partner << '\n'; + } + } + + return 0; +} + diff --git a/examples/random_walk.cpp b/examples/random_walk.cpp new file mode 100644 index 00000000000..2b771306419 --- /dev/null +++ b/examples/random_walk.cpp @@ -0,0 +1,82 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" +#include "alf/boost/mpi3/process.hpp" +#include "alf/boost/mpi3/detail/package_archive.hpp" + +#include + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +struct walker{ + int location; + int steps_left; + template + void serialize(Archive& ar, const unsigned int){ + ar & location & steps_left; + } +}; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + int domain_size = 20; + int max_walk_size = 100; + int num_walkers_per_proc = 10; + + // decompose domain + assert(world.size() <= domain_size); + int subdomain_start = domain_size/world.size()*world.rank(); + int subdomain_size = domain_size/world.size(); + if(world.rank() == world.size() - 1) subdomain_size += domain_size%world.size(); + + // initialize walkers + std::vector incomming_walkers(num_walkers_per_proc); + std::vector outgoing_walkers; + + std::random_device rd; + std::mt19937 gen(rd()*world.rank()); + std::uniform_int_distribution<> dis(0, max_walk_size); + std::generate( + incomming_walkers.begin(), incomming_walkers.end(), + [&]{return walker{subdomain_start, dis(gen)};} + ); + + cout << "process " << world.rank() << " initialized " << num_walkers_per_proc << " walkers in subdomain " << subdomain_start << " - " << subdomain_start + subdomain_size - 1 << std::endl; + + // determine the maximum amount of sends and receives needed to complete all walkers + int maximum_sends_recvs = max_walk_size/(domain_size/world.size()) + 1; + for(int m = 0; m != maximum_sends_recvs; ++m){ + // process all incomming_walkers + for(auto& w : incomming_walkers){ + while(w.steps_left){ + if(w.location == subdomain_start + subdomain_size){ + if(w.location == domain_size) w.location = 0; + outgoing_walkers.push_back(w); + break; + }else{ + --w.steps_left; + ++w.location; + } + } + } + cout << "process " << world.rank() << " sending " << outgoing_walkers.size() << " outgoing walkers to process " << (world.rank() + 1)%world.size() << std::endl; + if(world.rank()%2 == 0){ + world[(world.rank() + 1)%world.size()] << outgoing_walkers; + outgoing_walkers.clear(); + world[world.rank()?world.rank()-1:world.size()-1] >> incomming_walkers; + }else{ + world[world.rank()?world.rank()-1:world.size()-1] >> incomming_walkers; + world[(world.rank() + 1)%world.size()] << outgoing_walkers; + outgoing_walkers.clear(); + } + cout << "process " << world.rank() << " received " << incomming_walkers.size() << " incomming_walkers" << std::endl; + } + cout << "process " << world.rank() << " done" << std::endl; + return 0; +} + diff --git a/examples/rank.cpp b/examples/rank.cpp new file mode 100644 index 00000000000..199014ea6d0 --- /dev/null +++ b/examples/rank.cpp @@ -0,0 +1,35 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" +#include "alf/boost/mpi3/process.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +std::random_device rd; +std::mt19937 gen(rd()); +std::uniform_real_distribution<> dis; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + + double number = dis(gen); + std::vector numbers = world.gather_value(number, 0); + + std::vector ranks; + if(world.rank() == 0){ + assert( not numbers.empty() ); + ranks.resize(numbers.size()); + std::iota(ranks.begin(), ranks.end(), 0); + std::sort(ranks.begin(), ranks.end(), [&numbers](auto a, auto b){return numbers[a] < numbers[b];}); + } else assert( numbers.empty() ); + + int rank = world.scatter_value(ranks); + cout << "processor " << world.rank() << " has value " << number << " and ranks " << rank << std::endl; + return 0; +} + diff --git a/examples/reduce_average.cpp b/examples/reduce_average.cpp new file mode 100644 index 00000000000..eeaf8d70707 --- /dev/null +++ b/examples/reduce_average.cpp @@ -0,0 +1,54 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" +#include "alf/boost/mpi3/process.hpp" + +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +std::random_device rd; +std::mt19937 gen(rd()); +std::uniform_real_distribution<> dis; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + + int elements_per_proc = 5000; + + std::vector v(elements_per_proc); + std::generate(v.begin(), v.end(), [&](){return dis(gen);}); + + double local_sum = std::accumulate(v.begin(), v.end(), 0.); + cout << "local sum for proc " << world.rank() << " is " << local_sum << " average is " << local_sum/v.size() << std::endl; + + {// reduce +// double global_sum = world.reduce_value(local_sum, mpi3::sum, 0); + mpi3::optional global_sum = (world[0] += local_sum); + if(global_sum){ + assert(world.rank() == 0); + cout << "total sum is " << *global_sum << ", average is " << *global_sum/(world.size()*elements_per_proc) << std::endl; + } + } + + {// all reduce, all reduce is necessary to compute standard deviations, so everyone knows the mean + double global_sum = world.all_reduce_value(local_sum, mpi3::sum); + double mean = global_sum /(elements_per_proc*world.size()); + double local_squares = 0.; + for(auto& e : v){ + local_squares += (e - mean)*(e - mean); + } + // double global_squares = world.reduce_value(local_squares, mpi3::sum, 0); + mpi3::optional global_squares = (world[0] += local_squares); + if(global_squares){ + double stddev = sqrt( *global_squares / elements_per_proc / world.size() ); + cout << "mean = " << mean << ", standard deviation = " << stddev << std::endl; + } + } + + return 0; +} + diff --git a/examples/ring.cpp b/examples/ring.cpp new file mode 100644 index 00000000000..c0f5f9eb542 --- /dev/null +++ b/examples/ring.cpp @@ -0,0 +1,28 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x && time mpirun -np 8 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/process.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + int token = -1; + + if(world.rank() != 0){ + world[world.rank() - 1] << token; + cout << "proc " << world.rank() << " received token " << token << " from proc " << world.rank() -1 << '\n'; + }; + + world[(world.rank() + 1)%world.size()] << token; + + if(world.rank() == 0){ + world[world.size() - 1] >> token; + cout << "proc " << world.rank() << " received token " << token << " from proc " << world.size() - 1 << '\n'; + } + + return 0; +} + diff --git a/examples/send_receive.cpp b/examples/send_receive.cpp new file mode 100644 index 00000000000..1cb6e012cd2 --- /dev/null +++ b/examples/send_receive.cpp @@ -0,0 +1,26 @@ +#if COMPILATION_INSTRUCTIONS +export PATH=/home/correaa/prj/alf/boost/mpi3/fake:$PATH +mpic++ -O3 -std=c++14 -Wall -Wfatal-errors $0 -o $0x.x && time mpirun -np 1 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/process.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char* argv[], mpi3::communicator& world){ + if( world.size() < 2 ) throw std::runtime_error("needs 2 processes"); + + int number = 0; + if(world.rank() == 0){ + number = -1; + world[1] << number; + }else if(world.rank() == 1){ + world[0] >> number; + cout << "process 1 received number " << number << " from process 0\n"; + } + + return 0; +} + diff --git a/examples/shared_window.cpp b/examples/shared_window.cpp new file mode 100644 index 00000000000..bda652af5cf --- /dev/null +++ b/examples/shared_window.cpp @@ -0,0 +1,50 @@ +#if COMPILATION_INSTRUCTIONS +mpic++ -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/shared_window.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char *argv[], mpi3::communicator world){ + int rank = world.rank(); + int size = world.size(); + + mpi3::shared_window sw = world.make_shared_window(world.rank()?0:size); + int* shared = (int*)sw.base(0); + + if(rank==0){ + // sw.lock_exclusive(0); + for(int i = 0; i < size; i++) shared[i] = -1; + // sw.unlock(0); + } + + world.barrier(); + + std::vector local(size); + + sw.lock_shared(0); + for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); + cout << "processor " << world.rank() << " (before) : "; + for(int i = 0; i != size; ++i) cout << local[i] << " "; + cout << std::endl; + sw.unlock(0); + + sw.lock_exclusive(0); + sw.put_n(&rank, 1, 0, rank); + sw.unlock(0); + + sw.lock_shared(0); + for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); + cout << "processor " << world.rank() << " (after) : "; + for(int i = 0; i != size; ++i) cout << local[i] << ' '; + cout << std::endl; + sw.unlock(0); + + world.barrier(); + + return 0; +} + diff --git a/examples/shared_window_2.cpp b/examples/shared_window_2.cpp new file mode 100644 index 00000000000..d2b76aa3281 --- /dev/null +++ b/examples/shared_window_2.cpp @@ -0,0 +1,47 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/mutex.hpp" +#include "alf/boost/mpi3/shared_window.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char *argv[], mpi3::communicator& world){ + + mpi3::communicator node = world.split_shared(0); + + mpi3::shared_window win = node.make_shared_window(1); + win.lock_all(); + win.sync(); + + node.barrier(); + + std::vector b_size(node.size()); + std::vector b_disp(node.size()); + std::vector buf(node.size()); + +// mpi3::mutex m(node); + if(node.rank() != 0){ + for(int i = 0; i != node.size(); ++i){ + b_size[i] = win.size(i); + b_disp[i] = win.disp_unit(i); + buf[i] = static_cast(win.base(i)); + if(i == node.rank()){ + *buf[i] = node.rank() + 100; + cout << "node rank " << node.rank() << ": *buf[" << i << "] = " << *buf[i] << std::endl; + } + } + }else cout << "node rank " << node.rank() << " master just watches." << std::endl; + + win.unlock_all(); + + if(node.rank() != 0) + for(int i=0; i != node.size(); ++i) + if(buf[i]) cout << "node rank " << node.rank() << ", target = " << i << ", buf = " << *buf[i] << ", b_size = " << b_size[i] << ", b_disp = " << b_disp[i] << std::endl; + + return 0; +} + diff --git a/examples/shared_window_3.cpp b/examples/shared_window_3.cpp new file mode 100644 index 00000000000..f038ce42cb6 --- /dev/null +++ b/examples/shared_window_3.cpp @@ -0,0 +1,50 @@ +#if COMPILATION_INSTRUCTIONS +mpic++ -O3 -std=c++14 -Wfatal-errors -Wall $0 -o $0x.x && time mpirun -np 10 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/shared_window.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int argc, char *argv[], mpi3::communicator& world){ + int rank = world.rank(); + int size = world.size(); + + mpi3::shared_window sw = world.make_shared_window(world.rank()?0:size); + int* shared = (int*)sw.base(0); + + if(rank==0){ + // sw.lock_exclusive(0); + for(int i = 0; i < size; i++) shared[i] = -1; + // sw.unlock(0); + } + + world.barrier(); + + std::vector local(size); + + sw.lock_shared(0); + for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); + cout << "processor " << world.rank() << " (before) : "; + for(int i = 0; i != size; ++i) cout << local[i] << " "; + cout << std::endl; + sw.unlock(0); + + sw.lock_exclusive(0); + sw.put_n(&rank, 1, 0, rank); + sw.unlock(0); + + sw.lock_shared(0); + for(int i = 0; i != 10; i++) sw.get_n(&local[i], 1, 0, i); + cout << "processor " << world.rank() << " (after) : "; + for(int i = 0; i != size; ++i) cout << local[i] << ' '; + cout << std::endl; + sw.unlock(0); + + world.barrier(); + + return 0; +} + diff --git a/examples/shm_vector.cpp b/examples/shm_vector.cpp new file mode 100644 index 00000000000..4f25ffa3def --- /dev/null +++ b/examples/shm_vector.cpp @@ -0,0 +1,86 @@ +#if COMPILATION_INSTRUCTIONS +time mpicxx -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/shm/vector.hpp" + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" +#include "alf/boost/mpi3/mutex.hpp" + +#include +#include // lock_guard +#include +#include // sleep_for + +int rand(int lower, int upper){ + static std::random_device rd; + static std::mt19937 rng(rd()); + static std::uniform_int_distribution uni(lower, upper); + return uni(rng); +} +int rand(int upper = RAND_MAX){return rand(0, upper);} + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + + mpi3::shm::managed_shared_memory mpi3mshm(world); + + using elem = double; + + cout << "about to construct" << std::endl; + mpi3::shm::vector v(10, mpi3mshm.get_allocator()); + assert(v.data()); + v.resize(55); + cout << v.data() << std::endl; + assert(v.data()); + assert(not v.empty() and v.front() == 0 and std::equal(std::next(v.begin()), v.end(), v.begin()) ); + return 0; + + mpi3::mutex m(world); + std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); + { + std::lock_guard lock(m); // m.lock(); + for(int i = 0; i != v.size(); ++i){ + v[i] = world.rank(); + std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); + } // m.unlock(); + } + world.barrier(); + + if(world.rank() == 0){ + for(int i = 0; i != v.size(); ++i) cout << v[i] << " "; + cout << std::endl; + } + assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); + cout << "about to resize" << std::endl; + assert(v.data()); + v.resize(20); + assert(v.data()); + cout << "after resize" << std::endl; + world.barrier(); + assert(v.size()==20); + cout << "before assign" << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); + { + std::lock_guard lock(m); + for(int i = 0; i != v.size(); ++i){ + v[i] = world.rank(); + std::this_thread::sleep_for(std::chrono::milliseconds(rand(10))); + } + } + cout << "after assign" << std::endl; + world.barrier(); + cout << "after barrier" << std::endl; + + if(world.rank() == 0){ + // for(int i = 0; i != v.size(); ++i) cout << v[i] << " "; + cout << std::endl; + } +// assert( std::equal(std::next(v.begin()), v.end(), v.begin()) ); + cout << "end" << std::endl; + return 0; +} + diff --git a/examples/split.cpp b/examples/split.cpp new file mode 100644 index 00000000000..fb4e03aeaee --- /dev/null +++ b/examples/split.cpp @@ -0,0 +1,28 @@ +#if COMPILATION_INSTRUCTIONS +mpicxx -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 16 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/main.hpp" +#include "alf/boost/mpi3/communicator.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + + { + // int color = world.rank() / 4; + // mpi3::communicator row = world.split(color, world.rank()); + mpi3::communicator row = world / 4; // shortcut + cout << "world " << world.rank() << '/' << world.size() << " ---> row " << row.rank() << '/' << row.size() << '\n'; + } + { + // int color = world.rank() % 4; + // mpi3::communicator col = world.split(color, world.rank()); + mpi3::communicator col = world % 4; // shortcut + cout << "world " << world.rank() << '/' << world.size() << " ---> col " << col.rank() << '/' << col.size() << '\n'; + } + + return 0; +} + diff --git a/examples/test3.cpp b/examples/test3.cpp new file mode 100644 index 00000000000..c1ae4cd6fea --- /dev/null +++ b/examples/test3.cpp @@ -0,0 +1,137 @@ +#if COMPILATION_INSTRUCTIONS +mpic++ -O3 -std=c++14 -Wall `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit +#endif + +#include "alf/boost/mpi3/shared_main.hpp" + +#include + +#include +#include +#include + +void mySum(void *in_, void *out_, int *len, MPI_Datatype *dtype) { + double* in = reinterpret_cast(in_); + double* out = reinterpret_cast(out_); + MPI_Aint sz,lb; + MPI_Type_get_extent(*dtype,&lb,&sz); + for(int i=0, iend=sz/sizeof(double); i,2> A(extents[ni][nw*node.size()]); // 2x2 matrix + boost::multi_array,2> B(extents[ni][nw]); // 2x2 matrix + boost::multi_array,2> C(extents[ni][nw]); // 2x2 matrix + + std::vector nr{10000,100000,250000,500000,ni}; + + for(std::size_t n=0; n != nr.size(); n++) { + ni = nr[n]; + mpi3::type tA = mpi3::double_.vector(ni, 2*nw, 2*A.strides()[0]); +// MPI_Datatype typeA; +// MPI_Type_vector(ni, 2*nw, 2*A.strides()[0], MPI_DOUBLE, &typeA); + tA.commit(); +// MPI_Type_commit(&typeA); +// MPI_Op myOpSum; +// MPI_Op_create(mySum,true,&myOpSum); +// mpi3::operation my_sum = + mpi3::commutative_operation my_sum(&mySum); +// MPI_Allreduce(MPI_IN_PLACE,A.data(),2*ni*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + node.all_reduce_in_place_n(A.data(), A.num_elements(), std::plus<>{}); + + int nt = 4; + node.barrier(); +// MPI_Barrier(MPI_COMM_WORLD); + { + boost::timer::auto_cpu_timer t("All reduce"); +// t1=getTime(); + for(int i = 0; i != nt; i++) + node.all_reduce_in_place_n(A.data(), A.num_elements(), std::plus<>{}); +/// MPI_Allreduce(MPI_IN_PLACE,A.data(),2*ni*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + } +// t2=getTime(); +// double tv1=(t2-t1)/nt; + { + for(int r=0, p=0; r != node.size(); r++, p+=nw) +// if(rank==r) node.reduce_in_place_n(A.data() + p, 1, my_sum, + MPI_Reduce(MPI_IN_PLACE, A.data()+p, 1, tA.impl_, myOpSum, r, node.impl_); + else + MPI_Reduce(A.data()+p, NULL, 1, tA.impl_, myOpSum, r, MPI_COMM_WORLD); + } + node.barrier(); + { + boost::timer::auto_cpu_timer t; + for(int i=0; i != nt; i++) + for(int r=0, p=0; r != size; r++, p+=nw) + if(rank==r) + MPI_Reduce(MPI_IN_PLACE, A.data()+p, 1, A.impl_, myOpSum, r, node.impl_); + else + MPI_Reduce(A.data()+p, NULL, 1, typeA, myOpSum, r, node.impl_); + } + std::vector req(node.size()); +// std::vector st(node.size()); + for(int r=0; r != node.size(); r++) + if(node.rank()==r) + MPI_Ireduce(C.data(), B.data(),2*ni*nw,MPI_DOUBLE,MPI_SUM,r, node.impl_, &req[r].impl_); + else + MPI_Ireduce(C.data(), B.data(),2*ni*nw,MPI_DOUBLE,MPI_SUM,r, node.impl_, &req[r].impl_); + mpi3::wait_all(req); + node.barrier(); + + { + boost::timer::auto_cpu_timer t("multiple Reduce"); + for(int i=0; i!=nt; i++){ + for(int r=0; r != node.size(); r++) + if(node.rank() == r) + MPI_Ireduce(C.data(), B.data(), 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_, &req[r].impl_); + else + MPI_Ireduce(C.data(), B.data(), 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_, &req[r].impl_); + mpi3::wait_all(req); + } + } + + for(int r=0; r != node.size(); r++) + if(node.rank()==r) + MPI_Reduce(MPI_IN_PLACE, B.data(), 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_); + else + MPI_Reduce(B.data(), NULL, 2*ni*nw, MPI_DOUBLE, MPI_SUM, r, node.impl_); + + node.barrier(); + + { + boost::timer::auto_cpu_timer t("multiple Ireduce"); + for(int i=0; i != nt; i++) + for(int r=0; r +#include +#include +#include +#include +#include +#include + +double getTime() { + struct timeval tv; + gettimeofday(&tv, NULL); + return double(tv.tv_sec)+double(tv.tv_usec)/1000000.0; +} + +void mySum(void *in_, void *out_, int *len, MPI_Datatype *dtype) { + double* in = reinterpret_cast(in_); + double* out = reinterpret_cast(out_); + MPI_Aint sz,lb; + MPI_Type_get_extent(*dtype,&lb,&sz); + for(int i=0, iend=sz/sizeof(double); i1) blk=atoi(argv[1]); + + boost::multi_array,2> A(extents[ni][nw*size]); // 2x2 matrix + boost::multi_array,2> B(extents[ni][nw]); // 2x2 matrix + boost::multi_array,2> C(extents[ni][nw]); // 2x2 matrix + + std::vector nr{10000,100000,250000,500000,ni}; + + for(int n=0; n ni) continue; + int nblk = ni/blk; + ni = nblk*blk; + + MPI_Allreduce(MPI_IN_PLACE,A.data(),2*ni*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + + int nt=4; + MPI_Barrier(MPI_COMM_WORLD); + t1=getTime(); + for(int i=0; i req(size*nblk); + std::vector st(size*nblk); + for(int r=0; r0) { + MPI_Waitall(cnt,req.data(),st.data()); + cnt=0; + } + } + t2=getTime(); + double tv5=(t2-t1)/nt; + + if(rank==0) cout<<" n, times (Allreduce, multiple IReduce 1, multiple IReduce 2): " < +#include +#include +#include +#include +#include +#include + +double getTime() { + struct timeval tv; + gettimeofday(&tv, NULL); + return double(tv.tv_sec)+double(tv.tv_usec)/1000000.0; +} + +void mySum(void *in_, void *out_, int *len, MPI_Datatype *dtype) { + double* in = reinterpret_cast(in_); + double* out = reinterpret_cast(out_); + MPI_Aint sz,lb; + MPI_Type_get_extent(*dtype,&lb,&sz); + for(int i=0, iend=sz/sizeof(double); i1) blk=atoi(argv[1]); + std::vector nr{1,10,25,50,75}; + + boost::multi_array,2> A(extents[ni*75][nw*size]); // 2x2 matrix + boost::multi_array,2> A_(extents[ni*75][nw*size]); // 2x2 matrix + boost::multi_array,2> B(extents[ni*75][nw]); // 2x2 matrix + boost::multi_array,2> C(extents[ni*75][nw]); // 2x2 matrix + + + for(int n=0; n ni_) continue; + int nblk = ni_/blk; + ni_ = nblk*blk; + + MPI_Allreduce(A_.data(),A.data(),2*ni_*nw*size,MPI_DOUBLE,MPI_SUM,MPI_COMM_GROUP); + + int nt=4; + MPI_Barrier(MPI_COMM_WORLD); + t1=getTime(); + for(int i=0; i req(nmmax); + std::vector st(nmmax); + MPI_Barrier(MPI_COMM_WORLD); + t1=getTime(); + for(int i=0; i0) { + MPI_Waitall(cnt,req.data(),st.data()); + cnt=0; + } + } + MPI_Barrier(MPI_COMM_WORLD); + t2=getTime(); + double tv5=(t2-t1)/nt; + + if(global_rank==0) cout<<" n, times (Allreduce, multiple IReduce): " < +int main(int argc, char *argv[]) +{ + int rank, nprocs; + + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD,&nprocs); + MPI_Comm_rank(MPI_COMM_WORLD,&rank); + MPI_Barrier(MPI_COMM_WORLD); + printf("Hello, world. I am %d of %d\n", rank, nprocs);fflush(stdout); + MPI_Finalize(); + return 0; +} + diff --git a/fake/examples/MPI_Comm_call_errhandler.c b/fake/examples/MPI_Comm_call_errhandler.c new file mode 100644 index 00000000000..03a23511b6e --- /dev/null +++ b/fake/examples/MPI_Comm_call_errhandler.c @@ -0,0 +1,40 @@ +#include "mpi.h" +#include + +static int calls = 0; +static int errs = 0; +static MPI_Comm mycomm; + +void eh( MPI_Comm *comm, int *err, ... ) +{ + if (*err != MPI_ERR_OTHER) { + errs++; + printf( "Unexpected error code\n" );fflush(stdout); + } + if (*comm != mycomm) { + errs++; + printf( "Unexpected communicator\n" );fflush(stdout); + } + calls++; + return; +} + +int main( int argc, char *argv[] ) +{ + MPI_Comm comm; + MPI_Errhandler newerr; + + MPI_Init( &argc, &argv ); + comm = MPI_COMM_WORLD; + mycomm = comm; + MPI_Comm_create_errhandler( eh, &newerr ); + MPI_Comm_set_errhandler( comm, newerr ); + MPI_Comm_call_errhandler( comm, MPI_ERR_OTHER ); + MPI_Errhandler_free( &newerr ); + if (calls != 1) { + errs++; + printf( "Error handler not called\n" );fflush(stdout); + } + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Comm_compare.c b/fake/examples/MPI_Comm_compare.c new file mode 100644 index 00000000000..d22fd4a9807 --- /dev/null +++ b/fake/examples/MPI_Comm_compare.c @@ -0,0 +1,36 @@ +#include "mpi.h" +#include + +int main( int argc, char *argv[] ) +{ + int errs = 0; + int size, dims[2], periods[2], remain[2]; + int result; + MPI_Comm comm, newcomm; + + MPI_Init( &argc, &argv ); + + /* First, create a 1-dim cartesian communicator */ + periods[0] = 0; + MPI_Comm_size( MPI_COMM_WORLD, &size ); + dims[0] = size; + MPI_Cart_create( MPI_COMM_WORLD, 1, dims, periods, 0, &comm ); + + /* Now, extract a communicator with no dimensions */ + remain[0] = 0; + MPI_Cart_sub( comm, remain, &newcomm ); + + /* This should be congruent to MPI_COMM_SELF */ + MPI_Comm_compare( MPI_COMM_SELF, newcomm, &result ); + if (result != MPI_CONGRUENT) { + errs++; + printf( "cart sub to size 0 did not give self\n" );fflush(stdout); + } + + /* Free the new communicator */ + MPI_Comm_free( &newcomm ); + MPI_Comm_free( &comm ); + + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Comm_dup.c b/fake/examples/MPI_Comm_dup.c new file mode 100644 index 00000000000..ea7292436ca --- /dev/null +++ b/fake/examples/MPI_Comm_dup.c @@ -0,0 +1,25 @@ +#include +#include + +int main(int argc, char* argv[] ) +{ + MPI_Comm dup_comm_world, world_comm; + MPI_Group world_group; + int world_rank, world_size, rank, size; + + MPI_Init(&argc, &argv); + MPI_Comm_rank( MPI_COMM_WORLD, &world_rank ); + MPI_Comm_size( MPI_COMM_WORLD, &world_size ); + MPI_Comm_dup( MPI_COMM_WORLD, &dup_comm_world ); + /* Exercise Comm_create by creating an equivalent to dup_comm_world (sans attributes) */ + MPI_Comm_group( dup_comm_world, &world_group ); + MPI_Comm_create( dup_comm_world, world_group, &world_comm ); + MPI_Comm_rank( world_comm, &rank ); + if (rank != world_rank) { + printf( "incorrect rank in world comm: %d\n", rank );fflush(stdout); + MPI_Abort(MPI_COMM_WORLD, 3001 ); + } + MPI_Finalize(); + return 0; +} + diff --git a/fake/examples/MPI_Comm_free.c b/fake/examples/MPI_Comm_free.c new file mode 100644 index 00000000000..8179a075f10 --- /dev/null +++ b/fake/examples/MPI_Comm_free.c @@ -0,0 +1,90 @@ +#include "mpi.h" +#include +#include + +/* Test that communicators have reference count semantics */ +#define NELM 128 +#define NCOMM 1020 + +int main( int argc, char *argv[] ) +{ + int errs = 0; + int rank, size, source, dest, i; + MPI_Comm comm; + MPI_Comm tmpComm[NCOMM]; + MPI_Status status; + MPI_Request req; + int *buf=0; + + MPI_Init( &argc, &argv ); + MPI_Comm_dup( MPI_COMM_WORLD, &comm ); + /* This is similar to the datatype test, except that we post + an irecv on a simple data buffer but use a rank-reordered communicator. + In this case, an error in handling the reference count will most + likely cause the program to hang, so this should be run only + if (a) you are confident that the code is correct or (b) + a timeout is set for mpiexec + */ + MPI_Comm_rank( comm, &rank ); + MPI_Comm_size( comm, &size ); + if (size < 2) { + fprintf( stderr, "This test requires at least two processes." );fflush(stderr); + MPI_Abort( MPI_COMM_WORLD, 1 ); + } + source = 0; + dest = size - 1; + if (rank == dest) { + buf = (int *)malloc( NELM * sizeof(int) ); + for (i=0; i +int main(int argc, char *argv[]) +{ + int rank, nprocs; + + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD,&nprocs); + MPI_Comm_rank(MPI_COMM_WORLD,&rank); + printf("Hello, world. I am %d of %d\n", rank, nprocs);fflush(stdout); + MPI_Finalize(); + return 0; +} + diff --git a/fake/examples/MPI_Comm_set_errhandler.c b/fake/examples/MPI_Comm_set_errhandler.c new file mode 100644 index 00000000000..5f7ca5c0273 --- /dev/null +++ b/fake/examples/MPI_Comm_set_errhandler.c @@ -0,0 +1,45 @@ +#include "mpi.h" +#include +#include +static int calls = 0; +static int errs = 0; +static MPI_Comm mycomm; + +void eh( MPI_Comm *comm, int *err, ... ) +{ + printf( "eh called\n" );fflush(stdout); + if (*err != MPI_ERR_OTHER) { + errs++; + printf( "Unexpected error code\n" );fflush(stdout); + } + if (*comm != mycomm) { + errs++; + printf( "Unexpected communicator\n" );fflush(stdout); + } + calls++; + return; +} + +int main( int argc, char *argv[] ) +{ + MPI_Comm comm; + MPI_Errhandler newerr; + + MPI_Init( &argc, &argv ); + comm = MPI_COMM_WORLD; + mycomm = comm; + MPI_Comm_create_errhandler( eh, &newerr ); + int s = -1; + s = MPI_Comm_set_errhandler( comm, newerr ); // fatal error in OpenMPI + assert(s == MPI_SUCCESS); + int rank = -1; + MPI_Comm_rank( comm, &rank ); // error but handler not called + MPI_Comm_call_errhandler( comm, MPI_ERR_OTHER ); // ok, it seems + MPI_Errhandler_free( &newerr ); + if (calls != 1) { + errs++; + printf( "Error handler not called\n" );fflush(stdout); + } + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Gather.c b/fake/examples/MPI_Gather.c new file mode 100644 index 00000000000..d37a374bdf0 --- /dev/null +++ b/fake/examples/MPI_Gather.c @@ -0,0 +1,58 @@ +#include "mpi.h" +#include +#include + +/* Gather data from a vector to contiguous */ + +int main( int argc, char **argv ) +{ + MPI_Datatype vec; + MPI_Comm comm; + double *vecin, *vecout; + int minsize = 2, count; + int root, i, n, stride, errs = 0; + int rank, size; + + MPI_Init( &argc, &argv ); + comm = MPI_COMM_WORLD; + /* Determine the sender and receiver */ + MPI_Comm_rank( comm, &rank ); + MPI_Comm_size( comm, &size ); + + for (root=0; root + +int main(int argc, char *argv[]) +{ + int buffer[6]; + int rank, size, i; + int receive_counts[4] = { 0, 1, 2, 3 }; + int receive_displacements[4] = { 0, 0, 1, 3 }; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (size != 4) + { + if (rank == 0) + { + printf("Please run with 4 processes\n");fflush(stdout); + } + MPI_Finalize(); + return 0; + } + for (i=0; i + +int main(int argc, char *argv[]) +{ + int rank, size, i; + int buffer[10]; + MPI_Status status; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (size < 2) + { + printf("Please run with two processes.\n");fflush(stdout); + MPI_Finalize(); + return 0; + } + if (rank == 0) + { + for (i=0; i<10; i++) + buffer[i] = i; + MPI_Send(buffer, 10, MPI_INT, 1, 123, MPI_COMM_WORLD); + } + if (rank == 1) + { + for (i=0; i<10; i++) + buffer[i] = -1; + MPI_Recv(buffer, 10, MPI_INT, 0, 123, MPI_COMM_WORLD, &status); + for (i=0; i<10; i++) + { + if (buffer[i] != i) + printf("Error: buffer[%d] = %d but is expected to be %d\n", i, buffer[i], i); + } + fflush(stdout); + } + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Reduce.c b/fake/examples/MPI_Reduce.c new file mode 100644 index 00000000000..e2097bfbfe9 --- /dev/null +++ b/fake/examples/MPI_Reduce.c @@ -0,0 +1,43 @@ +#include "mpi.h" +#include +#include + +/* A simple test of Reduce with all choices of root process */ +int main( int argc, char *argv[] ) +{ + int errs = 0; + int rank, size, root; + int *sendbuf, *recvbuf, i; + int minsize = 2, count; + MPI_Comm comm; + + MPI_Init( &argc, &argv ); + + comm = MPI_COMM_WORLD; + /* Determine the sender and receiver */ + MPI_Comm_rank( comm, &rank ); + MPI_Comm_size( comm, &size ); + + for (count = 1; count < 130000; count = count * 2) { + sendbuf = (int *)malloc( count * sizeof(int) ); + recvbuf = (int *)malloc( count * sizeof(int) ); + for (root = 0; root < size; root ++) { + for (i=0; i + +int main(int argc, char *argv[]) +{ + int rank, size, i; + int buffer[10]; + MPI_Status status; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (size < 2) + { + printf("Please run with two processes.\n");fflush(stdout); + MPI_Finalize(); + return 0; + } + if (rank == 0) + { + for (i=0; i<10; i++) + buffer[i] = i; + MPI_Send(buffer, 10, MPI_INT, 1, 123, MPI_COMM_WORLD); + } + if (rank == 1) + { + for (i=0; i<10; i++) + buffer[i] = -1; + MPI_Recv(buffer, 10, MPI_INT, 0, 123, MPI_COMM_WORLD, &status); + for (i=0; i<10; i++) + { + if (buffer[i] != i) + printf("Error: buffer[%d] = %d but is expected to be %d\n", i, buffer[i], i); + } + fflush(stdout); + } + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Sendrecv_replace.c b/fake/examples/MPI_Sendrecv_replace.c new file mode 100644 index 00000000000..17f9574414b --- /dev/null +++ b/fake/examples/MPI_Sendrecv_replace.c @@ -0,0 +1,24 @@ +#include "mpi.h" +#include + +int main(int argc, char *argv[]) +{ + int myid, numprocs, left, right; + int buffer[10]; +/* MPI_Request request;*/ + MPI_Status status; + + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &numprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &myid); + + right = (myid + 1) % numprocs; + left = myid - 1; + if (left < 0) + left = numprocs - 1; + + MPI_Sendrecv_replace(buffer, 10, MPI_INT, left, 123, right, 123, MPI_COMM_WORLD, &status); + + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Type_contiguos.c b/fake/examples/MPI_Type_contiguos.c new file mode 100644 index 00000000000..d39081543c2 --- /dev/null +++ b/fake/examples/MPI_Type_contiguos.c @@ -0,0 +1,29 @@ +#include "mpi.h" +#include + +int main(int argc, char *argv[]) +{ + int myrank; + MPI_Status status; + MPI_Datatype type; + int buffer[100]; + + MPI_Init(&argc, &argv); + + MPI_Type_contiguous( 100, MPI_CHAR, &type ); + MPI_Type_commit(&type); + + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + if (myrank == 0) + { + MPI_Send(buffer, 1, type, 1, 123, MPI_COMM_WORLD); + } + else if (myrank == 1) + { + MPI_Recv(buffer, 1, type, 0, 123, MPI_COMM_WORLD, &status); + } + + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Type_dup.c b/fake/examples/MPI_Type_dup.c new file mode 100644 index 00000000000..af1cbcf9076 --- /dev/null +++ b/fake/examples/MPI_Type_dup.c @@ -0,0 +1,31 @@ +#include "mpi.h" +#include + +int main(int argc, char *argv[]) +{ +// int myrank; +// MPI_Status status; + MPI_Datatype type, type2; +// int buffer[100]; + + MPI_Init(&argc, &argv); + + MPI_Type_contiguous( 100, MPI_CHAR, &type ); + MPI_Type_commit(&type); + MPI_Type_dup(type, &type2); + MPI_Type_free(&type); + +/* MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + + if (myrank == 0) + { + MPI_Send(buffer, 1, type2, 1, 123, MPI_COMM_WORLD); + } + else if (myrank == 1) + { + MPI_Recv(buffer, 1, type2, 0, 123, MPI_COMM_WORLD, &status); + } +*/ + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Type_get_extent.c b/fake/examples/MPI_Type_get_extent.c new file mode 100644 index 00000000000..e4fcfd74521 --- /dev/null +++ b/fake/examples/MPI_Type_get_extent.c @@ -0,0 +1,62 @@ +#include "mpi.h" +#include +#include + +int main(int argc, char **argv) +{ + int /*mpi_err,*/ errs = 0, size; + MPI_Aint lb, ub, extent; + MPI_Datatype type; + struct { float a; int b; } foo; + + MPI_Init(&argc, &argv); + + type = MPI_INT; + MPI_Type_size(type, &size); + if (size != sizeof(int)) { + fprintf(stderr, "MPI_Type_size of MPI_INT incorrect size (%d); should be %d.\n", size, (int) sizeof(int));fflush(stderr); + errs++; + } + + MPI_Type_get_extent(type, &lb, &extent); + if (extent != sizeof(int)) { + fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(int));fflush(stderr); + errs++; + } + if (lb != 0) { + fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); + errs++; + } + + MPI_Type_ub(type, &ub); + if (ub != extent - lb) { + fprintf(stderr, "MPI_Type_ub of MPI_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); + errs++; + } + + type = MPI_FLOAT_INT; + MPI_Type_size(type, &size); + if (size != sizeof(float) + sizeof(int)) { + fprintf(stderr, "MPI_Type_size of MPI_FLOAT_INT returned incorrect size (%d); should be %d.\n", size, (int) (sizeof(float) + sizeof(int)));fflush(stderr); + errs++; + } + + MPI_Type_get_extent(type, &lb, &extent); + if (extent != sizeof(foo)) { + fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(foo));fflush(stderr); + errs++; + } + if (lb != 0) { + fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); + errs++; + } + + MPI_Type_ub(type, &ub); + if (ub != extent - lb) { + fprintf(stderr, "MPI_Type_ub of MPI_FLOAT_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); + errs++; + } + + MPI_Finalize(); + return errs; +} diff --git a/fake/examples/MPI_Type_size.c b/fake/examples/MPI_Type_size.c new file mode 100644 index 00000000000..e4fcfd74521 --- /dev/null +++ b/fake/examples/MPI_Type_size.c @@ -0,0 +1,62 @@ +#include "mpi.h" +#include +#include + +int main(int argc, char **argv) +{ + int /*mpi_err,*/ errs = 0, size; + MPI_Aint lb, ub, extent; + MPI_Datatype type; + struct { float a; int b; } foo; + + MPI_Init(&argc, &argv); + + type = MPI_INT; + MPI_Type_size(type, &size); + if (size != sizeof(int)) { + fprintf(stderr, "MPI_Type_size of MPI_INT incorrect size (%d); should be %d.\n", size, (int) sizeof(int));fflush(stderr); + errs++; + } + + MPI_Type_get_extent(type, &lb, &extent); + if (extent != sizeof(int)) { + fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(int));fflush(stderr); + errs++; + } + if (lb != 0) { + fprintf(stderr, "MPI_Type_get_extent of MPI_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); + errs++; + } + + MPI_Type_ub(type, &ub); + if (ub != extent - lb) { + fprintf(stderr, "MPI_Type_ub of MPI_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); + errs++; + } + + type = MPI_FLOAT_INT; + MPI_Type_size(type, &size); + if (size != sizeof(float) + sizeof(int)) { + fprintf(stderr, "MPI_Type_size of MPI_FLOAT_INT returned incorrect size (%d); should be %d.\n", size, (int) (sizeof(float) + sizeof(int)));fflush(stderr); + errs++; + } + + MPI_Type_get_extent(type, &lb, &extent); + if (extent != sizeof(foo)) { + fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect extent (%d); should be %d.\n", (int) extent, (int) sizeof(foo));fflush(stderr); + errs++; + } + if (lb != 0) { + fprintf(stderr, "MPI_Type_get_extent of MPI_FLOAT_INT returned incorrect lb (%d); should be 0.\n", (int) lb);fflush(stderr); + errs++; + } + + MPI_Type_ub(type, &ub); + if (ub != extent - lb) { + fprintf(stderr, "MPI_Type_ub of MPI_FLOAT_INT returned incorrect ub (%d); should be %d.\n", (int) ub, (int) (extent - lb));fflush(stderr); + errs++; + } + + MPI_Finalize(); + return errs; +} diff --git a/fake/examples/MPI_Type_vector.c b/fake/examples/MPI_Type_vector.c new file mode 100644 index 00000000000..17e28164220 --- /dev/null +++ b/fake/examples/MPI_Type_vector.c @@ -0,0 +1,45 @@ +#include "mpi.h" +#include + +int main(int argc, char *argv[]) +{ + int rank, size, i; + MPI_Datatype type, type2; + int buffer[24]; + MPI_Status status; + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (size < 2) + { + printf("Please run with 2 processes.\n"); + MPI_Finalize(); + return 1; + } + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + MPI_Type_contiguous(3, MPI_INT, &type2); + MPI_Type_commit(&type2); + MPI_Type_vector(3, 2, 3, type2, &type); + MPI_Type_commit(&type); + + if (rank == 0) + { + for (i=0; i<24; i++) + buffer[i] = i; + MPI_Send(buffer, 1, type, 1, 123, MPI_COMM_WORLD); + } + + if (rank == 1) + { + for (i=0; i<24; i++) + buffer[i] = -1; + MPI_Recv(buffer, 1, type, 0, 123, MPI_COMM_WORLD, &status); + for (i=0; i<24; i++) + printf("buffer[%d] = %d\n", i, buffer[i]); + fflush(stdout); + } + + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/MPI_Wtick.c b/fake/examples/MPI_Wtick.c new file mode 100644 index 00000000000..25356b25c91 --- /dev/null +++ b/fake/examples/MPI_Wtick.c @@ -0,0 +1,14 @@ +#include "mpi.h" +#include + +int main( int argc, char *argv[] ) +{ + double tick; + + MPI_Init( 0, 0 ); + tick = MPI_Wtick(); + printf("A single MPI tick is %0.9f seconds\n", tick); + fflush(stdout); + MPI_Finalize( ); + return 0; +} diff --git a/fake/examples/MPI_Wtime.c b/fake/examples/MPI_Wtime.c new file mode 100644 index 00000000000..c3a920e9264 --- /dev/null +++ b/fake/examples/MPI_Wtime.c @@ -0,0 +1,17 @@ +#include "mpi.h" +//#include +#include +#include + +int main( int argc, char *argv[] ) +{ + double t1, t2; + + MPI_Init( 0, 0 ); + t1 = MPI_Wtime(); + sleep(5); + t2 = MPI_Wtime(); + printf("MPI_Wtime measured a 1 second sleep to be: %1.2f\n", t2-t1);fflush(stdout); + MPI_Finalize( ); + return 0; +} diff --git a/fake/examples/Makefile b/fake/examples/Makefile new file mode 100644 index 00000000000..b3acdb995ce --- /dev/null +++ b/fake/examples/Makefile @@ -0,0 +1,25 @@ +all : abort.x MPI_Send.x MPI_Recv.x MPI_Comm_dup.x + mpirun -np 1 ./abort.x + mpirun -np 1 ./MPI_Send.x + mpirun -np 1 ./MPI_Recv.x + mpirun -np 1 ./MPI_Comm_dup.x + rm -f $^ + +abort.x : abort.cpp + mpic++ abort.cpp -o abort.x + +MPI_Send.x : MPI_Send.cpp + mpic++ MPI_Send.cpp -o MPI_Send.x + +MPI_Recv.x : MPI_Recv.cpp + mpic++ MPI_Recv.cpp -o MPI_Recv.x + +MPI_Comm_dup.x : MPI_Comm_dup.cpp + mpic++ MPI_Comm_dup.cpp -o MPI_Comm_dup.x + +clean: + rm -f *.x + +load: + module unload mpi + diff --git a/fake/examples/abort.cpp b/fake/examples/abort.cpp new file mode 100644 index 00000000000..3ef5a87a642 --- /dev/null +++ b/fake/examples/abort.cpp @@ -0,0 +1,10 @@ +#include +int main(int argc, char *argv[]) +{ + MPI_Init(NULL, NULL); + MPI_Abort(MPI_COMM_WORLD, 911); + /* No further code will execute */ + assert(0); + MPI_Finalize(); + return 0; +} diff --git a/fake/examples/testme b/fake/examples/testme new file mode 100755 index 00000000000..64b6838036a --- /dev/null +++ b/fake/examples/testme @@ -0,0 +1,3 @@ +$2mpicc $1 -g -o $1.x -Wall -Wno-unused-parameter `#-Wfatal-errors`; $2mpirun -np 1 $1.x; +#rm $1.x + diff --git a/fake/load b/fake/load new file mode 100755 index 00000000000..139597f9cb0 --- /dev/null +++ b/fake/load @@ -0,0 +1,2 @@ + + diff --git a/fake/mpi.h b/fake/mpi.h new file mode 100644 index 00000000000..9a621567b6e --- /dev/null +++ b/fake/mpi.h @@ -0,0 +1,2763 @@ +#ifndef FAKE_MPI_H//-*-indent-tabs-mode: t; c-basic-offset: 4; tab-width: 4;-*- +#define FAKE_MPI_H +// Fake MPI for serial execution + +#include // HOST_NAME_MAX +#include // gethostname +#include // strlen +#include // assert +#include // exit +#include // clock_gettime +#include // puts for debug + +#include +#ifdef __cplusplus +#include +#endif +#ifndef __cplusplus +#include +#endif + + +// Use weak linking on functions and variables to avoid multiple-definition +// errors at link time. + +#ifdef __cplusplus +#define WEAK inline +#else +#define WEAK __attribute__((weak)) +#endif + +#define WEAKVAR __attribute__((weak)) + + +typedef int MPI_Count; +typedef long long int MPI_Offset; + + +const int MPI_MAX_PROCESSOR_NAME = HOST_NAME_MAX; +const int MPI_MAX_INFO_KEY = 128; +const int MPI_MAX_INFO_VAL = 128; +const int MPI_MAX_ERROR_STRING = 128; + +enum {//: int { // error classes + MPI_SUCCESS, // No error + MPI_ERR_BUFFER, // Invalid buffer pointer + MPI_ERR_COUNT, // Invalid count argument + MPI_ERR_TYPE, // Invalid datatype argument + MPI_ERR_TAG, // Invalid tag argument + MPI_ERR_COMM, // Invalid communicator + MPI_ERR_RANK, // Invalid rank + MPI_ERR_REQUEST, // Invalid request (handle) + MPI_ERR_ROOT, // Invalid root + MPI_ERR_GROUP, // Invalid group + MPI_ERR_OP, // Invalid operation + MPI_ERR_TOPOLOGY, // Invalid topology + MPI_ERR_DIMS, // Invalid dimension argument + MPI_ERR_ARG, // Invalid argument of some other kind + MPI_ERR_UNKNOWN, // Unknown error + MPI_ERR_TRUNCATE, // Message truncated on receive + MPI_ERR_OTHER, // Known error not in this list + MPI_ERR_INTERN, // Internal MPI (implementation) error + MPI_ERR_IN_STATUS, // Error code is in status + MPI_ERR_PENDING, // Pending request + MPI_ERR_KEYVAL, // Invalid keyval has been passed + MPI_ERR_NO_MEM, // MPI_ALLOC_MEM failed because memory is exhausted + MPI_ERR_BASE, // Invalid base passed to MPI_FREE_MEM + MPI_ERR_INFO_KEY, // Key longer than MPI_MAX_INFO_KEY + MPI_ERR_INFO_VALUE, // Value longer than MPI_MAX_INFO_VAL + MPI_ERR_INFO_NOKEY, // Invalid key passed to MPI_INFO_DELETE + MPI_ERR_SPAWN, // Error in spawning processes + MPI_ERR_PORT, // Invalid port name passed to MPI_COMM_CONNECT + MPI_ERR_SERVICE, // Invalid service name passed to MPI_UNPUBLISH_NAME + MPI_ERR_NAME, // Invalid service name passed to MPI_LOOKUP_NAME + MPI_ERR_WIN, // Invalid win argument + MPI_ERR_SIZE, // Invalid size argument + MPI_ERR_DISP, // Invalid disp argument + MPI_ERR_INFO, // Invalid info argument + MPI_ERR_LOCKTYPE, // Invalid locktype argument + MPI_ERR_ASSERT, // Invalid assert argument + MPI_ERR_RMA_CONFLICT, // Conflicting accesses to window + MPI_ERR_RMA_SYNC, // Wrong synchronization of RMA calls + MPI_ERR_RMA_RANGE, // Target memory is not part of the window (in the case of a window created with MPI_WIN_CREATE_DYNAMIC, target memory is not attached) + MPI_ERR_RMA_ATTACH, // Memory cannot be attached (e.g., because of resource exhaustion) + MPI_ERR_RMA_SHARED, // Memory cannot be shared (e.g., some process in the group of the specified communicator cannot expose shared memory) + MPI_ERR_RMA_FLAVOR, // Passed window has the wrong flavor for the called function + MPI_ERR_FILE, // Invalid file handle + MPI_ERR_NOT_SAME, // Collective argument not identical on all processes, or collective routines called in a different order by different processes + MPI_ERR_AMODE, // Error related to the amode passed to MPI_FILE_OPEN + MPI_ERR_UNSUPPORTED_DATAREP, // Unsupported datarep passed to MPI_FILE_SET_VIEW + MPI_ERR_UNSUPPORTED_OPERATION, // Unsupported operation, such as seeking on a file which supports sequential access only + MPI_ERR_NO_SUCH_FILE, // File does not exist + MPI_ERR_FILE_EXISTS, // File exists + MPI_ERR_BAD_FILE, // Invalid file name (e.g., path name too long) + MPI_ERR_ACCESS, // Permission denied + MPI_ERR_NO_SPACE, // Not enough space + MPI_ERR_QUOTA, // Quota exceeded + MPI_ERR_READ_ONLY, // Read-only file or file system + MPI_ERR_FILE_IN_USE, // File operation could not be completed, as the file is currently open by some process + MPI_ERR_DUP_DATAREP, // Conversion functions could not be registered because a data representation identifier that was already defined was passed to MPI_REGISTER_DATAREP + MPI_ERR_CONVERSION, // An error occurred in a user supplied data conversion function. + MPI_ERR_IO, // Other I/O error + MPI_ERR_LASTCODE // Last error code +}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm + +enum { + MPI_PROC_NULL, + MPI_ANY_SOURCE, + MPI_ANY_TAG, + MPI_UNDEFINED, + MPI_BSEND_OVERHEAD, + MPI_KEYVAL_INVALID, + MPI_LOCK_EXCLUSIVE, + MPI_LOCK_SHARED, + MPI_ROOT +}; + +enum { + MPI_MODE_APPEND, + MPI_MODE_CREATE, + MPI_MODE_DELETE_ON_CLOSE, + MPI_MODE_EXCL, + MPI_MODE_NOCHECK, + MPI_MODE_NOPRECEDE, + MPI_MODE_NOPUT, + MPI_MODE_NOSTORE, + MPI_MODE_NOSUCCEED, + MPI_MODE_RDONLY, + MPI_MODE_RDWR, + MPI_MODE_SEQUENTIAL, + MPI_MODE_UNIQUE_OPEN, + MPI_MODE_WRONLY +}; + +// Forward declarations + +struct MPI_Comm_impl_; +typedef struct MPI_Comm_impl_* MPI_Comm; + +MPI_Comm MPI_COMM_NULL WEAKVAR = NULL;//{{&fatal_error_}}; +MPI_Comm MPI_COMM_WORLD WEAKVAR = NULL; +MPI_Comm MPI_COMM_SELF WEAKVAR = NULL; + +struct MPI_Datatype_impl_; +typedef struct MPI_Datatype_impl_* MPI_Datatype; + +int MPI_Comm_rank(MPI_Comm comm, int* rank); +int MPI_Comm_size(MPI_Comm comm, int *size); + +// Communicator split type constants +const int MPI_COMM_TYPE_SHARED = 1; + +// MPI Requests +typedef enum { + MPI_REQUEST_NULL +} MPI_Request; + +typedef struct { + int MPI_ERROR; + int MPI_SOURCE; + int MPI_TAG; +} MPI_Status; + +// Constants specifying empty or ignored input + +static MPI_Status *MPI_STATUS_IGNORE = NULL; +static MPI_Status *MPI_STATUSES_IGNORE = NULL; + +static char **MPI_ARGV_NULL = NULL; +static char ***MPI_ARGVS_NULL = NULL; +static int *MPI_ERRCODES_IGNORE = NULL; + +typedef struct {} MPI_Message; + +struct MPI_Group_impl_ {}; +typedef struct MPI_Group_impl_* MPI_Group; + + +static MPI_Group MPI_GROUP_NULL = NULL; +struct MPI_Group_impl_ MPI_GROUP_EMPTY_impl WEAKVAR; +MPI_Group MPI_GROUP_EMPTY WEAKVAR = &MPI_GROUP_EMPTY_impl; + +//struct MPI_Info{}; +//const struct MPI_Info MPI_INFO_NULL; + +typedef enum { //MPI_Info { + MPI_INFO_NULL +} MPI_Info ; + +//typedef struct {} MPI_Info; + +const void* MPI_IN_PLACE WEAKVAR = (const void*)(-1); + +typedef enum { // redefined operations are supplied for MPI_REDUCE + MPI_OP_NULL, // TODO: is this an operator? + MPI_MAX, // maximum + MPI_MIN, // minimum + MPI_SUM, // sum + MPI_PROD, // product + MPI_LAND, // logical and + MPI_BAND, // bitwise and + MPI_LOR, // logical or + MPI_BOR, // bitwise or + MPI_LXOR, // logical exclusive or (xor) + MPI_BXOR, // bitwise excluse or (xor) + MPI_MAXLOC, // max value and location + MPI_MINLOC, // min value and location +// MPI_REPLACE, // TODO: is this an operator? + MPI_NO_OP, // TODO: is this an operator? + MPI_OP_LASTCODE // not standard? +} MPI_Op; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node112.htm + + +// Forward declartions for Chapter 8 - MPI Environment Management + +typedef void MPI_Comm_errhandler_function(MPI_Comm *, int *, ...); + +#ifdef USE_MPI_REMOVED_FUNCTIONS +// Deprecated in MPI-2.0. Removed in MPI_3.0 +typedef MPI_Comm_errhandler_function MPI_Handler_function; +#endif + +struct MPI_Errhandler_impl_{ + MPI_Comm_errhandler_function* func_; +}; + +typedef struct MPI_Errhandler_impl_* MPI_Errhandler; + +struct MPI_Comm_impl_{ + MPI_Errhandler errhandler_; +}; + +MPI_Errhandler MPI_ERRORS_ARE_FATAL WEAKVAR; +MPI_Errhandler MPI_ERRORS_RETURN WEAKVAR; + +MPI_Errhandler MPI_ERRHANDLER_NULL WEAKVAR = NULL; + +int MPI_Comm_call_errhandler(MPI_Comm comm, int errorcode); + + +// ----------------------------------------------------------------------------- +// Chapter 9 - The Info Object +// ----------------------------------------------------------------------------- + +// MPI Info handling. +// Would not be too hard to create an std::map implementation + +int MPI_Info_create(MPI_Info *info); + +int MPI_Info_free(MPI_Info *info); + +int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo); + +int MPI_Info_delete(MPI_Info info, const char *key); + +int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, int *flag); + +int MPI_Info_get_nkeys(MPI_Info info, int *nkeys); + +int MPI_Info_get_nthkey(MPI_Info info, int n, char *key); + +int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen, int *flag); + +int MPI_Info_set(MPI_Info info, const char *key, const char *value); + +// ----------------------------------------------------------------------------- + + +enum { // level of thread support + MPI_THREAD_SINGLE, + MPI_THREAD_FUNNELED, + MPI_THREAD_SERIALIZED, + MPI_THREAD_MULTIPLE +}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm + +// ----------------------------------------------------------------------------- +// Chapter 3 - Point-to-Point Communication +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 3.2 Blocking Send and Receive Operations +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Send( // Performs a blocking send + const void *buf, // [in] initial address of send buffer (choice) + int count, // [in] number of elements in send buffer (nonnegat...) + MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) + int dest, // [in] rank of destination (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm // [in] communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node47.htm#Node47 + int rank = -1; MPI_Comm_rank(comm, &rank); + assert(rank != dest); + int size = -1; MPI_Comm_size(comm, &size); + assert(dest < size); // Invalid rank has value 1 but must be nonnegative and less than 1 + return MPI_SUCCESS; +} + +WEAK +int MPI_Recv( // Blocking receive for a message + void *buf, // [out] initial address of receive buffer (choice) + int count, // [in] maximum number of elements in receive buffer... + MPI_Datatype datatype, // [in] datatype of each receive buffer element... + int source, // [in] rank of source (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Status *status // [out] status object (Status) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node50.htm#Node50 + int rank = -1; MPI_Comm_rank(comm, &rank); + assert(rank != source); + assert(0); + return MPI_SUCCESS; +} + +WEAK +int MPI_Get_count( // Gets the number of "top level" elements + const MPI_Status *status, // [in] return status of receive operation (Status) + MPI_Datatype datatype, // [in] datatype of each receive buffer element (handle) + int *count // [out] number of received elements (integer) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node51.htm + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.2 Communication Modes +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Bsend( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Ssend( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Rsend( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.6 Buffer Allocation and Usage +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Buffer_attach( + void *buffer, + int size +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Buffer_detach( + void *buffer_addr, + int *size +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.10 Send-Receive +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Sendrecv( // Sends and receives a message + const void *sendbuf, // [in] initial address of send buffer (choice) + int sendcount, // [in] number of elements in send buffer (integer) + MPI_Datatype sendtype, // [in] type of elements in send buffer (handle) + int dest, // [in] rank of destination (integer) + int sendtag, // [in] send tag (integer) + void *recvbuf, // [out] initial address of receive buffer (choice) + int recvcount, // [in] number of elements in receive buffer (integer) + MPI_Datatype recvtype, // [in] type of elements in receive buffer (handle) + int source, // [in] rank of source (integer) + int recvtag, // [in] receive tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Status *status // [out] status object (Status). This refers to the receive operation. +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Sendrecv_replace( // Sends and receives a message + const void *buf, // [inout] address of buffer (choice) + int sendcount, // [in] number of elements in send buffer (integer) + MPI_Datatype datatype, // [in] type of elements in send buffer (handle) + int dest, // [in] rank of destination (integer) + int sendtag, // [in] send tag (integer) + int source, // [in] rank of source (integer) + int recvtag, // [in] receive tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Status *status // [out] status object (Status). This refers to the receive operation. +) +{ + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.7 Nonblocking Communication +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Isend( // Start a nonblocking send + const void *buf, // [in] initial address of send buffer (choice) + int count, // [in] number of elements in send buffer (nonnegat...) + MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) + int dest, // [in] rank of destination (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Request *request // [out] communication request (handle) +){ + return MPI_SUCCESS; +} + +WEAK +int MPI_Ibsend( // Start a nonblocking send in buffered mode + const void *buf, // [in] initial address of send buffer (choice) + int count, // [in] number of elements in send buffer (nonnegat...) + MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) + int dest, // [in] rank of destination (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Request *request // [out] communication request (handle) +){ + return MPI_SUCCESS; +} + +WEAK +int MPI_Issend( // Start a nonblocking send in synchronous mode + const void *buf, // [in] initial address of send buffer (choice) + int count, // [in] number of elements in send buffer (nonnegat...) + MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) + int dest, // [in] rank of destination (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Request *request // [out] communication request (handle) +){ + return MPI_SUCCESS; +} + +WEAK +int MPI_Irsend( // Start a nonblocking send in ready mode + const void *buf, // [in] initial address of send buffer (choice) + int count, // [in] number of elements in send buffer (nonnegat...) + MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) + int dest, // [in] rank of destination (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Request *request // [out] communication request (handle) +){ + return MPI_SUCCESS; +} + +WEAK +int MPI_Irecv( // Nonblocking receive for a message + void *buf, // [out] initial address of receive buffer (choice) + int count, // [in] maximum number of elements in receive buffer... + MPI_Datatype datatype, // [in] datatype of each receive buffer element... + int source, // [in] rank of source (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Request *request // [out] communication request (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node50.htm#Node50 + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.7.3 Communication Completion +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Wait( // Waits for an MPI request to complete + MPI_Request *request, // [in] request (handle) + MPI_Status *status // [out] status object (Status). May be MPI_STATUS_IGNORE. +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node64.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Test( + MPI_Request *request, + int *flag, + MPI_Status *status +){ + return MPI_SUCCESS; +} + +WEAK +int MPI_Request_free( + MPI_Request *request +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.7.5 Multiple Completions +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Waitany( + int count, + MPI_Request array_of_requests[], + int *index, + MPI_Status *status +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Testany( + int count, + MPI_Request array_of_requests[], + int *index, + int *flag, + MPI_Status *status +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Waitall( + int count, + MPI_Request array_of_requests[], + MPI_Status array_of_statuses[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Testall( + int count, + MPI_Request array_of_requests[], + int *flag, + MPI_Status array_of_statuses[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Waitsome( + int incount, + MPI_Request array_of_requests[], + int *outcount, + int array_of_indices[], + MPI_Status array_of_statuses[]) +{ + if (outcount) *outcount = 0; + return MPI_SUCCESS; +} + +WEAK +int MPI_Testsome( + int incount, + MPI_Request array_of_requests[], + int *outcount, + int array_of_indices[], + MPI_Status array_of_statuses[]) +{ + if (outcount) *outcount = 0; + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.7.6 Non-destructive Test of status +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Request_get_status( + MPI_Request request, + int *flag, + MPI_Status *status +) { + if (flag) *flag = -1; // true + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.8 Probe and Cancel +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 3.8.1 Probe +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Iprobe( // Nonblocking test for a message + int source, // [in] source rank, or MPI_ANY_SOURCE (integer) + int tag, // [in] tag value or MPI_ANY_TAG (integer) + MPI_Comm comm, // [in] communicator (handle) + int *flag, // [out] True if a message with the specified source, tag... + MPI_Status *status // [out] status object (Status) +) { + return MPI_SUCCESS; +} + +// Blocking test for a message +WEAK +int MPI_Probe( // like MPI_IMPROBE except that it is a blocking call that returns only after a matching message has been found. + int source, // [in] source rank, or MPI_ANY_SOURCE (integer) + int tag, // [in] tag value or MPI_ANY_TAG (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Status *status // [out] status object (Status) +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.8.2 Matching Probe +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Mprobe( + int source, // rank of source or MPI_ANY_SOURCE (integer) + int tag, // message tag or MPI_ANY_TAG (integer) + MPI_Comm comm, // communicator (handle) + MPI_Message *message, // returned message (handle) + MPI_Status *status // status object (Status) +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.8.3 Matched Receives +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Mrecv( + void* buf, // initial address of receive buffer (choice) + int count, // number of elements in receive buffer (non-negati) + MPI_Datatype datatype, // datatype of each receive buffer element (handle) + MPI_Message *message, // message (handle) + MPI_Status *status // status object (Status) +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.8.4 Cancel +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Cancel( + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Test_cancelled( + const MPI_Status *status, + int *flag +) { + if (flag) *flag = -1; // -1 true + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 3.9 Persistent Communication Requests +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Send_init( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Bsend_init( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Ssend_init( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Rsend_init( + const void *buf, + int count, + MPI_Datatype datatype, + int dest, + int tag, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Recv_init( + const void *buf, + int count, + MPI_Datatype datatype, + int source, + int tag, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Start( + MPI_Request *request +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Startall( + int count, + MPI_Request array_of_requests[] +) { + return MPI_SUCCESS; +} + + + + +// ----------------------------------------------------------------------------- +// Chapter 4 - Datatypes +// ----------------------------------------------------------------------------- + + +//typedef unsigned long long MPI_Aint; +typedef ssize_t MPI_Aint; + +struct MPI_Datatype_impl_{ +// MPI_Aint lb; +// MPI_Aint extent; + int bytes; // for basic types - needs to be first for gcc compilation of the DEFINE_FAKE_MPI_DATATYPE macros + bool is_basic; + int count; // [in] number of blocks (integer) -- also number of entries in arrays array_of_types , array_of_displacements and array_of_blocklengths + int* blocklens; // [in] number of elements in each block (array) + MPI_Aint* indices; // [in] byte displacement of each block (array) + MPI_Datatype* old_types; // [in] type of elements in each block (array of handles to datatype objects) +}; + +#define DEFINE_FAKE_MPI_DATATYPE(s) \ + struct MPI_Datatype_impl_ DEF##s WEAKVAR = {.bytes =0, .is_basic=true, .count=0, .blocklens=NULL, .indices=NULL, .old_types=NULL}; \ + MPI_Datatype s WEAKVAR =&DEF##s; \ + + +// MPI Datatype with associated C/C++ type +#define DEFINE_FAKE_MPI_DATATYPE2(s, t) \ + struct MPI_Datatype_impl_ DEF##s WEAKVAR = {.bytes=sizeof(t), .is_basic=true, .count=0, .blocklens=NULL, .indices=NULL, .old_types=NULL}; \ + MPI_Datatype s WEAKVAR =&DEF##s; + + +DEFINE_FAKE_MPI_DATATYPE(MPI_DATATYPE_NULL) +DEFINE_FAKE_MPI_DATATYPE2(MPI_CHAR, char) +DEFINE_FAKE_MPI_DATATYPE2(MPI_SHORT, short int) +DEFINE_FAKE_MPI_DATATYPE2(MPI_INT, int) +DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG, long) +DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG_LONG_INT, long long) +DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG_LONG, long long) +DEFINE_FAKE_MPI_DATATYPE2(MPI_SIGNED_CHAR, char) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_CHAR, unsigned char) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_SHORT, unsigned short) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED, unsigned) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_LONG, unsigned long) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UNSIGNED_LONG_LONG, unsigned long long) +DEFINE_FAKE_MPI_DATATYPE2(MPI_FLOAT, float) +DEFINE_FAKE_MPI_DATATYPE2(MPI_DOUBLE, double) +DEFINE_FAKE_MPI_DATATYPE2(MPI_LONG_DOUBLE, long double) +DEFINE_FAKE_MPI_DATATYPE2(MPI_WCHAR, wchar_t) +#ifdef __cplusplus +DEFINE_FAKE_MPI_DATATYPE(MPI_C_BOOL) +#else +DEFINE_FAKE_MPI_DATATYPE2(MPI_C_BOOL, _Bool) +#endif +DEFINE_FAKE_MPI_DATATYPE2(MPI_INT8_T, int8_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_INT16_T, int16_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_INT32_T, int32_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_INT64_T, int64_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT8_T, uint8_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT16_T, uint16_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT32_T, uint32_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_UINT64_T, uint64_t) +DEFINE_FAKE_MPI_DATATYPE2(MPI_AINT, MPI_Aint) +DEFINE_FAKE_MPI_DATATYPE2(MPI_COUNT, MPI_Count) +DEFINE_FAKE_MPI_DATATYPE2(MPI_OFFSET, MPI_Offset) +DEFINE_FAKE_MPI_DATATYPE2(MPI_C_COMPLEX, float _Complex) +DEFINE_FAKE_MPI_DATATYPE2(MPI_C_FLOAT_COMPLEX, float _Complex) +DEFINE_FAKE_MPI_DATATYPE2(MPI_C_DOUBLE_COMPLEX, double _Complex) +DEFINE_FAKE_MPI_DATATYPE(MPI_BYTE) +DEFINE_FAKE_MPI_DATATYPE(MPI_PACKED) +#ifdef __cplusplus +DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_BOOL, bool) +DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_FLOAT_COMPLEX, std::complex) +DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_DOUBLE_COMPLEX, std::complex) +DEFINE_FAKE_MPI_DATATYPE2(MPI_CXX_LONG_DOUBLE_COMPLEX, std::complex) +#else +DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_BOOL) +DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_FLOAT_COMPLEX) +DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_DOUBLE_COMPLEX) +DEFINE_FAKE_MPI_DATATYPE(MPI_CXX_LONG_DOUBLE_COMPLEX) +#endif +DEFINE_FAKE_MPI_DATATYPE(MPI_FLOAT_INT) +DEFINE_FAKE_MPI_DATATYPE(MPI_DOUBLE_INT) +DEFINE_FAKE_MPI_DATATYPE(MPI_LONG_INT) +DEFINE_FAKE_MPI_DATATYPE(MPI_2INT) +DEFINE_FAKE_MPI_DATATYPE(MPI_SHORT_INT) +DEFINE_FAKE_MPI_DATATYPE(MPI_LONG_DOUBLE_INT) +#ifdef USE_MPI_REMOVED_FUNCTIONS +DEFINE_FAKE_MPI_DATATYPE(MPI_LB) +DEFINE_FAKE_MPI_DATATYPE(MPI_UB) +#endif + +// ----------------------------------------------------------------------------- +// Chapter 4.1.2 Datatype Constructors +// ----------------------------------------------------------------------------- + +int MPI_Type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent); + +WEAK +int MPI_Type_contiguous( // Creates a contiguous datatype + int count, // [in] replication count (nonnegative integer) + MPI_Datatype oldtype, // [in] old datatype (handle) + MPI_Datatype *newtype // [out] new datatype (handle) +){ +// MPI_Aint oldlb; +// MPI_Aint oldextent; +// int s = MPI_Type_get_extent(oldtype, &oldlb, &oldextent); + *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); + if(oldtype->count == 1){ + (*newtype)->count = 1; + (*newtype)->blocklens = (int*)malloc(sizeof(int)); + (*newtype)->blocklens[0] = count; + (*newtype)->indices = (MPI_Aint*)malloc(sizeof(MPI_Aint)); + (*newtype)->indices[0] = 0; + (*newtype)->old_types = (MPI_Datatype*)malloc(count*sizeof(MPI_Datatype)); + (*newtype)->old_types[0] = MPI_BYTE; + }else assert(0); +// if(count < 0) return MPI_ERR_INTERN; +// if(!newtype) return MPI_ERR_INTERN; +// (*newtype)->lb = 0; +// if(s != MPI_SUCCESS) return MPI_ERR_TYPE; +// (*newtype)->extent = count*oldextent; + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_vector( // Creates a vector (strided) datatype + int count, // [in] number of blocks (nonnegative integer) + int blocklength, // [in] number of elements in each block (nonnegative integer) + int stride, // [in] number of elements between start of each block (integer) + MPI_Datatype old_type, // [in] old datatype (handle) + MPI_Datatype *newtype_p // [out] new datatype (handle) +){ + assert(0); + if(count < 0) return MPI_ERR_INTERN; + *newtype_p = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); + if(!newtype_p) return MPI_ERR_INTERN; +// (*newtype)->lb = 0; + MPI_Aint oldlb; + MPI_Aint oldextent; + int s = MPI_Type_get_extent(old_type, &oldlb, &oldextent); +// (*newtype) + if(s != MPI_SUCCESS) return MPI_ERR_TYPE; +// (*newtype) +// (*newtype)->extent = count*oldextent; + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_create_hvector( // Creates a vector (strided) datatype + int count, // [in] number of blocks (nonnegative integer) + int blocklength, // [in] number of elements in each block (nonnegative integer) + MPI_Aint stride, // [in] number of bytes between start of each block (integer) + MPI_Datatype old_type, // [in] old datatype (handle) + MPI_Datatype *newtype_p // [out] new datatype (handle) +) { + return MPI_SUCCESS; +} + +#ifdef USE_MPI_REMOVED_FUNCTIONS +#define MPI_Type_hvector MPI_Type_create_hvector +#endif + +WEAK +int MPI_Type_indexed( + int count, + const int array_of_blocklengths[], + const int array_of_displacements[], + MPI_Datatype oldtype, + MPI_Datatype *newtype +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_hindexed( + int count, + const int array_of_blocklengths[], + const MPI_Aint array_of_displacements[], + MPI_Datatype oldtype, + MPI_Datatype *newtype +) { + return MPI_SUCCESS; +} + + +WEAK +int MPI_Type_create_struct( + int count, + const int array_of_blocklengths[], + const MPI_Aint array_of_displacements[], + const MPI_Datatype array_of_types[], + MPI_Datatype *newtype) +{ + return MPI_SUCCESS; +} + + + +// Removed in 3.0 +#ifdef USE_MPI_REMOVED_FUNCTIONS +WEAK +int MPI_Type_struct( // Creates a struct datatype + int count, // [in] number of blocks (integer) -- also number of entries in arrays array_of_types , array_of_displacements and array_of_blocklengths + int blocklens[], // [in] number of elements in each block (array) + MPI_Aint indices[], // [in] byte displacement of each block (array) + MPI_Datatype old_types[], // [in] type of elements in each block (array of handles to datatype objects) + MPI_Datatype *newtype // [out] new datatype (handle) +){ + *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); + (*newtype)->count = count; + (*newtype)->blocklens = (int*)malloc(count*sizeof(int)); + (*newtype)->indices = (MPI_Aint*)malloc(count*sizeof(MPI_Aint)); + (*newtype)->old_types = (MPI_Datatype*)malloc(count*sizeof(MPI_Datatype)); + memcpy((*newtype)->blocklens, blocklens, count*sizeof(int)); + memcpy((*newtype)->indices, indices, count*sizeof(MPI_Aint)); + + return MPI_SUCCESS; +} +#endif + +// ----------------------------------------------------------------------------- +// Chapter 4.1.3 Subarray Datatype Constructor +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 4.1.4 Distributed Array Datatype Constructor +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 4.1.5 Address and Size Functions +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Get_address( + const void *location, + MPI_Aint *address) +{ + return MPI_SUCCESS; +} + +#ifdef USE_MPI_REMOVED_FUNCTIONS +#define MPI_Address MPI_Get_address +#endif + +WEAK +int MPI_Type_size( // Return the number of bytes occupied by entries in the datatype + MPI_Datatype datatype, // [in] datatype (handle) + int *size // [out] datatype size (integer) +){ +// assert(size); + if(datatype->is_basic){ + *size = datatype->bytes; + return MPI_SUCCESS; + } + + if(datatype->count == 1){ + int oldsize = -1; + MPI_Type_size(datatype->old_types[0], &oldsize); + *size = datatype->blocklens[0]*oldsize; + }else assert(0); +// *size = datatype->extent; +/* switch(datatype){ + MPI_INT : *size = sizeof(int); break; + MPI_FLOAT : *size = sizeof(float); break; + MPI_DOUBLE : *size = sizeof(double); break; + MPI_FLOAT_INT: *size = sizeof(float) + sizeof(int); break; + default: assert(0); + }*/ + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 4.1.7 Extent and Bounds of Datatypes +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Type_extent( // Returns the extent of a datatype, deprecated->MPI_Type_get_extent + MPI_Datatype datatype, // [in] datatype (handle) + MPI_Aint *extent // [out] datatype extent (integer) +){ + assert(0); + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_get_extent( // Get the lower bound and extent for a Datatype + MPI_Datatype datatype, // [in] datatype to get information on (handle) + MPI_Aint *lb, // [out] lower bound of datatype (integer) + MPI_Aint *extent // [out] extent of datatype (integer) +){ + if(!lb || !extent) return MPI_ERR_ARG; + if(datatype == MPI_BYTE){ + *lb = 0; + *extent = 1; + return MPI_SUCCESS; + } + if(datatype->count == 1){ + MPI_Aint oldlb = 0; + MPI_Aint oldextent = 0; + int s = MPI_Type_get_extent(datatype->old_types[0], &oldlb, &oldextent); + assert(s == MPI_SUCCESS); + *lb = 0; + *extent = datatype->blocklens[0]*oldextent; + }else assert(0); + return MPI_SUCCESS; +} + +// Removed from the 3.0 standard +#ifdef USE_MPI_REMOVED_FUNCTIONS +WEAK +int MPI_Type_lb( // Returns the lower bound of a datatype + MPI_Datatype datatype, // [in] datatype (handle) + MPI_Aint *displacement +){ + if(!displacement) return MPI_ERR_ARG; + if(!datatype) return MPI_ERR_TYPE; + MPI_Aint lb = -1; + MPI_Aint extent = -1; + MPI_Type_get_extent(datatype, &lb, &extent); + *displacement = lb + extent; + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_ub( // Returns the upper bound of a datatype + MPI_Datatype datatype, // [in] datatype (handle) + MPI_Aint *displacement +){ + if(!displacement) return MPI_ERR_ARG; + if(!datatype) return MPI_ERR_TYPE; + MPI_Aint lb = -1; + MPI_Aint extent = -1; + MPI_Type_get_extent(datatype, &lb, &extent); + *displacement = lb + extent; + return MPI_SUCCESS; +} +#endif + +// ----------------------------------------------------------------------------- +// Chapter 4.1.7 True Extent of Datatypes +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 4.1.9 Commit and Free +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Type_commit( // Commits the datatype + MPI_Datatype *datatype // [in] datatype (handle) +){ + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_free( // Frees the datatype + MPI_Datatype *datatype // [in] datatype that is freed (handle) +){ + int c; +// *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); +// (*newtype)->count = oldtype->count; +// ; = (MPI_Datatype*)malloc(oldtype->count*sizeof(MPI_Datatype)); + for(c = 0 ; c != (*datatype)->count; ++c){ + MPI_Type_free((*datatype)->old_types + c); + } + free((*datatype)->old_types); + free((*datatype)->indices); + free((*datatype)->blocklens); + free((*datatype)); + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 4.1.10 Duplicating a Datatype +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Type_dup( // MPI_Type_dup + MPI_Datatype oldtype, // [in] datatype (handle) + MPI_Datatype *newtype // [out] copy of type (handle) +){ + int c; + *newtype = (MPI_Datatype)malloc(sizeof(struct MPI_Datatype_impl_)); + (*newtype)->count = oldtype->count; + (*newtype)->blocklens = (int*)malloc(oldtype->count*sizeof(int)); + (*newtype)->indices = (MPI_Aint*)malloc(oldtype->count*sizeof(MPI_Aint)); + (*newtype)->old_types = (MPI_Datatype*)malloc(oldtype->count*sizeof(MPI_Datatype)); + for(c = 0 ; c != oldtype->count; ++c){ + (*newtype)->blocklens[c] = oldtype->blocklens[c]; + (*newtype)->indices[c] = oldtype->indices[c]; + MPI_Type_dup(oldtype->old_types[c], (*newtype)->old_types + c); + } + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 4.1.11 Use of General Datatypes in Communication +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Get_elements( + const MPI_Status *status, + MPI_Datatype datatype, + int *count +) { + return MPI_SUCCESS; + +} +WEAK +int MPI_Get_elements_x( + const MPI_Status *status, + MPI_Datatype datatype, + MPI_Count *count +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 4.1.13 Decoding a Datatype +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Type_get_envelope( + MPI_Datatype datatype, + int *num_integers, + int *num_addresses, + int *num_datatypes, + int *combiner +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Type_get_contents( + MPI_Datatype datatype, + int max_integers, + int max_addresses, + int max_datatypes, + int array_of_integers[], + MPI_Aint array_of_addresses[], + MPI_Datatype array_of_datatypes[] +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 4.2 Pack and Unpack +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Pack( + const void *inbuf, + int incount, + MPI_Datatype datatype, + void *outbuf, + int outsize, + int *position, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Unpack( + const void* inbuf, + int insize, + int *position, + void *outbuf, + int outcount, + MPI_Datatype datatype, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// Returns the upper bound on the amount of space needed to pack a message +WEAK +int MPI_Pack_size( + int incount, + MPI_Datatype datatype, + MPI_Comm comm, + int *size +) { + return MPI_SUCCESS; +} + + + + +// ----------------------------------------------------------------------------- +// Chapter 5 - Collective Communication +// ----------------------------------------------------------------------------- + + +// ----------------------------------------------------------------------------- +// Chapter 5.3 Barrier Synchronization +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Barrier( // Blocks until all processes in the communicator have reached this routine. + MPI_Comm comm // [in] communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node100.htm + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.4 Broadcast +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Bcast( // Broadcasts a message from the process with rank "root" to all other processes of the communicator + void* buffer, // starting address of buffer (choice) + int count, // number of entries in buffer (non-negative integer) + MPI_Datatype datatype, // data type of buffer (handle) + int root, // rank of broadcast root (integer) + MPI_Comm comm // communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node101.htm + if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.5 Gather +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Gather( // Gathers together values from a group of processes + const void *sendbuf, // [in] starting address of send buffer (choice) + int sendcnt, // [in] number of elements in send buffer (integer) + MPI_Datatype sendtype, // [in] data type of send buffer elements (handle) + void *recvbuf, // [out] address of receive buffer (choice, significant only at root) + int recvcnt, // [in] number of elements for any single receive (integer, significant only at root) + MPI_Datatype recvtype, // [in] data type of recv buffer elements (significant only at root) (handle) + int root, // [in] rank of receiving process (integer) + MPI_Comm comm // [in] communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node103.htm + if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; + assert(root == 0); + int sendsize = -1; + int recvsize = -1; +#define fake_mpi_max(a,b) ((a) > (b) ? (a) : (b)) + MPI_Type_size(sendtype, &sendsize); + MPI_Type_size(recvtype, &recvsize); + memcpy((char*)recvbuf, (const char*)sendbuf, fake_mpi_max(sendcnt*sendsize, recvcnt*recvsize)); + return MPI_SUCCESS; +#undef fake_mpi_max +} + +WEAK +int MPI_Gatherv( + const void *sendbuf, + int sendcount, + MPI_Datatype sendtype, + void * recvbuf, + const int recvcounts[], + const int displs[], + MPI_Datatype recvtype, + int root, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.6 Scatter +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Scatter( + const void* sendbuf, + int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + int recvcount, + MPI_Datatype recvtype, + int root, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Scatterv( + const void* sendbuf, + const int sendcounts[], + const int displs[], + MPI_Datatype sendtype, + void* recvbuf, + int recvcount, + MPI_Datatype recvtype, + int root, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.7 Gather-to-all +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Allgather( + const void* sendbuf, + int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + int recvcount, + MPI_Datatype recvtype, + MPI_Comm comm +) { + int sendsize = -1; + int recvsize = -1; +#define fake_mpi_max(a,b) ((a) > (b) ? (a) : (b)) + MPI_Type_size(sendtype, &sendsize); + MPI_Type_size(recvtype, &recvsize); + memcpy((char*)recvbuf, (const char*)sendbuf, fake_mpi_max(sendcount*sendsize, recvcount*recvsize)); + return MPI_SUCCESS; +#undef fake_mpi_max +} + +WEAK +int MPI_Allgatherv( + const void* sendbuf, + int sendcount, + MPI_Datatype sendtype, + void* recvbuf, + const int recvcounts[], + const int displs[], + MPI_Datatype recvtype, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.8 All-to-All Scatter/Gather +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Alltoall( + const void* sendbuf, + int sendcount, + MPI_Datatype sendtype, + void *recvbuf, + int recvcount, + MPI_Datatype recvtype, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Alltoallv( + const void* sendbuf, + const int sendcounts[], + const int sdispls[], + MPI_Datatype sendtype, + void *recvbuf, + const int recvcounts[], + const int rdispls[], + MPI_Datatype recvtype, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Alltoallw( + const void* sendbuf, + const int sendcounts[], + const int sdispls[], + MPI_Datatype sendtypes[], + void *recvbuf, + const int recvcounts[], + const int rdispls[], + const MPI_Datatype recvtypes[], + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.9.1 Reduce +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Reduce( + const void *sendbuf, + void *recvbuf, + int count, + MPI_Datatype datatype, + MPI_Op op, + int root, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.9.5 User-Defined Reduction Operators +// ----------------------------------------------------------------------------- + +typedef void (MPI_User_function)(void *a, void *b, int *len, MPI_Datatype *); + +WEAK +int MPI_Op_create( + MPI_User_function *user_fn, + int commute, + MPI_Op *op +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Op_free( + MPI_Op *op +) { + if (op) *op = MPI_NO_OP; + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.9.6 All-reduce +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Allreduce( // Combines values from all processes and distributes the result back to all processes + const void* sendbuf, // starting address of send buffer (choice) + void* recvbuf, // starting address of receive buffer (choice) + int count, // number of elements in send buffer (non-negative) + MPI_Datatype datatype, // data type of elements of send buffer (handle) + MPI_Op op, // operation (handle) + MPI_Comm comm // communicator (handle) +) // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node117.htm#Node117 +{ + assert(&comm != &MPI_COMM_NULL); + //assert(sendbuf == MPI_IN_PLACE); + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.10 Reduce-Scatter +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 5.10.1 MPI_REDUCE_SCATTER_BLOCK +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Reduce_scatter_block( + const void* sendbuf, + void *recvbuf, + int recvcount, + MPI_Datatype datatype, + MPI_Op op, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.10.2 MPI_REDUCE_SCATTER +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Reduce_scatter( + const void* sendbuf, + void *recvbuf, + const int recvcounts[], + MPI_Datatype datatype, + MPI_Op op, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.11 Scan +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 5.11.1 Inclusive Scan +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Scan( + const void *sendbuf, + void *recvbuf, + int count, + MPI_Datatype datatype, + MPI_Op op, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.11.2 Exclusive Scan +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Exscan( + const void *sendbuf, + void *recvbuf, + int count, + MPI_Datatype datatype, + MPI_Op op, + MPI_Comm comm +) { + return MPI_SUCCESS; +} + + + +// ----------------------------------------------------------------------------- +// Chapter 5.12 Nonblocking Collective Operations +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 5.12.2 Nonblocking Broadcast +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Ibcast( + void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + + +// ----------------------------------------------------------------------------- +// Chapter 5.12.3 Nonblocking Gather +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Igather( // Gathers together values from a group of processes + const void *sendbuf, // [in] starting address of send buffer (choice) + int sendcnt, // [in] number of elements in send buffer (integer) + MPI_Datatype sendtype, // [in] data type of send buffer elements (handle) + void *recvbuf, // [out] address of receive buffer (choice, significant only at root) + int recvcnt, // [in] number of elements for any single receive (integer, significant only at root) + MPI_Datatype recvtype, // [in] data type of recv buffer elements (significant only at root) (handle) + int root, // [in] rank of receiving process (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Request *request // [out] communicatio request (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node130.htm + if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; + assert(0); // TODO implementation + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 5.12.5 Nonblocking Gather-to-all +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Iallgather( + const void* sendbuf, + int sendcount, + MPI_Datatype sendtype, + void *recvbuf, + int recvcount, + MPI_Datatype recvtype, + MPI_Comm comm, + MPI_Request *request +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6 - Groups, Context, Communicators, and Caching +// ----------------------------------------------------------------------------- + +enum { + MPI_IDENT, + MPI_CONGRUENT, + MPI_SIMILAR, + MPI_UNEQUAL +}; + +// ----------------------------------------------------------------------------- +// Chapter 6.3 Group Management +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 6.3.1 Group Accessors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Group_size( // Returns the size of a group + MPI_Group group, // [in] group (handle) + int *size // [out] number of processes in the group (integer) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_rank( // Determines the rank of the calling process in the communicator + MPI_Group group, // group (handle) + int *rank // rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_translate_ranks( // Translates the ranks of processes in one group to those in another group + MPI_Group group1, // [in] group1 (handle) + int n, // [in] number of ranks in ranks1 and ranks2 arrays (integer) + const int ranks1[], // [in] array of zero or more valid ranks in group1 + MPI_Group group2, // [in] group2 (handle) + int ranks2[] // [out] array of corresponding ranks in group2, MPI_UNDEFINED when no correspondence exists. +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_compare( // Compares two groups + MPI_Group group1, // [in] group1 (handle) + MPI_Group group2, // [in] group2 (handle) + int *result // [out] integer which is MPI_IDENT if the order and members of the two groups are the same, MPI_SIMILAR if only the members are the same, and MPI_UNEQUAL otherwise +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm#Node151 + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6.3.2 Group Constructors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_group( + MPI_Comm comm, + MPI_Group *group +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_union( // Produces a group by combining two groups + MPI_Group group1, // first group (handle) + MPI_Group group2, // second group (handle) + MPI_Group *newgroup // union group (handle) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_intersection( // Produces a group as the intersection of two existing groups + MPI_Group group1, // [in] first group (handle) + MPI_Group group2, // [in] second group (handle) + MPI_Group *newgroup // [out] intersection group (handle) +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_difference( // Produces a group as the difference of two existing groups + MPI_Group group1, // [in] first group (handle) + MPI_Group group2, // [in] second group (handle) + MPI_Group *newgroup // [out] difference group (handle) +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_incl( // Produces a group by reordering an existing group and taking only listed members + MPI_Group group, // [in] group (handle) + int n, // [in] number of elements in array ranks (and size of newgroup ) (integer) + const int ranks[], + MPI_Group *newgroup // [in] ranks of processes in group to appear in newgroup (array of integers) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_excl( // Produces a group by reordering an existing group and taking only unlisted members + MPI_Group group, //[in] group (handle) + int n, // [in] number of elements in array ranks (integer) + const int ranks[], // [in] array of integer ranks in group not to appear in newgroup + MPI_Group *newgroup // [out] new group derived from above, preserving the order defined by group (handle) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_range_incl( // Creates a new group from ranges of ranks in an existing group + MPI_Group group, // [in] group (handle) + int n, // [in] number of triplets in array ranges (integer) + int ranges[][3], // [in] a one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in group or processes to be included in newgroup. + MPI_Group *newgroup // [out] new group derived from above, in the order defined by ranges (handle) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_range_excl( // Produces a group by excluding ranges of processes from an existing group + MPI_Group group, // [in] group (handle) + int n, // [in] number of elements in array ranks (integer) + int ranges[][3], // [in] a one-dimensional array of integer triplets of the form (first rank, last rank, stride), indicating the ranks in group of processes to be excluded from the output group newgroup . + MPI_Group *newgroup // [out] new group derived from above, preserving the order in group (handle) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Group_free( // Frees a group + MPI_Group *group // group (handle) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node153.htm + return MPI_SUCCESS; +} + + +// ----------------------------------------------------------------------------- +// Chapter 6.4 Communicator Management +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 6.4.1 Communicator Accessors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_size( // Determines the size of the group associated with a communicator + MPI_Comm comm, // communicator (handle) + int *size // number of processes in the group of comm (integer) +){ + //if(comm == MPI_COMM_NULL){ + // int error = MPI_ERR_COMM; + // comm->errhandler_->func_(&comm, &error); + // return error; + //} + *size = 1; + return MPI_SUCCESS; +} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node155.htm + +WEAK +int MPI_Comm_rank( // MPI_Group_rank Returns the rank of this process in the given group + MPI_Comm comm, // [in] group (handle) + int* rank // [out] rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer) +){ + //if(comm == MPI_COMM_NULL){ + // MPI_Comm_call_errhandler(comm, MPI_ERR_COMM); + // return MPI_ERR_COMM; + //} + *rank = 0; + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_compare( // Compares two communicators + MPI_Comm comm1, // [in] comm1 (handle) + MPI_Comm comm2, // [in] comm2 (handle) + int *result // [out] integer which is MPI_IDENT if the contexts and groups are the same, MPI_CONGRUENT if different contexts but identical groups, MPI_SIMILAR if different contexts but similar groups, and MPI_UNEQUAL otherwise +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node155.htm + if(&comm1 == &MPI_COMM_NULL || &comm2 == &MPI_COMM_NULL) return MPI_ERR_COMM; + if(result == NULL) return MPI_ERR_ARG; + *result = MPI_CONGRUENT; //MPI_IDENT; + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6.4.2 Communicator Constructors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_dup( // Duplicates an existing communicator with all its cached information + MPI_Comm comm, // communicator (handle) + MPI_Comm *newcomm // copy of comm (handle) +) +{ + if(&comm == &MPI_COMM_NULL) return MPI_ERR_COMM; + *newcomm = (struct MPI_Comm_impl_*)malloc(sizeof(struct MPI_Comm_impl_)); + assert(*newcomm != MPI_COMM_NULL); + (**newcomm).errhandler_ = comm->errhandler_; + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_create( // Creates a new communicator + MPI_Comm comm, // [in] communicator (handle) + MPI_Group group, // [in] group, which is a subset of the group of comm (handle) + MPI_Comm *newcomm // [out] new communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm + if(&comm == &MPI_COMM_NULL) return MPI_ERR_COMM; + if(group == MPI_GROUP_NULL) return MPI_ERR_GROUP; + newcomm = (MPI_Comm*)malloc(sizeof(MPI_Comm)); + return MPI_SUCCESS; +} + +int MPI_Comm_create_group( // must be called by all processes in group, which is a subgroup of the group of comm + MPI_Comm comm, // intracommunicator (handle) + MPI_Group group, // group, which is a subset of the group of comm (handle) + int tag, // tag (integer) + MPI_Comm *newcomm // new communicator (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm + +WEAK +int MPI_Comm_split( // Creates new communicators based on colors and keys + MPI_Comm comm, // [in] communicator (handle) + int color, // [in] control of subset assignment (integer) + int key, // [in] control of rank assigment (integer) + MPI_Comm *newcomm // [out] new communicator (handle) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6.4.3 Communicator Destructors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_free( // Marks the communicator object for deallocation + MPI_Comm *comm // [in] Communicator to be destroyed (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node157.htm + *comm = MPI_COMM_NULL; + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6.6 Inter-Communication +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 6.6.1 Inter-communicator Accessors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_test_inter( + MPI_Comm comm, + int *flag +) { + return MPI_SUCCESS; +} + +// Determines the size of the remote group associated with an inter-communictor +WEAK +int MPI_Comm_remote_size( + MPI_Comm comm, + int *size +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_remote_group( + MPI_Comm comm, + MPI_Group *group +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6.6.2 Inter-communicator Operations +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Intercomm_create( // Creates an intercommuncator from two intracommunicators + MPI_Comm local_comm, // [in] Local (intra)communicator + int local_leader, // [in] Rank in local_comm of leader (often 0) + MPI_Comm peer_comm, // [in] Communicator used to communicate between a designated process in the other communicator. Significant only at the process in local_comm with rank local_leader. + int remote_leader, // [in] Rank in peer_comm of remote leader (often 0) + int tag, // [in] Message tag to use in constructing intercommunicator; if multiple MPI_Intercomm_creates are being made, they should use different tags (more precisely, ensure that the local and remote leaders are using different tags for each MPI_intercomm_create). + MPI_Comm *newintercomm // [out] Created intercommunicator +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node168.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Intercomm_merge( + MPI_Comm intercomm, + int high, + MPI_Comm *newintracomm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 6.7 Caching +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 6.7.2 Communicators +// ----------------------------------------------------------------------------- + +typedef int MPI_Comm_copy_attr_function(MPI_Comm oldcomm, int comm_keyval, + void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag); + +typedef int MPI_Comm_delete_attr_function(MPI_Comm comm, int comm_keyval, + void *attribute_val, void *extra_state); + +WEAK +int MPI_Comm_create_keyval( + MPI_Comm_copy_attr_function *comm_copy_attr_fn, + MPI_Comm_delete_attr_function *comm_delete_attr_fn, + int *comm_keyval, + void *extra_state +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_free_keyval( + int *comm_keyval +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_set_attr( + MPI_Comm comm, + int comm_keyval, + void *attribute_val +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_get_attr( + MPI_Comm comm, + int comm_keyval, + void *attribute_val, + int *flag +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_delete_attr( + MPI_Comm comm, + int comm_keyval +) { + return MPI_SUCCESS; +} + +#ifdef USE_MPI_REMOVED_FUNCTIONS +#define MPI_Attr_get MPI_Comm_get_attr +#define MPI_Attr_put MPI_Comm_set_attr +#endif + +enum { + MPI_HOST, + MPI_IO, + MPI_WTIME_IS_GLOBAL, + MPI_APPNUM, + MPI_UNIVERSE_SIZE, + MPI_LASTUSEDCODE, + MPI_TAG_UB +}; + +// ----------------------------------------------------------------------------- +// Chapter 6.8 Naming Objects +// ----------------------------------------------------------------------------- + +const int MPI_MAX_OBJECT_NAME = 128; + +WEAK +int MPI_Comm_set_name( // Sets the print name for a communicator + MPI_Comm comm, // [in] communicator to name (handle) + const char *comm_name // [in] Name for communicator (string) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_get_name( // Return the print name from the communicator + MPI_Comm comm, // communicator whose name is to be returned (handle) + char *comm_name, // the name previously stored on the communicator, or an... + int *resultlen // length of returned name (integer) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm + return MPI_SUCCESS; +} + +inline int MPI_Type_get_name(MPI_Datatype datatype, char *type_name, int *resultlen) +{ + if (resultlen) *resultlen = 0; + return MPI_SUCCESS; +} + +inline int MPI_Type_set_name(MPI_Datatype datatype, const char *type_name) +{ + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 7 - Process Topologies +// ----------------------------------------------------------------------------- + + +enum { + MPI_GRAPH, + MPI_CART, + MPI_DIST_GRAPH +}; + +// ----------------------------------------------------------------------------- +// Chapter 7.5 Topology Constructors +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Cart_create( // Makes a new communicator to which topology information has been attached + MPI_Comm comm_old, // [in] input communicator (handle) + int ndims, // [in] number of dimensions of cartesian grid (integer) + int *dims, // [in] integer array of size ndims specifying the number of processes in each dimension + int *periods, // [in] logical array of size ndims specifying whether the grid is periodic (true) or not (false) in each dimension + int reorder, // [in] ranking may be reordered (true) or not (false) (logical) + MPI_Comm *comm_cart // [out] communicator with new cartesian topology (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node192.htm + if(dims == NULL) return MPI_ERR_DIMS; + for(int i = 0; i != ndims; ++i) if(dims[i] <= 0) return MPI_ERR_DIMS; + *comm_cart = comm_old; // TODO: allocate unique handle + return MPI_SUCCESS; +} + +WEAK +int MPI_Dims_create( + int nnodes, + int ndims, + int dims[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Graph_create( + MPI_Comm comm_old, + int nnodes, + const int index[], + const int edges[], + int reorder, + MPI_Comm *comm_graph +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Dist_graph_create_adjacent( + MPI_Comm comm_old, + int indegree, + const int sources[], + const int sourceweights[], + int outdegree, + const int destinations[], + const int destweights[], + MPI_Info info, + int reorder, + MPI_Comm *comm_dist_graph +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Dist_graph_create( + MPI_Comm comm_old, + int n, + const int sources[], + const int degrees[], + const int destinations[], + const int weights[], + MPI_Info info, + int reorder, + MPI_Comm *comm_dist_graph +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 7.5.5 Topology Inquiry Functions +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Topo_test( + MPI_Comm comm, + int *status +) { + if (status) *status = MPI_UNDEFINED; + return MPI_SUCCESS; +} + +WEAK +int MPI_Graphdims_get( + MPI_Comm comm, + int *nnodes, + int *nedges +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Graph_get( + MPI_Comm comm, + int maxindex, + int maxedges, + int index[], + int edges[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Cartdim_get( + MPI_Comm comm, + int *ndims +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Cart_get( + MPI_Comm comm, + int maxdims, + int dims[], + int periods[], + int coords[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Cart_rank( + MPI_Comm comm, + const int coords[], + int *rank +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Cart_coords( + MPI_Comm comm, + int rank, + int maxdims, + int coords[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Graph_neighbors_count( + MPI_Comm comm, + int rank, + int *nneighbors +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Graph_neighbors( + MPI_Comm comm, + int rank, + int maxneighbors, + int neighbors[] +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Dist_graph_neighbors_count( + MPI_Comm comm, + int *indegree, + int *outdegree, + int *weighted +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Dist_graph_neighbors( + MPI_Comm comm, + int maxindegree, + int sources[], + int sourceweights[], + int maxoutdegree, + int destinations[], + int destweights[] +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 7.5.6 Cartesian Shift Coordinates +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Cart_shift( + MPI_Comm comm, + int direction, + int disp, + int *rank_source, + int *rank_dest +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 7.5.7 Partitioning of Cartesian Structures +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Cart_sub( // Partitions a communicator into subgroups which form lower-dimensional cartesian subgrids + MPI_Comm comm, // [in] communicator with cartesian structure (handle) + int *remain_dims, // [in] the ith entry of remain_dims specifies whether the ith dimension is kept in the subgrid (true) or is dropped (false) (logical vector) + MPI_Comm *newcomm // [out] communicator containing the subgrid that includes the calling process (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node198.htm + if(&comm == &MPI_COMM_NULL) return MPI_ERR_COMM; + *newcomm = comm; // TODO: allocate unique handle + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 7.5.8 Low-Level Topology Functions +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Cart_map( + MPI_Comm comm, + int ndims, + const int dims[], + const int periods[], + int *newrank +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Graph_map( + MPI_Comm comm, + int nnodes, + const int index[], + const int edges[], + int *newrank +) { + return MPI_SUCCESS; +} + + +// ----------------------------------------------------------------------------- +// Chapter 8 - MPI Environmental Management +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 8.1 Implementation Information +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 8.1.1 Version Inquiries +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Get_version( // Return the version number of MPI + int* version, // [out] Version of MPI + int* subversion // [out] Suversion of MPI +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node209.htm + *version = 3; + *subversion = 1; + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 8.1.2 Environmental Inquiries +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Get_processor_name( // the name of the processor on which it was called at the moment of the call. + char *name, // A unique specifier for the actual (as opposed to virtual) node. + int *resultlen // Length (in printable characters) of the result returned in name +){ + if(gethostname(name, MPI_MAX_PROCESSOR_NAME) > 0) + return MPI_ERR_UNKNOWN; + *resultlen = strlen(name); + return MPI_SUCCESS; +} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node210.htm#Node215 + +// ----------------------------------------------------------------------------- +// Chapter 8.2 Memory Allocation +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Alloc_mem( + MPI_Aint size, + MPI_Info info, + void *baseptr +) { + //if (baseptr) *(void **)baseptr = malloc(size); + if (baseptr) *(void **)baseptr = 0; + return MPI_SUCCESS; +} + +WEAK +int MPI_Free_mem( + void *base +) { + //free(base); + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 8.3 Error Handling +// ----------------------------------------------------------------------------- + +// http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node218.htm + +WEAK +int MPI_Comm_create_errhandler( // Create a communicator error handler + MPI_Comm_errhandler_function *comm_errhandler_fn, // [in] user defined error handling procedure (function) + MPI_Errhandler *errhandler // [out] MPI error handler (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node218.htm + *errhandler = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); + (*errhandler)->func_ = comm_errhandler_fn; + return MPI_SUCCESS; +} + +WEAK +int MPI_Comm_set_errhandler( // Set the error handler for a communicator + MPI_Comm comm, // [in] communicator (handle) + MPI_Errhandler errhandler // [in] new error handler for communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node218.htm +// assert(comm != MPI_COMM_NULL); + if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; + comm->errhandler_ = errhandler; //->func_ = errhandler->func_; + return MPI_SUCCESS; +} + + +WEAK +int MPI_Comm_get_errhandler( // Get the error handler attached to a communicator + MPI_Comm comm, // [in] communicator (handle) + MPI_Errhandler *errhandler // [out] handler currently associated with communicator (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node220.htm + return MPI_SUCCESS; +} + + +void fatal_error_(MPI_Comm * comm, int * errcode, ...); +WEAK +void no_op_error_(MPI_Comm * comm, int * errcode, ...){} + +#ifdef USE_MPI_REMOVED_FUNCTIONS +WEAK +int MPI_Errhandler_create( + MPI_Handler_function *errhandler_fn, + MPI_Errhandler *errhandler +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Errhandler_set( + MPI_Comm comm, + MPI_Errhandler errhandler +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Errhandler_get( + MPI_Comm comm, + MPI_Errhandler *errhandler +) { + return MPI_SUCCESS; +} +#endif + + + +// ----------------------------------------------------------------------------- +// Chapter 8.3.4 Freeing Errohandlers and Retrieving Error Strings +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Errhandler_free( // Frees an MPI-style errorhandler + MPI_Errhandler *errhandler // [in-out] MPI error handler (handle) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node221.htm + *errhandler = MPI_ERRHANDLER_NULL; + return MPI_SUCCESS; +} + +WEAK +int MPI_Error_string( // Return a string for a given error code + int errorcode, // [in] Error code returned by an MPI routine or an MPI error class + char *string, // [out] Text that corresponds to the errorcode + int *resultlen // [out] Length of string +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node221.htm#Node221 + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 8.4 Error Codes and Classes +// ----------------------------------------------------------------------------- + +int MPI_Error_class( // Converts an error code into an error class + int errorcode, // Error code returned by an MPI routine + int *errorclass // Error class associated with errorcode +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm#Node222 + +// ----------------------------------------------------------------------------- +// Chapter 8.5 Error Classes, Error Codes, and Error Handlers +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_call_errhandler( // Call the error handler installed on a communicator + MPI_Comm comm, // [in] communicator with error handler (handle) + int errorcode // [in] error code (integer) +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node223.htm +// if(comm == MPI_COMM_NULL){ +// return MPI_ERR_COMM; +// } + comm->errhandler_->func_(&comm, &errorcode); + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 8.6 Timers and Synchronization +// ----------------------------------------------------------------------------- + +WEAK +double MPI_Wtime( // Returns an elapsed time on the calling processor +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm + struct timespec tw; + clock_gettime(CLOCK_MONOTONIC, &tw); + return 1.0*tw.tv_sec + 1e-9*tw.tv_nsec;; +} + +WEAK +double MPI_Wtick( // Returns the resolution of MPI_Wtime +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm + return 1e-9; +} + +// ----------------------------------------------------------------------------- +// Chapter 8.7 Startup +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Init( // Initialize the MPI execution environment + int *argc, // [in] Pointer to the number of arguments + char ***argv // [in] Pointer to the argument vector +){ + MPI_Comm_create_errhandler(&fatal_error_, &MPI_ERRORS_ARE_FATAL); + MPI_Comm_create_errhandler(&no_op_error_, &MPI_ERRORS_RETURN); +// MPI_ERRORS_ARE_FATAL = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_));//{&fatal_error_}; +// static struct MPI_Errhandler_impl_ MPI_ERRORS_RETURN = {&no_op_error_}; + +// MPI_COMM_NULL = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_)); +// MPI_COMM_NULL->errhandler_ = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); +// MPI_Comm_set_errhandler(MPI_COMM_NULL, MPI_ERRORS_ARE_FATAL); + + MPI_COMM_WORLD = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_)); + MPI_COMM_WORLD->errhandler_ = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_ARE_FATAL); +// MPI_COMM_WORLD->errhandler_ = MPI_ERRORS_ARE_FATAL; + + MPI_COMM_SELF = (MPI_Comm)malloc(sizeof(struct MPI_Comm_impl_)); + MPI_COMM_SELF->errhandler_ = (struct MPI_Errhandler_impl_*)malloc(sizeof(struct MPI_Errhandler_impl_)); + MPI_Comm_set_errhandler(MPI_COMM_SELF, MPI_ERRORS_ARE_FATAL); +// MPI_COMM_SELF->errhandler_ = MPI_ERRORS_ARE_FATAL; + + return MPI_SUCCESS; +} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm + + +WEAK +int MPI_Finalize( // Terminates MPI execution environment +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm + return MPI_SUCCESS; +} + +WEAK +int MPI_Initialized( // Indicates whether MPI_Init has been called. + int *flag // [out] Flag is true if MPI_Init or MPI_Init_thread has been called and false otherwise. +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node225.htm + if (flag) *flag = true; + return MPI_SUCCESS; +} + +WEAK +int MPI_Abort( // Terminates MPI execution environment + MPI_Comm comm, // [in] communicator of tasks to abort + int errorcode // [in] error code to return to invoking environment +){ // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node225.htm + exit(errorcode); +// return MPI_SUCCESS; // function never returns +} + + +WEAK +void fatal_error_(MPI_Comm * comm, int * errcode, ...){ + switch(*errcode){ + case MPI_ERR_COMM : puts("[] *** MPI_ERR_COMM: invalid communicator\n[] *** MPI_ERRORS_ARE_FATAL (will now abort)"); MPI_Abort(*comm, *errcode); + } +} + +// ----------------------------------------------------------------------------- +// Chapter 8.7.2 Determining Whether MPI Has Finished +// ----------------------------------------------------------------------------- + +int MPI_Finalized( // Indicates whether MPI_Finalize has been called + int *flag // [out] true if MPI was finalized (logical) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node227.htm + + + +// ----------------------------------------------------------------------------- +// Chapter 10 - Process Creation and Management +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 10.3 Process Manager Interface +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 10.3.2 Starting Processes and Establishing Communication +// ----------------------------------------------------------------------------- + +int MPI_Comm_spawn( // Spawn up to maxprocs instances of a single MPI application + const char *command, // [in] name of program to be spawned (string, at root) + char *argv[], // [in] arguments to command (array of strings, at root) + int maxprocs, // maximum number of processes to start(int at root) + MPI_Info info, // a set of key-value pairs telling the runtime system where and how to start the processes (handle, significant only at root) + int root, // rank of process in which previous arguments are examined (integer) + MPI_Comm comm, // intracommunicator containing group of spawning processes (handle) + MPI_Comm *intercomm, // [out] intercommunicator between original group and the newly spawned group (handle) + int array_of_errcodes[] // [out] one code per process (array of integer) +); + +int MPI_Comm_get_parent( // Return the parent communicator for this process + MPI_Comm *parent // [out] the parent communicator (handle) +); + +// ----------------------------------------------------------------------------- +// Chapter 10.4 Establishing Communication +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 10.4.2 Server Routines +// ----------------------------------------------------------------------------- + +const int MPI_MAX_PORT_NAME = 128; + +int MPI_Open_port( + MPI_Info info, + char *port_name +); + +int MPI_Close_port( + const char *port_name +); + +WEAK +int MPI_Comm_accept( + const char *port_name, + MPI_Info info, + int root, + MPI_Comm comm, + MPI_Comm *newcomm +) { + return MPI_SUCCESS; +} + + +// ----------------------------------------------------------------------------- +// Chapter 10.4.2 Client Routines +// ----------------------------------------------------------------------------- + + +WEAK +int MPI_Comm_connect( + const char *port_name, + MPI_Info info, + int root, + MPI_Comm comm, + MPI_Comm *newcomm +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 10.5.4 Releasing Connections +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Comm_disconnect( // MPI_Comm_disconnect + MPI_Comm *comm // [in] communicator (handle) +){ + if(*comm == MPI_COMM_NULL) return MPI_ERR_COMM; + *comm = MPI_COMM_NULL; + return MPI_SUCCESS; +} + + + +// ----------------------------------------------------------------------------- +// Chapter 12.2 Generalized Requests +// ----------------------------------------------------------------------------- + +typedef int MPI_Grequest_query_function(void *extra_state, MPI_Status *status); +typedef int MPI_Grequest_free_function(void *extra_state); +typedef int MPI_Grequest_cancel_function(void *extra_state, int complete); + +WEAK +int MPI_Grequest_start( // Start new generalized request + MPI_Grequest_query_function *query_fn, // [in] status query callback function + MPI_Grequest_free_function *free_fn, // [in] query free callback function + MPI_Grequest_cancel_function *cancel_fn, // [in] request cancel callback function + void *extra_state, // [in] extra state + MPI_Request *request // [out] generalized request (handle) +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Grequest_complete( + MPI_Request request // [in] generalized request (handle) +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 12.3 Associating Information with Status +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Status_set_elements( + MPI_Status *status, + MPI_Datatype datatype, + int count +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Status_set_elements_x( + MPI_Status *status, + MPI_Datatype datatype, + MPI_Count count +) { + return MPI_SUCCESS; +} + +WEAK +int MPI_Status_set_cancelled( + MPI_Status *status, + int flag +) { + return MPI_SUCCESS; +} + +// ----------------------------------------------------------------------------- +// Chapter 12.4 MPI and Threads +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 12.4.3 Initialization +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Init_thread( // Initialize the MPI execution environment + int *argc, // [in] Pointer to the number of arguments + char ***argv, // [in] Pointer to the argument vector + int required, // [in] Level of desired thread support + int *provided // [out] Level of provided thread support +){// http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm + *provided = MPI_THREAD_MULTIPLE; + return MPI_SUCCESS; +} + +WEAK +int MPI_Query_thread( // The following function can be used to query the current level of thread support. + int *provided // provided level of thread support (integer) +) { // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm#Node303 + return MPI_SUCCESS; +} + +int MPI_Is_thread_main( // This function can be called by a thread to determine if it is the main thread (the thread that called MPI_INIT or MPI_INIT_THREAD). + int *flag // true if calling thread is main thread, false otherwise (logical) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm + + +// ----------------------------------------------------------------------------- +// Chapter 14 - Tool Support +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +// Chapter 14.2.4 Miscellaneous Control of Profiling +// ----------------------------------------------------------------------------- + +WEAK +int MPI_Pcontrol( + int level, + ... +) { + return MPI_SUCCESS; +} + + +#endif diff --git a/fake/mpi_qmcpack_compile.h b/fake/mpi_qmcpack_compile.h new file mode 100644 index 00000000000..2ef4962b06f --- /dev/null +++ b/fake/mpi_qmcpack_compile.h @@ -0,0 +1,741 @@ +#ifndef FAKE_MPI_H +#define FAKE_MPI_H +// Fake MPI for serial execution + +#include // HOST_NAME_MAX +#include // gethostname +#include // strlen + +const int MPI_MAX_PROCESSOR_NAME = HOST_NAME_MAX; +const int MPI_MAX_INFO_KEY = 128; +const int MPI_MAX_INFO_VAL = 128; +const int MPI_MAX_ERROR_STRING = 128; + +enum : int { // error classes + MPI_SUCCESS, // No error + MPI_ERR_BUFFER, // Invalid buffer pointer + MPI_ERR_COUNT, // Invalid count argument + MPI_ERR_TYPE, // Invalid datatype argument + MPI_ERR_TAG, // Invalid tag argument + MPI_ERR_COMM, // Invalid communicator + MPI_ERR_RANK, // Invalid rank + MPI_ERR_REQUEST, // Invalid request (handle) + MPI_ERR_ROOT, // Invalid root + MPI_ERR_GROUP, // Invalid group + MPI_ERR_OP, // Invalid operation + MPI_ERR_TOPOLOGY, // Invalid topology + MPI_ERR_DIMS, // Invalid dimension argument + MPI_ERR_ARG, // Invalid argument of some other kind + MPI_ERR_UNKNOWN, // Unknown error + MPI_ERR_TRUNCATE, // Message truncated on receive + MPI_ERR_OTHER, // Known error not in this list + MPI_ERR_INTERN, // Internal MPI (implementation) error + MPI_ERR_IN_STATUS, // Error code is in status + MPI_ERR_PENDING, // Pending request + MPI_ERR_KEYVAL, // Invalid keyval has been passed + MPI_ERR_NO_MEM, // MPI_ALLOC_MEM failed because memory is exhausted + MPI_ERR_BASE, // Invalid base passed to MPI_FREE_MEM + MPI_ERR_INFO_KEY, // Key longer than MPI_MAX_INFO_KEY + MPI_ERR_INFO_VALUE, // Value longer than MPI_MAX_INFO_VAL + MPI_ERR_INFO_NOKEY, // Invalid key passed to MPI_INFO_DELETE + MPI_ERR_SPAWN, // Error in spawning processes + MPI_ERR_PORT, // Invalid port name passed to MPI_COMM_CONNECT + MPI_ERR_SERVICE, // Invalid service name passed to MPI_UNPUBLISH_NAME + MPI_ERR_NAME, // Invalid service name passed to MPI_LOOKUP_NAME + MPI_ERR_WIN, // Invalid win argument + MPI_ERR_SIZE, // Invalid size argument + MPI_ERR_DISP, // Invalid disp argument + MPI_ERR_INFO, // Invalid info argument + MPI_ERR_LOCKTYPE, // Invalid locktype argument + MPI_ERR_ASSERT, // Invalid assert argument + MPI_ERR_RMA_CONFLICT, // Conflicting accesses to window + MPI_ERR_RMA_SYNC, // Wrong synchronization of RMA calls + MPI_ERR_RMA_RANGE, // Target memory is not part of the window (in the case of a window created with MPI_WIN_CREATE_DYNAMIC, target memory is not attached) + MPI_ERR_RMA_ATTACH, // Memory cannot be attached (e.g., because of resource exhaustion) + MPI_ERR_RMA_SHARED, // Memory cannot be shared (e.g., some process in the group of the specified communicator cannot expose shared memory) + MPI_ERR_RMA_FLAVOR, // Passed window has the wrong flavor for the called function + MPI_ERR_FILE, // Invalid file handle + MPI_ERR_NOT_SAME, // Collective argument not identical on all processes, or collective routines called in a different order by different processes + MPI_ERR_AMODE, // Error related to the amode passed to MPI_FILE_OPEN + MPI_ERR_UNSUPPORTED_DATAREP, // Unsupported datarep passed to MPI_FILE_SET_VIEW + MPI_ERR_UNSUPPORTED_OPERATION, // Unsupported operation, such as seeking on a file which supports sequential access only + MPI_ERR_NO_SUCH_FILE, // File does not exist + MPI_ERR_FILE_EXISTS, // File exists + MPI_ERR_BAD_FILE, // Invalid file name (e.g., path name too long) + MPI_ERR_ACCESS, // Permission denied + MPI_ERR_NO_SPACE, // Not enough space + MPI_ERR_QUOTA, // Quota exceeded + MPI_ERR_READ_ONLY, // Read-only file or file system + MPI_ERR_FILE_IN_USE, // File operation could not be completed, as the file is currently open by some process + MPI_ERR_DUP_DATAREP, // Conversion functions could not be registered because a data representation identifier that was already defined was passed to MPI_REGISTER_DATAREP + MPI_ERR_CONVERSION, // An error occurred in a user supplied data conversion function. + MPI_ERR_IO, // Other I/O error + MPI_ERR_LASTCODE // Last error code +}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm + +struct MPI_Group{ + bool operator==(const MPI_Group&o) const { return true; } +}; + +const struct MPI_Group MPI_GROUP_EMPTY; +const struct MPI_Group MPI_GROUP_NULL; + +// MPI Info handling. +// Would not be too hard to create an std::map implementation +//struct MPI_Info{}; +//const struct MPI_Info MPI_INFO_NULL; + +enum MPI_Info { + MPI_INFO_NULL +}; + +int MPI_Info_create(MPI_Info *info); + +int MPI_Info_free(MPI_Info *info); + +int MPI_Info_dup(MPI_Info info, MPI_Info *newinfo); + +int MPI_Info_delete(MPI_Info info, const char *key); + +int MPI_Info_get(MPI_Info info, const char *key, int valuelen, char *value, int *flag); + +int MPI_Info_get_nkeys(MPI_Info info, int *nkeys); + +int MPI_Info_get_nthkey(MPI_Info info, int n, char *key); + +int MPI_Info_get_valuelen(MPI_Info info, const char *key, int *valuelen, int *flag); + +int MPI_Info_set(MPI_Info info, const char *key, const char *value); + +const int MPI_MAX_PORT_NAME = 128; + +int MPI_Open_port(MPI_Info info, char *port_name); + +int MPI_Close_port(const char *port_name); + +enum : int { // level of thread support + MPI_THREAD_SINGLE, + MPI_THREAD_FUNNELED, + MPI_THREAD_SERIALIZED, + MPI_THREAD_MULTIPLE +}; // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm + +typedef enum _MPI_Datatype { + MPI_DATATYPE_NULL = 0, + MPI_CHAR, + MPI_SHORT, + MPI_INT, + MPI_LONG, + MPI_LONG_LONG_INT, + MPI_LONG_LONG, + MPI_SIGNED_CHAR, + MPI_UNSIGNED_CHAR, + MPI_UNSIGNED_SHORT, + MPI_UNSIGNED, + MPI_UNSIGNED_LONG, + MPI_UNSIGNED_LONG_LONG, + MPI_FLOAT, + MPI_DOUBLE, + MPI_LONG_DOUBLE, + MPI_WCHAR, + MPI_C_BOOL, + MPI_INT8_T, + MPI_INT16_T, + MPI_INT32_T, + MPI_INT64_T, + MPI_UINT8_T, + MPI_UINT16_T, + MPI_UINT32_T, + MPI_UINT64_T, + MPI_AINT, + MPI_COUNT, + MPI_OFFSET, + MPI_C_COMPLEX, + MPI_C_FLOAT_COMPLEX, + MPI_C_DOUBLE_COMPLEX, + MPI_BYTE, + MPI_PACKED, + MPI_CXX_BOOL, + MPI_CXX_FLOAT_COMPLEX, + MPI_CXX_DOUBLE_COMPLEX, + MPI_CXX_LONG_DOUBLE_COMPLEX, + MPI_FLOAT_INT, + MPI_DOUBLE_INT, + MPI_LONG_INT, + MPI_2INT, + MPI_SHORT_INT, + MPI_LONG_DOUBLE_INT, +} MPI_Datatype; + +typedef unsigned long long MPI_Aint; + +inline int MPI_Type_dup(MPI_Datatype oldtype, MPI_Datatype *newtype) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_commit(MPI_Datatype *datatype) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_contiguous(int count, MPI_Datatype oldtype, MPI_Datatype *newtype) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_vector(int count, int blocklength, int stride, MPI_Datatype oldtype, MPI_Datatype *newtype) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_create_struct(int count, const int array_of_blocklengths[], const MPI_Aint array_of_displacements[], + const MPI_Datatype array_of_types[], MPI_Datatype *newtype) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_size(MPI_Datatype datatype, int *size) +{ + if (size) *size = 0; + return MPI_SUCCESS; +} + +const int MPI_MAX_OBJECT_NAME = 128; + +inline int MPI_Type_get_name(MPI_Datatype datatype, char *type_name, int *resultlen) +{ + if (resultlen) *resultlen = 0; + return MPI_SUCCESS; +} + +inline int MPI_Type_set_name(MPI_Datatype datatype, const char *type_name) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_get_extent(MPI_Datatype datatype, MPI_Aint *lb, MPI_Aint *extent) +{ + return MPI_SUCCESS; +} + +inline int MPI_Type_ub(MPI_Datatype datatype, MPI_Aint *displacement) +{ + return MPI_SUCCESS; +} + + +//struct MPI_Comm { +// bool operator!=(const MPI_Comm &o) const { return false; } +// bool operator==(const MPI_Comm &o) const { return true; } +//}; + +//const struct MPI_Comm MPI_COMM_WORLD; +//const struct MPI_Comm MPI_COMM_SELF; +//const struct MPI_Comm MPI_COMM_NULL; + +enum MPI_Comm : int{ + MPI_COMM_NULL = 0, + MPI_COMM_WORLD = 1, + MPI_COMM_SELF = 2, +}; + +static const void* MPI_IN_PLACE = reinterpret_cast(-1); + +enum MPI_Op{ + MPI_MAX, + MPI_MIN, + MPI_SUM, + MPI_PROD, + MPI_MAXLOC, + MPI_MINLOC, + MPI_BAND, + MPI_BOR, + MPI_BXOR, + MPI_LAND, + MPI_LOR, + MPI_LXOR, + MPI_REPLACE, + MPI_NO_OP, +}; + +struct MPI_Status { + int MPI_SOURCE; + int MPI_TAG; +}; +static MPI_Status *MPI_STATUS_IGNORE = 0; + +static char *MPI_ARGV_NULL[] = {}; +static int MPI_ERRCODES_IGNORE[] = {}; + +struct MPI_Message{}; + +inline int MPI_Allreduce( // Combines values from all processes and distributes the result back to all processes + const void* sendbuf, // starting address of send buffer (choice) + void* recvbuf, // starting address of receive buffer (choice) + int count, // number of elements in send buffer (non-negative) + MPI_Datatype datatype, // data type of elements of send buffer (handle) + MPI_Op op, // operation (handle) + MPI_Comm comm // communicator (handle) +) // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node117.htm#Node117 +{ + return MPI_SUCCESS; +} + +inline int MPI_Bcast( // Broadcasts a message from the process with rank "root" to all other processes of the communicator + void* buffer, // starting address of buffer (choice) + int count, // number of entries in buffer (non-negative integer) + MPI_Datatype datatype, // data type of buffer (handle) + int root, // rank of broadcast root (integer) + MPI_Comm comm // communicator (handle) +) // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node101.htm +{ + return MPI_SUCCESS; +} + +int MPI_Comm_create( // Creates a new communicator + MPI_Comm comm, // [in] communicator (handle) + MPI_Group group, // [in] group, which is a subset of the group of comm (handle) + MPI_Comm *newcomm // [out] new communicator (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm +// Determines the size of the remote group associated with an inter-communictor +int MPI_Comm_remote_size(MPI_Comm comm, int *size); +inline int MPI_Comm_size( // Determines the size of the group associated with a communicator + MPI_Comm comm, // communicator (handle) + int *size // number of processes in the group of comm (integer) +){ + if(comm == MPI_COMM_NULL) return MPI_ERR_COMM; + *size = comm > MPI_COMM_NULL; + return MPI_SUCCESS; +} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node155.htm +int MPI_Comm_spawn( // Spawn up to maxprocs instances of a single MPI application + const char *command, // [in] name of program to be spawned (string, at root) + char *argv[], // [in] arguments to command (array of strings, at root) + int maxprocs, // maximum number of processes to start(int at root) + MPI_Info info, // a set of key-value pairs telling the runtime system where and how to start the processes (handle, significant only at root) + int root, // rank of process in which previous arguments are examined (integer) + MPI_Comm comm, // intracommunicator containing group of spawning processes (handle) + MPI_Comm *intercomm, // [out] intercommunicator between original group and the newly spawned group (handle) + int array_of_errcodes[] // [out] one code per process (array of integer) +); +// Creates new communicators based on colors and keys +int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm); +int MPI_Comm_get_name( // Return the print name from the communicator + MPI_Comm comm, // communicator whose name is to be returned (handle) + char *comm_name, // the name previously stored on the communicator, or an... + int *resultlen // length of returned name (integer) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm +int MPI_Comm_get_parent( // Return the parent communicator for this process + MPI_Comm *parent // [out] the parent communicator (handle) +); +int MPI_Get_count( // Gets the number of "top level" elements + MPI_Status *status, // [in] return status of receive operation (Status) + MPI_Datatype datatype, // [out] number of received elements (integer) + int *count // [in] datatype of each receive buffer element (han +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node51.htm +int MPI_Comm_set_name( // Sets the print name for a communicator + MPI_Comm comm, // [in] communicator to name (handle) + const char *comm_name // [in] Name for communicator (string) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node179.htm#Node179 +int MPI_Comm_create_group( // must be called by all processes in group, which is a subgroup of the group of comm + MPI_Comm comm, // intracommunicator (handle) + MPI_Group group, // group, which is a subset of the group of comm (handle) + int tag, // tag (integer) + MPI_Comm *newcomm // new communicator (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node156.htm +inline int MPI_Comm_dup( // Duplicates an existing communicator with all its cached information + MPI_Comm comm, // communicator (handle) + MPI_Comm *newcomm // copy of comm (handle) +) +{ + *newcomm = comm; + return MPI_SUCCESS; +} +inline int MPI_Get_processor_name( // the name of the processor on which it was called at the moment of the call. + char *name, // A unique specifier for the actual (as opposed to virtual) node. + int *resultlen // Length (in printable characters) of the result returned in name +){ + if(gethostname(name, MPI_MAX_PROCESSOR_NAME) > 0) + return MPI_ERR_UNKNOWN; + *resultlen = strlen(name); + return MPI_SUCCESS; +} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node210.htm#Node215 +int MPI_Group_free( // Frees a group + MPI_Group *group // group (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node153.htm +int MPI_Group_incl( // Produces a group by reordering an existing group and taking only listed members + MPI_Group group, // [in] group (handle) + int n, // [in] number of elements in array ranks (and size of newgroup ) (integer) + const int ranks[], + MPI_Group *newgroup // [in] ranks of processes in group to appear in newgroup (array of integers) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm +int MPI_Group_excl( // Produces a group by reordering an existing group and taking only unlisted members + MPI_Group group, //[in] group (handle) + int n, // [in] number of elements in array ranks (integer) + const int ranks[], // [in] array of integer ranks in group not to appear in newgroup + MPI_Group *newgroup // [out] new group derived from above, preserving the order defined by group (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm +int MPI_Group_range_excl( // Produces a group by excluding ranges of processes from an existing group + MPI_Group group, // [in] group (handle) + int n, // [in] number of elements in array ranks (integer) + int ranges[][3], // [in] a one-dimensional array of integer triplets of the form (first rank, last rank, stride), indicating the ranks in group of processes to be excluded from the output group newgroup . + MPI_Group *newgroup // [out] new group derived from above, preserving the order in group (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm +int MPI_Group_range_incl( // Creates a new group from ranges of ranks in an existing group + MPI_Group group, // [in] group (handle) + int n, // [in] number of triplets in array ranges (integer) + int ranges[][3], // [in] a one-dimensional array of integer triplets, of the form (first rank, last rank, stride) indicating ranks in group or processes to be included in newgroup. + MPI_Group *newgroup // [out] new group derived from above, in the order defined by ranges (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm +int MPI_Group_rank( // Determines the rank of the calling process in the communicator + MPI_Group group, // group (handle) + int *rank // rank of the calling process in group, or MPI_UNDEFINED if the process is not a member (integer) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm +int MPI_Group_size( // Returns the size of a group + MPI_Group group, // [in] group (handle) + int *size // [out] number of processes in the group (integer) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm +int MPI_Group_compare( // Compares two groups + MPI_Group group1, // [in] group1 (handle) + MPI_Group group2, // [in] group2 (handle) + int *result // [out] integer which is MPI_IDENT if the order and members of the two groups are the same, MPI_SIMILAR if only the members are the same, and MPI_UNEQUAL otherwise +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm#Node151 +int MPI_Group_intersection( // Produces a group as the intersection of two existing groups + MPI_Group group1, // [in] first group (handle) + MPI_Group group2, // [in] second group (handle) + MPI_Group *newgroup // [out] intersection group (handle) +); +int MPI_Group_translate_ranks( // Translates the ranks of processes in one group to those in another group + MPI_Group group1, // [in] group1 (handle) + int n, // [in] number of ranks in ranks1 and ranks2 arrays (integer) + const int ranks1[], // [in] array of zero or more valid ranks in group1 + MPI_Group group2, // [in] group2 (handle) + int ranks2[] // [out] array of corresponding ranks in group2, MPI_UNDEFINED when no correspondence exists. +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node151.htm +int MPI_Group_union( // Produces a group by combining two groups + MPI_Group group1, // first group (handle) + MPI_Group group2, // second group (handle) + MPI_Group *newgroup // union group (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node152.htm +int MPI_Error_class( // Converts an error code into an error class + int errorcode, // Error code returned by an MPI routine + int *errorclass // Error class associated with errorcode +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node222.htm#Node222 +int MPI_Error_string( // Return a string for a given error code + int errorcode, // [in] Error code returned by an MPI routine or an MPI error class + char *string, // [out] Text that corresponds to the errorcode + int *resultlen // [out] Length of string +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node221.htm#Node221 +inline int MPI_Finalize( // Terminates MPI execution environment +){return MPI_SUCCESS;} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm +int MPI_Finalized( // Indicates whether MPI_Finalize has been called + int *flag // [out] true if MPI was finalized (logical) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node227.htm +inline int MPI_Init( // Initialize the MPI execution environment + int *argc, // [in] Pointer to the number of arguments + char ***argv // [in] Pointer to the argument vector +){return MPI_SUCCESS;} // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node28.htm +int MPI_Init_thread( // Initialize the MPI execution environment + int *argc, // [in] Pointer to the number of arguments + char ***argv, // [in] Pointer to the argument vector + int required, // [in] Level of desired thread support + int *provided // [out] Level of provided thread support +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm +int MPI_Initialized( // Indicates whether MPI_Init has been called. + int *flag // [out] Flag is true if MPI_Init or MPI_Init_thread has been called and false otherwise. +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node225.htm +int MPI_Intercomm_create( // Creates an intercommuncator from two intracommunicators + MPI_Comm local_comm, // [in] Local (intra)communicator + int local_leader, // [in] Rank in local_comm of leader (often 0) + MPI_Comm peer_comm, // [in] Communicator used to communicate between a designated process in the other communicator. Significant only at the process in local_comm with rank local_leader. + int remote_leader, // [in] Rank in peer_comm of remote leader (often 0) + int tag, // [in] Message tag to use in constructing intercommunicator; if multiple MPI_Intercomm_creates are being made, they should use different tags (more precisely, ensure that the local and remote leaders are using different tags for each MPI_intercomm_create). + MPI_Comm *newintercomm // [out] Created intercommunicator +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node168.htm +int MPI_Iprobe( // Nonblocking test for a message + int source, // [in] source rank, or MPI_ANY_SOURCE (integer) + int tag, // [in] tag value or MPI_ANY_TAG (integer) + MPI_Comm comm, // [in] communicator (handle) + int *flag, // [out] True if a message with the specified source, tag... + MPI_Status *status // [out] status object (Status) +); +int MPI_Is_thread_main( // This function can be called by a thread to determine if it is the main thread (the thread that called MPI_INIT or MPI_INIT_THREAD). + int *flag // true if calling thread is main thread, false otherwise (logical) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm +// Returns the upper bound on the amount of space needed to pack a message +int MPI_Pack_size( + int incount, + MPI_Datatype datatype, + MPI_Comm comm, + int *size +); +int MPI_Mprobe( + int source, // rank of source or MPI_ANY_SOURCE (integer) + int tag, // message tag or MPI_ANY_TAG (integer) + MPI_Comm comm, // communicator (handle) + MPI_Message *message, // returned message (handle) + MPI_Status *status // status object (Status) +); +int MPI_Mrecv( + void* buf, // initial address of receive buffer (choice) + int count, // number of elements in receive buffer (non-negati) + MPI_Datatype datatype, // datatype of each receive buffer element (handle) + MPI_Message *message, // message (handle) + MPI_Status *status // status object (Status) +); +// Blocking test for a message +int MPI_Probe( // like MPI_IMPROBE except that it is a blocking call that returns only after a matching message has been found. + int source, // [in] source rank, or MPI_ANY_SOURCE (integer) + int tag, // [in] tag value or MPI_ANY_TAG (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Status *status // [out] status object (Status) +); +int MPI_Query_thread( // The following function can be used to query the current level of thread support. + int *provided // provided level of thread support (integer) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node303.htm#Node303 +int MPI_Send( // Performs a blocking send + const void *buf, // [in] initial address of send buffer (choice) + int count, // [in] number of elements in send buffer (nonnegat...) + MPI_Datatype datatype, // [in] datatype of each send buffer element (handle) + int dest, // [in] rank of destination (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm // [in] communicator (handle) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node47.htm#Node47 +int MPI_Recv( // Blocking receive for a message + void *buf, // [out] initial address of receive buffer (choice) + int count, // [in] maximum number of elements in receive buffer... + MPI_Datatype datatype, // [in] datatype of each receive buffer element... + int source, // [in] rank of source (integer) + int tag, // [in] message tag (integer) + MPI_Comm comm, // [in] communicator (handle) + MPI_Status *status // [out] status object (Status) +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node50.htm#Node50 +double MPI_Wtime( // Returns an elapsed time on the calling processor +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm +double MPI_Wtick( // Returns the resolution of MPI_Wtime +); // http://mpi-forum.org/docs/mpi-3.1/mpi31-report/node37.htm#Node37 + + +inline int MPI_Status_set_cancelled(MPI_Status *status, int flag) +{ + return MPI_SUCCESS; +} + +inline int MPI_Test_cancelled(const MPI_Status *status, int *flag) +{ + if (flag) *flag = true; + return MPI_SUCCESS; +} + +typedef void (MPI_User_function)(void *a, void *b, int *len, MPI_Datatype *); +inline int MPI_Op_create(MPI_User_function *user_fn, int commute, MPI_Op *op) +{ + return MPI_SUCCESS; +} + +inline int MPI_Op_free(MPI_Op *op) +{ + if (op) *op = MPI_NO_OP; + return MPI_SUCCESS; +} + + +//const int MPI_MAX_PROCESSOR_NAME = 128; + + +inline int MPI_Get_address(const void *location, MPI_Aint *address) +{ + return MPI_SUCCESS; +} + + +// Memory handling + +//#include + +inline int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr) +{ + //if (baseptr) *(void **)baseptr = malloc(size); + if (baseptr) *(void **)baseptr = 0; + return MPI_SUCCESS; +} + +inline int MPI_Free_mem(void *base) +{ + //free(base); + return MPI_SUCCESS; +} + + +// MPI Requests + + +struct MPI_Request { + bool operator!=(const MPI_Request &o) { return false; } +}; +static struct MPI_Request MPI_REQUEST_NULL; + +inline int MPI_Request_get_status(MPI_Request request, int *flag, MPI_Status *status) +{ + if (flag) *flag = true; + return MPI_SUCCESS; +} + + +inline int MPI_Cancel(MPI_Request *request) +{ + return MPI_SUCCESS; +} + +inline int MPI_Request_free(MPI_Request *request) +{ + return MPI_SUCCESS; +} + +inline int MPI_Start(MPI_Request *request) +{ + return MPI_SUCCESS; +} + +inline int MPI_Wait(MPI_Request *request, MPI_Status *status) +{ + return MPI_SUCCESS; +} + +inline int MPI_Waitall(int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[]) +{ + return MPI_SUCCESS; +} + +inline int MPI_Testsome(int incount, MPI_Request array_of_requests[], int *outcount, int array_of_indices[], MPI_Status array_of_statuses[]) +{ + if (outcount) *outcount = 0; + return MPI_SUCCESS; +} + +static MPI_Status* MPI_STATUSES_IGNORE; + + +enum +{ + MPI_IDENT, + MPI_CONGRUENT, + MPI_SIMILAR, + MPI_UNEQUAL +}; + +enum { + MPI_GRAPH, + MPI_CART, + MPI_DIST_GRAPH +}; + +enum { + MPI_PROC_NULL, + MPI_ANY_SOURCE, + MPI_ANY_TAG, + MPI_UNDEFINED, + MPI_BSEND_OVERHEAD, + MPI_KEYVAL_INVALID, + MPI_LOCK_EXCLUSIVE, + MPI_LOCK_SHARED, + MPI_ROOT +}; + +enum { + MPI_MODE_APPEND, + MPI_MODE_CREATE, + MPI_MODE_DELETE_ON_CLOSE, + MPI_MODE_EXCL, + MPI_MODE_NOCHECK, + MPI_MODE_NOPRECEDE, + MPI_MODE_NOPUT, + MPI_MODE_NOSTORE, + MPI_MODE_NOSUCCEED, + MPI_MODE_RDONLY, + MPI_MODE_RDWR, + MPI_MODE_SEQUENTIAL, + MPI_MODE_UNIQUE_OPEN, + MPI_MODE_WRONLY + +}; + + +inline int MPI_Comm_group(MPI_Comm comm, MPI_Group *group) +{ + return MPI_SUCCESS; +} + +inline int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm) +{ + return MPI_SUCCESS; +} + +inline int MPI_Comm_compare(MPI_Comm comm1, MPI_Comm comm2, int *result) +{ + if (result) *result = MPI_UNEQUAL; + return MPI_SUCCESS; +} + +inline int MPI_Comm_test_inter(MPI_Comm comm, int *flag) +{ + if (flag) *flag = true; + return MPI_SUCCESS; +} + + +inline int MPI_Topo_test(MPI_Comm comm, int *status) +{ + if (status) *status = MPI_UNDEFINED; + return MPI_SUCCESS; +} + +inline int MPI_Comm_rank(MPI_Comm comm, int *rank) +{ + if (rank) *rank = 0; + return MPI_SUCCESS; +} + +// const should not be there +inline int MPI_Comm_disconnect(const MPI_Comm *comm) +{ + //if (comm) *comm = MPI_COMM_NULL; + return MPI_SUCCESS; +} + +// const on last arg should not be there +inline int MPI_Comm_accept(const char *port_name, MPI_Info info, int root, MPI_Comm comm, const MPI_Comm *newcomm) +{ + return MPI_SUCCESS; +} + +// const on last arg should not be there +inline int MPI_Comm_connect(const char *port_name, MPI_Info info, int root, MPI_Comm comm, const MPI_Comm *newcomm) +{ + return MPI_SUCCESS; +} + +inline int MPI_Abort(MPI_Comm comm, int errorcode) +{ + // should call exit or something here + return MPI_SUCCESS; +} + +inline int MPI_Comm_call_errhandler(MPI_Comm comm, int errorcode) +{ + return MPI_SUCCESS; +} + +inline int MPI_Barrier(MPI_Comm comm) +{ + return MPI_SUCCESS; +} + +inline int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm) +{ + return MPI_SUCCESS; +} + + + +#endif diff --git a/fake/mpic++ b/fake/mpic++ new file mode 100755 index 00000000000..b0170c6b023 --- /dev/null +++ b/fake/mpic++ @@ -0,0 +1,2 @@ +c++ -I/home/correaa/prj/alf/boost/mpi3/fake $@ + diff --git a/fake/mpicc b/fake/mpicc new file mode 100755 index 00000000000..2e86f25c50a --- /dev/null +++ b/fake/mpicc @@ -0,0 +1,2 @@ +cc -I/home/correaa/prj/alf/boost/mpi3/fake $@ + diff --git a/fake/mpirun b/fake/mpirun new file mode 100755 index 00000000000..8f1104b73c0 --- /dev/null +++ b/fake/mpirun @@ -0,0 +1,2 @@ +$3 + diff --git a/fake/test/CMakeLists.txt b/fake/test/CMakeLists.txt new file mode 100644 index 00000000000..0e0f1635d3f --- /dev/null +++ b/fake/test/CMakeLists.txt @@ -0,0 +1,33 @@ + +cmake_minimum_required(VERSION 3.6.0) + +enable_testing() +include(CTest) + +SET(FAKE_MPI_DIR "..") +SET(MPI_WRAPPER_DIR "../../..") + +set(TEST_SRCS + test_c_mpi_init.c + test_mpi_init.cpp + test_wrapper_mpi_init.cpp +) + +foreach(TEST_FILE ${TEST_SRCS}) + SET(TEST_EXE "${TEST_FILE}x.x") + add_executable(${TEST_EXE} ${TEST_FILE}) + target_include_directories(${TEST_EXE} PUBLIC "..") + if (TEST_FILE MATCHES ".cpp$") + target_include_directories(${TEST_EXE} PUBLIC ${MPI_WRAPPER_DIR}) + endif() + add_test(NAME ${TEST_EXE} COMMAND ./${TEST_EXE}) +endforeach() + + +# Compile-time check for multiply defined symbols +add_library(library_check ${MPI_WRAPPER_DIR}/test/library_check.cpp) +add_executable(library_main ${MPI_WRAPPER_DIR}/test/library_main.cpp) +target_link_libraries(library_main library_check) +target_include_directories(library_check PUBLIC ${FAKE_MPI_DIR} ${MPI_WRAPPER_DIR}) +target_include_directories(library_main PUBLIC ${FAKE_MPI_DIR} ${MPI_WRAPPER_DIR}) + diff --git a/fake/test/test_c_mpi_init.c b/fake/test/test_c_mpi_init.c new file mode 100644 index 00000000000..d9db60b13ab --- /dev/null +++ b/fake/test/test_c_mpi_init.c @@ -0,0 +1,11 @@ + +#include + +/* Test MPI initialization in C */ + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + MPI_Finalize(); + return 0; +} diff --git a/fake/test/test_mpi_init.cpp b/fake/test/test_mpi_init.cpp new file mode 100644 index 00000000000..a706df68b84 --- /dev/null +++ b/fake/test/test_mpi_init.cpp @@ -0,0 +1,11 @@ + +#include + +// Test MPI initialization in C++ + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + MPI_Finalize(); + return 0; +} diff --git a/fake/test/test_wrapper_mpi_init.cpp b/fake/test/test_wrapper_mpi_init.cpp new file mode 100644 index 00000000000..fb283605789 --- /dev/null +++ b/fake/test/test_wrapper_mpi_init.cpp @@ -0,0 +1,11 @@ + +#include +#include + +namespace mpi3 = boost::mpi3; + +int main(int argc, char **argv) +{ + mpi3::environment(argc, argv); + return 0; +} diff --git a/include/mpi3/FILE.hpp b/include/mpi3/FILE.hpp new file mode 100644 index 00000000000..dfa945c9600 --- /dev/null +++ b/include/mpi3/FILE.hpp @@ -0,0 +1,145 @@ +#ifndef ALF_BOOST_MPI3_FILE_HPP +#define ALF_BOOST_MPI3_FILE_HPP + +namespace boost{ +namespace mpi3{ + +struct FILE{ + MPI_File impl_; + static void delete_(std::string const& filename){MPI_File_delete(filename.c_str(), MPI_INFO_NULL);} + int amode() const{ + int ret; + MPI_File_get_amode(impl_, &ret); + return ret; + } + bool atomicity() const{ + int ret; + MPI_File_get_atomicity(impl_, &ret); + return ret; + } + void atomicity(bool flag){MPI_File_set_atomicity(impl_, flag);} + MPI_Offset byte_offset(MPI_Offset offset){ + MPI_Offset disp; + MPI_File_get_byte_offset(impl_, offset, &disp); + return disp; + } + info hints() const{ + info ret; + MPI_File_get_info(impl_, &ret.impl_); + return ret; + } + void hints(info set){MPI_File_set_info(impl_, set.impl_);} + MPI_Offset position() const{ + MPI_Offset offset; + MPI_File_get_position(impl_, &offset); + return offset; + } + MPI_Offset position_shared() const{ + MPI_Offset offset; + MPI_File_get_position_shared(impl_, &offset); + return offset; + } + MPI_Offset size() const{ + MPI_Offset ret; + MPI_File_get_size(impl_, &ret); + return ret; + } + MPI_Aint extent(boost::mpi3::type const& t) const{ + MPI_Aint ret; + MPI_File_get_type_extent(impl_, t.impl_, &ret); + return ret; + } + void view(); // int MPI_File_get_view + template::value_type, class datatype = detail::datatype> + auto read_n(ContiguousIterator O, Size count) const{ + status s; + MPI_File_read(impl_, std::addressof(*O), count, datatype{}, &s.impl_); + return O + count; + } + template::value_type, class datatype = detail::datatype> + auto read_all_n(ContiguousIterator O, Size count) const{ + status s; + MPI_File_read_all(impl_, std::addressof(*O), count, datatype{}, &s.impl_); + return O + count; + } + // MPI_File_read_all_begin + // MPI_File_read_all_end + template::value_type, class datatype = detail::datatype> + auto read_at_n(MPI_Offset offset, ContiguousIterator O, Size count) const{ + status s; + MPI_File_read_at(offset, impl_, std::addressof(*O), count, datatype{}, &s.impl_); + return O + count; + } + template::value_type, class datatype = detail::datatype> + auto read_at_all_n(MPI_Offset offset, ContiguousIterator O, Size count) const{ + status s; + MPI_File_read_at_all(offset, impl_, std::addressof(*O), count, datatype{}, &s.impl_); + return O + count; + } + // MPI_File_read_at_all_begin + // MPI_File_read_at_all_end + template::value_type, class datatype = detail::datatype> + auto read_ordered_n(ContiguousIterator O, Size count) const{ + status s; + MPI_File_read_ordered(impl_, std::addressof(*O), count, datatype{}, &s.impl_); + return O + count; + } + // MPI_File_read_ordered_begin + // MPI_File_read_ordered_end + template::value_type, class datatype = detail::datatype> + auto read_shared_n(ContiguousIterator O, Size count) const{ + status s; + MPI_File_read_ordered(impl_, std::addressof(*O), count, datatype{}, &s.impl_); + return O + count; + } + void seek(MPI_Offset offset, int whence){MPI_File_seek(impl_, offset, whence);} + void seek_set(MPI_Offset offset){seek(offset, MPI_SEEK_SET);} + void seek_current(MPI_Offset offset){seek(offset, MPI_SEEK_CUR);} + void seek_end(MPI_Offset offset = 0){seek(offset, MPI_SEEK_END);} + + void seek_shared(MPI_Offset offset, int whence){MPI_File_seek_shared(impl_, offset, whence);} + //MPI_File_set_errhandler + template::value_type, class datatype = detail::datatype> + request iread_n(ContiguousIterator I, Size count); + template::value_type, class datatype = detail::datatype> + request iread_at_n(MPI_Offset offset, ContiguousIterator I, Size count); + template::value_type, class datatype = detail::datatype> + request iread_shared_n(ContiguousIterator I, Size count); + template::value_type, class datatype = detail::datatype> + status write_n(ContiguousIterator I, Size count) const{ + status ret; + MPI_File_write(impl_, std::addressof(*I), count, datatype{}, &ret.impl_); + return ret; + } + template::value_type, class datatype = detail::datatype> + status write_all_n(ContiguousIterator I, Size count) const{ + status ret; + MPI_File_write_all(impl_, std::addressof(*I), count, datatype{}, &ret.impl_); + return ret; + } + template::value_type, class datatype = detail::datatype> + request iwrite_n(ContiguousIterator I, Size count); + template::value_type, class datatype = detail::datatype> + request iwrite_at_n(MPI_Offset offset, ContiguousIterator I, Size count); + template::value_type, class datatype = detail::datatype> + request iwrite_shared_n(ContiguousIterator I, Size count); + void preallocate(MPI_Offset size){MPI_File_preallocate(impl_, size);} + void resize(MPI_Offset size){MPI_File_set_size(impl_, size);} + void sync(){MPI_File_sync(impl_);} +}; + +int ftell(FILE* stream){return stream->position();} +int fseek(FILE* stream, MPI_Offset offset, int whence/*origin*/){ + return MPI_File_seek(stream->impl_, offset, whence); +} + +FILE* communicator::fopen(const char* filename, int amode = MPI_MODE_RDWR | MPI_MODE_CREATE){ + FILE* ret; + MPI_File_open(impl_, filename, amode, MPI_INFO_NULL, &(ret->impl_)); + return ret; +} + + +}} +#endif + diff --git a/include/mpi3/address.hpp b/include/mpi3/address.hpp new file mode 100644 index 00000000000..3d4b0ada580 --- /dev/null +++ b/include/mpi3/address.hpp @@ -0,0 +1,53 @@ +/* -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;-*- */ +//#if COMPILATION +//mpic++ -D_TEST_BOOST_MPI3_ADDRESS -x c++ $0 -o $0x -lboost_serialization&&mpirun --oversubscribe -n 4 $0x&&rm $0x;exit +//#endif +// Copyright 2018-2021 Alfredo A. Correa + +#ifndef BOOST_MPI3_ADDRESS_HPP +#define BOOST_MPI3_ADDRESS_HPP + +#include "../mpi3/detail/call.hpp" +#include "../mpi3/types.hpp" + +// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 +// #include + +namespace boost{ +namespace mpi3{ + +inline address get_address(void const* location){ + address ret; // NOLINT(cppcoreguidelines-init-variables) : delayed init + // this function requires an initialized environment, TODO should be a (static?) member of environment? + MPI_(Get_address)(location, &ret); // MPI_Address is deprecated + return ret; +} + +template +address addressof(T const& t){return get_address(std::addressof(t));} + +} // end namespace mpi3 +} // end namespace boost + +//#ifdef _TEST_BOOST_MPI3_ADDRESS + +//#include "../mpi3/main.hpp" + +//using std::cout; +//namespace mpi3 = boost::mpi3; + +//int mpi3::main(int, char*[], mpi3::communicator world){ + +// std::cout << "dsadsad" << std::endl; + +// std::vector v(10); +// mpi3::address a1 = mpi3::addressof(v[0]); +// mpi3::address a2 = mpi3::addressof(v[1]); +// assert( a2 - a1 == sizeof(int) ); + +// return 0; +//} + +//#endif +#endif + diff --git a/include/mpi3/allocator.hpp b/include/mpi3/allocator.hpp new file mode 100644 index 00000000000..d96fd09f37c --- /dev/null +++ b/include/mpi3/allocator.hpp @@ -0,0 +1,137 @@ +// Copyright 2018-2023 Alfredo A. Correa + +#ifndef BOOST_MPI3_ALLOCATOR_HPP +#define BOOST_MPI3_ALLOCATOR_HPP + +#include "../mpi3/address.hpp" + +#include + +#include +#include + +namespace boost{ +namespace mpi3{ + +struct /*__attribute__((aligned(0)))*/ bad_alloc : std::bad_alloc{using std::bad_alloc::bad_alloc;}; + +inline void* malloc(mpi3::size_t size) { + void* ret; // NOLINT(cppcoreguidelines-init-variables) delayed init +#if not defined(EXAMPI) + int const s = MPI_Alloc_mem(size, MPI_INFO_NULL, &ret); + if(s != MPI_SUCCESS) {return nullptr;} //s throw bad_alloc();//"cannot allocate " + std::to_string(size) + " bytes"); +#else + ret = std::malloc(size); +#endif + return ret; +} + +inline void free(void* ptr){ +#if not defined(EXAMPI) + MPI_(Free_mem)(ptr); +#else + std::free(ptr); +#endif +} + +template +struct /*__attribute__((aligned(0)))*/ allocator{ + using size_type = mpi3::size_t; + using value_type = T; + using pointer = T*; + + allocator() = default; + + // cppcheck-suppress noExplicitConstructor + template allocator(allocator const&/*other*/) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) : allocator convention + + auto allocate(size_type n) { + if(void* ptr = mpi3::malloc(n * static_cast(sizeof(T)))) { + return static_cast(ptr); + } + throw bad_alloc(); + } + void deallocate(pointer p, std::size_t /*size*/) { mpi3::free(p); } + static size_type max_size() { return std::numeric_limits::max(); } +}; + +template +struct uallocator : allocator{ + template void construct(U* /*unused*/){ + static_assert( + std::is_trivially_destructible{}, + "uallocator cannot be used with non trivial types" + ); + } +}; + +template< class T1, class T2 > constexpr +bool operator==(allocator const&/*self*/, uallocator const&/*other*/) { // TODO(correaa) check that both(?) are trivial? + return true; +} + +template< class T1, class T2 > constexpr +bool operator==(uallocator const&/*self*/, allocator const&/*other*/) { + return true; +} + +template +constexpr std::add_const_t& as_const(T& t) noexcept{return t;} + +} // end namespace mpi3 +} // end namespace boost + +//#ifdef _TEST_BOOST_MPI3_ALLOCATOR + +//#include "../mpi3/main.hpp" + +//#include +//#include + +//#include + +//namespace mpi3 = boost::mpi3; +//using std::cout; + +//int mpi3::main(int argc, char* argv[], mpi3::communicator world){ + +// std::vector> v(1000000); +// std::vector> uv(1000000); +// std::iota(v.begin(), v.end(), 0.); +// using boost::mpi3::data; +// assert( data(uv.begin()) == &*uv.begin() ); +// assert( std::accumulate(v.begin(), v.end(), 0.) == (v.size()*(v.size() - 1))/2 ); +// return 0; +// +// { +// boost::container::flat_set, mpi3::allocator > fs; +// fs.insert(5.); +// fs.insert(3.); +// auto it = fs.begin(); +// assert(*it == 3.); +// ++it; +// assert(*it == 5.); +// } +// { +// boost::container::flat_set, std::allocator_traits>::rebind_alloc> fs; +// fs.insert(5); +// fs.insert(3); +// auto it = fs.begin(); +// assert(*it == 3); +// ++it; +// assert(*it == 5); +// } +// { +// boost::container::flat_set, std::less>, mpi3::allocator>> fsp; +// fsp.insert({1.,2.}); +// fsp.insert({3.,4.}); +// auto it = fsp.begin(); +// assert(*it == std::make_pair(1.,2.)); +// ++it; +// assert(*it == std::make_pair(3.,4.)); +// } +// return 0; +//} + +//#endif +#endif diff --git a/include/mpi3/buffer.hpp b/include/mpi3/buffer.hpp new file mode 100644 index 00000000000..f973524ee3b --- /dev/null +++ b/include/mpi3/buffer.hpp @@ -0,0 +1,111 @@ +#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ +(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wall -Wfatal-errors -D_TEST_BOOST_MPI3_BUFFER $0x.cpp -o $0x.x && time mpirun -n 4 $0x.x $@ && rm -f $0x.x $0x.cpp; exit +#endif +#ifndef BOOST_MPI3_BUFFER_HPP +#define BOOST_MPI3_BUFFER_HPP + +#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 +#include + +#include +#include + +namespace boost{ +namespace mpi3{ + +template +void buffer_attach_n(T* data, std::size_t n){ + static_assert(sizeof(T)%sizeof(char) == 0, ""); + int status = MPI_Buffer_attach((void*)data, n*sizeof(T)/sizeof(char)); + if(status != MPI_SUCCESS) throw std::runtime_error("cannot attach buffer"); +} +template +void buffer_attach(T* first, T* last){ + buffer_attach_n(first, std::distance(first, last)); +} + +template +void attach_n(T* data, Size n){return buffer_attach_n(data, n);} + +template +void attach(T* first, T* last){return buffer_attach(first, last);} + +template +void attach(C& c){return buffer_attach(c);} + +std::pair buffer_detach(){ + char* buffer = 0; + int size = -1; + int s = MPI_Buffer_detach(&buffer, &size); + if(s != MPI_SUCCESS) throw std::runtime_error("cannot buffer detach"); + return {buffer, size}; +} +std::pair detach(){ + return buffer_detach(); +} + +template +struct scoped_buffer_attach{ + scoped_buffer_attach(T* data, int size){ + buffer_attach_n(data, size); + } + ~scoped_buffer_attach(){ + buffer_detach(); + } +}; + +template +struct scoped_buffer{ + T* buf_; + int size_; + scoped_buffer(int size) : size_(size){ + buf_ = new T[size]; + buffer_attach_n(buf_, size_*sizeof(T)); + } + ~scoped_buffer(){ + std::pair check = buffer_detach(); + assert(check.first == (char*)(buf_)); + // assert(check.second/sizeof(T) == size_); + delete[] buf_; + } +}; + +}} + +#ifdef _TEST_BOOST_MPI3_BUFFER + +#include "../mpi3/main.hpp" +#include + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator world){ + + std::vector a(10); + + mpi3::scoped_buffer buf(2000); + + for(int j = 0; j !=10; ++j){ + auto r = world.bsend_init_n(a.data(), a.size(), 0, 27 + j); + for(int i = 0; i != 10; ++i) a[i] = (world.rank() + 10*j)*world.size() + i; + r.start(); + // r.wait(); // requests wait automatically on destruction (they don't start automatically) + } + + std::vector b(10); + + if(world.root()) + for(int i = 0; i != world.size(); ++i) + for(int j = 0; j != 10; ++j){ + world.receive(b.data(), i, 27 + j); + for(int k = 0; k != 10; ++k) + if(b[k] != (i + 10*j)*world.size() + k) assert(0); + } + + return 0; +} + +#endif +#endif + diff --git a/include/mpi3/cartesian_communicator.hpp b/include/mpi3/cartesian_communicator.hpp new file mode 100644 index 00000000000..f993cc2aa56 --- /dev/null +++ b/include/mpi3/cartesian_communicator.hpp @@ -0,0 +1,387 @@ +// Copyright 2018-2023 Alfredo A. Correa + +#ifndef BOOST_MPI3_CARTESIAN_COMMUNICATOR_HPP +#define BOOST_MPI3_CARTESIAN_COMMUNICATOR_HPP + +#include +#include + +#include + +namespace boost::mpi3 { + +using dimensionality_type = int; + +static constexpr dimensionality_type dynamic_extent = -1; + +template struct cartesian_communicator; + +template<> +struct cartesian_communicator : communicator { + + cartesian_communicator() = default; + + cartesian_communicator(cartesian_communicator const&) = delete; + cartesian_communicator(cartesian_communicator&&) = default; + // vvv--- this is an unusual "duplicate" constructor + cartesian_communicator(cartesian_communicator& other) : communicator{other} {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) cannot be defaulted because bug in nvcc 11 + + template + cartesian_communicator(communicator& comm_old, Shape const& s, Period const& p) { + assert(s.size() == p.size()); + using dimensionality_type = int; + MPI_(Cart_create)(comm_old.get(), static_cast(s.size()), s.data(), p.data(), /*reorder*/ true, &impl_); + // assert(impl_ != MPI_COMM_NULL); // null communicator is a valid outcome + // TODO(correaa) try with mpich, WAS: there is an bug in mpich, in which if the remaining dim are none then the communicator is not well defined. + } + + template + cartesian_communicator(communicator& old, Shape const& s) + : cartesian_communicator{old, s, std::vector(s.size(), true)} {} + + cartesian_communicator(communicator& comm_old, std::initializer_list shape) + : cartesian_communicator(comm_old, std::vector(shape)) {} + + cartesian_communicator(communicator& comm_old, std::initializer_list shape, std::initializer_list period) + : cartesian_communicator(comm_old, std::vector(shape), std::vector(period)) {} + + +#if not defined(EXAMPI) + [[deprecated("use dimensionality() instead of dimension")]] + int dimension() const { + int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init // TODO(correaa) + MPI_(Cartdim_get)(impl_, &ret); + return ret; + } +#endif + + cartesian_communicator& operator=(cartesian_communicator const&) = delete; + cartesian_communicator& operator=(cartesian_communicator&&) = default; + // vvv nvcc 11 workaround, needs explicit definition of duplicate assigment + [[deprecated]] cartesian_communicator& operator=(cartesian_communicator& other) { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) "duplicate" assignment + if(this == std::addressof(other)) { + return *this; + } // lints cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator + if(not (compare(other) == boost::mpi3::detail::congruent)) {throw std::logic_error{"assignment is going to be deprecated"};} + // communicator::operator=(other); + return *this; + } + + ~cartesian_communicator() = default; + +#if not defined(EXAMPI) + int dimensionality() const { + int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_(Cartdim_get)(impl_, &ret); + return ret; + } + + std::vector coordinates() const { + std::vector ret(static_cast::size_type>(dimensionality())); + MPI_(Cart_coords)(impl_, rank(), dimensionality(), ret.data()); + return ret; + } + + auto topology() const { + auto const maxdims = static_cast(dimensionality()); // TODO(correaa) use safe cast + class topology_t { + std::vector dimensions_; + std::vector periods_; + std::vector coordinates_; + friend mpi3::cartesian_communicator; + + public: + explicit topology_t(std::size_t n) : dimensions_(n), periods_(n), coordinates_(n) {} + + auto const& dimensions() const { return dimensions_; } + auto const& periods() const { return periods_; } + auto const& coordinates() const { return coordinates_; } + } ret(maxdims); + + MPI_(Cart_get)(impl_, static_cast(maxdims), ret.dimensions_.data(), ret.periods_.data(), ret.coordinates_.data()); + + assert(ret.coordinates() == coordinates()); + return ret; + } + + std::vector shape() const { return topology().dimensions(); } + + std::vector periods() const { + auto ps = topology().periods(); + return {ps.begin(), ps.end()}; + } +#endif + + auto num_elements() const { return size(); } + + template + auto operator()(Coord const& coord) { + int rank = -1; + MPI_(Cart_rank)(impl_, coord.data(), &rank); + return (*this)[rank]; + // return operator[](rank); + } + +#if not defined(EXAMPI) + // int MPI_Cart_map not implemented + cartesian_communicator sub_aux(std::vector const& remain_dims) { + assert(static_cast(remain_dims.size()) == dimensionality()); + cartesian_communicator ret; + MPI_(Cart_sub)(impl_, remain_dims.data(), &ret.impl_); + return ret; + } + + template> + cartesian_communicator sub(RemainDim const& remain_dims) { + return sub_aux(std::vector(remain_dims.begin(), remain_dims.end())); + } + cartesian_communicator sub() { + assert(dimensionality() > 1); + std::vector remain(static_cast(dimensionality()), 1 /*true*/); + remain[0] = 0 /*false*/; + return sub_aux(remain); + } +#endif +}; + +enum fill_t { + fill = 0, + _ = 0 +}; + +struct circular_communicator; + +template +struct cartesian_communicator : cartesian_communicator<> { + cartesian_communicator() = default; + + cartesian_communicator(cartesian_communicator& other) : cartesian_communicator<>{other} {} + cartesian_communicator(cartesian_communicator const&) = delete; + cartesian_communicator(cartesian_communicator&&) noexcept = default; + + ~cartesian_communicator() = default; + +// #if not defined(EXAMPI) + static std::array division(int nnodes, std::array suggest = {}) { + MPI_(Dims_create)(nnodes, D, suggest.data()); + return suggest; + } +// #endif + + constexpr static dimensionality_type dimensionality = D; + + cartesian_communicator( + communicator& other, + std::array dims, + std::array periods + ) + try + : cartesian_communicator<>{ + other, + division(other.size(), dims), + std::apply([](auto... e) { return std::array{e...}; }, periods) + } {} + catch(std::runtime_error& e) { + std::ostringstream ss; + std::copy(dims.begin(), dims.end(), std::ostream_iterator{ss, " "}); + throw std::runtime_error{"cannot create cartesian communicator with constrains " + ss.str() + " from communicator of size " + std::to_string(other.size()) + " because " + e.what()}; + } + + cartesian_communicator( + communicator& other, + std::array dims + ) : cartesian_communicator( + other, + dims, + std::apply([](auto... e) { return std::array{(static_cast(e), true)...}; }, std::array{}) + ) {} + + explicit cartesian_communicator( + communicator& other + ) : cartesian_communicator(other, std::array{}) {} + + auto topology() const { + struct topology_t { + std::array dimensions, periods, coordinates; + } ret = {}; + MPI_(Cart_get)( + impl_, dimensionality, + ret.dimensions.data(), ret.periods.data(), ret.coordinates.data() + ); + return ret; + } + + constexpr auto dimensions() const { return topology().dimensions; } + + cartesian_communicator& operator=(cartesian_communicator const&) = delete; + cartesian_communicator& operator=(cartesian_communicator&&) noexcept = default; + // vvv nvcc 11 workaround, needs explicit definition of duplicate assigment + [[deprecated]] cartesian_communicator& operator=(cartesian_communicator& other) { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) duplicate assignment + if(this == std::addressof(other)) { + return *this; + } // lints cert-oop54-cpp + if(not (compare(other) == boost::mpi3::detail::congruent)) {throw std::logic_error{"assignment is going to be deprecated"};} + // cartesian_communicator<>::operator=(other); // NOLINT(clang-diagnostic-deprecated-declarations) + return *this; + } + + cartesian_communicator<1> axis(int d) { // TODO(correaa) return a circular_communicator + assert(d >= 0); + assert(d < D); + cartesian_communicator<1> ret; + std::array remains{}; + remains.fill(false); + remains[static_cast(d)] = true; // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index) + MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); + return ret; + } + + template auto axis() -> circular_communicator; + + auto plane(int d1, int d2) { + assert(d1 >= 0 and d2 >= 0); + + auto const d1s = static_cast(d1); + auto const d2s = static_cast(d2); + + assert(d2s < D and d1s < d2s); + + std::array remains{}; + remains.at(d1s) = true; + remains.at(d2s) = true; + + cartesian_communicator<2> ret; + MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); + return ret; + } + + template + auto plane() { + static_assert(D1 < D2); + std::array remains{}; + remains.fill(false); + std::get(remains) = true; + std::get(remains) = true; + + cartesian_communicator<2> ret; + MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); + return ret; + } + + template auto shift(int displacement) const { + std::pair source_dest; + MPI_(Cart_shift)(impl_, Direction, displacement, &source_dest.first, &source_dest.second); + return source_dest; + } + + template + auto send_receive_shift(As... as, int displacement = 1) { + send_receive(as..., shift(displacement)); + } + + using coordinates_type = std::array; + + using cartesian_communicator<>::rank; +#if not defined(EXAMPI) + auto rank(coordinates_type cs) const -> int { + auto const ps = periods(); + auto const s = shape(); + for(std::size_t i = 0; i != D; ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm + if(ps[i] == false) { + assert(cs[i] >= 0); + assert(cs[i] < s[i]); + } + } + return MPI_(Cart_rank)(impl_, cs.data()); + } +#endif + auto coordinates(int r) const -> coordinates_type { + coordinates_type ret; + MPI_(Cart_coords)(impl_, r, D, ret.data()); + return ret; + } + auto coordinates() const -> coordinates_type { return coordinates(rank()); } + template + auto operator()(Indices... idx) { return (*this)[rank(std::array{idx...})]; } + + cartesian_communicator hyperplane(int d) { + static_assert(D != 0, "hyperplane not possible for 0D communicators"); +#if defined(MPICH_VERSION) + static_assert(D != 1, "hyperplane not possible for 1D communicators"); // they work in openMPI but do not work in MPICH +#endif + assert(d >= 0); + assert(d < D); + + cartesian_communicator ret; + std::array remains{}; + remains.fill(true); + + remains[static_cast(d)] = false; // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index) + MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); + return ret; + } +}; + +struct circular_communicator : cartesian_communicator<1> { + circular_communicator() = default; + + circular_communicator(circular_communicator& other) : cartesian_communicator<1>{other} {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) icpc needs this definition explicitly + circular_communicator(circular_communicator const&) = delete; + circular_communicator(circular_communicator&&) noexcept = default; + + ~circular_communicator() = default; + + explicit circular_communicator(communicator& other) + : cartesian_communicator<1>{other, {}, {true}} {} + + auto operator=(cartesian_communicator const&) -> circular_communicator& = delete; + auto operator=(circular_communicator&& other) noexcept -> circular_communicator& { + cartesian_communicator<1>::operator=(std::move(other)); + return *this; + } + // vvv nvcc 11 workaround, needs explicit definition of duplicate assigment + [[deprecated]] auto operator=(circular_communicator& other) -> circular_communicator& { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) duplicate assignment + if(this == std::addressof(other)) { + return *this; + } // lints cert-oop54-cpp + if(not (compare(other) == boost::mpi3::detail::congruent)) {throw std::logic_error{"assignment is going to be deprecated"};} + // cartesian_communicator<1>::operator=(other); // NOLINT(clang-diagnostic-deprecated-declarations) + return *this; + } + + auto coordinate() const { return std::get<0>(this->coordinates()); } + auto coordinate(int rank) const { return std::get<0>(this->coordinates(rank)); } + + using cartesian_communicator<1>::rank; + +#if not defined(EXAMPI) + auto rank(int coordinate) const { return cartesian_communicator<1>::rank({coordinate}); } +#endif + + template + auto rotate(As... as, int displacement) { return this->send_receive(as..., this->shift<0>(-displacement)); } + template + auto rotate(As... as) { return this->send_receive(as..., this->shift<0>(-1)); } + + template + auto unrotate(As... as, int displacement) { return this->send_receive(as..., this->shift<0>(+displacement)); } + template + auto unrotate(As... as) { return this->send_receive(as..., this->shift<0>(+1)); } +}; + +template using torus = cartesian_communicator; + +using ring = circular_communicator; + +template template +auto cartesian_communicator::axis() -> circular_communicator { + circular_communicator ret; + std::array remains{}; + remains.fill(false); + std::get
(remains) = true; + MPI_(Cart_sub)(impl_, remains.data(), &ret.get()); + return ret; +} + +} // end namespace boost::mpi3 +#endif diff --git a/include/mpi3/communication_mode.hpp b/include/mpi3/communication_mode.hpp new file mode 100644 index 00000000000..2d3c44d8696 --- /dev/null +++ b/include/mpi3/communication_mode.hpp @@ -0,0 +1,69 @@ +// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- +// Copyright 2018-2022 Alfredo A. Correa + +#ifndef MPI3_DETAIL_COMMUNICATION_MODE +#define MPI3_DETAIL_COMMUNICATION_MODE + +#include // if you get an error here probably you need to compile with and MPI compiler wrapper + +#include // forward + +namespace boost { +namespace mpi3 { + +struct blocking_mode {}; +struct nonblocking_mode {}; + +struct standard_communication_mode { + template static int Send(As... as) { return MPI_Send(as...); } // NOLINT(readability-identifier-naming) + template static int Recv(As... as) { return MPI_Recv(as...); } // NOLINT(readability-identifier-naming) + template static int ISend(As... as) { return MPI_Isend(as...); } // NOLINT(readability-identifier-naming) + template static int IRecv(As... as) { return MPI_Irecv(as...); } // NOLINT(readability-identifier-naming) +}; + +struct buffered_communication_mode { + template static int Send(As... as) { return MPI_Bsend(as...); } // NOLINT(readability-identifier-naming) + template static int Recv(As... as) { return MPI_Brecv(as...); } // NOLINT(readability-identifier-naming) + template static int ISend(As... as) { return MPI_Ibsend(as...); } // NOLINT(readability-identifier-naming) + template static int IRecv(As... as) { return MPI_Ibrecv(as...); } // NOLINT(readability-identifier-naming) +}; + +struct synchronous_communication_mode { + template static int Send(As... as) { return MPI_Ssend(as...); } // NOLINT(readability-identifier-naming) + template static int Recv(As... as) { return MPI_Srecv(as...); } // NOLINT(readability-identifier-naming) + template static int ISend(As... as) { return MPI_Issend(as...); } // NOLINT(readability-identifier-naming) + template static int IRecv(As... as) { return MPI_Isrecv(as...); } // NOLINT(readability-identifier-naming) +}; + +struct ready_communication_mode { + template static int Send(As... as) { return MPI_Rsend(as...); } // NOLINT(readability-identifier-naming) + template static int Recv(As... as) { return MPI_Rrecv(as...); } // NOLINT(readability-identifier-naming) + template static int ISend(As... as) { return MPI_Irsend(as...); } // NOLINT(readability-identifier-naming) + template static int IRecv(As... as) { return MPI_Irrecv(as...); } // NOLINT(readability-identifier-naming) +}; + +struct gather_mode { + template int operator()(As... as) const { return MPI_Gather(as...); } +}; +struct igather_mode { + template int operator()(As... as) const { return MPI_Igather(as...); } +}; + +struct all_gather_mode { + template int operator()(As... as) const { return MPI_Allgather(as...); } +}; + +struct reduce_mode { + template int operator()(As... as) const { return MPI_Reduce(as...); } +}; +struct ireduce_mode { + template int operator()(As... as) const { return MPI_Ireduce(as...); } +}; + +struct all_reduce_mode { + template int operator()(As... as) const { return MPI_Allreduce(as...); } +}; + +} // end namespace mpi3 +} // end namespace boost +#endif diff --git a/include/mpi3/communicator.hpp b/include/mpi3/communicator.hpp new file mode 100644 index 00000000000..57afb77d065 --- /dev/null +++ b/include/mpi3/communicator.hpp @@ -0,0 +1,3329 @@ +// Copyright 2018-2023 Alfredo A. Correa + +#ifndef MPI3_COMMUNICATOR_HPP +#define MPI3_COMMUNICATOR_HPP + +#include "../mpi3/communication_mode.hpp" +#include "../mpi3/error.hpp" +#include "../mpi3/generalized_request.hpp" +#include "../mpi3/group.hpp" +#include "../mpi3/handle.hpp" +#include "../mpi3/info.hpp" +#include "../mpi3/message.hpp" +#include "../mpi3/operation.hpp" +#include "../mpi3/port.hpp" +#include "../mpi3/request.hpp" +#include "../mpi3/status.hpp" +#include "../mpi3/type.hpp" + +#include "../mpi3/detail/basic_communicator.hpp" +#include "../mpi3/detail/buffer.hpp" +#include "../mpi3/detail/datatype.hpp" +#include "../mpi3/detail/equality.hpp" +#include "../mpi3/detail/iterator.hpp" +#include "../mpi3/detail/value_traits.hpp" + +#include "../mpi3/detail/package.hpp" + +#include "../mpi3/config/NODISCARD.hpp" + +//#define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 +#include + +#define BOOST_PACKAGE_ARCHIVE_SOURCE + +#include +#include + +#include +#include +#include +//#include // must be the last header + +#include +#include +#include +#include + +#include + +#include +#include + +// use this to avoid need for linking -lserialization +#ifdef _MAKE_BOOST_SERIALIZATION_HEADER_ONLY +//#include +#if BOOST_VERSION > 106000 && BOOST_VERSION < 106600 +#include "../mpi3/serialization_hack/singleton.cpp" +#endif +#if BOOST_VERSION < 105900 +#define BOOST_ARCHIVE_DECL +#define BOOST_SERIALIZATION_DECL +#endif +// NOLINTBEGIN(hicpp-use-auto,misc-const-correctness,modernize-use-auto) external code +#include "../mpi3/serialization_hack/archive_exception.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/basic_archive.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/basic_iarchive.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/basic_iserializer.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/basic_oarchive.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/basic_oserializer.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/extended_type_info.cpp" // NOLINT(bugprone-suspicious-include) hack +#include "../mpi3/serialization_hack/extended_type_info_typeid.cpp" // NOLINT(bugprone-suspicious-include,misc-const-correctness) hack +// NOLINTEND(hicpp-use-auto,misc-const-correctness,modernize-use-auto) + +#endif + +#include "../mpi3/package_archive.hpp" + +#include +#include +#include // iterator_traits +#include +#include +#include // std::accumulate +#include +#include +#include // is_same +#include + +namespace boost { +namespace mpi3 { + +#define SAFE_MPI_(F) check_mpi_(MPI_##F) // NOLINT(cppcoreguidelines-macro-usage) : name concatenation + +#if !defined(OPEN_MPI) || (OMPI_MAJOR_VERSION < 2) +#define OMPI_COMM_TYPE_NODE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_HWTHREAD MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_CORE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_L1CACHE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_L2CACHE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_L3CACHE MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_SOCKET MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_NUMA MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_BOARD MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_HOST MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_CU MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#define OMPI_COMM_TYPE_CLUSTER MPI_COMM_TYPE_SHARED // NOLINT(cppcoreguidelines-macro-usage) +#endif + +// https://www.open-mpi.org/doc/v4.0/man3/MPI_Comm_split_type.3.php#toc8 +enum class communicator_type : int { +#if not defined(EXAMPI) + shared = MPI_COMM_TYPE_SHARED ,/*synomym*/ node = OMPI_COMM_TYPE_NODE, + hw_thread = OMPI_COMM_TYPE_HWTHREAD, + core = OMPI_COMM_TYPE_CORE , + l1_cache = OMPI_COMM_TYPE_L1CACHE , + l2_cache = OMPI_COMM_TYPE_L2CACHE , + l3_cache = OMPI_COMM_TYPE_L3CACHE , + socket = OMPI_COMM_TYPE_SOCKET , + numa = OMPI_COMM_TYPE_NUMA , + board = OMPI_COMM_TYPE_BOARD , + host = OMPI_COMM_TYPE_HOST , + cu = OMPI_COMM_TYPE_CU ,/*synomym*/ cpu = OMPI_COMM_TYPE_CU , + cluster = OMPI_COMM_TYPE_CLUSTER +#else +}; +auto const shared = {static_cast(MPI_COMM_TYPE_SHARED) +#endif +}; + +#if defined(EXAMPI) +inline +#endif +enum constant : int { +#if defined(EXAMPI) +} const +#endif + undefined = static_cast(MPI_UNDEFINED ), + process_null = static_cast(MPI_PROC_NULL ), + any_source = static_cast(MPI_ANY_SOURCE) +#if not defined(EXAMPI) +} +#endif +; + +#if defined(EXAMPI) +inline +#endif +enum key : int { // for attributes +#if defined(EXAMPI) +} +#endif + tag_ub = static_cast(MPI_TAG_UB) +#if not defined(EXAMPI) + , host = static_cast(MPI_HOST) + , io = static_cast(MPI_IO) + , wtime_is_global = static_cast(MPI_WTIME_IS_GLOBAL) + , application_number = static_cast(MPI_APPNUM) + , universe_size = static_cast(MPI_UNIVERSE_SIZE) + , last_used_code = static_cast(MPI_LASTUSEDCODE) +} +#endif +; + +template struct overload_priority : overload_priority{ +// using overload_priority::overload_priority; +}; +template<> struct overload_priority<0>{}; + +class environment; +class group; + +using std::optional; + +struct error_handler; + +template +struct shm_pointer; + +template +struct pointer; + +using address = MPI_Aint; +using intptr_t = MPI_Aint; +using size_t = MPI_Aint; +using ptrdiff_t = std::make_signed_t; +struct request; + +struct send_request; +struct receive_request; + +struct FILE; + +class process; + +struct ostream; + +struct message_header{ + int tag; +}; + +struct graph_communicator; +struct shared_communicator; // intracommunicator + +using std::any; +using std::any_cast; + +//class communicator_ptr{}; + +template class window; + +class communicator : protected detail::basic_communicator { // in mpich MPI_Comm == int + friend struct detail::package; + friend class window; + + protected: + bool is_null() const {return MPI_COMM_NULL == impl_;} + friend class mpi3::environment; + detail::equality compare(communicator const& other) const { + int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_(Comm_compare)(impl_, other.impl_, &ret); + return static_cast(ret); + } + + public: + communicator(communicator const& o, group const& g); + + communicator(group const& g, int tag); + explicit communicator(group const& g); + + using detail::basic_communicator::basic_communicator; + + communicator() = default; + + communicator(communicator const&) = delete;//default; + communicator(communicator& other) : basic_communicator{other} {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default) intel and nvcc 11 need this (not =default) + communicator(communicator&&) = default; + + communicator& operator=(communicator const&) = delete; + [[deprecated("duplicate assignment is a flawed operation")]] + auto operator=(communicator& other) -> communicator& { // NOLINT(cppcoreguidelines-c-copy-assignment-signature,misc-unconventional-assign-operator) duplicate assigment + communicator tmp{other}; + operator=(std::move(tmp)); + // swap(tmp); + return *this; + } + auto operator=(communicator && other) noexcept -> communicator& { // TODO(correaa) tidy this operator, consider removing it + if(impl_ != MPI_COMM_NULL) { + try { + #if not defined(EXAMPI) + MPI_(Comm_disconnect)(&impl_); //this will wait for communications to finish communications, if it gets to this point is probably an error anyway <-- not true, it is necessary to synchronize the flow + #else + MPI_(Comm_free )(&impl_); + #endif + } catch(std::exception& e) { std::cerr<< e.what() < iterator_t& {++rank_; return *this;} +//// auto operator--() -> iterator_t& {--rank_; return *this;} +//// auto operator*() const -> reference; + +//// private: +//// communicator* commP_ = nullptr; +//// int rank_ = MPI_PROC_NULL; + +//// friend class communicator; +//// iterator_t(communicator* self, int rank) : commP_{self}, rank_{rank} {} +// }; +// using iterator = iterator_t; + +// auto begin() -> iterator {return {this, 0 };} +// auto end () -> iterator {return {this, size()};} + + auto& handle() {return impl_;} + auto get_mutable() {return impl_;} + auto get() const {return impl_;} // TODO(correaa) deprecate + impl_t& get() {return this->impl_;} + + class ptr { // cppcheck-suppress noConstructor ; bug in cppcheck 2.3 + communicator* ptr_; + + public: + explicit ptr(communicator* ptr) : ptr_{ptr} {} + operator MPI_Comm() const {return ptr_->get_mutable();} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + explicit operator communicator *() const {return ptr_;} + // explicit operator communicator const*() const{return ptr_;} + friend bool operator==(ptr const& a, ptr const& b) {return a.ptr_ == b.ptr_;} + friend bool operator!=(ptr const& a, ptr const& b) {return a.ptr_ != b.ptr_;} + }; + + ptr operator&() & {return ptr{this};} // NOLINT(google-runtime-operator) + communicator const* operator&() const& {return this;} // NOLINT(google-runtime-operator) + communicator * operator&() && {return this;} // NOLINT(google-runtime-operator) + + ~communicator() { + if(impl_ != MPI_COMM_WORLD and impl_ != MPI_COMM_NULL and impl_ != MPI_COMM_SELF) { + try { + #if not defined(EXAMPI) + MPI_(Comm_disconnect)(&impl_); //this will wait for communications to finish communications, if it gets to this point is probably an error anyway <-- not true, it is necessary to synchronize the flow + #else + MPI_Comm_free(&impl_); + #endif + } catch(std::exception& e) { std::cerr<< e.what() <(size()); } + + public: + using size_type = int; + int size() const { + if(is_null()) {return 0;} + int const size = MPI_(Comm_size)(impl_); + assert(size > 0); + return size; + } + + [[nodiscard]] // ("empty is not an action")]] + bool empty() const {return is_empty();} + bool is_empty() const {return is_null();} + + void abort(int errorcode = 0) const {MPI_Abort(impl_, errorcode);} + + explicit operator group() const; + + communicator duplicate() { // note that this function is non-const + communicator ret; + MPI_(Comm_dup)(impl_, &ret.impl_); + return ret; + } + + using detail::basic_communicator::send_receive_n; + #if not defined(EXAMPI) + using detail::basic_communicator::matched_probe; + #endif + + template + auto send_n( + It first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + int dest, int tag + ) { + MPI_(Send)( + detail::data(first), static_cast(count), // TODO(correaa) use safe cast + mpi3::datatype::value_type>{}(), + dest, tag, impl_ + ); + } + template + mpi3::request isend_n( + It first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + int dest, int tag + ) { + return MPI_I(send)( + detail::data(first), count, + datatype::value_type>{}(), + dest, tag, impl_ + ); + } + template + void send_n( + It first, + detail::forward_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + Size count, + int dest, int tag + ) { + detail::package p(*this); + package_oarchive poa(p); + std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); + // while(count--) {poa << *first++;} + send_n(p.begin(), p.size(), dest, tag); // p.send(dest, tag); + } + template + auto isend_n(It first, Size count, int dest, int tag = 0){ + return isend_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + static_cast(count), // TODO(correaa) use safe cast + dest, tag + ); + } + template + auto isend_n( + It first, + detail::forward_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + Size count, + int dest, int tag + ) { + detail::package p(*this); + package_oarchive poa(p); + std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); + // while(count--) {poa << *first++;} + return isend_n(p.begin(), p.size(), dest, tag); + } + template static std::true_type has_dimensionality_aux(T const&); + static std::false_type has_dimensionality_aux(...); + + template struct has_dimensionality : decltype(has_dimensionality_aux(T{})) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + + template{})> > + void send_n(It first, Size count, int dest, int tag = 0) { + return send_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + dest, tag + ); + } + template + auto send( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int dest, int tag + ) { + return send_n(first, std::distance(first, last), dest, tag); + } + template + auto send( + It first, It last, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int dest, int tag + ) { + return send_n(first, std::distance(first, last), dest, tag); + } + template + auto send( + It first, It last, + detail::input_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int dest, int tag + ) { + mpi3::vector::value_type> buffer(first, last); + return send_n(buffer.begin(), buffer.size(), dest, tag); + } + template + auto send( + It first, It last, + /**/ detail::input_iterator_tag /*tag*/, + /**/ detail::value_unspecified_tag /*tag*/, + int dest, int tag + ) { + detail::package p(*this); + package_oarchive poa(p); + std::copy(first, last, package_oarchive::iterator::value_type>(poa)); + // while(first!=last) {poa << *first++;} + send_n(p.begin(), p.size(), dest, tag); // p.send(dest, tag); + } + + template + auto send_n(MultiIt first, typename MultiIt::difference_type count, int dest, int tag = 0) + ->decltype( MPI_Send (mpi3::base(first), count, mpi3::type{first}, dest, tag, impl_), first + count) { + return MPI_(Send)(mpi3::base(first), count, mpi3::type{first}, dest, tag, impl_), first + count; } + + template + auto receive_n(MultiIt first, typename MultiIt::difference_type count, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) + ->decltype( MPI_Recv (mpi3::base(first), count, mpi3::type{first}, source, tag, impl_, MPI_STATUS_IGNORE), first + count) { // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro + return MPI_(Recv)(mpi3::base(first), count, mpi3::type{first}, source, tag, impl_, MPI_STATUS_IGNORE), first + count; } // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro + + template + auto isend( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int dest, int tag + ) { + return isend_n(first, std::distance(first, last), dest, tag); + } + template + auto send(It first, It last, int dest, int tag = 0) { + return send( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + dest, tag + ); + } + + using rank_index = int; + + template + auto isend(It first, It last, rank_index dest, int tag = 0) { + return isend( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + dest, tag + ); + } + + bool is_intercommunicator() const { + int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_Comm_test_inter(impl_, &flag); + return flag != 0; + } + + communicator split(int color, int key) { + communicator ret; + MPI_(Comm_split)(impl_, color, key, &ret.impl_); + if(ret) { ret.set_name(name() + std::to_string(color)); } + #if not defined(EXAMPI) + if(ret) { ret.attribute("color") = color; } + #endif + return ret; + } + communicator split(int color = MPI_UNDEFINED) { + return split(color, rank()); + } + + communicator keep(bool cond) {return split(cond?0:mpi3::undefined);} + + shared_communicator split_shared(int key = 0); + shared_communicator split_shared(communicator_type t, int key = 0); + + int remote_size() const {return MPI_(Comm_remote_size)(impl_);} + + communicator reversed() {return split(0, size() - rank());} + + #if not defined(EXAMPI) + int cartesian_map(std::vector const& dims, std::vector const& periods) const { + assert(dims.size() == periods.size()); + return MPI_(Cart_map)(impl_, static_cast(dims.size()), dims.data(), periods.data()); // TODO(correaa) use safe cast + } + int cartesian_map(std::vector const& dimensions) const { + return cartesian_map(dimensions, std::vector(dimensions.size(), 0)); + } + #endif + + pointer malloc(MPI_Aint size) const; + template void deallocate_shared(pointer p); + template void deallocate(pointer& p, MPI_Aint size = 0); + void free(pointer& p) const; + + bool similar(communicator const& o) const {return compare(o)==detail::equality::similar;} + template//, typename = typename std::enable_if{}>::type> + communicator subcomm(Vector const& v) const { + MPI_Group old_g; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_Comm_group(impl_, &old_g); + MPI_Group new_g; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_Group_incl(old_g, static_cast(v.size()), v.data(), &new_g); // TODO(correaa) safe cast + communicator ret; MPI_Comm_create(impl_, new_g, &ret.impl_); + MPI_Group_free(&new_g); + MPI_Group_free(&old_g); + return ret; + } + communicator subcomm(std::initializer_list l) const { + return subcomm(std::vector(l)); + } + + int rank() const { + assert(not is_empty()); // an empty communicator doesn't have ranks + int rank; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_(Comm_rank)(impl_, &rank); + return rank; + } + int right() const { + int const s = size(); assert(s != 0); + return (rank() + 1) % s; + } + int left() const { + int const s = size(); assert(s != 0); + int left = rank() - 1; + if(left < 0) {left = s - 1;} + return left; + } + int next(int n = 1) const { + assert(rank() + n < size()); + return rank() + n; + } + int prev(int n = 1) const { + assert(rank() - n > 0); + return rank() - n; + } + #if not defined(EXAMPI) + communicator accept(port const& p, int root = 0) const { + communicator ret; + MPI_Comm_accept(p.name_.c_str(), MPI_INFO_NULL, root, impl_, &ret.impl_); + return ret; + } + #endif + + [[deprecated("call non const version")]] + void barrier() const { MPI_( Barrier)(get() ) ;} + void barrier() { MPI_( Barrier)(handle()) ;} +#if not defined(EXAMPI) + auto ibarrier() {request ret; MPI_(Ibarrier)(handle(), &ret.impl_); return ret;} +#endif + +#if not defined(EXAMPI) + communicator connect(port const& p, int root = 0) const { + communicator ret; + MPI_(Comm_connect)(p.name_.c_str(), MPI_INFO_NULL, root, impl_, &ret.impl_); + return ret; + } +#endif + + bool root() const {return (not empty()) and (rank() == 0);} + bool is_root() const {return root();} + bool at_root() const {return root();} + + void set_error_handler(error_handler const& eh); + error_handler get_error_handler() const; + + auto operator[](int rank) -> reference; + + protected: +#if not defined(EXAMPI) + template void set_attribute(int kv_idx, T const& t) { + MPI_(Comm_set_attr)(impl_, kv_idx, new T{t}); // NOLINT(readability-implicit-bool-conversion, cppcoreguidelines-owning-memory) TODO(correaa) + } + inline void delete_attribute(int kv_idx){ + MPI_Comm_delete_attr(impl_, kv_idx); + } + void* get_attribute(int kvidx) const { + void* v = nullptr; + int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_(Comm_get_attr)(impl_, kvidx, &v, &flag); + if(flag == 0) {assert(!v); return nullptr;} + return v; + } + bool has_attribute(int kvidx) const { + void* v = nullptr; + int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_Comm_get_attr(impl_, kvidx, &v, &flag); + return flag != 0; + } +#endif + + public: + +#if not defined(EXAMPI) + template + class keyval { + static int delete_fn(MPI_Comm /*comm*/, int /*keyval*/, void *attr_val, void */*extra_state*/){ + delete static_cast(attr_val); // NOLINT(cppcoreguidelines-owning-memory) + // attr_val = nullptr; + return MPI_SUCCESS; + } + static int copy_fn( + MPI_Comm /*oldcomm*/, int /*keyval*/, + void * /*extra_state*/, void *attribute_val_in, // cppcheck-suppress constParameterCallback ; C-function callback + void *attribute_val_out, int *flag + ) { + *static_cast(attribute_val_out) = static_cast(new T{*(static_cast(attribute_val_in))}); + assert(flag); *flag = 1; + return MPI_SUCCESS; + } + + public: + int impl_ = {}; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) + + using mapped_type = T; + + keyval() { // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) + MPI_(Comm_create_keyval)(copy_fn, delete_fn, &impl_, nullptr); + } + + keyval(keyval const&) = delete; + keyval(keyval &&) = delete; + + keyval& operator=(keyval const&) = delete; + keyval& operator=(keyval &&) = delete; + + ~keyval() noexcept {MPI_Comm_free_keyval(&impl_);} + }; + + template void + set_attribute(keyval const& k, TT const& t = {}) {set_attribute(k.impl_, t);} + template + inline void delete_attribute(keyval const& k) {delete_attribute(k.impl_);} + template + T const& get_attribute(keyval const& kv) const {return *static_cast(get_attribute(kv.impl_));} + template + T& get_attribute(keyval const& kv) {return *static_cast(get_attribute(kv.impl_));} + template + bool has_attribute(keyval const& kv) const {return has_attribute(kv.impl_);} + template + T& attribute(keyval const& kv) { + if(not has_attribute(kv)) {set_attribute(kv);} + return get_attribute(kv); + } + + mpi3::any& attribute(std::string const& s); +#endif + +#if not defined(EXAMPI) + void call_error_handler(int errorcode) noexcept { + auto const s = MPI_Comm_call_errhandler(impl_, errorcode); (void)s; + assert(s == MPI_SUCCESS); + } + void error(mpi3::error const& e) noexcept { + auto const s = MPI_Comm_call_errhandler(impl_, static_cast(e)); (void)s; + assert(s == MPI_SUCCESS); + } +#endif + + communicator divide_low(int n) { + assert(n != 0); + return split( + (rank() < size()/n*(n-size()%n))? + rank()/(size()/n): + n-size()%n + (rank() - (n-size()%n)*(size()/n))/((size()/n)+1) + ); + } + communicator divide_high(int n) { + int const bat=size()/n; int const residue = size()%n; + int i = 0; + for(int last = 0; ; i++) { // NOLINT(altera-unroll-loops) TODO(correaa) + last += bat + ((i < residue)?1:0); + if(rank() < last) {break;} + } + return split(i); + } + communicator operator/(int n) { + assert(n!=0); + if(n > 0) {return divide_high(n);} + return divide_low(n); + } + communicator operator%(int n) {return split(rank()%n);} + communicator divide_even(int n) { + return split(2*(rank()%n) > n?mpi3::undefined:rank()/n); + } + + communicator operator< (int n) {return split((rank() < n)?0:MPI_UNDEFINED);} + communicator operator<=(int n) {return split((rank() <= n)?0:MPI_UNDEFINED);} + communicator operator> (int n) {return split((rank() > n)?0:MPI_UNDEFINED);} + communicator operator>=(int n) {return split((rank() >= n)?0:MPI_UNDEFINED);} + communicator operator==(int n) {return split((rank() == n)?0:MPI_UNDEFINED);} + + template + void send_value(T const& t, int dest, int tag = 0) { + send(std::addressof(t), std::next(std::addressof(t)), dest, tag); + } + template + auto isend_value(T const& t, int dest, int tag = 0) { + return isend(std::addressof(t), std::next(std::addressof(t)), dest, tag); + } + template + void send_value(T(&t)[N], int dest, int tag = 0) { // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) compatibility + send(std::addressof(t[0]), std::next(std::addressof(t[N-1])), dest, tag); + } + template + request send_init_n(ContIt first, Size count, int dest, int tag = 0); + template + request send_init(ContIt first, ContIt last, int dest, int tag = 0); + + template + receive_request receive_init_n(ContIt first, Size count, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG); + template + receive_request receive_init(ContIt first, ContIt last, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG); + + template< + class InputIt, class Size, + typename V = typename std::iterator_traits::value_type, class Datatype = datatype + > + auto unpack_from_n(InputIt first, Size count, char const* buffer) { + int position = 0; + using detail::data; + MPI_(Unpack)(buffer, count*pack_size(), &position, data(first), count, Datatype{}(), impl_); + return std::next(buffer, position); + } + + template + auto pack( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + uvector& b, int pos + ) { + return pack_n(first, std::distance(first, last), b, pos); + } + template + auto pack(It first, It last, uvector& b, int pos) { + return pack( + first, last, + detail::iterator_category_t{}, + b, pos + ); + } + +#if 0 +#ifdef MPICH_NUMVERSION +#if MPICH_NUMVERSION >= 30400000 + private: + template + [[nodiscard]] auto isend_receive_replace_n(It first, Size count, It2 d_first, int dest, int source = MPI_ANY_SOURCE) + -> decltype(detail::data(first), std::declval()){ + auto const sz = size(); + assert(sz != 0); + mpi3::request r; + MPI_I(sendrecv_replace)(detail::data(first), count/sz, datatype::value_type>{}(), dest, 0, source, MPI_ANY_TAG, &*this, &r.impl_); + return r; + } + + public: +#endif +#endif +#endif + +#if not defined(EXAMPI) + template + auto send_receive_replace_n( + It first, Size size, + int dest, int source, // = MPI_ANY_SOURCE, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + using value_type = typename std::iterator_traits::value_type; + return send_receive_replace_n( + first, + detail::iterator_category_t{}, + detail::value_category_t{}, + static_cast(size), + dest, source, sendtag, recvtag + ); + } + template + It send_receive_replace_n( + It first, + detail::random_access_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, int dest, int source, int sendtag, int recvtag + ) { + mpi3::status const s = MPI_(Sendrecv_replace)( + detail::data(first), count, + datatype::value_type>{}(), + dest, sendtag, source, recvtag, impl_ + ); + return first + s.count::value_type>(); + } + + template + auto send_receive_replace_n( + It first, + detail::forward_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, int dest, int source, int sendtag, int recvtag + ) { + uvector::value_type> v(static_cast(count)); + std::copy_n(first, count, v.begin()); + send_receive_replace_n(v.begin(), v.size(), dest, source, sendtag, recvtag); + return std::copy_n(v.begin(), v.size(), first); + } + +#endif // not defined(EXAMPI) + + template + auto send_receive_n( + It first, Size count, int dest, + It2 d_first, Size d_count, int source, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + int sendtag, int recvtag + ) { + status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init + MPI_(Sendrecv)( + detail::data( first), static_cast( count), datatype::value_type>{}(), dest , sendtag, // TODO(correaa) use safe cast + detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), source, recvtag, // TODO(correaa) use safe cast + impl_, &ret.impl_ + ); + assert( static_cast(ret.count::value_type>()) == d_count ); + return d_first + static_cast::difference_type>(d_count); + } + + template + auto send_receive_n( + It1 first, Size count, int dest, + It2 d_first, Size d_count, int source, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_n( + first, count, dest, + d_first, d_count, source, + detail::iterator_category_t{}, // It2??? + detail::value_category_t::value_type>{}, + sendtag, recvtag + ); + } + +#if not defined(EXAMPI) + template + auto send_receive_replace_n( + It first, + /**/ detail::forward_iterator_tag /*tag*/, + /**/ detail::value_unspecified_tag /*tag*/, + Size count, + int dest, int source, + int sendtag, int recvtag + ) { + detail::package p(*this); + package_oarchive poa(p); + auto first_copy = first; + std::copy_n(first_copy, count, package_oarchive::iterator::value_type>(poa) ); + // while(count--) {poa << *first_copy++;} // TODO(correaa) remove first_copy + auto s = p.size(); + send_receive_replace_n(&s, 1, dest, source, sendtag, recvtag); + detail::package p2(*this); + p2.resize(s); + auto st = send_receive_n( + p.begin(), p.size(), dest, + p2.begin(), p2.size(), + source, sendtag, recvtag + ); + (void)st; + package_iarchive pia(p2); + // while(p2) {pia >> *first++;} + // return first; + return std::copy_n( + package_iarchive::iterator::value_type>(pia), + count, first + ); + } +#endif // not defined(EXAMPI) + + template + auto send_receive_n( + It1 first, Size count, int dest, + It2 d_first, int source = MPI_ANY_SOURCE, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_n( + first, count, dest, + d_first, source, + detail::iterator_category_t{}, // It2??? TODO(correaa) + detail::value_category_t::value_type>{}, // It2??? TODO(correaa) + sendtag, recvtag + ); + } + +// private: + +// public: +// template +// auto isend_receive_replace_n( +// It first, Size size, +// int dest, int source, // = MPI_ANY_SOURCE, +// int sendtag = 0, int recvtag = MPI_ANY_TAG +// ) { +// using value_type = typename std::iterator_traits::value_type; +// return isend_receive_replace_n( +// first, +// detail::iterator_category_t{}, +// detail::value_category_t{}, +// size, +// dest, source, sendtag, recvtag +// ); +// } + + private: + template::value_type + , class V2 = typename std::iterator_traits::value_type + > + auto send_receive_n( + It1 first, Size count, int dest, + It2 d_first, int source, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + int sendtag, int recvtag + ) { + static_assert( std::is_same{}, "source and destination need to be same type" ); + status const ret = MPI_(Sendrecv)( + detail::data(first), static_cast(count), datatype{}(), + dest, sendtag, + detail::data(d_first), std::numeric_limits::max() /*unlim in receiving*/, datatype{}(), + source, recvtag, + impl_ //, &ret.impl_ // status refers to the receive operation. + ); + return d_first + ret.count(); + } + + public: + template + auto send_receive_n( + It first, Size size, + int dest, int source, // = MPI_ANY_SOURCE, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_replace_n( + first, size, + dest, source, sendtag, recvtag + ); + } + + private: + template + auto send_receive( + It1 first, It1 last, int dest, + It2 d_first, It2 d_last, int source, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int sendtag, int recvtag + ) { + return send_receive_n( + first, std::distance(first, last), dest, + d_first, std::distance(d_first, d_last), source, + sendtag, recvtag + ); + } + + template + auto send_receive( + It1 first, It1 last, int dest, + It2 d_first, int source, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int sendtag, int recvtag + ) { + return send_receive_n( + first, std::distance(first, last), dest, + d_first, source, + sendtag, recvtag + ); + } + + public: + template + auto send_receive( + It1 first, It1 last, int dest, + It2 d_first, It2 d_last, int source = MPI_ANY_SOURCE, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ){ + return send_receive( + first, last, dest, + d_first, d_last, source, + detail::iterator_category_t{}, // It2??? + detail::value_category_t::value_type>{}, // It2??? + sendtag, recvtag + ); + } + + template + auto send_receive( + It1 first, It1 last, int dest, + It2 d_first + ){ + return send_receive( + first, last, dest, + d_first, MPI_ANY_SOURCE, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + /*sendtag*/ 0, /*recvtag*/ MPI_ANY_TAG + ); + } + + template + auto send_receive_n(It first, Size n, std::pair dest_source, std::pair send_recv_tag = {0, MPI_ANY_TAG}) { + return send_receive_n(first, n, dest_source.first, dest_source.second, send_recv_tag.first, send_recv_tag.second); + } + template + auto send_receive (It first, It last, std::pair dest_source, std::pair send_recv_tag = {0, MPI_ANY_TAG}) { + return send_receive (first, last, dest_source.first, dest_source.second, send_recv_tag.first, send_recv_tag.second); + } + + status probe(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + status s; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init + MPI_(Probe)(source, tag, impl_, &s.impl_); + return s; + } + auto iprobe(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + status s; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init + int flag; // NOLINT(cppcoreguidelines-init-variables) delayed init + MPI_(Iprobe)(source, tag, impl_, &flag, &s.impl_); + if(flag == 0) {return optional();} + return optional(s); + } + template + auto send_receive_replace( + It first, It last, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int dest, int source, int sendtag, int recvtag + ) { + return send_receive_replace_n( + first, std::distance(first, last), + dest, source, sendtag, recvtag + ); + } + template + auto send_receive_replace( + It first, It last, + detail::forward_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int dest, int source, int sendtag, int recvtag + ) { + mpi3::vector::value_type> buffer(first, last); + send_receive_replace_n(buffer.begin(), buffer.size(), dest, source, sendtag, recvtag); + return std::copy(buffer.begin(), buffer.end(), first); + } + template + auto send_receive_replace( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int dest, int source, int sendtag, int recvtag + ) { + return send_receive_replace_n( + first, std::distance(first, last), + dest, source, sendtag, recvtag + ); + } + template + auto send_receive_replace( + It first, It last, + int dest, + int source = MPI_ANY_SOURCE, int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_replace( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + dest, source, sendtag, recvtag + ); + } + template + auto send_receive( + It first, It last, + int dest, + int source = MPI_ANY_SOURCE, int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_replace(first, last, dest, source, sendtag, recvtag); + } + + void send_packed_n(void const* begin, int n, int dest, int tag = 0) { + std::cout<<"sending packet of size "<< n <(); + // receive_packed_n(begin, n, source, tag); + return static_cast(std::next(static_cast(begin), count)); + } + + template + auto receive_n( + It dest, + detail::forward_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + Size count, + int source, int tag + ){ + detail::package p(*this); + p.receive(source, tag); + package_iarchive pia(p); + return std::copy_n(package_iarchive::iterator::value_type>{pia}, count, dest); + } + + template + auto receive( + It dest, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int source, int tag + ) { + match m = matched_probe(source, tag); + auto count = m.count::value_type>(); + m.receive_n(dest, count); + return dest + count; + } + + template + [[deprecated]] auto receive( + It dest, + /**/ detail::forward_iterator_tag /*tag*/, + /**/ detail::value_unspecified_tag /*tag*/, + int source, int tag + ) { + detail::package p(*this); + p.receive(source, tag); + package_iarchive const pia(p); // TODO(correaa) investigate + while(p) {pia >> *dest++;} // NOLINT(altera-unroll-loops) deprecating + return dest; + } + + template + auto receive( + It dest, + detail::forward_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int source, int tag + ) { + return matched_probe(source, tag).receive_n(dest); + } + + template::value_type> + auto dynamic_receive(InputIt first, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + // auto count = probe(source, tag).count(); + // return receive(first, first + count, source, tag); + MPI_Status status; + MPI_Message msg; // NOLINT(cppcoreguidelines-init-variables) delayed init + int count = -1; + MPI_Mprobe(source, tag, impl_, &msg, &status); + MPI_Get_count(&status, datatype{}(), &count); + using detail::data; + MPI_Mrecv(data(first), count, datatype{}(), &msg, MPI_STATUS_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro + } +#endif + + template + auto receive_n( + It dest, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + int source, int tag + ) { + status sta; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed init + MPI_(Recv)( + detail::data(dest), static_cast(count), + datatype::value_type>{}(), + source, tag, impl_, &sta.impl_ + ); + assert(sta.count::value_type>() == static_cast(count)); + return dest + sta.count::value_type>(); + } + template + auto ireceive_n( + It dest, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + Size count, + int source, int tag + ) { + mpi3::request r; + MPI_(Irecv)( + detail::data(dest), static_cast(count), + datatype::value_type>{}(), + source, tag, impl_, &r.impl_ + ); + return r; + } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // MPI_Wait called on destructor of ret + + template{}, int> =0// or (not detail::is_basic::value_type>{}), int> =0 // needed by intel commpiler + > + auto receive_n(It dest, Size n, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + return receive_n( + dest, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + n, + source, tag + ); + } + + template + mpi3::request ireceive_n( + It dest, Size n, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG + ) { + return ireceive_n( + dest, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + n, + source, tag + ); + } + + template + [[deprecated]] auto receive(It dest, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + return receive( + dest, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + source, tag + ); + } + template + auto receive( + It d_first, It d_last, + /**/ detail::random_access_iterator_tag /*tag*/, + /**/ detail::value_unspecified_tag /*tag*/, + int source, int tag + ) { + return receive_n(d_first, std::distance(d_first, d_last), source, tag); + } + template + auto receive( + It d_first, It d_last, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int source, int tag + ) { + return receive_n(std::addressof(*d_first), std::distance(d_first, d_last), source, tag); + // return std::copy(buffer.begin(), buffer.end(), d_first); + } + + template + auto receive( + It d_first, It d_last, + /**/ detail::forward_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + int source, int tag + ) { + mpi3::uvector::value_type> buffer(std::distance(d_first, d_last)); + receive_n(buffer.begin(), buffer.size(), source, tag); + return std::copy(buffer.begin(), buffer.end(), d_first); + } +// class ir_req{ +// boost::mpi3::status query(){ +// boost::mpi3::status ret; +// ret.set_source(MPI_UNDEFINED); +// ret.set_tag(MPI_UNDEFINED); +// ret.set_cancelled(); +// ret.set_elements(0); +// return ret; +// } +// static void free(){ +// std::cout << "free" << std::endl; +// } +// static void cancel(int complete) { +// std::cout << "cancel " << complete << std::endl; +// } +// }; +// template +// struct receive_args { +// communicator* commP; +// It d_first; +// // It d_last; +// int source; +// int tag; +// MPI_Request* requestP; +// }; +// struct receive_state{ +// int cancelled = 0; +// int source = MPI_UNDEFINED; +// int tag = MPI_UNDEFINED; +// }; +// template +// inline static void* receive_thread(void* ptr) { +// receive_args* args = (receive_args*)ptr; +// args->commP->receive(args->d_first, args->source, args->tag);//, /*args->d_last,*/ ); +// MPI_Grequest_complete(*args->requestP); +// ::free(ptr); +// return nullptr; +// } +// inline static int query_fn(void* extra_state, MPI_Status *status){ +// auto* rs = static_cast(extra_state); +// /* always send just one int */ +// MPI_Status_set_elements(status, MPI_INT, 1); +// /* can never cancel so always true */ +// MPI_Status_set_cancelled(status, rs->cancelled); +// /* choose not to return a value for this */ +// status->MPI_SOURCE = rs->source; +// /* tag has not meaning for this generalized request */ +// status->MPI_TAG = rs->tag; +// /* this generalized request never fails */ +// return MPI_SUCCESS; +// } +// inline static int free_fn(void* extra_state) { +// /* this generalized request does not need to do any freeing */ +// /* as a result it never fails here */ +// ::free(extra_state); +// return MPI_SUCCESS; +// } +// inline static int cancel_fn(void* /*extra_state*/, int complete) { +// /* This generalized request does not support cancelling. +// Abort if not already done. If done then treat as if cancel failed. */ +// if(not (complete == 0)) { +// std::cerr<< "Cannot cancel generalized request - aborting program" < +// auto ireceive(It d_first, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { +// // based on http://liinwww.ira.uka.de/courses/spprakt/mpi2-html-doc/node157.html +// mpi3::request ret; /* receive_args* args = (receive_args*)::malloc(sizeof(receive_args)); args->commP = this; args->d_first = d_first; // args->d_last = d_last; args->source = source; args->tag = tag; args->requestP = &ret.impl_;*/ +// receive_state* rs = (receive_state*)::malloc(sizeof(receive_state)); +// rs->cancelled = 0; +// rs->source = source; +// rs->tag = tag; +// MPI_Grequest_start(query_fn, free_fn, cancel_fn, rs, &ret.impl_);//args->requestP); +// std::thread( // static_cast(receive_thread), args +// [this, d_first, source, tag, &ret](){ +// this->receive(d_first, source, tag); // receive_args* args = (receive_args*)ptr; // args->commP->receive(args->d_first, args->source, args->tag);//, /*args->d_last,*/ ); +// MPI_Grequest_complete(ret.impl_); // MPI_Grequest_complete(*args->requestP); // ::free(ptr); +// } +// ).detach(); // t.detach(); // pthread_t thread; // pthread_create(&thread, NULL, static_cast(receive_thread), args); // pthread_detach(thread); +// return ret; +// } + template + auto ireceive( + It d_first, It d_last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int source, int tag + ) { + return ireceive_n(d_first, std::distance(d_first, d_last), source, tag); + } + template + auto receive(It d_first, It d_last, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + return receive( + d_first, d_last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + source, tag + ); + } + template::value_type> + auto bsend_init_n( + InputIt first, Size count, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int dest, int tag + ) { + request ret; + int s = MPI_Bsend_init( + std::addressof(*first), count, datatype{}(), + dest, tag, impl_, &ret.impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot bsend init"};} + return ret; + } + template + auto bsend_init_n(It first, Size count, int dest, int tag = 0){ + return bsend_init_n( + first, count, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + dest, tag + ); + } + template + auto bsend_init( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int dest, int tag + ){ + return bsend_init_n(first, std::distance(first, last), dest, tag); + } + template + auto bsend_init(It first, It last, int dest, int tag = 0){ + bsend_init( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + dest, tag + ); + } + template::iterator_category> + auto bsend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ + return send(buffered_communication_mode{}, blocking_mode{}, It1, It2, dest, tag); + } + + template::iterator_category> + auto breceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ + return receive(buffered_communication_mode{}, blocking_mode{}, It1, It2, source, tag); + } + template::iterator_category> + auto ibsend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ + return send(buffered_communication_mode{}, nonblocking_mode{}, It1, It2, dest, tag); + } + template::iterator_category> + auto ibreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ + return receive(buffered_communication_mode{}, nonblocking_mode{}, It1, It2, source, tag); + } + + template::iterator_category> + auto ssend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ + return send(synchronous_communication_mode{}, blocking_mode{}, It1, It2, dest, tag); + } + template::iterator_category> + auto sreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG){ + return receive(synchronous_communication_mode{}, blocking_mode{}, It1, It2, source, tag); + } + template::iterator_category> + auto issend(InputIterator It1, InputIterator It2, int dest, int tag = 0){ + return send(synchronous_communication_mode{}, nonblocking_mode{}, It1, It2, dest, tag); + } + template + auto isend_n( + CommunicationMode /*mode*/, + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, int dest, int tag + ) { + mpi3::request ret; + CommunicationMode{}.ISend( + std::addressof(*first), count, datatype::value_type>{}(), dest, tag, impl_, &ret.impl_ + ); + return ret; + } + template + auto issend_n(It1 first, Size count, int dest, int tag = 0) { + return isend_n( + synchronous_communication_mode{}, + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + dest, tag + ); + } + template::iterator_category> + auto isreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + return receive(synchronous_communication_mode{}, nonblocking_mode{}, It1, It2, source, tag); + } + + template::iterator_category> + auto rsend(InputIterator It1, InputIterator It2, int dest, int tag = 0) { + return send(ready_communication_mode{}, blocking_mode{}, It1, It2, dest, tag); + } + template::iterator_category> + auto rreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + return receive(ready_communication_mode{}, blocking_mode{}, It1, It2, source, tag); + } + template::iterator_category> + auto irsend(InputIterator It1, InputIterator It2, int dest, int tag = 0) { + return send(ready_communication_mode{}, nonblocking_mode{}, It1, It2, dest, tag); + } + template::iterator_category> + auto irreceive(Iterator It1, Iterator It2, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) { + return receive(ready_communication_mode{}, nonblocking_mode{}, It1, It2, source, tag); + } + template + auto send_category(CommunicationMode cm, BlockingMode bm, std::input_iterator_tag, + InputIterator first, InputIterator last, int dest, int tag + ); + template< + class CommunicationMode, class BlockingMode, class InputIterator, + class Category = typename std::iterator_traits::iterator_category + > + auto send(CommunicationMode cm, BlockingMode bm, InputIterator It1, InputIterator It2, int dest, int tag = 0) { + return send_category(cm, bm, Category{}, It1, It2, dest, tag); + } + + private: + template< + class CommunicationMode, class ContiguousIterator, + class Size, class ValueType = typename std::iterator_traits::value_type, + class Datatype = datatype + > + request send_n_randomaccess_contiguous_builtin( + CommunicationMode cm, nonblocking_mode /*mode*/, + std::true_type /*true*/, + ContiguousIterator first, Size n, int dest, int tag + ) { + request r; + int s = cm.ISend(detail::data(first), n, Datatype{}(), dest, tag, impl_, &r.impl_); + if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot send random access iterators"};} + return r; + } + template< + class CommunicationMode, class ContiguousIterator, + class Size, class ValueType = typename std::iterator_traits::value_type, + class Datatype = datatype + > + request receive_n_randomaccess_contiguous_builtin( + CommunicationMode cm, nonblocking_mode /*mode*/, std::true_type /*true*/, + ContiguousIterator first, Size n, int dest, int tag + ) { + request r; + int s = cm.IRecv(detail::data(first), n, Datatype{}(), dest, tag, impl_, &r.impl_); + if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot send random access iterators"};} + return r; + } + + template + auto all_to_all_n( + It1 first, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + Size count, + It2 d_first, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/ + ) { + // auto const sz = static_cast(size()); + // assert(sz != 0 and count % sz == 0); + using count_type = int; + MPI_(Alltoall)( + detail::data( first), static_cast(count/* / sz*/), datatype::value_type>{}(), + detail::data(d_first), static_cast(count/* / sz*/), datatype::value_type>{}(), + impl_ + ); + return d_first + count; + } + +#if not defined(EXAMPI) + using in_place_type = decltype(MPI_IN_PLACE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general + + template + auto all_to_all_n( + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count + ) + ->decltype(MPI_(Alltoall)( + std::declval(), 0*count/*/size()*/, + datatype::value_type>{}(), + detail::data(first), count/*/size()*/, + datatype::value_type>{}(), + impl_ + ), first + count*size()) { + // auto const sz = size(); + // assert(sz > 0); + // assert( count % sz == 0 ); + auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general // TODO(correaa) define constant globally for the library + MPI_(Alltoall)( + in_place, 0*count/*/sz*/, + datatype::value_type>{}(), + detail::data(first), count/*/sz*/, + datatype::value_type>{}(), + impl_ + ); + return first + count*size(); + } +#endif + + public: + +#if not defined(EXAMPI) + template + auto all_to_all_inplace_n(It1 first, Size count) { + using count_type = int; + // auto const sz = static_cast(size()); // TODO(correaa) safe cast + // assert(sz > 0); + // assert( static_cast(count) % sz == 0 ); + auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general // TODO(correaa) define constant globally for the library + using count_type = int; + MPI_(Alltoall)( + in_place , 0*static_cast(count)/*/sz*/, datatype::value_type>{}(), + detail::data(first), static_cast(count)/*/sz*/, datatype::value_type>{}(), + impl_ + ); + return first + count; + } +#endif + + template + auto all_to_all_n(It1 first, Size count, It2 d_first) { + // using count_type = int; + // assert( static_cast(count) % size() == 0 ); // NOLINT(clang-analyzer-core.DivideZero) TODO(correaa) add size cache to immutable communicator + return + all_to_all_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{} + ) + ; + } + template + auto all_to_all_n(It1 first, Size count) + ->decltype( + all_to_all_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count + )) + { + assert( count % size() == 0 ); // NOLINT(clang-analyzer-core.DivideZero) TODO(correaa) add size cache to immutable communicator + return + all_to_all_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count + );} + template + auto all_to_all(It1 first, It2 d_first){return all_to_all_n(first, size(), d_first);} + template + auto all_to_all(It1 first) + ->decltype(all_to_all_n(first, size())){ + return all_to_all_n(first, size());} + + template + auto operator()(T&& t) + ->decltype(communicator::all_to_all(begin(std::forward(t))), void()){ + assert(t.size() == size()); + auto e = communicator::all_to_all(begin(std::forward(t))); + using std::end; + assert( e == end(t) ); + return std::forward(t); + } + template + [[nodiscard]] // ("do not ignore result when argument is const")]] + auto operator()(T const& t) + ->decltype(communicator::all_to_all(t.begin(), std::declval().begin()), T(communicator::size())){ + assert(t.size() == communicator::size()); + T ret(communicator::size()); + communicator::all_to_all(t.begin(), ret.begin()); + return ret; + } + + private: + template + auto scatter_builtin_q( + std::true_type /*true*/, + It1 first, It1 last, It2 d_first, int root + ) { + return scatter_builtin( + detail::iterator_category{}, + detail::iterator_category{}, first, last, d_first, root + ); + } + template< + class It1, class It2, + class Sendtype = datatype::value_type>, + class Recvtype = datatype::value_type> + > + auto scatter_builtin( + detail::contiguous_iterator_tag /*tag*/, + detail::contiguous_iterator_tag /*tag*/, + It1 first, It1 last, It2 d_first, int root + ) { + MPI_(Scatter)( + detail::data( first), std::distance(first, last), + Sendtype{}(), + detail::data(d_first), std::distance(first, last), + Recvtype{}(), + root, + impl_ + ); + } + template + auto scatter_builtin_q(std::false_type, Iterator1 first, Iterator2 last, Iterator1 d_first, int root) +// { TODO implement } + ; + + public: + template + void broadcast_value(T& t, int root = 0) {broadcast_n(std::addressof(t), 1, root);} + + template + void broadcast_value(std::vector& t, int root = 0){ + auto t_size = t.size(); + broadcast_value(t_size, root); + t.resize(t_size); + broadcast_n(t.data(), t.size(), root); + } + + template + auto ibroadcast_n( + It first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + int root + ) { + request r; + MPI_(Ibcast)( + detail::data(first), static_cast(count), datatype::value_type>{}(), + root, impl_, &r.impl_ + ); + return r; + } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // MPI_Wait called on destructor of ret + template + auto broadcast_n( + It first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + int root + ) { + MPI_(Bcast)( + detail::data(first), static_cast(count), // TODO(correaa) use safe cast + datatype::value_type>{}(), + root, impl_ + ); + return first + static_cast::difference_type>(count); + } + template + auto ibroadcast( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + int root + ) { + return ibroadcast_n( + first, std::distance(first, last), root + ); + } + template + void broadcast( + It first, It last, + detail::random_access_iterator_tag /*tag*/, + int root + ){broadcast_n(first, std::distance(first, last), root);} + + template + auto ibroadcast_n(It first, Size count, int root = 0){ + return ibroadcast_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + root + ); + } + template + auto broadcast_n(It first, Size count, int root = 0){ + return broadcast_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + root + ); + } + template + auto ibroadcast(It first, It last, int root = 0){ + return ibroadcast( + first, last, + detail::iterator_category_t{}, + root + ); + } + template + void broadcast(It first, It last, int root = 0){ + return broadcast( + first, last, + detail::iterator_category_t{}, + root + ); + } + template > + void reduce_value(T const& t, T& ret, Op op = {}, int root = 0){ + reduce_n(std::addressof(t), 1, std::addressof(ret), op, root); + } + template > + auto reduce_value(T const& t, Op op = {}, int root = 0){ + auto ret = static_cast(0); + reduce_value(t, ret, op, root); // if(rank() == root) return optional(ret); + return ret; + } + template + It2 reduce_n( + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + It2 d_first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Op /*operation*/, // TODO(correaa) why value is not used? + int root + ) { + static_assert(std::is_same::value_type, typename std::iterator_traits::value_type>{}); + static mpi3::operation::value_type, typename std::iterator_traits::pointer> const combine{Op{}}; // will leak? + MPI_(Reduce)( + detail::data(first), detail::data(d_first), static_cast(count), + datatype::value_type>{}(), &combine, + root, impl_ + ); + return rank() == root?d_first + static_cast::difference_type>(count):d_first; + } + + template > + It2 reduce_n(It1 first, Size count, It2 d_first, Op op = {}, int root = 0) { + return reduce_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + op, + // predefined_operation{}, + root + ); + } + + template > + void all_reduce_value(T const& t, T& ret, Op op = {}){ + auto* e = all_reduce_n(std::addressof(t), 1, std::addressof(ret), op); (void)e; + assert( e == std::next(std::addressof(ret)) ); + } + template, typename = decltype(T{T(0)})> + auto all_reduce_value(T const& t, Op op = {}){ + auto ret = static_cast(0); + all_reduce_value(t, ret, op); // if(rank() == root) return optional(ret); + return ret; + } + template> + bool all_reduce_value(bool t, Op op={}){ + int ret = 0; + all_reduce_value(int{t}, ret, op); + return ret != 0; + } + + template + auto max(T const& t) { + auto ret = std::numeric_limits::lowest(); + all_reduce_value(t, ret, mpi3::max<>{}); return ret; + } + template + auto min(T const& t) { + auto ret = std::numeric_limits::max(); + all_reduce_value(t, ret, mpi3::min<>{}); return ret; + } + template + auto max_loc(T const& t) { + mpi3::vlp const in{t, rank()}; + mpi3::vlp ret{std::numeric_limits::lowest(), -1}; + all_reduce_value(in, ret, mpi3::max_loc<>{}); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return ret; + } + template + std::pair max_location(T const& t); + + template + T* max_element(T& t) { + auto const ml = max_loc(t); + if(ml.location == rank()) {assert(t == ml.value);} + return ml.location == rank()? &t : nullptr; + } + + template, typename = decltype(static_cast(detail::data(std::declval())))> + auto all_reduce_n( + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size count, + It2 d_first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Op /*operation*/ // TODO(correaa) why is not used? + ) { + static_assert(std::is_same::value_type, typename std::iterator_traits::value_type>{}); + using count_type = int; + static mpi3::operation::value_type, typename std::iterator_traits::pointer> const combine{Op{}}; // will leak? + MPI_(Allreduce)( + detail::data(first), detail::data(d_first), static_cast(count), datatype::value_type>{}(), // TODO(correaa) use safe cast + &combine, impl_ + ); + return d_first + static_cast::difference_type>(count); + } + + template< + class It1, class Size, class It2, class Op = std::plus<>, + class VC1 = detail::value_category_t::value_type>, + class VC2 = detail::value_category_t::value_type>, + class = decltype(std::declval::reference>() = std::declval()(typename std::iterator_traits::value_type{}, typename std::iterator_traits::value_type{})) + > + It2 all_reduce_n(It1 first, Size count, It2 d_first, Op op = {}) { + return all_reduce_n( + first, + detail::iterator_category_t{}, + VC1{}, + count, + d_first, + detail::iterator_category_t{}, + VC2{}, + op + ); + } + template> + It2 all_reduce(It1 first, It1 last, It2 d_first, Op op = {}){ + return all_reduce_n(first, std::distance(first, last), d_first, op); + } + + private: + template static auto data_adl(T&& t){ + using detail::data; + return data(std::forward(t)); + } + + public: +#if not defined(EXAMPI) + template< + class It1, class Size, class Op = std::plus<>, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(data_adl(It1{})), + class = std::enable_if_t()(std::declval(), std::declval()))>> + > + auto all_reduce_in_place_n(It1 first, Size count, Op /*op*/) { + auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general + static mpi3::operation::value_type, typename std::iterator_traits::pointer> const combine{Op{}}; // will leak? + MPI_(Allreduce)(in_place, data_adl(first), static_cast(count), datatype{}(), &combine, impl_); + } + + template< + class It1, class Size, class Op, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(data_adl(It1{})), + class PredefinedOp = predefined_operation + > + auto reduce_in_place_n(It1 first, Size count, Op /*op*/, int root = 0) { + auto const in_place = MPI_IN_PLACE; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast,llvm-qualified-auto,readability-qualified-auto,performance-no-int-to-ptr) openmpi #defines this as (void*)1, it may not be a pointer in general + (rank() == root)?MPI_(Reduce)(in_place , data_adl(first), count, datatype{}(), PredefinedOp{}, root, impl_): + MPI_(Reduce)(data_adl(first), nullptr , count, datatype{}(), PredefinedOp{}, root, impl_) + ; + } +#endif + + template< + class It1, class Size, class Op = std::plus<>, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(data_adl(It1{})) + > + auto all_reduce_n(It1 first, Size count, Op op = {}) + ->decltype(all_reduce_in_place_n(first, count, op)) { + return all_reduce_in_place_n(first, count, op); } + + template< + class It1, class Size, class Op = std::plus<>, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), + class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) + > + auto reduce_n(It1 first, Size count, Op op = {}, int root = 0) { + if(rank() == root) {return reduce_in_place_n(first, count, op, root);} + return reduce_n(first, 0, nullptr, op, root); + } + template< + class It1, class Op, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), + class PredefinedOp = predefined_operation + > + auto reduce_in_place(It1 first, It1 last, Op op, int root = 0) { + return reduce_in_place_n(first, std::distance(first, last), op, root); + } + + template< + class It1, class Op = std::plus<>, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), + class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) + > + auto reduce(It1 first, It1 last, Op op = {}, int root = 0) { + return reduce_n(first, std::distance(first, last), op, root); + } + + template< + class It1, class It2, class Op = std::plus<>, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), + class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) + > + auto reduce(It1 first, It1 last, It2 d_first, Op op = {}, int root = 0) { + return reduce_n(first, std::distance(first, last), d_first, op, root); + } + + template< + class It1, class Op = std::plus<>, + class V1 = typename std::iterator_traits::value_type, class P1 = decltype(detail::data(It1{})), + class = decltype(std::declval::reference>() = std::declval()(V1{}, V1{})) + > + auto all_reduce(It1 first, It1 last, Op op = {}) { + return all_reduce_in_place_n(first, std::distance(first, last), op); + } + + private: + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto reduce_n( + ReducePolicy rp, + It1 first, Size count, It2 d_first, + Op op, int root = 0 + ) { + return reduce_n_dispatch(rp, + std::integral_constant{} and detail::is_basic{} and detail::is_contiguous{} and detail::is_contiguous{}>{}, + first, count, d_first, op, root + ); + } + template + auto reduce_category( + ReducePolicy rp, std::random_access_iterator_tag /*tag*/, + RandomAccessIt1 first, RandomAccessIt1 last, It2 d_first, + Op op, int root + ) { + return reduce_n(rp, first, std::distance(first, last), d_first, op, root); + } + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto reduce_n_dispatch( + all_reduce_mode rp, std::true_type /*true*/, + It1 first, Size count, It2 d_first, + Op op, int /*root*/ = 0 + ) { + int s = rp( + detail::data(first) , + detail::data(d_first), count, datatype{}(), + op.impl_, impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot reduce");} + } + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto reduce_n_dispatch( + reduce_mode rp, std::true_type /*true*/, + It1 first, Size count, It2 d_first, + Op op, int root = 0 + ) { + int s = rp( + detail::data(first) , + detail::data(d_first), count, datatype{}(), + op.impl_, + root, impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} + } + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto reduce_n_dispatch( + ireduce_mode rp, std::true_type /*true*/, + It1 first, Size count, It2 d_first, + Op op, int root = 0 + ) { + request ret; + int s = rp( + detail::data(first) , + detail::data(d_first), count, datatype{}(), + op.impl_, + root, impl_, &ret.impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} + return ret; + } + + public: + template + auto scatter_n( + CIt1 first, Size n, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + CIt2 d_first, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + int root + ) { + auto const s = size(); + if(s == 0) {throw std::runtime_error{"invalid empty communicator for scatter_n"};} + assert( n%s == 0 ); + MPI_(Scatter)( + detail::data( first), static_cast(n/s), datatype::value_type>{}(), + detail::data(d_first), static_cast(n/s), datatype::value_type>{}(), + root, impl_ + ); + if(rank() == root) {std::advance(d_first, n);} + return d_first; + } + template + It2 scatter_n( + In1 first, Size n, detail::input_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + It2 d_first, Any2 /*unused*/, Any3 /*unused*/, + int root + ) { + auto const s = size(); + if(s == 0) { + throw std::runtime_error{"invalid empty communicator for scatter_n"}; + } + assert(n % s == 0); + vector::value_type> buff; + buff.reserve(static_cast(n)); + using std::copy_n; + copy_n(first, n, std::back_inserter(buff)); + scatter_n(buff.begin(), n, d_first, root); + std::advance(d_first, n / s); + return d_first; + } + template::value_type, class V2 = typename std::iterator_traits::value_type> + auto scatter_n(It1 first, Size n, It2 d_first, int root = 0){ + return scatter_n( + first, n, detail::iterator_category_t{}, detail::value_category_t{}, + d_first, detail::iterator_category_t{}, detail::value_category_t{}, + root + ); + } + template + auto scatter_n_from(It2 d_first, Size n, It1 first, int root = 0) { + return scatter_n(first, n*size(), d_first, root); + } + template + auto scatter( + RA1 first, RA1 last, + detail::random_access_iterator_tag /*tag*/, + It2 d_first, int root + ) { + return scatter_n(first, std::distance(first, last), d_first, root); + } + template + auto scatter(It1 first, It1 last, It2 d_first, int root = 0){ + return scatter(first, last, detail::iterator_category_t{}, d_first, root); + } + template + auto scatter_from( + RA2 first, RA2 last, + detail::random_access_iterator_tag /*tag*/, + It1 d_first, int root + ) { + return scatter_n_from(first, std::distance(first, last), d_first, root); // NOLINT(readability-suspicious-call-argument) TODO(correaa) range based passing + } + template + auto scatter_value_from(V& v, It1 first, int root = 0) { + return scatter_n_from(std::addressof(v), 1, first, root); + } + template::value_type> + V scatter(It1 first, [[maybe_unused]] It1 last, int root = 0) { + if(rank()==root) {assert(std::distance(first, last) == size());} + V v; + scatter_value_from(v, first, root); + return v; + } + template::value_type> + V scatter(It first, int root = 0) { + V v; + scatter_value_from(v, first, root); + return v; + } + template::value_type> + V scatter(Container c, int root = 0, void* /*unused*/ = nullptr) { // TODO(correaa) find name for parameter + assert( (int)c.size() == (rank()==root?size():0) ); + using std::begin; + return scatter(begin(c), root); + } + + template + auto scatter_from(It2 d_first, It2 d_last, It1 first, int root = 0){ + return scatter_from(d_first, d_last, detail::iterator_category_t{}, first, root); // NOLINT(readability-suspicious-call-argument) TODO(correaa) range based passing + } + + template + void scatterv_n( + CIt1 first, int* counts, int* displs, + detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + CIt2 d_first, Size n, + detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + int root = 0 + ) { + MPI_(Scatterv)( + detail::data( first), counts, displs, + datatype::value_type>{}(), + detail::data(d_first), n, + datatype::value_type>{}(), + root, impl_ + ); + } + template + auto scatterv_n( + CItCIt1 citcit1, CItN ns, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + CIt2 it2, detail::contiguous_iterator_tag /*tag*/, detail::basic_tag /*tag*/, + int root + ) { + std::vector counts(ns, ns + size()); + std::vector displs(usize()); + for(int i = 0; i != size(); ++i) { // NOLINT(altera-unroll-loops) TODO(correaa) use algorithm + displs[i+1] = detail::data(citcit1[i+1]) - detail::data(citcit1[i]); + } // adjacent_difference doesn't work here because of type incompatibility + // std::adjacent_difference(citcit1, citcit1 + size(), begin(displs) + 1, [](auto& a, auto& b){return detail::data(a) - detail::data(b);}); + int n = scatter(counts.begin(), counts.end()); + scatterv_n( + detail::data(*citcit1), counts.data(), displs.data(), detail::contiguous_iterator_tag{}, detail::basic_tag{}, + it2, n , detail::contiguous_iterator_tag{}, detail::basic_tag{}, + root + ); + return it2; + } + template + auto scatterv_n(CItIt1 citit1, CItN ns, It2 it2, int root = 0){ + scatterv_n( + citit1, ns, detail::iterator_category_t::value_type>{}, detail::basic_tag{}, + it2 , detail::iterator_category_t{}, detail::basic_tag{}, + root + ); + } + template + auto scatterv(Container const& c, It2 it2, int root = 0) { + assert( (int)c.size() == ((rank()==root)?size():0) ); + if(rank() == root) { + std::cerr<< "in scatterv " << detail::data(c[1].begin()) - detail::data(c[0].begin()) << " " << detail::data(c[2].begin()) - detail::data(c[0].begin()) << std::endl; + } + using std::begin; using std::end; + std::vector displs(c.size()); + for(std::vector::size_type i = 0; i != displs.size(); ++i) { // NOLINT(altera-id-dependent-backward-branch,altera-unroll-loops) TODO(correaa) use an algorithm + std::ptrdiff_t d = detail::data(c[i].begin()) - detail::data(c[0].begin()); + assert( d <= static_cast(std::numeric_limits::max()) ); + displs[i] = static_cast(d); + } + if(rank() == root) { + std::cerr<<"in scatterv 2 "<< displs[0] <<" "<< displs[1] <<" "<< displs[2] < counts(c.size()); + std::transform( + counts.begin(), counts.end(), begin(c), counts.begin(), + [](auto& /*unused*/, auto const& b){return std::distance(begin(b), end(b));} + ); + int n = scatter(counts); + scatterv_n( + detail::data(begin(*begin(c))), counts.data(), displs.data(), detail::contiguous_iterator_tag{}, detail::basic_tag{}, + detail::data(it2), n , detail::contiguous_iterator_tag{}, detail::basic_tag{}, + root + ); + return it2 + n; + } + template class id {using type = T;}; + + template=1)> > + auto gather_n(MultiIt first, Size count, MultiIt2 d_first, int root = 0) + ->decltype( MPI_Gather (mpi3::base(first), count, mpi3::type{first}, mpi3::base(d_first), count, mpi3::type{d_first}, root, impl_), d_first+count) { + return MPI_(Gather)(mpi3::base(first), count, mpi3::type{first}, mpi3::base(d_first), count, mpi3::type{d_first}, root, impl_), d_first+count; } + + using count_type = int; + + template + It2 gather_n( + It1 first, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + Size1 count, + It2 d_first, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + Size2 d_count, + int root + ) { + MPI_(Gather) + ( + detail::data( first), static_cast( count), datatype::value_type>{}(), + detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), + root, impl_ + ); + return d_first + ((rank() == root) ? static_cast::difference_type>(d_count * static_cast(size())) : 0); + } + + template + auto gather_n(It1 first, Size1 count, It2 d_first, Size2 d_count, int root) { + return gather_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_count, + root + ); + } + template{})>> + auto gather_n(It1 first, Size1 count, It2 d_first, int root = 0) { + return gather_n(first, count, d_first, count, root); + } + + template + auto scatterv_n_from(It2 d_first, Size n, It1 first, int root = 0) { + std::vector counts(usize()); + int nn = n; + gather_n(std::addressof(nn), 1, counts.data(), root); + std::vector displs(usize()); + partial_sum(counts.begin(), counts.end(), displs.begin()+1); + scatterv_n( + first, counts.data(), displs.data(), + detail::iterator_category_t{}, detail::value_category_t::value_type>{}, + d_first, n, + detail::iterator_category_t{}, detail::value_category_t::value_type>{}, + root + ); + std::advance(first, rank()==root?displs.back() + counts.back():0); + return first; + } + template + auto scatterv_from( + RA2 d_first, RA2 d_last, + detail::random_access_iterator_tag /*tag*/, It1 first, int root + ) { + return scatterv_n_from(d_first, std::distance(d_first, d_last), first, root); + } + template + auto scatterv_from(It2 d_first, It2 d_last, It1 first, int root = 0){ + return scatterv_from(d_first, d_last, detail::iterator_category_t{}, first, root); + } + + template + void all_gather_value(T const& t, It first){all_gather_n(std::addressof(t), 1, first);} + template std::vector + all_gather_value(T const& t) { + std::vector ret(size()); + all_gather_value(t, ret.begin()); + return ret; + } + + template + It gather_value(T const& t, It first, int root) { + return gather_n(std::addressof(t), 1, first, root); + } + template + std::vector gather_value(T const& t, int root = 0) { + std::vector ret((rank() == root) ? static_cast(size()) : 0); + gather_value(t, ret.begin(), root); + return ret; + } + + protected: + template + void advance(It& it, Size count) { std::advance(it, count); } + + template + void advance(It& it, Size s, int r) { std::advance(it, rank() == r ? s : 0); } + + template< + class GatherMode, + typename It1, typename Size, typename It2, + typename = std::enable_if_t< + std::is_same< + typename std::iterator_traits::value_type, + typename std::iterator_traits::value_type + >{} + >, class... Root + > + auto a_gather_n( + GatherMode gm, + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::memcopyable_tag /*tag*/, + Size count, + It2 d_first, + detail::contiguous_iterator_tag /*tag*/, + detail::memcopyable_tag /*tag*/, + Root... root + ) { + int s = gm( + detail::data(first), count*sizeof(*first), MPI_BYTE, + detail::data(d_first), count*sizeof(*d_first), MPI_BYTE, + root..., impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot gather");} + advance(d_first, count*size(), root...); + // std::advance(d_first, count); + return d_first; + } + + public: + template + auto all_gather_n( + It1 first, Size1 count, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + It2 d_first, Size2 d_count, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/ + ) { + MPI_(Allgather)( + detail::data( first), static_cast( count), datatype::value_type>{}(), // TODO(correaa) use safe cast + detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), // TODO(correaa) use safe cast + impl_ + ); + return d_first + static_cast(d_count) * usize(); + } + template + auto all_gather_n(It1 first, Size count, It2 d_first, Size d_count) { + return all_gather_n( + first, count, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_first, d_count, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{} + ); + } + template + It2 all_gather( + It1 first, It1 last, + /**/ detail::random_access_iterator_tag /*tag*/, + It2 d_first + ) {return all_gather_n(first, std::distance(first, last), d_first);} + template + It2 all_gather(It1 first, It1 last, It2 d_first) { + return all_gather(first, last, detail::iterator_category_t{}, d_first); + } + template + auto all_gatherv_n( + It1 first, Size count, + It2 d_first, + detail::output_iterator_tag /*tag*/, + CountsIt counts, DisplsIt displs + ) { + auto const s = static_cast(std::accumulate(counts, counts + size(), typename std::iterator_traits::value_type{0})); + std::vector::value_type> buff(s); + auto e = all_gatherv_n(first, count, buff.data(), counts, displs); + assert( e == std::next(buff.data(), static_cast(buff.size())) ); + using std::move; + return move(buff.begin(), buff.end(), d_first); // cppcheck-suppress returnDanglingLifetime ; cppcheck 2.3 bug + } + template + auto all_gatherv_n( + It1 first, Size count, + It2 d_first, + detail::forward_iterator_tag /*tag*/, + CountsIt counts, DisplsIt displs + ) { + return all_gatherv_n( + first, count, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + counts, displs); + } + template + auto all_gatherv_n(It1 first, Size count, It2 d_first, CountsIt counts, DisplsIt displs) { + return all_gatherv_n( + first, count, + d_first, + detail::iterator_category_t{}, + counts, displs + ); + } + template + auto all_gatherv_n( + It1 first, Size count, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + It2 d_first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + CountsIt counts, DisplsIt displs + ) { + MPI_(Allgatherv)( + detail::data( first), static_cast(count) , datatype::value_type>{}(), // TODO(correaa) use safe cast + detail::data(d_first), detail::data(counts), detail::data(displs), datatype::value_type>{}(), + impl_ + ); + return d_first + detail::data(displs)[size()-1] + detail::data(counts)[size()-1]; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic) + } + template + auto all_gather_n( + It1 first, Size count, + detail::forward_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + It2 d_first, Size d_count, + detail::forward_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/ + ) { + detail::package po(*this); + package_oarchive poa(po); + std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); + // while(count--) {poa << *(first++);} // TODO(correaa) remove comment + auto const posize = static_cast(po.size()); + std::vector sizes (usize()); + std::vector displs(usize()); + all_gather_n(&posize, 1, sizes.begin(), 1); + partial_sum(sizes.begin(), sizes.end(), displs.begin()+1); + detail::package pi(*this); + auto const total = static_cast(std::accumulate(sizes.begin(), sizes.end(), 0)); + pi.resize(total); + all_gatherv_n(po.data(), po.size(), pi.data(), sizes.data(), displs.data()); + package_iarchive pia(pi); + d_count *= size(); + return std::copy_n(package_iarchive::iterator::value_type>(pia), d_count, d_first); + } + template + auto all_gather_n(It1 first, Size count, It2 d_first) { + return all_gather_n(first, count, d_first, count); + } + template + auto all_gather_v(V const& v, It2 d_first) { + return all_gather_n(std::addressof(v), 1, d_first); + } + template + auto all_gather_as(V const& v) { + Vector ret(usize()); + all_gather_v(v, ret.begin()); + return ret; + } + template + auto all_gatherv_n(It1 first, Size count, It2 d_first) { + std::vector counts(usize() ); + std::vector displs(usize() + 1); + int c = static_cast(count); + all_gather_n(&c, 1, counts.begin()); + partial_sum(counts.begin(), counts.end(), displs.begin()+1); + return all_gatherv_n(first, count, d_first, counts.begin(), displs.begin()); + } + template + auto gatherv_n(It1 first, Size count, It2 d_first) { + std::vector counts(usize() ); + std::vector displs(usize() + 1); + int c = count; + all_gather_n(&c, 1, counts.begin()); + partial_sum(counts.begin(), counts.end(), displs.begin()+1); + return gatherv_n(first, count, d_first, counts.begin(), displs.begin()); + } + template + auto all_gather_n( + It1 first, + /**/ detail::forward_iterator_tag /*tag*/, + /**/ detail::value_unspecified_tag /*tag*/, + Size1 count, + It2 d_first, + /**/ detail::forward_iterator_tag /*tag*/, + /**/ detail::value_unspecified_tag /*tag*/, + Size2 d_count + ) { + detail::package po(*this); + package_oarchive poa(po); + std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); + // while(count--) {poa << *(first++);} + auto const posize = static_cast(po.size()); + std::vector sizes (usize()); + std::vector displs(usize()); + all_gather_n(&posize, 1, sizes.begin(), 1); + partial_sum(sizes.begin(), sizes.end(), displs.begin()+1); + detail::package pi(*this); + auto total = static_cast(std::accumulate(sizes.begin(), sizes.end(), 0)); + pi.resize(total); + all_gatherv_n(po.data(), po.size(), pi.data(), sizes.data(), displs.data()); + package_iarchive pia(pi); + d_count *= size(); + return std::copy_n(package_iarchive::iterator::value_type>(pia), d_count, d_first); + } + template + auto all_gatherv( + It1 first, It1 last, + detail::random_access_iterator_tag /*tag*/, It2 d_first + ) { + return all_gatherv_n(first, std::distance(first, last), d_first); + } + template + auto all_gatherv(It1 first, It1 last, It2 d_first) { + return all_gatherv( + first, last, + detail::iterator_category_t{}, + d_first + ); + } + + template + auto gatherv_n( + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size1 count, + It2 d_first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Itc counts, + detail::contiguous_iterator_tag /*tag*/, + Itd displs, + detail::contiguous_iterator_tag /*tag*/, + int root + ) { + MPI_(Gatherv)( + detail::data( first), static_cast(count), datatype::value_type>{}(), // TODO(correaa) use safe cast + detail::data(d_first), detail::data(counts), detail::data(displs), datatype::value_type>{}(), + root, impl_ + ); + } + template + auto gatherv_n( + It1 first, Size1 count, + It2 d_first, Itc counts, Itd displs = 0, + int root = 0 + ){ + return gatherv_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + counts, + detail::iterator_category_t{}, + displs, + detail::iterator_category_t{}, + root + ); + } + template + auto igather_n( + It1 first, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + Size1 count, + It2 d_first, + /**/ detail::contiguous_iterator_tag /*tag*/, + /**/ detail::basic_tag /*tag*/, + Size2 d_count, + int root + ) { + request ret; + MPI_(Igather)( + detail::data(first) , static_cast( count), datatype::value_type>{}(), + detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), + root, impl_, &ret.impl_ + ); + return ret; + } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) MPI_Wait called on destructor of ret + template + auto iall_gather_n( + It1 first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size1 count, + It2 d_first, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + Size2 d_count + ) { + request ret; + MPI_(Iallgather)( + detail::data(first) , static_cast( count), datatype::value_type>{}(), // TODO(correaa) use safe cast + detail::data(d_first), static_cast(d_count), datatype::value_type>{}(), // TODO(correaa) use safe cast + impl_, &ret.impl_ + ); + return ret; + } // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // wait is in request destructor + template + auto gather_n( + It1 first, + detail::input_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + Size1 count, + It2 d_first, + detail::input_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + Size2 d_count, + int root + ) { + detail::package po(*this); + package_oarchive poa(po); + std::copy_n(first, count, package_oarchive::iterator::value_type>(poa)); + // while(count--) {poa << *(first++);} TODO(correaa) remove comment + int posize = static_cast(po.size()); + std::vector sizes(rank()==root?usize():0); + gather_n(&posize, 1, sizes.begin(), 1, root); + std::vector displs(sizes.size()+1); + partial_sum(sizes.begin(), sizes.end(), displs.begin()+1); + detail::package pi(*this); + auto const total = static_cast(std::accumulate(sizes.begin(), sizes.end(), 0)); + pi.resize(total); + gatherv_n(po.data(), po.size(), pi.data(), sizes.data(), displs.data(), root); + if(rank() == root) { + package_iarchive pia(pi); + d_count *= size(); + return std::copy_n( + package_iarchive::iterator::value_type>(pia), + d_count, + d_first + ); + // while(d_count--) {pia >> *(d_first++);} + } + return d_first; + } + template + auto igather_n(It1 first, Size1 count, It2 d_first, Size2 d_count, int root){ + return igather_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_count, + root + ); + } + template + auto iall_gather_n(It1 first, Size1 count, It2 d_first, Size2 d_count) { + return iall_gather_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + d_first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_count + ); + } + template + auto igather_n(It1 first, Size1 count, It2 d_first, int root = 0) { + return igather_n(first, count, d_first, count, root); + } + template + auto iall_gather_n(It1 first, Size1 count, It2 d_first) { + return iall_gather_n(first, count, d_first, count); + } + template + auto gather( + It1 first, It1 last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + It2 d_first, int root + ) { + return gather_n(first, std::distance(first, last), d_first, root); + } + template + auto gather( + It1 first, It1 last, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + It2 d_first, int root + ) { + return gather_n(first, std::distance(first, last), d_first, root); + } + template + auto igather( + It1 first, It1 last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + It2 d_first, int root + ) { + return igather_n(first, std::distance(first, last), d_first, root); + } + template + auto iall_gather( + It1 first, It1 last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + It2 d_first + ) { + return iall_gather_n(first, std::distance(first, last), d_first); + } + template + auto gather( + It1 first, It1 last, + detail::input_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + It2 d_first, int root + ){ + mpi3::vector::value_type> buffer(first, last); + return gather_n(buffer.data(), buffer.size(), d_first, root); + } + template + auto gather(It1 first, It1 last, It2 d_first, int root = 0){ + return gather( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_first, + root + ); + } + template + auto igather(It1 first, It1 last, It2 d_first, int root){ + return igather( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_first, + root + ); + } + template + auto iall_gather(It1 first, It1 last, It2 d_first) { + return iall_gather( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_first + ); + } + template + auto gather( + It1 first, It1 last, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + It2 d_first, It2 d_last, + detail::contiguous_iterator_tag /*tag*/, + detail::basic_tag /*tag*/, + int root + ) { + return gather_n( + first, std::distance(first, last), + d_first, std::distance(d_first, d_last), + root + ); + } + template + auto gather( + It1 first, It1 last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + It2 d_first, It2 d_last, + detail::random_access_iterator_tag /*tag*/, + detail::value_unspecified_tag /*tag*/, + int root + ) { + return gather_n( + first, std::distance(first, last), + d_first, std::distance(d_first, d_last), + root + ); + } + template + auto gather( + It1 first, It1 last, + detail::forward_iterator_tag /*forward*/, + detail::basic_tag /*basic*/, + It2 d_first, It2 d_last, + detail::random_access_iterator_tag /*random_access*/, + detail::value_unspecified_tag /*value_unspecified*/, + int root + ) { + mpi3::vector::value_type> v(first, last); + return gather_n( + v.data(), v.size(), + d_first, std::distance(d_first, d_last), + root + ); + } + + template + auto gather(It1 first, It1 last, It2 d_first, It2 d_last, int root) { + return gather( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + d_first, d_last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + root + ); + } + + private: + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto gather_n_dispatch( + gather_mode g, std::true_type /*true*/, + It1 first, Size count, It2 d_first, int root = 0 + ) { + int s = g( + detail::data(first) , count, datatype{}(), + detail::data(d_first), count, datatype{}(), + root, impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} + } + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto gather_n_dispatch( + gather_mode g, std::false_type /*false*/, + It1 first, Size count, It2 d_first, int root = 0 + ) { + int s = g( + detail::data(first) , count, datatype{}(), + detail::data(d_first), count, datatype{}(), + root, impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} + } + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto gather_n_dispatch( + igather_mode g, std::true_type /*true*/, + It1 first, Size count, It2 d_first, int root = 0 + ) { + request r; + int s = g( + detail::data(first) , count, datatype{}(), + detail::data(d_first), count, datatype{}(), + root, impl_, &r.impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} + return r; + } + template::value_type, + class V2 = typename std::iterator_traits::value_type + > + auto gather_n_dispatch( + all_gather_mode g, std::true_type /*true*/, + It1 first, Size count, It2 d_first, int /*root*/= 0 + ) { + int s = g( + detail::data(first) , count, datatype{}(), + detail::data(d_first), count, datatype{}(), + impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot scatter");} + } + + public: + std::string get_name() const { + std::string ret(MPI_MAX_OBJECT_NAME, '\0'); + int len; // NOLINT(cppcoreguidelines-init-variables) : delayed initialization + MPI_(Comm_get_name)(impl_, ret.data(), &len); + ret.resize(static_cast(len)); + return ret; + } + void set_name(std::string const& s) {MPI_(Comm_set_name)(impl_, s.c_str());} + std::string name() const {return get_name();} + + [[deprecated]] void name(std::string const& s) {set_name(s);} + +#if not defined(EXAMPI) + static mpi3::communicator& parent() { + static_assert(sizeof(MPI_Comm) == sizeof(mpi3::communicator), "!"); + static_assert(std::is_same{}, "!"); + MPI_Comm* p{}; MPI_Comm_get_parent(p); assert(p); + return reinterpret_cast(*p); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) : TODO(correaa) avoid reinterpret_cast + } + + static communicator spawn(std::string const& argv0, int np) { + communicator intercomm; + MPI_Comm_spawn(argv0.data(), MPI_ARGV_NULL, np, MPI_INFO_NULL, 0, MPI_COMM_SELF, &intercomm.impl_, MPI_ERRCODES_IGNORE ); + return intercomm; + } + + communicator intercommunicator_create(int local_leader, communicator const& peer, int remote_leader, int tag = 0) const{ + communicator ret; + MPI_(Intercomm_create)(impl_, local_leader, peer.impl_, remote_leader, tag, &ret.impl_); + return ret; + } + + communicator create(int local_leader, communicator const& peer, int remote_leader, int tag = 0) const{ + return intercommunicator_create(local_leader, peer, remote_leader, tag); + } +#endif + + communicator create(group const& g) const; + communicator create_group(group const& g, int tag) const; + FILE* fopen(char const* filename, int amode = unsigned{ + #if not defined(EXAMPI) + MPI_MODE_RDWR} | unsigned{MPI_MODE_CREATE + #endif + }); + + class topology { + int impl_; + + public: + explicit topology(int impl) noexcept : impl_{impl} {} + + static topology const undefined; + static topology const graph; + static topology const cartesian; + + bool operator==(topology const& other) const {return impl_ == other.impl_;} + bool operator!=(topology const& other) const {return impl_ != other.impl_;} + bool operator< (topology const& other) const {return impl_ < other.impl_;} + }; + + inline static auto name(communicator::topology const& t) -> std::string const& { + static std::map const names = { + {communicator::topology::undefined, "undefined"}, + {communicator::topology::graph , "graph"}, + {communicator::topology::cartesian, "cartesian"}}; + return names.find(t)->second; + } + + template + friend T operator+=(communicator& comm, T const& t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator + return comm.all_reduce_value(t, std::plus<>{}); + } + + template + friend T operator&=(communicator& comm, T const& t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator + return comm.all_reduce_value(t, std::bit_and<>{}); + } + + friend bool operator&=(communicator& comm, bool t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator + bool ret = true; + comm.all_reduce_n(&t, 1, &ret, std::logical_and<>{}); + return ret; + } + friend bool operator|=(communicator& comm, bool t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator + bool ret = false; + comm.all_reduce_n(&t, 1, &ret, std::logical_or<>{}); + return ret; + } + + template + friend communicator& operator<<(communicator& comm, T const& t) { // NOLINT(fuchsia-overloaded-operator) : experimental operator + comm.send_value(t); + return comm; + } +}; + +inline communicator::topology const communicator::topology::undefined{MPI_UNDEFINED}; // NOLINT(fuchsia-statically-constructed-objects) see if EXAMPI will allow it to be constexpr +inline communicator::topology const communicator::topology::graph {MPI_GRAPH}; // NOLINT(fuchsia-statically-constructed-objects) see if EXAMPI will allow it to be constexpr +inline communicator::topology const communicator::topology::cartesian{MPI_CART}; // NOLINT(fuchsia-statically-constructed-objects) see if EXAMPI will allow it to be constexpr + +inline void barrier(communicator& self) { self. barrier();} +#if not defined(EXAMPI) +inline auto ibarrier(communicator& self) {return self.ibarrier();} +#endif + +inline communicator::communicator(group const& g, int tag){ + MPI_(Comm_create_group)(MPI_COMM_WORLD, &const_cast(g), tag, &impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with +} + +inline communicator::communicator(group const& g){ + MPI_(Comm_create)(MPI_COMM_WORLD, &const_cast(g), &impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with +} +// https://www.open-mpi.org/doc/v3.0/man3/MPI_Comm_create_group.3.php +// MPI_Comm_create_group is similar to MPI_Comm_create; however, MPI_Comm_create must be called by all processes in the group of comm, whereas MPI_Comm_create_group must be called by all processes in group, which is a subgroup of the group of comm. In addition, MPI_Comm_create_group requires that comm is an intracommunicator. MPI_Comm_create_group returns a new intracommunicator, newcomm, for which the group argument defines the communication group. No cached information propagates from comm to newcomm. + +inline communicator::communicator(communicator const& o, group const& g){ + MPI_(Comm_create)(o.impl_, &const_cast(g), &impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with +} + +inline communicator::operator group() const{ + group ret; MPI_(Comm_group)(impl_, &ret.impl_); return ret; +} +//inline group::group(communicator const& c){MPI_(Comm_group)(&const_cast(c), &impl_);} + +inline communicator communicator::create(group const& g) const{ + communicator ret; + int const s = MPI_Comm_create(impl_, &const_cast(g), &ret.impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with + if(s != MPI_SUCCESS) {throw std::runtime_error{"cannot crate group"};} + return ret; +} + +inline communicator communicator::create_group(class group const& g, int tag = 0) const{ + communicator ret; + MPI_(Comm_create_group)(impl_, &const_cast(g), tag, &ret.impl_); // NOLINT(cppcoreguidelines-pro-type-const-cast) : TODO(correaa) consider using non-const argument to begin with + return ret; +} + +template +inline void communicator::deallocate_shared(pointer /*unused*/){ +// MPI_Free_mem(p.base_ptr(rank())); +} + +template +inline void communicator::deallocate(pointer& /*p*/, MPI_Aint /*size*/) { // TODO(correaa) should be called free? +// p.pimpl_->fence(); +// MPI_Free_mem(p.local_ptr()); +// MPI_Win_free(&p.pimpl_->impl_); +// delete p.pimpl_; +// p.pimpl_ == nullptr; +} + +#if 0 +template +inline window communicator::make_window(mpi3::size_t size){ + mpi3::info inf; + void* ptr; + window ret; + int s = MPI_Win_allocate(size*sizeof(T), sizeof(T), inf.impl_, this->impl_, &ptr, &ret.impl_); + if(s != MPI_SUCCESS) throw std::runtime_error("cannot window_allocate"); + return ret; +} +#endif + +class strided_range { + int first_; + int last_; + int stride_ = 1; + + public: + strided_range(int f, int l) : first_{f}, last_{l} {} // NOLINT(bugprone-easily-swappable-parameters) + strided_range(int f, int l, int s) : first_{f}, last_{l}, stride_{s} {} // NOLINT(bugprone-easily-swappable-parameters) + + int front() const {return first_;} + int back() const {return last_ - 1;} + int size() const {return (last_ - first_) / stride_;} +}; + +template +auto operator/(Range const& r, communicator& self) // NOLINT(fuchsia-overloaded-operator) : experimental operator overloading + ->decltype(self.scatter(begin(r), end(r))) { + return self.scatter(begin(r), end(r)); } + + +inline mpi3::communicator& deref(MPI_Comm const& handle) { + return reinterpret_cast(const_cast(handle)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-type-const-cast) +} + + +inline mpi3::communicator& grip_communicator(MPI_Comm const& handle) { + return reinterpret_cast(const_cast(handle)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-type-const-cast) +} + +[[deprecated]] inline mpi3::communicator& grip(MPI_Comm const& handle) { + return grip_communicator(handle); +} + +} // end namespace mpi3 +} // end namespace boost + +//BOOST_SERIALIZATION_REGISTER_ARCHIVE(boost::mpi3::package_oarchive) +//BOOST_SERIALIZATION_USE_ARRAY_OPTIMIZATION(boost::mpi3::detail::package_oarchive) +//BOOST_SERIALIZATION_USE_ARRAY_OPTIMIZATION(boost::mpi3::detail::package_iarchive) + +//#if not __INCLUDE_LEVEL__ + +//#include "../mpi3/main.hpp" +//#include "../mpi3/version.hpp" + +//#include + +//using std::cout; +//namespace mpi3 = boost::mpi3; + +//class V{ +// mpi3::communicator comm_; +// public: +// V(mpi3::communicator const& c) : comm_(c){} +// V(mpi3::communicator&& c) : comm_(std::move(c)){} +//}; + +//int mpi3::main(int, char*[], mpi3::communicator world){ +// std::cout << mpi3::undefined << std::endl; + +// static_assert(std::is_nothrow_constructible::value, "MyType should be noexcept MoveConstructible"); + +//// auto worldcopy1 = world; +//// auto worldcopy2 = std::move(worldcopy1); +//// V v(worldcopy); +//// V v2(std::move(v)); + +// if(world.rank() == 0) cout << "MPI version " << mpi3::version() << '\n'; +//// if(world.rank() == 0) cout << "Topology: " << name(world.topo()) << '\n'; + +// cout << "MPI_ERR_COMM = " << MPI_ERR_COMM << '\n'; + +// mpi3::communicator comm; +// assert(!comm); +//// cout << comm.rank() << '\n'; + +// mpi3::communicator comm2 = world; +// assert(comm2); +// assert(comm2.size() == world.size()); +// assert(comm2 == world); +// assert(&comm2 != &world); + +// mpi3::communicator comm3 = world;//.duplicate(); +// assert(comm3); +// assert(comm3 == world); +// assert(&comm3 != &world); +// comm = comm2; +// assert(&comm != &comm2); + +//// world2 = world; + +// return 0; +//#if 0 +//// boost::mpi3::communicator newcomm = world; +// { +// int color = world.rank()/3; +// communicator row_comm; +// row_comm = world.split(color); +// world.barrier(); +// std::cout << std::to_string(world.rank()) + " " + std::to_string(row_comm.rank()) + "\n";// << std::endl; +// world.barrier(); +// } +// { +// communicator row_comm = world/3; +// world.barrier(); +// std::cout << std::to_string(world.rank()) + " " + std::to_string(row_comm.rank()) + "\n";// << std::endl; +// world.barrier(); +// } + +// world.barrier(); +// if(world.rank() == 0) cout << "prime communicator" << '\n'; +// world.barrier(); + +// { +// // group world_group(world); +// // const int ranks[4] = {2, 3, 5, 7}; +// // group prime = world_group.include(ranks, ranks + 4); +// // communicator prime_comm(world, prime); +// auto prime_comm = world.subcomm({2,3,5,7}); +// cout << world.rank() << " -> " << prime_comm.rank() << "/" << prime_comm.size() << '\n'; +//#if 0 +// if(communicator::null != prime_comm){ +// cout << world.rank() << " -> " << prime_comm.rank() << "/" << prime_comm.size() << '\n'; +// }else{ +// cout << world.rank() << " not in prime comm\n"; +// } +//#endif +// } + +// world.barrier(); +// if(world.rank() == 0) cout << "prime communicator" << '\n'; +// world.barrier(); + +// if(0){ +// auto prime = world.subcomm({2,3,5,7}); +// if(prime.is_empty()){ +// // if (communicator::null != prime){ +// cout << world.rank() << " -> " << prime.rank() << "/" << prime.size() << '\n'; +// }else{ +// cout << world.rank() << " not in prime comm\n"; +// } +// } +//#endif +//} + +//#endif +#endif + diff --git a/include/mpi3/communicator/operatorators.hpp b/include/mpi3/communicator/operatorators.hpp new file mode 100644 index 00000000000..de7322e76e7 --- /dev/null +++ b/include/mpi3/communicator/operatorators.hpp @@ -0,0 +1,35 @@ +#if COMPILATION_INSTRUCTIONS +(echo "#include\""$0"\"" > $0x.cpp) && mpicxx -O3 -std=c++17 -Wfatal-errors -D_TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS $0x.cpp -o $0x.x && time mpirun -np 3 $0x.x $@ && rm -f $0x.x $0x.cpp; exit +#endif +#ifndef BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP +#define BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP + +#include "../../mpi3/main.hpp" +#include "../../mpi3/communicator.hpp" + +namespace boost{ +namespace mpi3{ +// communicator operator/(communicator const& comm, int n){ +// int color = comm.rank()*n/comm.size(); +// return comm.split(color); +// } +}} + +#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS +#include "../../mpi3/main.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator& world){ + auto hemi = world/2; + cout + << "I am " << world.name() << " " << world.rank() + << " also I am " << hemi.name() << " " << hemi.rank() + << std::endl + ; +} + +#endif +#endif + diff --git a/include/mpi3/communicator/operators.hpp b/include/mpi3/communicator/operators.hpp new file mode 100644 index 00000000000..bc18b7e818c --- /dev/null +++ b/include/mpi3/communicator/operators.hpp @@ -0,0 +1,36 @@ +#if COMPILATION_INSTRUCTIONS +(echo "#include\""$0"\"" > $0x.cpp) && mpicxx -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS $0x.cpp -o $0x.x && time mpirun -np 3 $0x.x $@ && rm -f $0x.x $0x.cpp; exit +#endif +#ifndef BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP +#define BOOST_MPI3_COMMUNICATOR_OPERATORS_HPP + +#include "../../mpi3/main.hpp" +#include "../../mpi3/communicator.hpp" + +namespace boost{ +namespace mpi3{ +// communicator operator/(communicator const& comm, int n){ +// int color = comm.rank()*n/comm.size(); +// return comm.split(color); +// } +}} + +#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_OPERATORS +#include "../../mpi3/main.hpp" + +namespace mpi3 = boost::mpi3; +using std::cout; + +int mpi3::main(int, char*[], mpi3::communicator world){ + auto hemi = world/2; + assert(hemi); + cout + << "I am " << world.name() << " " << world.rank() + << " also I am " << hemi.name() << " " << hemi.rank() + << std::endl + ; +} + +#endif +#endif + diff --git a/include/mpi3/communicator_fwd.hpp b/include/mpi3/communicator_fwd.hpp new file mode 100644 index 00000000000..9a6cea91583 --- /dev/null +++ b/include/mpi3/communicator_fwd.hpp @@ -0,0 +1,24 @@ +#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ +(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -g -std=c++14 `#-Wfatal-errors` -D_TEST_BOOST_MPI3_COMMUNICATOR_FWD $0x.cpp -o $0x.x && time mpirun -np 1 $0x.x $@ && rm -f $0x.x $0x.cpp; exit +#endif +#ifndef BOOST_MPI3_COMMUNICATOR_FWD_HPP +#define BOOST_MPI3_COMMUNICATOR_FWD_HPP + +namespace boost{ +namespace mpi3{ + +struct communicator; + +}} + +#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_FWD + +#include +using std::cout; + +int main(){} + +#endif +#endif + + diff --git a/include/mpi3/communicator_iterator.hpp b/include/mpi3/communicator_iterator.hpp new file mode 100644 index 00000000000..5b88f7736ec --- /dev/null +++ b/include/mpi3/communicator_iterator.hpp @@ -0,0 +1,55 @@ +#if COMPILATION_INSTRUCTIONS /* -*- indent-tabs-mode: t -*- */ +(echo "#include\""$0"\"" > $0x.cpp) && mpic++ -O3 -std=c++14 -Wfatal-errors -D_TEST_BOOST_MPI3_COMMUNICATOR_ITERATOR $0x.cpp -o $0x.x && time mpirun -n 5 $0x.x $@ && rm -f $0x.x $0x.cpp; exit +#endif +#ifndef BOOST_MPI3_COMMUNICATOR_ITERATOR_HPP +#define BOOST_MPI3_COMMUNICATOR_ITERATOR_HPP + +#include "../mpi3/communicator.hpp" +#include "../mpi3/process.hpp" + +namespace boost{ +namespace mpi3{ + +struct communicator_iterator { + int n_; + communicator* commptr_; + communicator_iterator() : n_(-1), commptr_(nullptr){} + communicator_iterator(void*** p) : communicator_iterator(){} +// communicator_iterator(communicator_iterator const& other) = default; + process operator*() const{return commptr_->operator[](n_);} + communicator_iterator& operator++(){n_++; return *this;} + communicator_iterator& operator--(){n_--; return *this;} + communicator_iterator& operator+=(int n){n_+=n; return *this;} + communicator_iterator& operator-=(int n){n_-=n; return *this;} +}; + +communicator_iterator next_periodic(communicator_iterator it){ + it.n_++; + it.n_ = it.n_ % it.commptr_->size(); + return it; +} + +communicator_iterator prior_periodic(communicator_iterator it){ + it.n_--; + if(it.n_ < 0) it.n_ += it.commptr_->size(); + return it; +} + +}} + +#ifdef _TEST_BOOST_MPI3_COMMUNICATOR_ITERATOR + +#include "../mpi3/main.hpp" + +namespace mpi3 = boost::mpi3; + +int mpi3::main(int, char*[], mpi3::communicator world){ +// for(auto&& p : world) +// std::periodic_next(p) << world.rank()*10; + return 0; +} + +#endif +#endif + + diff --git a/include/mpi3/config/NODISCARD.hpp b/include/mpi3/config/NODISCARD.hpp new file mode 100644 index 00000000000..f83a9851f5d --- /dev/null +++ b/include/mpi3/config/NODISCARD.hpp @@ -0,0 +1,27 @@ +// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4-*- +// © Alfredo A. Correa 2019-2021 +#ifndef MPI3_CONFIG_NODISCARD_HPP +#define MPI3_CONFIG_NODISCARD_HPP + +//#ifndef __has_cpp_attribute +//#define __has_cpp_attribute(name) 0 +//#endif + +//#ifndef BMPI3_NODISCARD +//#if defined(__NVCC__) +// #define BMPI3_NODISCARD(MsG) +//#elif (__has_cpp_attribute(nodiscard) and (__cplusplus>=201703L)) +// #if (__has_cpp_attribute(nodiscard)>=201907) and (__cplusplus>201703L) +// #define BMPI3_NODISCARD(MsG) [[nodiscard(MsG)]] +// #else +// #define BMPI3_NODISCARD(MsG) [[nodiscard]] +// #endif +//#elif __has_cpp_attribute(gnu::warn_unused_result) +// #define BMPI3_NODISCARD(MsG) [[gnu::warn_unused_result]] // NOLINT(cppcoreguidelines-macro-usage) : replaced by `nodiscard` in C++17 +//#else +// #define BMPI3_NODISCARD(MsG) +//#endif +//#endif + +#endif + diff --git a/include/mpi3/core.hpp b/include/mpi3/core.hpp new file mode 100644 index 00000000000..538203eefe4 --- /dev/null +++ b/include/mpi3/core.hpp @@ -0,0 +1,36 @@ +// -*- indent-tabs-mode:t;c-basic-offset:4;tab-width:4; -*- +// Copyright 2018-2022 Alfredo A. Correa + +#ifndef BOOST_MPI3_CORE_HPP +#define BOOST_MPI3_CORE_HPP + +#include + +#include // if you get a compilation error here it means that 1) you need to compile or defined your CXX as mpic++ or 2) have not setup the compilation flags to find MPI headers, or 3) not installed an MPI implementation (e.g. `apt install libopenmpi-dev openmpi-bin`) + +namespace boost { +namespace mpi3 { + +inline bool initialized() { + return MPI_(Initialized)(); +} + +inline bool finalized() { + return MPI_(Finalized)(); +} + +} // end namespace mpi3 +} // end namespace boost + +#if not __INCLUDE_LEVEL__ // _TEST_BOOST_MPI3_ENVIRONMENT + +#include + +namespace mpi3 = boost::mpi3; + +int main(){ + assert(not mpi3::initialized() ); +} + +#endif +#endif \ No newline at end of file diff --git a/include/mpi3/detail/basic_communicator.hpp b/include/mpi3/detail/basic_communicator.hpp new file mode 100644 index 00000000000..ec3569082a4 --- /dev/null +++ b/include/mpi3/detail/basic_communicator.hpp @@ -0,0 +1,343 @@ +// Copyright 2018-2023 Alfredo A. Correa + +#ifndef BMPI3_DETAIL_BASIC_COMMUNICATOR_HPP +#define BMPI3_DETAIL_BASIC_COMMUNICATOR_HPP + +#include "../../mpi3/vector.hpp" + +#include "../../mpi3/detail/buffer.hpp" +#include "../../mpi3/detail/iterator_traits.hpp" +#include "../../mpi3/detail/value_traits.hpp" + +#include "../../mpi3/match.hpp" +#include + +#include + +#include + +namespace boost { +namespace mpi3 { +namespace detail { + +class basic_communicator{ + protected: + using impl_t = MPI_Comm; + impl_t impl_ = MPI_COMM_NULL; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes,misc-non-private-member-variables-in-classes) : TODO(correaa) make private + + explicit basic_communicator(MPI_Comm impl) noexcept : impl_(impl){} + + public: + basic_communicator() noexcept = default; //: impl_(MPI_COMM_NULL){} + ~basic_communicator() = default; + + auto operator=(basic_communicator const&) -> basic_communicator& = delete; + auto operator=(basic_communicator&&) -> basic_communicator& = delete; + + basic_communicator(basic_communicator const&) = delete; // communicators are not copyable, only duplicable, if you know what you are doing, use `mutable` + basic_communicator(basic_communicator& other) { + if(MPI_COMM_NULL != other.impl_) {MPI_(Comm_dup)(other.impl_, &impl_);} + } + basic_communicator(basic_communicator&& other) noexcept : + impl_{std::exchange(other.impl_, MPI_COMM_NULL)} {} + + // [[deprecated]] void swap(basic_communicator& o) noexcept {std::swap(impl_, o.impl_);} + // [[deprecated]] friend void swap(basic_communicator& self, basic_communicator& other) noexcept {self.swap(other);} + + template + int pack_size(int count, detail::basic_tag /*tag*/) { + int size = -1; + MPI_(Pack_size)(count, detail::basic_datatype{}, impl_, &size); + return size; + } + template + int pack_size(int count) { + return pack_size(count, detail::value_category_t{}); + } + template + auto pack_n( + It first, + detail::contiguous_iterator_tag /*contiguous*/, + detail::basic_tag /*basic*/, + Size count, + uvector& p, int pos // NOLINT(misc-unused-parameters) bug in clang-tidy 12 + ) { + using value_type = typename std::iterator_traits::value_type; + MPI_(Pack)( + detail::data(first), static_cast(count), detail::basic_datatype{}, + p.data(), static_cast(p.size()), // TODO(correaa) use safe cast + &pos, impl_ + ); + return pos; + } + template + auto pack_n( + It first, Size count, + uvector& b, int pos + ){ + return pack_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + b, pos + ); + } + template + auto pack_n(It first, Size count, uvector& p) { + assert(p.size() < std::numeric_limits::max()); + int const pos = static_cast(p.size()); + p.resize(p.size() + static_cast(pack_size::value_type>(static_cast(count)))); + return pack_n(first, count, p, pos); + } + template + auto pack( + It first, It second, + detail::random_access_iterator_tag /*random_access*/, + detail::value_unspecified_tag /*value_unspecified*/, + uvector& b + ) { + return pack_n(first, std::distance(first, second), b); + } + + template + auto pack( + It first, It last, + /**/ detail::input_iterator_tag /*input*/, + uvector& b + ) { + std::for_each(first, last, [this, &b](auto& e) { pack_n(std::addressof(e), 1, b); }); + return b.size(); + } + + template + auto pack(It first, It second, uvector& b){ + return pack( + first, second, + typename detail::iterator_category::type{}, + detail::value_category_t::value_type>{}, + b + ); + } + template + auto unpack_n( + uvector& b, int pos, + It first, + /**/ detail::contiguous_iterator_tag /*contiguous*/, + /**/ detail::basic_tag /*basic*/, + Size count + ){ + using value_type = typename std::iterator_traits::value_type; + MPI_(Unpack)( + b.data(), static_cast(b.size()), &pos, + detail::data(first), static_cast(count), // TODO(correaa) use safe cast + detail::basic_datatype{}, + impl_ + ); + return pos; + } + template + auto unpack( + uvector& b, int pos, + It first, It last, + detail::random_access_iterator_tag /*random_access*/ + ) { + return unpack_n(b, pos, first, std::distance(first, last)); + } + + template + auto unpack( + uvector& b, int pos, + It first, It last, + detail::forward_iterator_tag /*forward*/ + ) { + std::for_each(first, last, [&b, &pos, this](auto& e) {pos = unpack_n(b, pos, std::addressof(e), 1);}); + // while(first != last){ + // pos = unpack_n(b, pos, std::addressof(*first), 1); + // ++first; + // } + return pos; + } + template + auto unpack( + uvector& b, int pos, + It first, It last + ) { + return unpack( + b, pos, + first, last, + /**/ detail::iterator_category_t{} + ); + } + template + auto unpack(detail::buffer& b, It first, It last){ + return b.pos = unpack(b, b.pos, first, last); + } + template + auto unpack_n(uvector& b, int pos, It first, Size count){ + return unpack_n( + b, pos, + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count + ); + } + template + auto unpack_n(detail::buffer& b, It first, Size count) { + // assert(0); + b.pos = unpack_n(b, b.pos, first, count); + return b.pos; + } + template + auto send_n( + It first, + detail::contiguous_iterator_tag /*contiguous*/, + detail::basic_tag /*basic*/, + Size count, + int dest, int tag + ) { + MPI_(Send)( + detail::data(first), static_cast(count), // TODO(correaa) use safe cast + detail::basic_datatype::value_type>{}, + dest, tag, + impl_ + ); + } + template + auto send_n(It first, Size count, int dest, int tag = 0) { + return send_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + count, + dest, tag + ); + } + template + auto send( + It first, It last, + detail::random_access_iterator_tag /*random_access*/, + detail::value_unspecified_tag /*value_unspecified*/, + int dest, int tag + ) { + return send_n(first, std::distance(first, last), dest, tag); + } + template + auto send( + It first, It last, + detail::input_iterator_tag /*input*/, + detail::basic_tag /*basic*/, + int dest, int tag + ) { + mpi3::uvector::value_type> buffer(first, last); + return send_n(buffer.data(), buffer.size(), dest, tag); + } + template + auto send(It first, It last, int dest, int tag = 0) { + return send( + first, last, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + dest, tag + ); + } + auto send(uvector const& p, int dest, int tag = 0) { + return send_n(p.data(), p.size(), dest, tag); + } + + template + auto receive_n( + It dest, + /**/ detail::forward_iterator_tag /*forward*/, + /**/ detail::basic_tag /*basic*/, + Size n, + int source, int tag + ) { + mpi3::uvector::value_type> buffer(n); + receive_n(buffer.data(), buffer.size(), source, tag); + return std::copy_n(buffer.begin(), n, dest); + } + + #if not defined(EXAMPI) + match matched_probe(int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) const { + match m; + MPI_(Mprobe)(source, tag, impl_, &m.message::impl_, &m.status::impl_); + return m; + } + + auto receive(uvector& b, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) const { + match m = matched_probe(source, tag); + auto const count = static_cast(m.count()); + auto const size = static_cast(b.size()); + b.resize(b.size() + count); + return m.receive_n(std::next(b.data(), size), count); + } + #endif + + #if not defined(EXAMPI) + auto receive(detail::buffer& b, int source = MPI_ANY_SOURCE, int tag = MPI_ANY_TAG) const { + return receive(static_cast&>(b), source, tag); + } + #endif + template + auto send_receive_replace_n( + It first, + detail::forward_iterator_tag /*forward*/, + detail::basic_tag /*basic*/, + Size size, Meta... meta + ) { + using value_type = typename std::iterator_traits::value_type; + mpi3::uvector buffer(size); + std::copy_n(first, buffer.size(), buffer.begin()); + send_receive_replace_n(buffer.begin(), buffer.size(), meta...); + return std::copy_n(buffer.begin(), buffer.size(), first); + } + + template + auto send_receive_replace_n( + It first, + detail::contiguous_iterator_tag /*contiguous*/, + detail::basic_tag /*basic*/, + Size size, + int dest, int source, + int sendtag, int recvtag + ) { + using value_type = typename std::iterator_traits::value_type; + status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization + int s = MPI_Sendrecv_replace( + detail::data(first), size, detail::basic_datatype{}, + dest, sendtag, source, recvtag, impl_, &ret.impl_ + ); + if(s != MPI_SUCCESS) {throw std::runtime_error("cannot send_receive");} + return first + size; + } + + template + auto send_receive_replace_n( + It first, Size size, + int dest, int source, // = MPI_ANY_SOURCE, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_replace_n( + first, + detail::iterator_category_t{}, + detail::value_category_t::value_type>{}, + size, + dest, source, sendtag, recvtag + ); + } + template + auto send_receive_n( + It first, Size size, + int dest, int source, // = MPI_ANY_SOURCE, + int sendtag = 0, int recvtag = MPI_ANY_TAG + ) { + return send_receive_replace_n(first, size, dest, source, sendtag, recvtag); + } +}; + +} // end namespace detail +} // end namespace mpi3 +} // end namespace boost +#endif diff --git a/include/mpi3/detail/buffer.hpp b/include/mpi3/detail/buffer.hpp new file mode 100644 index 00000000000..03440cfb1ea --- /dev/null +++ b/include/mpi3/detail/buffer.hpp @@ -0,0 +1,42 @@ +#ifndef MPI3_DETAIL_BUFFER_HPP +#define MPI3_DETAIL_BUFFER_HPP + +//#include "../../mpi3/communicator_fwd.hpp" +//#include "../../mpi3/detail/iterator.hpp" +#include "../../mpi3/detail/datatype.hpp" // packed +#include "../../mpi3/vector.hpp" + + +namespace boost{ +namespace mpi3{ +namespace detail{ + +struct buffer : mpi3::uvector{ + int pos = 0; // NOLINT(misc-non-private-member-variables-in-classes) TODO(correaa) : make private + buffer() = default; + explicit buffer(std::size_t r) {reserve(r);} + buffer(buffer const&) = delete; + buffer(buffer&&) = delete; + buffer& operator=(buffer const&) = delete; + buffer& operator=(buffer&&) = delete; + ~buffer() = default; +}; + +} // end namespace detail +} // end namespace mpi3 +} // end namespace boost + +//#ifdef _TEST_MPI3_BUFFER + +//#include "../mpi3/communicator.hpp" +//#include "../mpi3/main.hpp" + +//namespace mpi3 = boost::mpi3; +//using std::cout; + +//int mpi3::main(int, char*[], mpi3::communicator world){ +//} + +//#endif +#endif + diff --git a/include/mpi3/detail/call.hpp b/include/mpi3/detail/call.hpp new file mode 100644 index 00000000000..b3e81844cb9 --- /dev/null +++ b/include/mpi3/detail/call.hpp @@ -0,0 +1,76 @@ +// © Alfredo A. Correa 2019-2021 +#ifndef BOOST_MPI3_DETAIL_CALL_HPP +#define BOOST_MPI3_DETAIL_CALL_HPP + +#include "../error.hpp" +#include "../status.hpp" + +#include "../config/NODISCARD.hpp" + +#include // MPI_MAX_PROCESSOR_NAME + +#include + +namespace boost { +namespace mpi3 { +namespace detail { + +template +int call() { + int ret; // NOLINT(cppcoreguidelines-init-variables) delayed init + auto const e = static_cast((*F)(&ret)); + if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} + return ret; +} + +template +std::string call() { + int len = -1; + std::string name(MPI_MAX_PROCESSOR_NAME, '\0'); // std::array name{}; + auto const e = static_cast((*F)(name.data(), &len)); + assert(len >= 0); + name.resize(static_cast(len)); + if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} + return name; +} + +template((*F)(std::declval()...)))* = nullptr> +void call(Args... args) { + auto const e = static_cast((*F)(args...)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor + if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} +} + +template((*F)(std::declval()..., std::declval())))* = nullptr> +mpi3::status call(Args... args) { + mpi3::status ret; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) delayed initialization + auto const e = static_cast((*F)(args..., &ret.impl_)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor + if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} + return ret; +} + +template((*F)(std::declval()..., std::declval(), std::declval())))* = nullptr> +[[nodiscard]] auto call(Args... args) { + std::pair ret; // NOLINT(cppcoreguidelines-init-variables) delayed initialization + auto const e = static_cast((*F)(args..., &ret.first, &ret.second)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor + if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} + // cppcheck-suppress uninitvar + return ret; +} + +template((*F)(std::declval()..., std::declval())))* = nullptr> +[[nodiscard]] int call(Args... args) { + int ret; // NOLINT(cppcoreguidelines-init-variables) delayed initialization + auto const e = static_cast((*F)(args..., &ret)); // NOLINT(clang-analyzer-optin.mpi.MPI-Checker) // non-blocking calls have wait in request destructor + if(e != mpi3::error::success) {throw std::system_error{e, "cannot call function " + std::string{__PRETTY_FUNCTION__}};} + // cppcheck-suppress uninitvar + return ret; +} + +#define MPI3_CALL(F) detail::call // NOLINT(cppcoreguidelines-macro-usage) +#define MPI_(F) MPI3_CALL(MPI_##F) // NOLINT(cppcoreguidelines-macro-usage): name concatenation + +} // end namespace detail +} // end namespace mpi3 +} // end namespace boost + +#endif diff --git a/include/mpi3/detail/datatype.hpp b/include/mpi3/detail/datatype.hpp new file mode 100644 index 00000000000..4e5797fbfc2 --- /dev/null +++ b/include/mpi3/detail/datatype.hpp @@ -0,0 +1,234 @@ +// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- +// Copyright 2017-2023 Alfredo A. Correa + +#ifndef BOOST_MPI3_DETAIL_DATATYPE_HPP +#define BOOST_MPI3_DETAIL_DATATYPE_HPP + +// #define OMPI_SKIP_MPICXX 1 // https://github.com/open-mpi/ompi/issues/5157 +#include + +#if defined(__NVCC__) +#include +#endif + +#include +#include +#include // std::byte +#include +#include +#include // pair + +namespace boost { +namespace mpi3 { + +template +struct vlp { // value location pair + T value; // NOLINT(misc-non-private-member-variables-in-classes) + int location; // NOLINT(misc-non-private-member-variables-in-classes) + bool operator<(vlp const& other) const { + // if(value == other.value) {return location < other.location;} // partial order on purpose? + return value < other.value; + } +}; + +namespace detail { + +using float_int = std::pair; +using long_int = std::pair; // NOLINT(google-runtime-int) : long <-> int64 +using double_int = std::pair; +using short_int = std::pair; // NOLINT(google-runtime-int) : short <-> int16 +using int_int = std::pair; + +using long_double_int = std::pair; + +using float_float = std::pair; +using double_double = std::pair; +using long_double_long_double = std::pair; + +using cxx_float_complex = std::complex; +using cxx_double_complex = std::complex; +using cxx_long_double_complex = std::complex; + +// using cxx_2double_complex = std::pair, std::complex>; + +using cxx_bool = bool; + +using wchar = wchar_t; + +#if(__cplusplus >= 201703L) +using byte = std::byte; +#endif + +class packed { + unsigned char t{}; + + public: + explicit packed(unsigned char t_) : t{t_} {}; + packed() = default; + + packed& operator=(unsigned char const& rhs) {t = rhs; return *this;} + + explicit operator const unsigned char&() const {return t;} + explicit operator unsigned char&() {return t;} + + bool operator==(packed const& rhs) const {return t == rhs.t;} + bool operator< (packed const& rhs) const {return t < rhs.t;} +}; + +template struct basic_datatype; + + +#if defined(MPI_DOUBLE_COMPLEX) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define MPI3_DECLARE_DATATYPE(TypE, MpiiD) \ +template<> struct basic_datatype { \ +/* constexpr*/ operator MPI_Datatype() const { \ + assert(MPI_DOUBLE_COMPLEX != MPI_DATATYPE_NULL ); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert)*/ \ + assert( (MpiiD) != MPI_DATATYPE_NULL ); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) in some MPI distros this is not constexpr */ /*this system doesn't support this type*/ \ + return MpiiD; \ + } \ + auto get() const -> MPI_Datatype {return MpiiD;} \ +/* static constexpr MPI_Datatype value = MpiiD;*/ \ +} +#else +#define MPI3_DECLARE_DATATYPE(TypE, MpiiD) \ +template<> struct basic_datatype { \ +/* constexpr*/ operator MPI_Datatype() const { \ + assert( (MpiiD) != MPI_DATATYPE_NULL ); /* NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) in some MPI distros this is not constexpr */ /*this system doesn't support this type*/ \ + return MpiiD; \ + } \ + auto get() const -> MPI_Datatype {return MpiiD;} \ +/* static constexpr MPI_Datatype value = MpiiD;*/ \ +} +#endif + +// basic data types http://beige.ucs.indiana.edu/I590/node100.html +MPI3_DECLARE_DATATYPE(char , MPI_CHAR); +MPI3_DECLARE_DATATYPE(unsigned char , MPI_UNSIGNED_CHAR); + +#if(__cplusplus >= 201703L) +MPI3_DECLARE_DATATYPE(byte , MPI_BYTE); +#endif +MPI3_DECLARE_DATATYPE(wchar , MPI_WCHAR); + +MPI3_DECLARE_DATATYPE(short , MPI_SHORT); +MPI3_DECLARE_DATATYPE(unsigned short , MPI_UNSIGNED_SHORT); +MPI3_DECLARE_DATATYPE(int , MPI_INT); +MPI3_DECLARE_DATATYPE(unsigned int , MPI_UNSIGNED); +MPI3_DECLARE_DATATYPE(long , MPI_LONG); +MPI3_DECLARE_DATATYPE(unsigned long , MPI_UNSIGNED_LONG); +MPI3_DECLARE_DATATYPE(float , MPI_FLOAT); +MPI3_DECLARE_DATATYPE(double , MPI_DOUBLE); +MPI3_DECLARE_DATATYPE(long double , MPI_LONG_DOUBLE); +MPI3_DECLARE_DATATYPE(long long int , MPI_LONG_LONG_INT); + +MPI3_DECLARE_DATATYPE(bool , MPI_C_BOOL); // C++ binding not used MPI_CXX_BOOL); + +// MPI_INT8_T int8_t +// MPI_INT16_T int16_t +// MPI_INT32_T int32_t +// MPI_INT64_T int64_t +// MPI_UINT8_T uint8_t +// MPI_UINT16_T uint16_t +// MPI_UINT32_T uint32_t +// MPI_UINT64_T uint64_t +#if defined(MPI_C_FLOAT_COMPLEX) +MPI3_DECLARE_DATATYPE(cxx_float_complex , MPI_C_FLOAT_COMPLEX); +MPI3_DECLARE_DATATYPE(cxx_double_complex , MPI_C_DOUBLE_COMPLEX); +MPI3_DECLARE_DATATYPE(cxx_long_double_complex, MPI_C_LONG_DOUBLE_COMPLEX); +#else +MPI3_DECLARE_DATATYPE(cxx_float_complex , MPI_CXX_FLOAT_COMPLEX); +MPI3_DECLARE_DATATYPE(cxx_double_complex , MPI_CXX_DOUBLE_COMPLEX); +MPI3_DECLARE_DATATYPE(cxx_long_double_complex, MPI_CXX_LONG_DOUBLE_COMPLEX); +#endif + +// MPI3_DECLARE_DATATYPE(cxx_2double_complex , MPI_2DOUBLE_COMPLEX); // not available in mpich + +// TODO(correaa) these types below probably don't behave correctly for reductions with multiplication + +#if defined(MPI_COMPLEX) +MPI3_DECLARE_DATATYPE(float_float , MPI_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); +MPI3_DECLARE_DATATYPE(double_double , MPI_DOUBLE_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); +MPI3_DECLARE_DATATYPE(decltype(std::tuple{}), MPI_DOUBLE_COMPLEX); // TODO(correaa) is this correct? reduce (specially multiplication) will not give correct result +MPI3_DECLARE_DATATYPE(long_double_long_double, MPI_DOUBLE_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); +#else +MPI3_DECLARE_DATATYPE(float_float , MPI_CXX_FLOAT_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); +MPI3_DECLARE_DATATYPE(double_double , MPI_CXX_DOUBLE_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); +MPI3_DECLARE_DATATYPE(decltype(std::tuple{}), MPI_CXX_DOUBLE_COMPLEX); // TODO(correaa) is this correct? reduce (specially multiplication) will not give correct result +MPI3_DECLARE_DATATYPE(long_double_long_double, MPI_CXX_DOUBLE_COMPLEX); static_assert(sizeof(std::pair) == sizeof(std::complex), "checking that complex mem layout maps to pair"); +#endif + +#if defined(__NVCC__) +MPI3_DECLARE_DATATYPE(thrust::complex, MPI_DOUBLE_COMPLEX); +#endif + +MPI3_DECLARE_DATATYPE(float_int , MPI_FLOAT_INT); +MPI3_DECLARE_DATATYPE(long_int , MPI_LONG_INT); +MPI3_DECLARE_DATATYPE(double_int , MPI_DOUBLE_INT); +MPI3_DECLARE_DATATYPE(short_int , MPI_SHORT_INT); +MPI3_DECLARE_DATATYPE(int_int , MPI_2INT); +MPI3_DECLARE_DATATYPE(long_double_int , MPI_LONG_DOUBLE_INT); + +MPI3_DECLARE_DATATYPE(vlp , MPI_FLOAT_INT); +MPI3_DECLARE_DATATYPE(vlp , MPI_LONG_INT); +MPI3_DECLARE_DATATYPE(vlp , MPI_DOUBLE_INT); +MPI3_DECLARE_DATATYPE(vlp , MPI_SHORT_INT); +MPI3_DECLARE_DATATYPE(vlp , MPI_2INT); +MPI3_DECLARE_DATATYPE(vlp , MPI_LONG_DOUBLE_INT); + +//BOOST_MPI3_DECLARE_DATATYPE(std::intptr_t , MPI_AINT); +//BOOST_MPI3_DECLARE_DATATYPE(std::size_t , MPI_AINT); +MPI3_DECLARE_DATATYPE(void* , MPI_AINT); + +MPI3_DECLARE_DATATYPE(packed , MPI_PACKED); + +// LB +// UB + +#undef BOOST_MPI3_DECLARE_DATATYPE + +template{})> +std::true_type is_basic_aux(T ); +std::false_type is_basic_aux(...); + +template +struct is_basic : decltype(is_basic_aux(std::declval())) {}; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + +template constexpr bool is_basic_v = is_basic::value; + +} // end namespace detail + +template class datatype; + +template class default_datatype { + public: + template{}.get())> + R operator()() const { return boost::mpi3::detail::basic_datatype{}.get(); } + template{}.get())> + R get() const { return boost::mpi3::detail::basic_datatype{}.get(); } +}; + +template +auto datatype_detect(...) -> default_datatype; + +template +auto datatype_detect(U const&) -> default_datatype>; + +// support enums +template> +auto datatype_detect(U const&) -> default_datatype
v(world.root()?v_local.size()*world.size():0); - auto last = world.gather(begin(v_local), end(v_local), begin(v)); - if(world.root()){ - assert(last == end(v)); - assert(( v[ 0] == dd{0.,1.} )); - assert(( v[10] == dd{1.,2.} )); - assert(( v[20] == dd{2.,3.} )); - }else{ - assert( last == begin(v) ); - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_grip_handle.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_grip_handle.cpp deleted file mode 100644 index 8e0c5115f8b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_grip_handle.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" - -#include -#include -#include - -namespace bmpi3 = boost::mpi3; - -int main(int argc, char** argv) try { - - MPI_Init(&argc, &argv); - MPI_Comm W{}; - MPI_Comm_dup(MPI_COMM_WORLD, &W); - { - bmpi3::communicator& w = bmpi3::grip_communicator(W); - // cppcheck-suppress[assertWithSideEffect,unmatchedSuppression] - assert(w.handle() == W); - - std::vector const xsend(10, 5.); - std::vector xrecv(10, -1.); - - switch( w.rank() ) { - case 0: { - w.receive(begin(xrecv), end(xrecv), 1); - assert(xrecv[5] == 5.); - break; - } - case 1: { - w.send(begin(xsend), end(xsend), 0); - assert(xrecv[5] == -1.); - break; - } - } - } - MPI_Comm_free(&W); - MPI_Finalize(); - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_ibroadcast.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_ibroadcast.cpp deleted file mode 100644 index 8fa147ab16a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_ibroadcast.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - assert(world.size() > 2); - - mpi3::ostream wout(world); - - std::vector large(10); - if(world.root()) { - iota(large.begin(), large.end(), 0); - } - - wout << "before:" << std::endl; - std::copy(large.begin(), large.end(), std::ostream_iterator(wout, " ")); - wout << std::endl; - - { - auto req = world.ibroadcast(large.begin(), large.end(), 0); - using namespace std::chrono_literals; - std::this_thread::sleep_for(5s); - } - - wout << "after:" << std::endl; - std::copy(large.begin(), large.end(), std::ostream_iterator(wout, " ")); - wout << std::endl; - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_igather.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_igather.cpp deleted file mode 100644 index 7f23014eb96..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_igather.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - - assert( world.size() > 1); - - { - std::vector small(10, 5.); - iota(begin(small), end(small), world.rank()); - std::vector large; - if(world.rank() == 0) { - large.resize(small.size() * static_cast(world.size()), -1.0); - } - { - auto req = world.igather_n(small.begin(), small.size(), large.begin(), 0); - using namespace std::chrono_literals; - std::this_thread::sleep_for(2s); - // req.wait(); - } - if(world.rank() == 0) { - assert(equal(begin(large), begin(large) + static_cast(small.size()), begin(small))); - } - } - { - std::vector small(10, 5.); - std::vector large(small.size() * static_cast(world.size()), -1.0); - { - auto req = world.iall_gather_n(small.begin(), small.size(), large.begin()); - using namespace std::chrono_literals; - std::this_thread::sleep_for(5s); - } - assert(std::all_of(large.begin(), large.end(), [](auto& e) { return 5. == e; })); - } - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_iprobe.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_iprobe.cpp deleted file mode 100644 index cb04d393558..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_iprobe.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wextra `#-Wfatal-errors` $0 -o $0x.x && time mpirun -np 4 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright Alfredo A. Correa 2018. - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - int send_message = 123; - int receive_message = 0; - - if(world.rank() == 0){ - mpi3::request r = world.isend(&send_message, &send_message + 1, 0, 0); - while(not world.iprobe(0, 0) ){}; - assert( world.iprobe(0, 0)->count() ); - world.receive(&receive_message, &receive_message + 1, 0, 0); - assert( receive_message == send_message ); - // r.wait(); // wait is called on the desctructor now - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_ireceive.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_ireceive.cpp deleted file mode 100644 index d1895e73154..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_ireceive.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wall -Wextra $0 -o $0x.x -D_MAKE_BOOST_SERIALIZATION_HEADER_ONLY && mpirun -n 3 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world)->int try{ - - assert( world.size() > 1 ); - - using T = double; - std::vector const cbuffer = {0, 1, 2}; - std::vector buffer(3); // TODO(correaa): test with list - - int right = world.right(); - int left = world.left(); - { - auto req = world.ireceive_n(buffer.begin(), buffer.size(), left); - world.send(cbuffer.begin(), cbuffer.end(), right); - cout <<"waiting ireceive in rank "<< world.rank() << std::endl; - req.wait(); - cout <<"ireceive completed in rank "<< world.rank() << std::endl; - } - assert( std::equal(cbuffer.begin(), cbuffer.end(), buffer.begin()) ); - - return 0; -}catch(...){ - return 1; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_issend.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_issend.cpp deleted file mode 100644 index 36ea84359d8..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_issend.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -Wfatal-errors -Wall -Wextra -Wunused-result $0 -o $0x.x && time mpirun -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - assert( world.size() == 2); - - int right = world.rank() + 1; if(right >= world.size()) right -= world.size(); - int left = world.rank() - 1; if(left < 0 ) left += world.size(); - - std::vector buffer1(10); - std::vector buffer2(10); - iota(begin(buffer2), end(buffer2), 0); - - mpi3::request r1 = world.ireceive_n(buffer1.begin(), buffer1.size(), left , 123); - mpi3::request r2 = world.issend_n (buffer2.begin(), buffer2.size(), right, 123); - r1.wait(); - r2.wait(); - - assert( buffer1 == buffer2 ); - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_list.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_list.cpp deleted file mode 100644 index d21dba0bea6..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_list.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" - -#include -#include - -namespace mpi3 = boost::mpi3; - -struct projector { - explicit projector(mpi3::communicator& comm) : comm_{comm} {} - private: - mutable mpi3::communicator comm_; -}; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - { - std::list v; - v.emplace_back(world); - v.emplace_back(world); - } -#if 0 - { // doesn't compile, communicator is not copiable - std::vector v = {world, world}; - v.emplace_back(world); - v.emplace_back(world); - } -#endif - { - std::vector v = {projector{world}, projector{world}}; - v.emplace_back(world); - v.emplace_back(world); - } - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp deleted file mode 100644 index 8a8c32548fe..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// #if COMPILATION_INSTRUCTIONS -// mpic++ -O3 -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --suppressions=$0.openmpi.supp $0x)&&rm $0x;exit -// #mpic++ -O3 -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --gen-suppressions=all $0x 2>&1|grep -v '==' > $0.openmpi.supp )&& -// #mpic++ -g -O3 -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --gen-suppressions=all $0x 2>&1|grep -v '==' > $0-g.openmpi.supp )&& -// #mpic++ -g -Wall -Wextra $0 -o $0x&&(mpirun -n 3 valgrind --leak-check=full --show-reachable=yes --error-limit=no --gen-suppressions=all $0x 2>&1|grep -v '==' > $0-O0.openmpi.supp )&& -// #cat $0-g.openmpi.supp $0-O0.openmpi.supp >> $0.openmpi.supp&&rm $0x;exit -// #endif -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argv*/, char** /*argc*/, mpi3::communicator world) -> int try { - assert( world.size() == 3 ); - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp.openmpi.supp b/external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp.openmpi.supp deleted file mode 100644 index 2c5b26b051a..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_main.cpp.openmpi.supp +++ /dev/null @@ -1,6558 +0,0 @@ -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:mca_pml_base_pml_check_selected - obj:* - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:mca_pml_base_pml_check_selected - obj:* - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:mca_pml_base_pml_check_selected - obj:* - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:mca_pml_base_pml_check_selected - obj:* - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:initialize - fun:initialize - fun:environment - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Param - writev(vector[...]) - fun:__writev - fun:writev - fun:pmix_ptl_base_send_handler - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:/usr/lib/x86_64-linux-gnu/pmix/lib/libpmix.so.2.2.25 - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:mca_pml_base_pml_check_selected - obj:* - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - fun:mca_base_framework_components_register - fun:mca_base_framework_register - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:ompi_pml_v_output_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libmpi.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - fun:_dl_catch_exception - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:mca_pml_base_pml_check_selected - obj:* - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_get_supported_methods - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:/usr/lib/x86_64-linux-gnu/libevent-2.1.so.7.0.0 - fun:event_base_loop - obj:* - fun:start_thread - fun:clone -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:strdup - fun:_dl_load_cache_lookup - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evsig_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:/usr/lib/x86_64-linux-gnu/libevent_pthreads-2.1.so.7.0.0 - fun:evutil_secure_rng_global_setup_locks_ - fun:event_global_setup_locks_ - fun:evthread_use_pthreads - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:event_config_avoid_method - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_comm_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:event_config_new - fun:opal_event_init - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:opal_init - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mtl_base_select - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_proc_complete_init_single - fun:ompi_proc_complete_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:mca_base_framework_components_open - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_check_map_versions - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - fun:add_to_global_resize - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - obj:/usr/lib/x86_64-linux-gnu/openmpi/lib/libopen-pal.so.40.20.3 - fun:mca_base_component_repository_open -} -{ - - Memcheck:Leak - match-leak-kinds: reachable - fun:calloc - fun:_dl_new_object - fun:_dl_map_object_from_fd - fun:_dl_map_object - fun:openaux - fun:_dl_catch_exception - fun:_dl_map_object_deps - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:opal_free_list_grow_st - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN5boost4mpi310initializeERiRPPc - fun:_ZN5boost4mpi311environment10initializeEiPPc - fun:_ZN5boost4mpi311environmentC1ERiRPPc - fun:main -} -{ - - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* -} -{ - - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - obj:* - fun:ompi_mpi_init -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_mutable.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_mutable.cpp deleted file mode 100644 index e3df54565f1..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_mutable.cpp +++ /dev/null @@ -1,161 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -#include -#include - -namespace mpi3 = boost::mpi3; - -struct projector { - projector() = default; - - explicit projector(mpi3::communicator comm) : n_{5}, comm_{std::move(comm)} {} - - projector(projector const&) = default; - projector(projector &&) = default; - - auto operator=(projector const&) -> projector& = default; // NOLINT(clang-diagnostic-deprecated-declarations) - auto operator=(projector &&) -> projector& = default; -// auto operator=(projector &) -> projector& = default; - - friend auto operator==(projector const& a, projector const& b) {return a.n_ == b.n_;} // a.comm_ == b.comm_;} - friend auto operator!=(projector const& a, projector const& b) {return a.n_ != b.n_;} // a.comm_ == b.comm_;} - - decltype(auto) get_comm() const {return comm_;} - auto get_n() const -> int{return n_;} - ~projector() = default; - - private: - int n_ = 0; - mutable mpi3::communicator comm_; -}; - -struct projector2 { - projector2() = default; - - explicit projector2(mpi3::communicator comm) : n_{5}, comm_{std::move(comm)} {} - - projector2(projector2 const&) = default; - projector2(projector2 &&) = default; - - auto operator=(projector2 const& other) -> projector2& { - if(this == &other) {return *this;} - assert(comm_ == other.comm_); - n_ = other.n_; - return *this; - } - auto operator=(projector2 &&) -> projector2& = default; -// auto operator=(projector2 &) -> projector2& = default; - - friend auto operator==(projector2 const& a, projector2 const& b) {return a.n_ == b.n_;} // a.comm_ == b.comm_;} - friend auto operator!=(projector2 const& a, projector2 const& b) {return a.n_ != b.n_;} // a.comm_ == b.comm_;} - - decltype(auto) get_comm() const { return comm_; } - auto get_n() const -> int { return n_; } - ~projector2() = default; - - private: - int n_ = 0; - mutable mpi3::communicator comm_; -}; - -struct projector3 { // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) - projector3() = default; - - explicit projector3(mpi3::communicator comm) : n_{5}, comm_{std::move(comm)} {} - - projector3(projector3 const&) = default; - projector3(projector3 &&) = default; - - auto operator=(projector3 other) noexcept -> projector3& { - swap(other); - return *this; - } -// auto operator=(projector3 &&) -> projector3& = default; -// auto operator=(projector3 &) -> projector3& = default; - - friend auto operator==(projector3 const& a, projector3 const& b) {return a.n_ == b.n_;} // a.comm_ == b.comm_;} - friend auto operator!=(projector3 const& a, projector3 const& b) {return a.n_ != b.n_;} // a.comm_ == b.comm_;} - - decltype(auto) get_comm() const { return comm_; } - auto get_n() const -> int { return n_; } - ~projector3() = default; - - private: - int n_ = 0; - mutable mpi3::communicator comm_; - void swap(projector3& other) noexcept { - std::swap(n_, other.n_); - std::swap(comm_, other.comm_); - } -}; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { -// { -// projector const p{world}; -// projector p2; -// p2 = p; -// } - { - std::list v; - v.emplace_back(world); - v.emplace_back(world); - } -// { // doesn't compile, communicator is not copiable -// std::vector v = {world, world}; -// v.emplace_back(world); -// v.emplace_back(world); -// } - { // but this works because the member is mutable - std::vector v = {projector{world}, projector{world}}; - v.emplace_back(world); - v.emplace_back(world); - v.emplace_back(world); - v[1] = v[0]; - assert( v[1] == v[0] ); - v[1] = std::move(v[0]); - assert( v[0] == v[0] ); - assert( v[0].get_comm().is_empty() ); - v[1] = static_cast(v[2]); - assert( v[2].get_comm() == world ); - } - { // but this works because the member is mutable - std::vector v = {projector2{world}, projector2{world}}; - v.emplace_back(world); - v.emplace_back(world); - v.emplace_back(world); - v[1] = v[0]; - assert( v[1] == v[0] ); - v[1] = std::move(v[0]); - assert( v[0] == v[0] ); - assert( v[0].get_comm().is_empty() ); - v[1] = static_cast(v[2]); - assert( v[2].get_comm() == world ); - } - { // but this works because the member is mutable - std::vector v = {projector3{world}, projector3{world}}; - v.emplace_back(world); - v.emplace_back(world); - v.emplace_back(world); - v[1] = v[0]; - assert( v[1] == v[0] ); - v[1] = std::move(v[0]); - assert( v[0] == v[0] ); - assert( v[0].get_comm().is_empty() ); - v[1] = static_cast(v[2]); - assert( v[2].get_comm() == world ); - } - { - mpi3::communicator comm; - assert( not comm ); - assert( comm.is_empty() ); - } - - return 0; -} catch(...) { - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_operator.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_operator.cpp deleted file mode 100644 index b24188ff005..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_operator.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -#include //literals -#include //sleep_for - -namespace mpi3 = boost::mpi3; -using std::cout; -using namespace std::chrono_literals; - - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - - std::vector inbuf(100); - std::vector outbuf(100); - - switch(world.rank()) { - case 0: { - iota(begin(outbuf), end(outbuf), 0.0); - std::this_thread::sleep_for(2s); - cout <<"world["<< world.rank() <<"] about to isent"<< std::endl; - mpi3::request r = world.isend(begin(outbuf), end(outbuf), 1); - cout <<"comm["<< world.rank() <<"] isent"<< std::endl; - // r.wait(); - }; break; - case 1: { - cout <<"comm["<< world.rank() <<"] about to ireceive"<< std::endl; - mpi3::request r;//= world.ireceive_n(inbuf.begin(), inbuf.size(), 0); - MPI_Irecv( - inbuf.data(), static_cast(inbuf.size()), - detail::basic_datatype{}, - MPI_ANY_SOURCE, MPI_ANY_TAG, world.get(), &r.impl_ - ); - cout <<"comm["<< world.rank() <<"] ireceived"<< std::endl; - MPI_Wait(&r.impl_, MPI_STATUS_IGNORE); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) for macro - // r.wait(); - }; break; - default: break; - } - cout <<"comm["<< world.rank() <<"] completed op"<< std::endl; - - if(world.rank() == 1) {assert( inbuf[9] == 9. );} - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_ostream.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_ostream.cpp deleted file mode 100644 index 570359af2d5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_ostream.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" -#include "../../mpi3/ostream.hpp" - -#include -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argv*/, char** /*argc*/, mpi3::communicator world) -> int try { - assert( world.size() > 2 ); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis{-1.0, +1.0}; - - world.set_name("world"); - - mpi3::ostream wout{world, std::cout}; - -// wout << mpi3::set_communicator_logging; // TODO - - wout << "Program starts" << std::endl; - wout << "Hello! for world using "<< world.size() <<" processes" << std::endl; - - wout << "Hello! I am rank "<< world.rank()<< " in " << world.name() << std::endl; - - wout << (world.root()?"this precess is root":"this process is NOT root") << std::endl; - - wout << "rank " << world.rank() << '\t' << std::flush; - wout << "small random " << dis(gen) << '\t' << std::flush; - wout << "large random " << dis(gen)*std::numeric_limits::max() << '\t' << std::flush; - - wout << "-------------------" << std::endl; - - wout << "raw_stuff " << world.rank() << std::flush; - wout << "\nsomething random" << std::flush; - - wout << "Program Ends" << std::endl; - - if(mpi3::communicator firsttwo = (world < 2) ) { - firsttwo.set_name("firsttwo"); - mpi3::ostream fout(firsttwo); - fout - <<"Hola! I am rank "<< firsttwo.rank() <<" in "<< firsttwo.name() - <<" and also rank "<< world.rank() <<" in "<< world.name() - < -#include - -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - - { - mpi3::detail::package p(world); - - int i = 123; - std::vector c(10); iota(begin(c), end(c), 0); - - p.pack_n(&i, 1); - std::istringstream iss("0 1 2 3 4 5 6 7 8 9"); - p.pack(std::istream_iterator(iss), std::istream_iterator()); - int i2; - std::list c2(10); - p.unpack_n(&i2, 1); - p.unpack(begin(c2), end(c2)); - assert(i2 == i); - assert(equal(begin(c2), end(c2), begin(c))); - } - { - assert(world.size() > 1); - - mpi3::detail::package p(world); - - int i; - std::vector c(100); - - switch(world.rank()){ - case 0: - i = 123; - iota(c.begin(), c.end(), 0); - p.pack_n(&i, 1); - p.pack(begin(c), end(c)); - p.send(1); - break; - case 1: - p.receive(0); - p.unpack_n(&i, 1); - p.unpack(begin(c), end(c)); - assert(i == 123); - std::vector c2(100); - iota(begin(c2), end(c2), 0); - assert(c == c2); - break; - } - - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_passby.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_passby.cpp deleted file mode 100644 index bcc1ee15b04..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_passby.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" - -#include -#include - -namespace mpi3 = boost::mpi3; - -void f(mpi3::communicator & /*comm*/) {} -void g(mpi3::communicator /*comm*/) {} -void h(mpi3::communicator const& /*comm*/) {} - -auto ovrld(mpi3::communicator /*comm*/) -> std::string {return "by value";} -auto ovrld(mpi3::communicator & /*comm*/) -> std::string {return "by reference";} -//auto ovrld(mpi3::communicator const& /*comm*/) -> std::string {return "by const reference";} - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - - f(world); - // f(world.duplicate()); // doesn't work, good - - g(world.duplicate()); - g(world); // works, implicit calls to duplicate - - h(world); - h(world.duplicate()); - -// assert( ovrld(world) == "by ???" ); // ambiguous, not much to do, overload by reference can never called -// assert( ovrld(std::ref(world)) == "by ???" ); // ambiguous - assert( ovrld(world.duplicate()) == "by value" ); - assert( ovrld(mpi3::communicator{world}) == "by value" ); - - return 0; -} catch(...) {return 1;} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_reduce.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_reduce.cpp deleted file mode 100644 index 416dcf94e31..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_reduce.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" -#include "../../mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; - -void part1(mpi3::communicator& world) { - std::size_t const count = 120; - std::vector send_buffer(count); - iota(send_buffer.begin(), send_buffer.end(), 0); - - std::vector recv_buffer; - if(world.rank() == 0) { - recv_buffer.resize(count, -1); - } - - world.reduce_n(send_buffer.begin(), send_buffer.size(), recv_buffer.begin(), std::plus<>{}, 0); - if(world.rank() == 0) { - for(std::size_t i = 0; i != recv_buffer.size(); ++i) { // NOLINT(altera-unroll-loops) use algorithm - assert(std::size_t(recv_buffer[i]) == i * static_cast(world.size())); - } - } -} - -void part2(mpi3::communicator& world) { - double const v = world.rank(); - double total = 0; - - double const* const it = world.reduce_n(&v, 1, &total, std::plus<>{}); - if(world.rank() == 0) { - assert(total == static_cast(world.size() * (world.size() - 1)) / 2); - } else { - assert(total == 0); - } - if(world.rank() == 0) { - assert(it != &total); - } else { - assert(it == &total); - } -} - -void part3(mpi3::communicator& world) { - mpi3::optional total = (world[0] += world.rank()); - if(world.rank() == 0) { - assert(total); - } - if(world.rank() != 0) { - assert(not total); - } - if(total) { - assert(*total == static_cast(world.size() * (world.size() - 1)) / 2); - } -} - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - assert(world.size() > 1); - - part1(world); - part2(world); // TODO(correaa) fix this - part3(world); - - return 0; -} catch(...) { return 1; } diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_reduce_in_place.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_reduce_in_place.cpp deleted file mode 100644 index 4c7dc7e388c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_reduce_in_place.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018-2021 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - - { - int n = 1; - world.reduce_in_place_n(&n, 1, std::plus<>{}); - if(world.rank() == 0) {assert(n == world.size());} - } - { - int n = 1; - world.all_reduce_in_place_n(&n, 1, std::plus<>{}); - assert(n == world.size()); - } - { - int n = 1; - world.all_reduce_n(&n, 1, std::plus<>{}); - assert(n == world.size()); - } - { - int n = 1; - world.all_reduce_n(&n, 1); - assert(n == world.size()); - } - { - int n = 1; - auto const m = world.all_reduce_value(n); - assert( n == 1 ); - assert(m == world.size()); - } - { - int n = 1; - auto const m = (world += n); - assert( n == 1 ); - assert(m == world.size()); - } -// { -// int n = 1; -// auto const m = (world + n); -// assert( n == 1 ); -// assert(m == world.size()); -// } - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_scatter.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_scatter.cpp deleted file mode 100644 index 0904f096d77..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_scatter.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include -#include - -#include -#include // for std::iota - -namespace mpi3 = boost::mpi3; - -auto main() -> int try { - mpi3::environment env; - mpi3::communicator world = env.world(); - - { - std::vector v(static_cast(world.size())); - iota(begin(v), end(v), 0); - std::vector w(1); - world.scatter(begin(v), end(v), begin(w), 0); - assert( w[0] == world.rank() ); - } - { - std::vector v(world.root()?static_cast(world.size()):0); - iota(begin(v), end(v), 0); - std::vector w(1); - // auto e = - world.scatter_from(begin(w), end(w), begin(v), 0); - // assert( e == end(v) ); - assert( w[0] == world.rank() ); - } - { - std::vector v(world.root()?static_cast(world.size()):0); - iota(begin(v), end(v), 0); - double w = -1; - // auto e = - world.scatter_value_from(w, begin(v), 0); - // assert( e == end(v) ); - assert( w == world.rank() ); - } - { - std::vector v(world.root()?static_cast(world.size()):0); - iota(begin(v), end(v), 0); - double w = world.scatter(begin(v), end(v), 0); - assert( w == world.rank() ); - } - { - std::vector v(world.root()?static_cast(world.size()):0); - iota(begin(v), end(v), 0); - double w = v / world; - assert( w == world.rank() ); - } - { - std::list l(world.root()?static_cast(world.size()):0); - iota(begin(l), end(l), 0); - double w = l / world; - assert( w == world.rank() ); - } -#if 0 - { - std::vector v = {1, 2, 2, 3, 3, 3}; if(!world.root()) v.clear(); - std::vector w(world.rank() + 1); - auto e = world.scatterv_from(begin(w), end(w), begin(v), 0); - assert( end(v) == e ); - switch(world.rank()){ - case 0: assert((w==std::vector{1} )); break; - case 1: assert((w==std::vector{2,2} )); break; - case 2: assert((w==std::vector{3,3,3})); break; - } - } - { - if(auto duo = (world < 2)){ - assert( duo.size() == 2 ); - // std::vector> vs = { {1}, {2, 2} }; - std::vector v = {1, 2, 2}; - std::vector ns = {1, 2}; if(!duo.root()) ns.clear(); - std::vector::iterator> its = {v.begin(), v.begin()+1}; if(!world.root()) its.clear(); - std::vector w(duo.rank()+1); - duo.scatterv_n(begin(its), begin(ns), begin(w)); - switch(duo.rank()){ - case 0: assert(( w == std::vector{1} )); break; - case 1: std::cerr <<"w2:" << w[0] << ", " << w[1] << std::endl; break; - } - } - } - { - if(auto duo = (world < 2)){ - assert( duo.size() == 2 ); - std::vector> vs = { {1}, {2, 2} }; if(!duo.root()) vs.clear(); - std::vector w(duo.rank()+1); - duo.scatterv(begin(vs), begin(w)); - // switch(duo.rank()){ - // case 0: assert(( w == std::vector{1} )); break; - // case 1: std::cerr <<"w2:" << w[0] << ", " << w[1] << std::endl; break; - // } - } - } -#endif - return 0; -} catch(...) { return 1; } diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_scatterv.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_scatterv.cpp deleted file mode 100644 index 5ef43c26d1e..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_scatterv.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -I$HOME/include -Wall -Wextra -Wpedantic `#-Wfatal-errors` $0 -o $0x.x && time mpirun -n 8 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/environment.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" -#include // iota -#include - -//#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -int mpi3::main(int, char*[], mpi3::communicator world){ - std::cerr << sizeof(MPI_Offset) << " " << sizeof(int) << std::endl; - #if 0 - if(0){ - if(auto duo = (world < 2)){ - assert( duo.size() == 2 ); - std::vector v = {1, 2, 2}; - std::vector ns = {1, 2}; if(!duo.root()) ns.clear(); - std::vector::iterator> its = {v.begin(), v.begin()+1}; if(!world.root()) its.clear(); - std::vector w(duo.rank()+1); - duo.scatterv_n(begin(its), begin(ns), begin(w)); - switch(duo.rank()){ - case 0: assert(( w == std::vector{1} )); break; - case 1: std::cerr <<"w2:" << w[0] << ", " << w[1] << std::endl; break; - } - } - } - #endif - #if 1 - { - if(auto quintet = (world < 5)){ - std::vector> vs = { {1}, {2, 2}, {3, 3, 3}, {4, 4, 4, 4}, {5, 5, 5, 5, 5} }; if(!quintet.root()) vs.clear(); - std::vector w( quintet.rank() + 1 ); - auto e = quintet.scatterv(vs, begin(w)); assert( e == end(w) ); - assert( w == std::vector(quintet.rank() + 1, quintet.rank() + 1) ); - } - } - #endif - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_send.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_send.cpp deleted file mode 100644 index 0ad8361648b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_send.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include -#include - -#include -#include -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - assert(world.size() > 1); - - switch(world.rank()) { - case 0: { - std::list b = {3, 4, 5}; - world.send(cbegin(b), cend(b), 1); - break; - }; - case 1: { - std::vector b2(3); - world.receive(begin(b2), end(b2)); - assert(b2[1] == 4.0); - break; - }; - } - // switch(world.rank()){ - // case 0: { - // std::vector b = {"hola", "blah", "chau"}; - // world.send(cbegin(b), cend(b), 1); - // }; break; - // case 1: { - // std::list b2(3); - // world.receive(begin(b2), end(b2)); // TODO(correaa) invesigate why it doesn't work - // assert( *begin(b2) == "hola" and *rbegin(b2) == "chau" ); - // }; break; - // } - switch(world.rank()) { - case 0: { - std::istringstream iss{"1 2 3"}; - world.send(std::istream_iterator{iss}, {}, 1); - break; - }; - case 1: { - std::vector out(3); - world.receive(begin(out), end(out)); - assert((out == std::vector{1, 2, 3})); - break; - } - } - - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_send_async.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_send_async.cpp deleted file mode 100644 index ea829788c3d..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_send_async.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -O3 -Wall -Wextra $0 -o $0x.x -D_MAKE_BOOST_SERIALIZATION_HEADER_ONLY `#-lboost_serialization` -pthread && time mpirun -n 2 $0x.x $@ && rm -f $0x.x; exit -#endif -// (C) Copyright Alfredo A. Correa 2018. - -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/main.hpp" - -#include -#include // async -#include -#include - -namespace mpi3 = boost::mpi3; -using std::cout; - -using namespace std::chrono_literals; - -int mpi3::main(int, char*[], mpi3::communicator world){ - assert( world.size() > 1 ); - - { - switch(world.rank()){ - case 0: { - std::list b = {3, 4, 5}; - world.send(cbegin(b), cend(b), 1); - }; break; - case 1: { - std::vector b2(3); - auto e = world.receive(begin(b2), 0); - assert( e == end(b2) ); - assert( b2[1] == 4. ); - }; break; - } - } - { - switch(world.rank()){ - case 0: { - std::list b = {3, 4, 5}; - std::this_thread::sleep_for(10s); - auto req = std::async([&](){return world.send(cbegin(b), cend(b), 1);}); - }; break; - case 1: { - std::vector b2(3); - auto req = std::async([&](){return world.receive(begin(b2), 0);}); - std::this_thread::sleep_for(2s); - std::cout << "rank " << world.rank() << " has the req" << std::endl; - assert( req.get() == end(b2) ); - assert( b2[1] == 4. ); - }; break; - } - } - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_send_class.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_send_class.cpp deleted file mode 100644 index ee522c25062..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_send_class.cpp +++ /dev/null @@ -1,198 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#include -#include -//#include // serialize std::pair -#include - -#include - -namespace mpi3 = boost::mpi3; - -struct A { // NOLINT(readability-identifier-naming) example name - private: - std::string name_ = "unnamed"; - std::size_t n_ = 0; - std::unique_ptr data_; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) notation - - public: - A() = default; - explicit A(std::size_t n) : n_{n}, data_{new double[n_]} {} - A(A const& other) : name_{other.name_}, n_{other.n_}, data_{new double[n_]} {} - A(A&&) = delete; - ~A() = default; - - // auto operator=(A&&) = delete; - auto operator=(A const& other) -> A& { - if(this == &other) { - return *this; - } - name_ = other.name_; - n_ = other.n_; - data_.reset(new double[other.n_]); // NOLINT(cppcoreguidelines-owning-memory) - std::copy_n(other.data_.get(), n_, data_.get()); - return *this; - } - auto operator=(A&&) -> A& = default; - - auto operator[](std::ptrdiff_t i) -> double& { return data_.get()[i]; } - // intrusive serialization - template - void save(Archive& ar, unsigned int const /*version*/) const { - ar << name_ << n_ << boost::serialization::make_array(data_.get(), n_); - } - template - void load(Archive& ar, unsigned int const /*version*/) { - ar >> name_ >> n_; - data_.reset(new double[n_]); // NOLINT(cppcoreguidelines-owning-memory) - ar >> boost::serialization::make_array(data_.get(), n_); - } - BOOST_SERIALIZATION_SPLIT_MEMBER() -}; - -struct B { // NOLINT(readability-identifier-naming) example name - std::string name_ = "unnamed"; // NOLINT(misc-non-private-member-variables-in-classes) exposed for serialization - std::size_t n_ = 0; // NOLINT(misc-non-private-member-variables-in-classes) - std::unique_ptr data; // NOLINT(misc-non-private-member-variables-in-classes, cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - - B() = default; - explicit B(std::size_t n) : n_{n}, data{std::make_unique(n_)} {} // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - B(B const& other) : name_{other.name_}, n_{other.n_}, data{std::make_unique(n_)} {} // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - B(B&&) = delete; - - auto operator=(B const& other) -> B& { - if(this == &other) { - return *this; - } - name_ = other.name_; - n_ = other.n_; - data = std::make_unique(other.n_); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - std::copy_n(other.data.get(), n_, data.get()); - return *this; - } - auto operator=(B&& other) -> B& = default; - - auto operator[](std::ptrdiff_t i) const -> double& { return data.get()[i]; } - ~B() = default; -}; - -// nonintrusive serialization -template -void save(Archive& ar, B const& b, unsigned int const /*version*/) { - ar << b.name_ << b.n_ << boost::serialization::make_array(b.data.get(), b.n_); -} -template -void load(Archive& ar, B& b, unsigned int const /*version*/) { // NOLINT(google-runtime-references): serialization protocol - ar >> b.name_ >> b.n_; - b.data = std::make_unique(b.n_); // NOLINT(cppcoreguidelines-owning-memory,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - ar >> boost::serialization::make_array(b.data.get(), b.n_); -} -BOOST_SERIALIZATION_SPLIT_FREE(B) - -template<> struct mpi3::datatype< - std::pair, std::complex> -> : struct_< - std::complex, - std::complex -> {}; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - - assert(world.size() > 1); - - switch(world.rank()) { - case 0: { - std::vector> buffer(10, std::vector(20)); - buffer[4][5] = 6.1; - world.send(buffer.begin(), buffer.end(), 1, 123); - break; - }; - case 1: { - std::vector> buffer(10, std::vector(20)); - world.receive(buffer.begin(), buffer.end(), 0, 123); - assert(buffer[4][5] == 6.1); - break; - }; - } - switch(world.rank()) { - case 0: { - std::vector buffer(10); - iota(begin(buffer), end(buffer), 0.0); - world.send(begin(buffer), end(buffer), 1); - break; - }; - case 1: { - std::vector v(10); - auto it = world.receive_n(begin(v), 10, 0); - assert(it == end(v) and v[3] == 3.0); - break; - }; - } - switch(world.rank()) { - case 0: { - std::map> m; - m[2] = std::vector(2); - m[5] = std::vector(5); - world.send(begin(m), end(m), 1, 123); - break; - }; - case 1: { - std::vector>> v(2); - world.receive(begin(v), end(v), 0, 123); - assert((v[1] == std::pair>{5, std::vector(5)})); - break; - }; - } - - switch(world.rank()) { - case 0: { - std::vector v(5, A(3)); - v[2][2] = 3.14; - world.send(begin(v), end(v), 1, 123); - break; - }; - case 1: { - std::vector v(5); - world.receive(begin(v), end(v), 0, 123); - assert(v[2][2] == 3.14); - break; - }; - } - - switch(world.rank()) { - case 0: { - std::vector v(5, B(3)); - v[2][2] = 3.14; - world.send(begin(v), end(v), 1, 123); - break; - }; - case 1: { - std::vector v(5); - world.receive(begin(v), end(v), 0, 123); - assert(v[2][2] == 3.14); - break; - }; - } - - switch(world.rank()) { - case 0: { - std::vector, std::complex> > v(5); - v[2] = std::make_pair(std::complex{3.14, 6.28}, std::complex{4.0, 5.0}); - world.send(begin(v), end(v), 1); - break; - }; - case 1: { - std::vector, std::complex> > v(5); - world.receive(begin(v), end(v), 0); - assert( v[2] == std::make_pair(std::complex{3.14, 6.28}, std::complex{4.0, 5.0}) ); - break; - }; - } - - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_send_class_nonintrusive.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_send_class_nonintrusive.cpp deleted file mode 100644 index b894bc40b3b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_send_class_nonintrusive.cpp +++ /dev/null @@ -1,92 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#include -#include -#include - -namespace mpi3 = boost::mpi3; - -// nontrivial nonpod class -class B { // NOLINT(readability-identifier-naming) example name - std::string name_ = "unnamed"; - int n_ = 0; - template friend void save(Archive & ar, B const& b, unsigned int/*version*/); - template friend void load(Archive & ar, B & b, unsigned int/*version*/); - std::unique_ptr data_; // NOLINT(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays) // test code - - public: - auto data() & -> double* { return data_.get(); } - auto data() const& -> double const* { return data_.get(); } - - auto operator[](std::size_t i) & -> double& { return data_.get()[i]; } - - auto name() & -> std::string& { return name_; } - auto name() const& -> std::string const& { return name_; } - - B() = default; - explicit B(int n) : n_{n}, data_{std::make_unique(static_cast(n))} {// NOLINT(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays) // test code - std::fill_n(data_.get(), n_, 0.); - } - B(B const& other) : name_{other.name_}, n_{other.n_}, data_{std::make_unique(static_cast(other.n_))} { // NOLINT(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays) // test code - std::copy_n(other.data_.get(), n_, data_.get()); - } - B(B&&) = default; - auto operator=(B const& other) -> B& { - if(this == &other) {return *this;} - name_ = other.name_; - n_ = other.n_; - data_ = std::make_unique(static_cast(other.n_)); // NOLINT(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays) // test code - std::copy_n(other.data_.get(), n_, data_.get()); - return *this; - } - B& operator=(B&&) = default; - ~B() = default; -}; - -// nonintrusive serialization -template -void save(Archive& ar, B const& b, unsigned int const /*version*/) { - ar << b.name() << b.n_ << boost::serialization::make_array(b.data_.get(), b.n_); -} -template -void load(Archive& ar, B& b, unsigned int const /*version*/) { - ar >> b.name() >> b.n_; - b.data_ = std::make_unique(static_cast(b.n_)); // NOLINT(modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays) // test code - ar >> boost::serialization::make_array(b.data_.get(), b.n_); -} -BOOST_SERIALIZATION_SPLIT_FREE(B) - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - assert( world.size() > 1 ); - - switch(world.rank()){ - case 0 : { - std::vector v(5, B(3)); - v[2][2] = 3.14; - world.send(v.begin(), v.end(), 1, 123); - }; break; - case 1 : { - std::vector v(5); - world.receive(v.begin(), v.end(), 0, 123); - assert(v[2][2] == 3.14); - }; break; - } - switch(world.rank()){ - case 0 : { - B b1(4); b1[2] = 4.5; - world[1] << b1; - }; break; - case 1 : { - B b2; - world[0] >> b2; - assert( b2[2] == 4.5 ); - }; break; - } - - return 0; -}catch(...){ - return 1; -} - - diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_send_receive.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_send_receive.cpp deleted file mode 100644 index 08e3c800699..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_send_receive.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpic++ -O3 -std=c++14 -O3 -Wall -Wextra `#-Wfatal-errors` $0 -o $0x.x -lboost_serialization && time mpirun -n 1 $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "../../mpi3/main.hpp" -#include "../../mpi3/cartesian_communicator.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - - assert( not world.is_empty() ); - auto const s = world.size(); - assert( s != 0 ); - - auto right = (world.rank() + 1 + s ) % s; - auto left = (world.rank() - 1 + s ) % s; - - { - using Container = std::vector; - Container c(10, world.rank()); - world.send_receive_n(c.begin(), c.size(), left, right); - assert( c.front() == right ); - } - { - using Container = std::vector; - Container c(10, world.rank()); - world.send_receive(c.begin(), c.end(), left, right); - assert( c.front() == right ); - } - { - using Container = std::list; - Container c(10, world.rank()); - world.send_receive(c.begin(), c.end(), left, right); - assert( c.front() == right ); - } - { - using Container = std::list; - Container c(10, std::to_string(world.rank())); - world.send_receive_n(c.begin(), c.size(), left, right); - assert( c.front() == std::to_string(right) ); - } - { - std::array buffer {}; buffer [5] = world.rank(); - std::array buffer2{}; buffer2[5] = -1; - world.send_receive_n( - buffer .data(), 10, left , - buffer2.data() , right - ); - assert(buffer2[5] == right); - } - { - std::vector buffer (10); buffer [5] = world.rank(); - std::vector buffer2(10); buffer2[5] = -1; - world.send_receive( - buffer .begin(), buffer .end(), left , - buffer2.begin(), buffer2.end(), right - ); - assert(buffer2[5] == right); - } - { - std::list> b(10, std::complex{});//std::to_string(1.*world.rank())); - world.send_receive_n(b.begin(), b.size(), left, right); - // assert( *b.begin() == std::to_string(1.*right) ); - } - { - std::vector buffer (10); buffer [5] = world.rank(); - auto right = world.rank() + 1; if(right >= world.size()) {right = 0 ;} - auto left = world.rank() - 1; if(left < 0) {left = world.size() - 1;} - world.send_receive_replace(buffer.begin(), buffer.end(), left, right); - // MPI_Sendrecv_replace(buffer, 10, MPI_INT, left, 123, right, 123, MPI_COMM_WORLD, &status); - assert( buffer[5] == right ); - } - { - std::vector buffer (10); buffer [5] = world.rank(); - world.send_receive(buffer.begin(), buffer.end(), world.rank(), world.rank()); - assert( buffer[5] == world.rank() ); - } - { - std::vector buffer (10); buffer [5] = world.rank(); - auto right = world.rank() + 1; if(right >= world.size()) {right = 0 ;} - auto left = world.rank() - 1; if(left < 0) {left = world.size() - 1;} - world.send_receive(buffer.begin(), buffer.end(), left, right); - assert( buffer[5] == right ); - } - { - mpi3::circular_communicator circle{world}; - std::vector buffer(10); buffer [5] = circle.coordinate(); - circle.send_receive(buffer.begin(), buffer.end(), circle.rank(circle.coordinate() - 1), circle.rank(circle.coordinate() + 1)); - assert( buffer[5] == right ); - } - try { - mpi3::ring circle{world}; - std::vector buffer(10); buffer [5] = circle.coordinate(); - circle.rotate(buffer.begin(), buffer.end()); - // assert( buffer[5] == circle.rank(circle.coordinate() - 1) ); - } catch(std::exception& e) {std::cout << e.what() <int try{ - - world.set_error_handler(mpi3::error_handler::code); // default, internal function returns codes - double d = 5.; - try{ - world.send_n(&d, 1, 100); - }catch(...){ - cout << "catched exception" << std::endl; - return 0; - } - -// world.set_error_handler(mpi3::error_handler::fatal); // fail immediately -// world.send(&d, &d + 1, 100); - - return 1; -} catch(...) {return 911;} diff --git a/external_codes/mpi_wrapper/mpi3/test/communicator_split.cpp b/external_codes/mpi_wrapper/mpi3/test/communicator_split.cpp deleted file mode 100644 index e05e397851c..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/communicator_split.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018-2022 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/ostream.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) -> int try { - - mpi3::ostream wout(world); - - mpi3::communicator third = world/3; // or other division - mpi3::communicator leaders = world.keep(third.root()); // same as world.split(third.root()?0:mpi3::undefined); - - wout << "I am 'world' rank "<(third.attribute("color")); - }else{ - wout<<" and not in 'third'"; - } - if(leaders){ - wout<<" and 'leader:'"<< leaders.name() <<"'s rank "<< leaders.rank() <<" with color attribute "<< mpi3::any_cast(third.attribute("color")); - }else{ - wout <<" and not in 'leader'"; - } - wout << std::endl; - - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/compare.cpp b/external_codes/mpi_wrapper/mpi3/test/compare.cpp deleted file mode 100644 index 9f65e1ff31b..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/compare.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#if COMPILATION_INSTRUCTIONS -mpicxx -O3 -std=c++14 -Wfatal-errors $0 -o $0x.x && time mpirun -np 8s $0x.x $@ && rm -f $0x.x; exit -#endif - -#include "alf/boost/mpi3/environment.hpp" -#include "alf/boost/mpi3/communicator.hpp" -#include "alf/boost/mpi3/cartesian_communicator.hpp" - -namespace mpi3 = boost::mpi3; -using std::cout; - -int main(int argc, char* argv[]){ - mpi3::environment env(argc, argv); - std::vector dims = {env.world().size()}; - mpi3::cartesian_communicator comm(env.world(), dims); - { - std::vector remain(1); - remain[0] = true; - mpi3::cartesian_communicator newcomm = comm.sub(remain); - assert( env.self().compare(newcomm) == boost::mpi3::unequal ); - } - - return 0; -} - diff --git a/external_codes/mpi_wrapper/mpi3/test/datatype.cpp b/external_codes/mpi_wrapper/mpi3/test/datatype.cpp deleted file mode 100644 index f491f34b2f4..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/datatype.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2022-2023 Alfredo A. Correa - -#include -#include - -#include -#include -#include - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator /*world*/) -> int try { - using mpi3::detail::is_basic_v; - - static_assert( is_basic_v ); - static_assert( is_basic_v ); - static_assert( is_basic_v ); - static_assert( is_basic_v> ); - - static_assert( not is_basic_v ); - - assert( mpi3::detail::basic_datatype{} == MPI_DOUBLE ); - - return 0; -} catch(...) { - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/datatype_struct_vector3.cpp b/external_codes/mpi_wrapper/mpi3/test/datatype_struct_vector3.cpp deleted file mode 100644 index 0d9cdc87c98..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/datatype_struct_vector3.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2023 Alfredo A. Correa - -#include -#include - -// #include - -namespace mpi3 = boost::mpi3; - -struct vec3 { - double x; // NOLINT(misc-non-private-member-variables-in-classes) - double y; // NOLINT(misc-non-private-member-variables-in-classes) - double z; // NOLINT(misc-non-private-member-variables-in-classes) - - constexpr bool operator==(vec3 const& other) const {return x == other.x and y == other.y and z == other.z;} - constexpr bool operator!=(vec3 const& other) const {return x != other.x or y != other.y or z != other.z;} - - constexpr auto operator+(vec3 const& other) const {return vec3{x + other.x, y + other.y, z + other.z};} -}; - -template<> struct mpi3::datatype : mpi3::struct_< - double, - double, - double -> {}; - -mpi3::environment mpienv; // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp,cppcoreguidelines-avoid-non-const-global-variables) experiment with global environment - -auto main(int /*argc*/, char** /*argv*/) -> int try { // NOLINT(bugprone-exception-escape) - - mpi3::communicator world{mpienv.world()}; - - switch(world.rank()) { - case 0: { - std::vector v(5); - v[2] = vec3{1.0, 2.0, 3.0}; - world.send_n(begin(v), 5, 1); - break; - }; - case 1: { - std::vector v(5); - world.receive_n(begin(v), 5, 0); - assert(( v[2] == vec3{1.0, 2.0, 3.0} )); - break; - }; - } - - { - std::vector w = { vec3{1.0, 2.0, 3.0}, vec3{4.0, 5.0, 6.0} }; - std::vector sum(2); - - world.all_reduce_n(w.begin(), w.size(), sum.begin()); - - assert(sum[0].x == w[0].x*world.size() ); - assert(sum[1].y == w[1].y*world.size() ); - } - { - std::vector w = { vec3{1.0, 2.0, 3.0}, vec3{4.0, 5.0, 6.0} }; - std::vector sum(2); - - world.all_reduce(w.begin(), w.end(), sum.begin()); - - assert(sum[0].x == w[0].x*world.size() ); - assert(sum[1].y == w[1].y*world.size() ); - } - { - std::vector w = { vec3{1.0, 2.0, 3.0}, vec3{4.0, 5.0, 6.0} }; - std::vector sum(2); - - world.all_reduce_n(w.begin(), w.size(), sum.begin()); - - assert(sum[0].x == w[0].x*world.size() ); - assert(sum[1].y == w[1].y*world.size() ); - } - { - std::vector w = { vec3{1.0, 2.0, 3.0}, vec3{4.0, 5.0, 6.0} }; - std::vector sum(2); - - world.all_reduce(w.begin(), w.end(), sum.begin()); - - assert(sum[0].x == w[0].x*world.size() ); - assert(sum[1].y == w[1].y*world.size() ); - } - { - std::vector w = { vec3{1.0, 2.0, 3.0}, vec3{4.0, 5.0, 6.0} }; - auto w_copy = w; - - world.all_reduce_n(w.begin(), w.size()); - - assert(w[0].x == w_copy[0].x*world.size() ); - assert(w[1].y == w_copy[1].y*world.size() ); - } - { - std::vector w = { vec3{1.0, 2.0, 3.0}, vec3{4.0, 5.0, 6.0} }; - auto w_copy = w; - - world.all_reduce(w.begin(), w.end()); - - assert(w[0].x == w_copy[0].x*world.size() ); - assert(w[1].y == w_copy[1].y*world.size() ); - } - static_assert(boost::mpi3::has_datatype{}); -} catch(...) { - throw; - return 1; -} diff --git a/external_codes/mpi_wrapper/mpi3/test/deino_all_to_all.cpp b/external_codes/mpi_wrapper/mpi3/test/deino_all_to_all.cpp deleted file mode 100644 index 12d2d2d3e38..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/deino_all_to_all.cpp +++ /dev/null @@ -1,117 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2022 Alfredo A. Correa - -// based on http://mpi.deino.net/mpi_functions/MPI_Alltoall.html - -#include -#include - -#include "mpi.h" - -namespace mpi3 = boost::mpi3; - -//template -//void do_all_to_all_n(mpi3::communicator& comm, InputIt first, Size count, OutputIt result) { -// #ifndef MPICH_VERSION -// comm.all_to_all_n(first, count, result); -// #else -// if(first != result) { -// comm.all_to_all_n(first, count, result); -// } else { -// std::vector reqs(comm.size(), MPI_REQUEST_NULL); - -// assert(count % comm.size() == 0); -// for(int iproc = 0; iproc < comm.size(); iproc++) { -// MPI_Isendrecv_replace(first, count/comm.size(), mpi3::detail::basic_datatype::value_type>(), iproc, 0, MPI_ANY_SOURCE, MPI_ANY_TAG, &comm, &reqs[iproc]); -// } - -// MPI_Waitall(static_cast(reqs.size()), reqs.data(), MPI_STATUSES_IGNORE); -// } -// #endif -//} - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::communicator world) try { - std::size_t chunk = 5; - - auto sb = std::vector(static_cast(world.size()) * chunk); - std::iota(sb.begin(), sb.end(), 40000 + world.rank()*100); - - auto rb = std::vector(static_cast(world.size()) * chunk); - - auto sz = static_cast(world.size()); assert( sz != 0 ); - assert( sb.size() % sz == 0); - - world.all_to_all_n(sb.data(), sb.size()/sz, rb.data()); - // do_all_to_all_n(world, sb.data(), sb.size(), rb.data()); - // do_all_to_all_n(world, sb.data(), sb.size(), rb.data()); - - mpi3::ostream wout(world); - std::copy(sb.begin(), sb.end(), std::ostream_iterator(wout<<"sb = ", ", ")); wout<(wout<<"rb = ", ", ")); wout<(wout<<"sb (inplace) = ", ", ")); wout< send_buff = { - world.rank() * 10 + 0, - world.rank() * 10 + 1, - world.rank() * 10 + 2, - world.rank() * 10 + 3, - }; - - assert((int)send_buff.size() == world.size()); - - std::vector recv_buff(world.size(), 99.); - { - - assert((int)recv_buff.size() == world.size()); - world.all_to_all(send_buff.begin(), recv_buff.begin()); - - if(world.rank() == 0) - assert(recv_buff[0] == 00 and recv_buff[1] == 10 and recv_buff[2] == 20 and recv_buff[3] == 30); - - if(world.rank() == 1) - assert(recv_buff[0] == 01 and recv_buff[1] == 11 and recv_buff[2] == 21 and recv_buff[3] == 31); - } - std::vector recv_buff2(world.size(), 99.); - for(std::size_t i = 0; i != send_buff.size(); ++i) - world[i] << send_buff[i] >> recv_buff2[i]; - - assert(recv_buff == recv_buff2); - - { - auto const& csend_buff = send_buff; - auto const r = world & csend_buff; - assert(recv_buff == r); - } - { - auto const& csend_buff = send_buff; - auto const r = world(csend_buff); - assert(recv_buff == r); - } - { - world& send_buff; - assert(recv_buff == send_buff); - } - - return 0; -} -#endif \ No newline at end of file diff --git a/external_codes/mpi_wrapper/mpi3/test/deino_broadcast.cpp b/external_codes/mpi_wrapper/mpi3/test/deino_broadcast.cpp deleted file mode 100644 index 6e6eca0c179..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/deino_broadcast.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2023 Alfredo A. Correa - -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/process.hpp" - -namespace mpi3 = boost::mpi3; - -void syntax_test(mpi3::communicator& world) { - { - bool b = world.root(); - world.broadcast_value(b); - assert( b == true ); - } - { - bool b = world.root(); - world.broadcast_n(&b, 1); - assert( b == true ); - } - { - bool b = world.root(); - world[0] || b; - assert( b == true ); - } -} - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try { - std::vector sizes = {100L, 64L*1024L};//, 128L*1024L}; // TODO check larger number (fails with openmpi 4.0.5) - std::size_t NUM_REPS = 5; - - using value_type = int; - std::vector buf(128L*1024L); - - for(std::size_t n=0; n != sizes.size(); ++n) { - // if(world.root()) cout<<"bcasting "<< sizes[n] <<" ints "<< NUM_REPS <<" times.\n"; - - for(std::size_t reps = 0; reps != NUM_REPS; ++reps) { - if(world.root()) { - for(std::size_t i = 0; i != sizes[n]; ++i) { // NOLINT(altera-unroll-loops) - buf[i] = static_cast(1000000.0 * static_cast(n * NUM_REPS + reps) + static_cast(i)); - } - } else { - for(std::size_t i = 0; i != sizes[n]; ++i) { // NOLINT(altera-unroll-loops) - buf[i] = static_cast(-(n * NUM_REPS + reps) - 1); - } - } - - world.broadcast_n(buf.begin(), sizes[n]); - // world.broadcast(buf.begin(), buf.begin() + sizes[n], 0); - - for(std::size_t i = 0; i != sizes[n]; ++i) { // NOLINT(altera-unroll-loops) - assert( fabs(buf[i] - (1000000.0*static_cast(n * NUM_REPS + reps) + static_cast(i))) < 1e-4 ); - } - } - } - syntax_test(world); - return 0; -} catch(...) {return 1;} diff --git a/external_codes/mpi_wrapper/mpi3/test/deino_op_create.cpp b/external_codes/mpi_wrapper/mpi3/test/deino_op_create.cpp deleted file mode 100644 index da72acc46a5..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/deino_op_create.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2023 Alfredo A. Correa - -// #include - -#include -#include - -namespace mpi3 = boost::mpi3; - -struct singleton { - int value; // NOLINT(misc-non-private-member-variables-in-classes) - bool operator==(singleton const& other) const {return value == other.value;} - bool operator!=(singleton const& other) const {return value != other.value;} - constexpr auto operator+(singleton const& other) const {return singleton{value + other.value};} -}; - -template<> struct mpi3::datatype : mpi3::struct_ {}; - -int mpi3::main(int /*argc*/, char** /*argv*/, mpi3::environment& mpie) { // NOLINT(bugprone-exception-escape) - auto world{mpie.world()}; - - { - auto data = singleton{world.rank()}; - - singleton result{}; - world.reduce_n(&data, 1, &result); - world.broadcast_n(&result, 1); - - singleton correct_result{0}; - for(int i = 0; i != world.size(); ++i) {correct_result = correct_result + singleton{i};} // NOLINT(altera-unroll-loops,altera-id-dependent-backward-branch) - assert(result == correct_result); - } - { - auto data = std::vector{{ {world.rank()}, {world.rank()}, {world.rank()} }}; - - std::vector result(3); - world.reduce_n(data.begin(), 3, result.begin()); - world.broadcast_n(result.begin(), 3); - - singleton correct_result{0}; - for(int i = 0; i != world.size(); ++i) {correct_result = correct_result + singleton{i};} // NOLINT(altera-unroll-loops,altera-id-dependent-backward-branch) - - assert(result[1].value == correct_result.value); - } - { - auto data = std::vector{{ {world.rank()}, {world.rank()}, {world.rank()} }}; - - std::vector result(3); - world.reduce(data.begin(), data.end(), result.begin()); - world.broadcast(result.begin(), result.end()); - - singleton correct_result{0}; - for(int i = 0; i != world.size(); ++i) {correct_result = correct_result + singleton{i};} // NOLINT(altera-unroll-loops,altera-id-dependent-backward-branch) - - assert(result[1].value == correct_result.value); - } - - return 0; -} \ No newline at end of file diff --git a/external_codes/mpi_wrapper/mpi3/test/empty_main.cpp b/external_codes/mpi_wrapper/mpi3/test/empty_main.cpp deleted file mode 100644 index b43c50ee1cb..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/empty_main.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2021-2022 Alfredo A. Correa - -#include "../../mpi3/environment.hpp" - -namespace mpi3 = boost::mpi3; -//mpi3::environment env; - -auto main() -> int try { - mpi3::environment env; -} catch(...) {} diff --git a/external_codes/mpi_wrapper/mpi3/test/environment_self.cpp b/external_codes/mpi_wrapper/mpi3/test/environment_self.cpp deleted file mode 100644 index 377ffea6b19..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/environment_self.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// -*-indent-tabs-mode:t;c-basic-offset:4;tab-width:4;autowrap:nil;-*- -// Copyright 2018-2022 Alfredo A. Correa - -#include - -using std::cout; -namespace mpi3 = boost::mpi3; - -int main() { - assert(not mpi3::initialzed()); - assert(not mpi3::finalized()); - { - mpi3::environment env; - assert(mpi3::initialzed()); - assert(not mpi3::finalized()); - - cout << "us " << env.get_world_instance().get_attribute_as(mpi3::universe_size) << std::endl; - - return 0; - auto self = env.self(); - assert(self.size() == 1); - assert(self.rank() == 0); - cout << "I am process " << self.rank() << " in communicator " << self.name() << std::endl; - - auto world = env.world(); - world.barrier(); - assert(world.size() == 4); - assert(world.rank() < 4); - cout << "I am process " << world.rank() << " in communicator " << world.name() << std::endl; - } - assert(mpi3::finalized()); - - /* output: - I am process 0 in communicator MPI_COMM_SELF - I am process 0 in communicator MPI_COMM_SELF - I am process 0 in communicator MPI_COMM_SELF - I am process 0 in communicator MPI_COMM_SELF - I am process 3 in communicator MPI_COMM_WORLD - I am process 0 in communicator MPI_COMM_WORLD - I am process 1 in communicator MPI_COMM_WORLD - I am process 2 in communicator MPI_COMM_WORLD - */ -} diff --git a/external_codes/mpi_wrapper/mpi3/test/environment_thread.cpp b/external_codes/mpi_wrapper/mpi3/test/environment_thread.cpp deleted file mode 100644 index d099642a248..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/environment_thread.cpp +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2018-2021 Alfredo A. Correa -#include "../../mpi3/environment.hpp" - -namespace mpi3 = boost::mpi3; - -mpi3::environment const mpienv{mpi3::thread::serialized}; // NOLINT(fuchsia-statically-constructed-objects,cert-err58-cpp) - -int main() try { - - assert( mpienv.thread_support() == mpi3::thread::single or mpienv.thread_support() == mpi3::thread::funneled or mpienv.thread_support() == mpi3::thread::serialized ); - assert( mpienv.thread_support() <= mpi3::thread::serialized ); - assert( mpienv.thread_support() < mpi3::thread::multiple ); - - assert( mpienv.is_thread_main() ); -} catch(...) {return 0;} diff --git a/external_codes/mpi_wrapper/mpi3/test/gather2.cpp b/external_codes/mpi_wrapper/mpi3/test/gather2.cpp deleted file mode 100644 index ffbf1331e94..00000000000 --- a/external_codes/mpi_wrapper/mpi3/test/gather2.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "../../mpi3/main.hpp" -#include "../../mpi3/communicator.hpp" -#include "../../mpi3/detail/iterator.hpp" - -namespace mpi3 = boost::mpi3; - -auto mpi3::main(int/*argc*/, char**/*argv*/, mpi3::communicator world) -> int try{ - - using dd = std::tuple; - - std::vector