From 7d30fff5636da8de4d6cdc77d4273f6a6fc1da38 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 4 Oct 2023 12:08:15 +0200 Subject: [PATCH 1/3] add distmem_n_vctrs --- src/impl/particles_impl.ipp | 12 ++++-- src/impl/particles_impl_bcnd.ipp | 2 +- src/impl/particles_impl_pack.ipp | 42 ++++++++++++------- .../particles_impl_reserve_hskpng_npart.ipp | 2 +- src/impl/particles_impl_unpack.ipp | 12 ++++-- 5 files changed, 46 insertions(+), 24 deletions(-) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index b07d3332c..71da6de44 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -270,11 +270,15 @@ namespace libcloudphxx // ids of sds to be copied with distmem thrust_device::vector &lft_id, &rgt_id; + // --- containters with vector pointers to help resize and copy vectors --- + // real_t vectors copied in distributed memory case - std::set*> distmem_real_vctrs; + std::set*> distmem_real_vctrs; + std::set*> distmem_n_vctrs; +// std::set*> distmem_size_vctrs; // no size vectors copied? - // methods + // --- methods --- // fills u01 with n random real numbers uniformly distributed in range [0,1) void rand_u01(thrust_size_t n) { rng.generate_n(u01, n); } @@ -384,7 +388,6 @@ namespace libcloudphxx } // initializing distmem_real_vctrs - list of real_t vectors with properties of SDs that have to be copied/removed/recycled when a SD is copied/removed/recycled - // TODO: add to that list vectors of other types (e.g integer pimpl->n) // NOTE: this does not include chemical stuff due to the way chem vctrs are organized! multi_CUDA / MPI does not work with chemistry as of now typedef thrust_device::vector* ptr_t; ptr_t arr[] = {&rd3, &rw2, &kpa, &vt}; @@ -418,6 +421,9 @@ namespace libcloudphxx if(opts_init.diag_incloud_time) distmem_real_vctrs.insert(&incloud_time); + + // initializing distmem_n_vctrs - list of n_t vectors with properties of SDs that have to be copied/removed/recycled when a SD is copied/removed/recycled + distmem_n_vctrs.insert(&n); } void sanity_checks(); diff --git a/src/impl/particles_impl_bcnd.ipp b/src/impl/particles_impl_bcnd.ipp index 6eebda9ee..4f28f8e43 100644 --- a/src/impl/particles_impl_bcnd.ipp +++ b/src/impl/particles_impl_bcnd.ipp @@ -124,7 +124,7 @@ namespace libcloudphxx arg::_1 >= opts_init.x1 ) - rgt_id.begin(); - const int no_of_n_vctrs_copied(int(1)); + const int no_of_n_vctrs_copied(distmem_n_vctrs.size()); const int no_of_real_vctrs_copied(distmem_real_vctrs.size()); if(lft_count*no_of_n_vctrs_copied > in_n_bfr.size() || rgt_count*no_of_n_vctrs_copied > in_n_bfr.size()) diff --git a/src/impl/particles_impl_pack.ipp b/src/impl/particles_impl_pack.ipp index 5c8226bf0..5b72c6ea3 100644 --- a/src/impl/particles_impl_pack.ipp +++ b/src/impl/particles_impl_pack.ipp @@ -29,25 +29,37 @@ namespace libcloudphxx template void particles_t::impl::pack_n_lft() { - assert(out_n_bfr.size() >= lft_count); - assert(in_n_bfr.size() >= lft_count); - thrust::copy( - thrust::make_permutation_iterator(n.begin(), lft_id.begin()), - thrust::make_permutation_iterator(n.begin(), lft_id.begin()) + lft_count, - out_n_bfr.begin() - ); + assert(out_n_bfr.size() >= lft_count * distmem_n_vctrs.size()); + assert(in_n_bfr.size() >= lft_count * distmem_n_vctrs.size()); + + auto it = distmem_n_vctrs.begin(); + while (it != distmem_n_vctrs.end()) + { + thrust::copy( + thrust::make_permutation_iterator((*it)->begin(), lft_id.begin()), + thrust::make_permutation_iterator((*it)->begin(), lft_id.begin()) + lft_count, + out_n_bfr.begin() + std::distance(distmem_n_vctrs.begin(), it) * lft_count + ); + it++; + } } template void particles_t::impl::pack_n_rgt() { - assert(out_n_bfr.size() >= rgt_count); - assert(in_n_bfr.size() >= rgt_count); - thrust::copy( - thrust::make_permutation_iterator(n.begin(), rgt_id.begin()), - thrust::make_permutation_iterator(n.begin(), rgt_id.begin()) + rgt_count, - out_n_bfr.begin() - ); + assert(out_n_bfr.size() >= rgt_count * distmem_n_vctrs.size()); + assert(in_n_bfr.size() >= rgt_count * distmem_n_vctrs.size()); + + auto it = distmem_n_vctrs.begin(); + while (it != distmem_n_vctrs.end()) + { + thrust::copy( + thrust::make_permutation_iterator((*it)->begin(), rgt_id.begin()), + thrust::make_permutation_iterator((*it)->begin(), rgt_id.begin()) + rgt_count, + out_n_bfr.begin() + std::distance(distmem_n_vctrs.begin(), it) * rgt_count + ); + it++; + } } template @@ -57,7 +69,6 @@ namespace libcloudphxx assert(in_real_bfr.size() >= lft_count * distmem_real_vctrs.size()); auto it = distmem_real_vctrs.begin(); - while (it != distmem_real_vctrs.end()) { thrust::copy( @@ -76,7 +87,6 @@ namespace libcloudphxx assert(in_real_bfr.size() >= rgt_count * distmem_real_vctrs.size()); auto it = distmem_real_vctrs.begin(); - while (it != distmem_real_vctrs.end()) { thrust::copy( diff --git a/src/impl/particles_impl_reserve_hskpng_npart.ipp b/src/impl/particles_impl_reserve_hskpng_npart.ipp index 063969e7f..27ec4369c 100644 --- a/src/impl/particles_impl_reserve_hskpng_npart.ipp +++ b/src/impl/particles_impl_reserve_hskpng_npart.ipp @@ -78,7 +78,7 @@ namespace libcloudphxx // done using resize, because _bfr.end() is never used and we want to assert that buffer is large enough using the .size() function if(distmem()) { - const int no_of_n_vctrs_copied(int(1)); + const int no_of_n_vctrs_copied(distmem_n_vctrs.size()); const int no_of_real_vctrs_copied(distmem_real_vctrs.size()); in_n_bfr.resize(no_of_n_vctrs_copied * opts_init.n_sd_max / opts_init.nx / config.bfr_fraction); // for n diff --git a/src/impl/particles_impl_unpack.ipp b/src/impl/particles_impl_unpack.ipp index be379de55..49cc8ef08 100644 --- a/src/impl/particles_impl_unpack.ipp +++ b/src/impl/particles_impl_unpack.ipp @@ -56,8 +56,15 @@ namespace libcloudphxx return; assert(opts_init.n_sd_max >= n_part); - n.resize(n_part); - thrust::copy(in_n_bfr.begin(), in_n_bfr.begin() + n_copied, n.begin() + n_part_old); + + auto it = distmem_n_vctrs.begin(); + while (it != distmem_n_vctrs.end()) + { + (*it)->resize(n_part); + auto distance = std::distance(distmem_n_vctrs.begin(), it); + thrust::copy( in_n_bfr.begin() + distance * n_copied, in_n_bfr.begin() + (distance+1) * n_copied, (*it)->begin() + n_part_old); + it++; + } } template @@ -67,7 +74,6 @@ namespace libcloudphxx return; auto it = distmem_real_vctrs.begin(); - while (it != distmem_real_vctrs.end()) { (*it)->resize(n_part); From 4a12da5102a0e261d7cbc9a942dcc426d0012685 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 4 Oct 2023 13:02:53 +0200 Subject: [PATCH 2/3] resize vectors using distmem_real_vctrs --- src/detail/config.hpp | 3 ++ src/impl/particles_impl.ipp | 46 +++++++++++------------ src/impl/particles_impl_hskpng_remove.ipp | 4 +- src/impl/particles_impl_hskpng_resize.ipp | 35 ++++------------- src/impl/particles_impl_pack.ipp | 8 ++-- src/impl/particles_impl_rcyc.ipp | 2 +- src/impl/particles_impl_unpack.ipp | 4 +- src/particles_init.ipp | 4 +- 8 files changed, 44 insertions(+), 62 deletions(-) diff --git a/src/detail/config.hpp b/src/detail/config.hpp index 44c01d5bd..8fade90d5 100644 --- a/src/detail/config.hpp +++ b/src/detail/config.hpp @@ -39,6 +39,9 @@ namespace libcloudphxx eps_tolerance(sizeof(real_t) * 8 / 4) {} }; + + // just some constant, not related to config but had to put them somewhere + enum { invalid = -1, no_initial_value = -44 }; }; }; }; diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 71da6de44..3e35f3e26 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -23,12 +23,6 @@ namespace libcloudphxx { namespace lgrngn { - namespace detail - { - enum { invalid = -1 }; - - }; - // pimpl stuff template struct particles_t::impl @@ -272,10 +266,12 @@ namespace libcloudphxx // --- containters with vector pointers to help resize and copy vectors --- - // real_t vectors copied in distributed memory case - std::set*> distmem_real_vctrs; + // vectors copied between distributed memories (MPI, multi_CUDA), these are SD attributes + std::set*, real_t>> distmem_real_vctrs; // pair of vector and its initial value std::set*> distmem_n_vctrs; // std::set*> distmem_size_vctrs; // no size vectors copied? + // vetors that need to be removed/resizes/recycled when the number of SDs changes + std::set*> resize_real_vctrs; // --- methods --- @@ -389,38 +385,40 @@ namespace libcloudphxx // initializing distmem_real_vctrs - list of real_t vectors with properties of SDs that have to be copied/removed/recycled when a SD is copied/removed/recycled // NOTE: this does not include chemical stuff due to the way chem vctrs are organized! multi_CUDA / MPI does not work with chemistry as of now - typedef thrust_device::vector* ptr_t; - ptr_t arr[] = {&rd3, &rw2, &kpa, &vt}; - distmem_real_vctrs = std::set(arr, arr + sizeof(arr) / sizeof(ptr_t) ); + distmem_real_vctrs.insert({&rd3, detail::no_initial_value}); + distmem_real_vctrs.insert({&rw2, detail::no_initial_value}); + distmem_real_vctrs.insert({&kpa, detail::no_initial_value}); + + distmem_real_vctrs.insert({&vt, detail::invalid}); - if (opts_init.nx != 0) distmem_real_vctrs.insert(&x); - if (opts_init.ny != 0) distmem_real_vctrs.insert(&y); - if (opts_init.nz != 0) distmem_real_vctrs.insert(&z); + if (opts_init.nx != 0) distmem_real_vctrs.insert({&x, detail::no_initial_value}); + if (opts_init.ny != 0) distmem_real_vctrs.insert({&y, detail::no_initial_value}); + if (opts_init.nz != 0) distmem_real_vctrs.insert({&z, detail::no_initial_value}); if(allow_sstp_cond && opts_init.exact_sstp_cond) { - distmem_real_vctrs.insert(&sstp_tmp_rv); - distmem_real_vctrs.insert(&sstp_tmp_th); - distmem_real_vctrs.insert(&sstp_tmp_rh); + distmem_real_vctrs.insert({&sstp_tmp_rv, detail::no_initial_value}); + distmem_real_vctrs.insert({&sstp_tmp_th, detail::no_initial_value}); + distmem_real_vctrs.insert({&sstp_tmp_rh, detail::no_initial_value}); // sstp_tmp_p needs to be added if a constant pressure profile is used, but this is only known after init - see particles_init } if(opts_init.turb_adve_switch) { - if(opts_init.nx != 0) distmem_real_vctrs.insert(&up); - if(opts_init.ny != 0) distmem_real_vctrs.insert(&vp); - if(opts_init.nz != 0) distmem_real_vctrs.insert(&wp); + if(opts_init.nx != 0) distmem_real_vctrs.insert({&up, 0}); + if(opts_init.ny != 0) distmem_real_vctrs.insert({&vp, 0}); + if(opts_init.nz != 0) distmem_real_vctrs.insert({&wp, 0}); } if(opts_init.turb_cond_switch) { - distmem_real_vctrs.insert(&wp); - distmem_real_vctrs.insert(&ssp); - distmem_real_vctrs.insert(&dot_ssp); + distmem_real_vctrs.insert({&wp, 0}); + distmem_real_vctrs.insert({&ssp, 0}); + distmem_real_vctrs.insert({&dot_ssp, 0}); } if(opts_init.diag_incloud_time) - distmem_real_vctrs.insert(&incloud_time); + distmem_real_vctrs.insert({&incloud_time, detail::no_initial_value}); // initializing distmem_n_vctrs - list of n_t vectors with properties of SDs that have to be copied/removed/recycled when a SD is copied/removed/recycled distmem_n_vctrs.insert(&n); diff --git a/src/impl/particles_impl_hskpng_remove.ipp b/src/impl/particles_impl_hskpng_remove.ipp index ae31eee6b..a8bb6517f 100644 --- a/src/impl/particles_impl_hskpng_remove.ipp +++ b/src/impl/particles_impl_hskpng_remove.ipp @@ -57,8 +57,8 @@ namespace libcloudphxx for(auto vec: distmem_real_vctrs) { thrust::remove_if( - vec->begin(), - vec->begin() + n_part, + vec.first->begin(), + vec.first->begin() + n_part, n.begin(), arg::_1 == 0 ); diff --git a/src/impl/particles_impl_hskpng_resize.ipp b/src/impl/particles_impl_hskpng_resize.ipp index d9f494bb7..80ccd68ed 100644 --- a/src/impl/particles_impl_hskpng_resize.ipp +++ b/src/impl/particles_impl_hskpng_resize.ipp @@ -8,7 +8,7 @@ namespace libcloudphxx { if(n_part > opts_init.n_sd_max) throw std::runtime_error(detail::formatter() << "n_sd_max (" << opts_init.n_sd_max << ") < n_part (" << n_part << ")"); { - thrust_device::vector *vec[] = {&rw2, &rd3, &kpa, &tmp_device_real_part}; + thrust_device::vector *vec[] = {&tmp_device_real_part}; for(int i=0; i<4; ++i) { vec[i]->resize(n_part); @@ -25,30 +25,10 @@ namespace libcloudphxx tmp_device_n_part.resize(n_part); tmp_device_size_part.resize(n_part); - vt.resize(n_part, detail::invalid); - if (opts_init.nx != 0) i.resize(n_part); if (opts_init.ny != 0) j.resize(n_part); if (opts_init.nz != 0) k.resize(n_part); - if (opts_init.nx != 0) x.resize(n_part); - if (opts_init.ny != 0) y.resize(n_part); - if (opts_init.nz != 0) z.resize(n_part); - - if(opts_init.turb_adve_switch) - { - if (opts_init.nx != 0) up.resize(n_part, 0); - if (opts_init.ny != 0) vp.resize(n_part, 0); - if (opts_init.nz != 0) wp.resize(n_part, 0); - } - - if(opts_init.turb_cond_switch) - { - wp.resize(n_part, 0); - ssp.resize(n_part, 0); - dot_ssp.resize(n_part, 0); - } - if(opts_init.chem_switch || allow_sstp_cond || n_dims >= 2) { tmp_device_real_part1.resize(n_part); @@ -62,18 +42,19 @@ namespace libcloudphxx { tmp_device_real_part3.resize(n_part); tmp_device_real_part4.resize(n_part); - sstp_tmp_rv.resize(n_part); - sstp_tmp_th.resize(n_part); - sstp_tmp_rh.resize(n_part); if(const_p) { tmp_device_real_part5.resize(n_part); - sstp_tmp_p.resize(n_part); } } - if(opts_init.diag_incloud_time) - incloud_time.resize(n_part); + for(auto &pair: distmem_real_vctrs) + { + if(pair.second == detail::no_initial_value) + pair.first->resize(n_part); + else + pair.first->resize(n_part, pair.second); + } } }; }; diff --git a/src/impl/particles_impl_pack.ipp b/src/impl/particles_impl_pack.ipp index 5b72c6ea3..915f0e873 100644 --- a/src/impl/particles_impl_pack.ipp +++ b/src/impl/particles_impl_pack.ipp @@ -72,8 +72,8 @@ namespace libcloudphxx while (it != distmem_real_vctrs.end()) { thrust::copy( - thrust::make_permutation_iterator((*it)->begin(), lft_id.begin()), - thrust::make_permutation_iterator((*it)->begin(), lft_id.begin()) + lft_count, + thrust::make_permutation_iterator((*it).first->begin(), lft_id.begin()), + thrust::make_permutation_iterator((*it).first->begin(), lft_id.begin()) + lft_count, out_real_bfr.begin() + std::distance(distmem_real_vctrs.begin(), it) * lft_count ); it++; @@ -90,8 +90,8 @@ namespace libcloudphxx while (it != distmem_real_vctrs.end()) { thrust::copy( - thrust::make_permutation_iterator((*it)->begin(), rgt_id.begin()), - thrust::make_permutation_iterator((*it)->begin(), rgt_id.begin()) + rgt_count, + thrust::make_permutation_iterator((*it).first->begin(), rgt_id.begin()), + thrust::make_permutation_iterator((*it).first->begin(), rgt_id.begin()) + rgt_count, out_real_bfr.begin() + std::distance(distmem_real_vctrs.begin(), it) * rgt_count ); it++; diff --git a/src/impl/particles_impl_rcyc.ipp b/src/impl/particles_impl_rcyc.ipp index 053cf559a..450d59556 100644 --- a/src/impl/particles_impl_rcyc.ipp +++ b/src/impl/particles_impl_rcyc.ipp @@ -98,7 +98,7 @@ namespace libcloudphxx // for each property... for(auto vec: distmem_real_vctrs) - detail::copy_prop(vec->begin(), sorted_id, n_flagged); + detail::copy_prop(vec.first->begin(), sorted_id, n_flagged); // ... chemical properties only if chem enabled if (opts_init.chem_switch){ diff --git a/src/impl/particles_impl_unpack.ipp b/src/impl/particles_impl_unpack.ipp index 49cc8ef08..1daa205d0 100644 --- a/src/impl/particles_impl_unpack.ipp +++ b/src/impl/particles_impl_unpack.ipp @@ -76,9 +76,9 @@ namespace libcloudphxx auto it = distmem_real_vctrs.begin(); while (it != distmem_real_vctrs.end()) { - (*it)->resize(n_part); + (*it).first->resize(n_part); auto distance = std::distance(distmem_real_vctrs.begin(), it); - thrust::copy( in_real_bfr.begin() + distance * n_copied, in_real_bfr.begin() + (distance+1) * n_copied, (*it)->begin() + n_part_old); + thrust::copy( in_real_bfr.begin() + distance * n_copied, in_real_bfr.begin() + (distance+1) * n_copied, (*it).first->begin() + n_part_old); it++; } diff --git a/src/particles_init.ipp b/src/particles_init.ipp index ecf938e52..bbbf09e1f 100644 --- a/src/particles_init.ipp +++ b/src/particles_init.ipp @@ -5,7 +5,7 @@ * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) * @brief initialisation routine for super droplets */ -#include +#include namespace libcloudphxx { @@ -35,7 +35,7 @@ namespace libcloudphxx pimpl->const_p = !p.is_null(); // if pressure comes from a profile, sstp_tmp_p also needs to be copied between distributed memories if(pimpl->const_p && pimpl->allow_sstp_cond && pimpl->opts_init.exact_sstp_cond) - pimpl->distmem_real_vctrs.insert(&pimpl->sstp_tmp_p); + pimpl->distmem_real_vctrs.insert({&pimpl->sstp_tmp_p, detail::no_initial_value}); // initialising Eulerian-Lagrangian coupling pimpl->init_sync(); // also, init of ambient_chem vectors From e7c22d385866c2b8742336d8741fa514089aee6d Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 4 Oct 2023 13:37:37 +0200 Subject: [PATCH 3/3] add resize_t_vctrs and use it in resize() --- src/impl/particles_impl.ipp | 29 ++++++++++++- src/impl/particles_impl_hskpng_resize.ipp | 50 +++++------------------ src/impl/particles_impl_mpi_exchange.ipp | 4 +- 3 files changed, 40 insertions(+), 43 deletions(-) diff --git a/src/impl/particles_impl.ipp b/src/impl/particles_impl.ipp index 3e35f3e26..29b5c9ef8 100644 --- a/src/impl/particles_impl.ipp +++ b/src/impl/particles_impl.ipp @@ -268,10 +268,13 @@ namespace libcloudphxx // vectors copied between distributed memories (MPI, multi_CUDA), these are SD attributes std::set*, real_t>> distmem_real_vctrs; // pair of vector and its initial value - std::set*> distmem_n_vctrs; + std::set*> distmem_n_vctrs; // std::set*> distmem_size_vctrs; // no size vectors copied? - // vetors that need to be removed/resizes/recycled when the number of SDs changes +// + // vetors that are not in distmem_real_vctrs that need to be resized when the number of SDs changes, these are helper variables std::set*> resize_real_vctrs; +// std::set*> resize_n_vctrs; + std::set*> resize_size_vctrs; // --- methods --- @@ -422,6 +425,28 @@ namespace libcloudphxx // initializing distmem_n_vctrs - list of n_t vectors with properties of SDs that have to be copied/removed/recycled when a SD is copied/removed/recycled distmem_n_vctrs.insert(&n); + + // real vctrs that need to be resized but do need to be copied in distmem + resize_real_vctrs.insert(&tmp_device_real_part); + if(opts_init.chem_switch || allow_sstp_cond || n_dims >= 2) + resize_real_vctrs.insert(&tmp_device_real_part1); + if((allow_sstp_cond && opts_init.exact_sstp_cond) || n_dims==3 || opts_init.turb_cond_switch) + resize_real_vctrs.insert(&tmp_device_real_part2); + if(allow_sstp_cond && opts_init.exact_sstp_cond) + { + resize_real_vctrs.insert(&tmp_device_real_part3); + resize_real_vctrs.insert(&tmp_device_real_part4); + if(const_p) + resize_real_vctrs.insert(&tmp_device_real_part5); + } + + resize_size_vctrs.insert(&ijk); + resize_size_vctrs.insert(&sorted_ijk); + resize_size_vctrs.insert(&sorted_id); + resize_size_vctrs.insert(&tmp_device_size_part); + if (opts_init.nx != 0) resize_size_vctrs.insert(&i); + if (opts_init.ny != 0) resize_size_vctrs.insert(&j); + if (opts_init.nz != 0) resize_size_vctrs.insert(&k); } void sanity_checks(); diff --git a/src/impl/particles_impl_hskpng_resize.ipp b/src/impl/particles_impl_hskpng_resize.ipp index 80ccd68ed..15828069d 100644 --- a/src/impl/particles_impl_hskpng_resize.ipp +++ b/src/impl/particles_impl_hskpng_resize.ipp @@ -7,46 +7,9 @@ namespace libcloudphxx void particles_t::impl::hskpng_resize_npart() { if(n_part > opts_init.n_sd_max) throw std::runtime_error(detail::formatter() << "n_sd_max (" << opts_init.n_sd_max << ") < n_part (" << n_part << ")"); - { - thrust_device::vector *vec[] = {&tmp_device_real_part}; - for(int i=0; i<4; ++i) - { - vec[i]->resize(n_part); - } - } - { - thrust_device::vector *vec[] = {&ijk, &sorted_id, &sorted_ijk}; - for(int i=0; i<3; ++i) - { - vec[i]->resize(n_part); - } - } - n.resize(n_part); - tmp_device_n_part.resize(n_part); - tmp_device_size_part.resize(n_part); - if (opts_init.nx != 0) i.resize(n_part); - if (opts_init.ny != 0) j.resize(n_part); - if (opts_init.nz != 0) k.resize(n_part); - - if(opts_init.chem_switch || allow_sstp_cond || n_dims >= 2) - { - tmp_device_real_part1.resize(n_part); - } - if((allow_sstp_cond && opts_init.exact_sstp_cond) || n_dims==3 || opts_init.turb_cond_switch) - { - tmp_device_real_part2.resize(n_part); - } - - if(allow_sstp_cond && opts_init.exact_sstp_cond) - { - tmp_device_real_part3.resize(n_part); - tmp_device_real_part4.resize(n_part); - if(const_p) - { - tmp_device_real_part5.resize(n_part); - } - } + for(auto &vec: distmem_n_vctrs) + vec->resize(n_part); for(auto &pair: distmem_real_vctrs) { @@ -55,6 +18,15 @@ namespace libcloudphxx else pair.first->resize(n_part, pair.second); } + + for(auto &vec: resize_real_vctrs) + vec->resize(n_part); + + for(auto &vec: resize_size_vctrs) + vec->resize(n_part); + + // its unsigned int vector, probably only one we will use, hence no resize_t_vctrs helper used + tmp_device_n_part.resize(n_part); } }; }; diff --git a/src/impl/particles_impl_mpi_exchange.ipp b/src/impl/particles_impl_mpi_exchange.ipp index 248463d6a..274cc5967 100644 --- a/src/impl/particles_impl_mpi_exchange.ipp +++ b/src/impl/particles_impl_mpi_exchange.ipp @@ -74,7 +74,7 @@ namespace libcloudphxx // start async copy of n buffer to the left MPI_CHECK(MPI_Isend( out_n_bfr.data().get(), // raw pointer to the buffer - lft_count, // no of values to send + lft_count * distmem_n_vctrs.size(), // no of values to send detail::get_mpi_type(), // type lft_rank, // dest comm detail::tag_n_lft, // message tag @@ -218,7 +218,7 @@ namespace libcloudphxx // start async copy of n buffer to the right MPI_CHECK(MPI_Isend( out_n_bfr.data().get(), // raw pointer to the buffer - rgt_count, // no of values to send + rgt_count * distmem_n_vctrs.size(), // no of values to send detail::get_mpi_type(), // type rgt_rank, // dest comm detail::tag_n_rgt, // message tag