Bunch of changes to support MLMG performance analysis and tuning (AMReX-Codes/amrex#388)

Weiqun/perf

some minor changes

diff --git a/Src/Base/AMReX.cpp b/Src/Base/AMReX.cpp
index d42c0cca2d4..ec0a2f92e94 100644
--- a/Src/Base/AMReX.cpp
+++ b/Src/Base/AMReX.cpp
@@ -20,6 +20,8 @@

 #include <AMReX_Gpu.H>

+#include <AMReX_Machine.H>
+
 #ifdef AMREX_USE_EB
 #include <AMReX_EB2.H>
 #endif
@@ -533,6 +535,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse,
 #endif
     BL_PROFILE_INITPARAMS();
 #endif
+    machine::Initialize();

     if (double(std::numeric_limits<long>::max()) < 9.e18)
     {
diff --git a/Src/Base/AMReX_DistributionMapping.H b/Src/Base/AMReX_DistributionMapping.H
index 7e399dab103..b7c7f38ed15 100644
--- a/Src/Base/AMReX_DistributionMapping.H
+++ b/Src/Base/AMReX_DistributionMapping.H
@@ -156,7 +156,9 @@ class DistributionMapping
     static DistributionMapping makeRoundRobin (const MultiFab& weight);
     static DistributionMapping makeSFC        (const MultiFab& weight, bool sort=true);

-    static std::vector<std::vector<int> > makeSFC (const BoxArray& ba);
+    // if use_box_vol is true, weight boxes by their volume in Distribute
+    // otherwise, all boxes will be treated with equal weight
+    static std::vector<std::vector<int> > makeSFC (const BoxArray& ba, bool use_box_vol=true);

 private:

diff --git a/Src/Base/AMReX_DistributionMapping.cpp b/Src/Base/AMReX_DistributionMapping.cpp
index 009f265a004..29a2fe32e55 100644
--- a/Src/Base/AMReX_DistributionMapping.cpp
+++ b/Src/Base/AMReX_DistributionMapping.cpp
@@ -24,6 +24,10 @@
 #include <cstring>
 #include <iomanip>

+namespace {
+int flag_verbose_mapper;
+}
+
 namespace amrex {

     bool initialized = false;
@@ -111,14 +115,16 @@ DistributionMapping::Initialize ()
     sfc_threshold    = 0;
     max_efficiency   = 0.9;
     node_size        = 0;
+    flag_verbose_mapper = 0;

     ParmParse pp("DistributionMapping");

-    pp.query("v"      ,          verbose);
-    pp.query("verbose",          verbose);
-    pp.query("efficiency",       max_efficiency);
-    pp.query("sfc_threshold",    sfc_threshold);
-    pp.query("node_size",        node_size);
+    pp.query("v"      ,             verbose);
+    pp.query("verbose",             verbose);
+    pp.query("efficiency",          max_efficiency);
+    pp.query("sfc_threshold",       sfc_threshold);
+    pp.query("node_size",           node_size);
+    pp.query("verbose_mapper",      flag_verbose_mapper);

     std::string theStrategy;

@@ -192,21 +198,10 @@ DistributionMapping::LeastUsedCPUs (int         nprocs,
     BL_PROFILE("DistributionMapping::LeastUsedCPUs()");

     AMREX_ASSERT(nprocs <= ParallelContext::NProcsSub());
-    Vector<long> bytes(ParallelContext::NProcsSub());

+    Vector<long> bytes(ParallelContext::NProcsSub());
     long thisbyte = amrex::TotalBytesAllocatedInFabs()/1024;
-
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::BeforeCall(),
-                    BLProfiler::NoTag());
-    MPI_Allgather(&thisbyte,
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  bytes.dataPtr(),
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  ParallelContext::CommunicatorSub());
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::AfterCall(),
-                    BLProfiler::NoTag());
+    ParallelAllGather::AllGather(thisbyte, bytes.dataPtr(), ParallelContext::CommunicatorSub());

     std::vector<LIpair> LIpairV;

@@ -225,6 +220,13 @@ DistributionMapping::LeastUsedCPUs (int         nprocs,
     {
         result[i] = LIpairV[i].second;
     }
+
+    if (flag_verbose_mapper) {
+        Print() << "LeastUsedCPUs:" << std::endl;
+        for (const auto &p : LIpairV) {
+            Print() << "  Rank " << p.second << " contains " << p.first << std::endl;
+        }
+    }
 #else
     for (int i(0); i < nprocs; ++i)
     {
@@ -244,21 +246,9 @@ DistributionMapping::LeastUsedTeams (Vector<int>        & rteam,

     AMREX_ALWAYS_ASSERT(ParallelContext::CommunicatorSub() == ParallelDescriptor::Communicator());

-    Vector<long> bytes(ParallelDescriptor::NProcs());
-
+    Vector<long> bytes(ParallelContext::NProcsSub());
     long thisbyte = amrex::TotalBytesAllocatedInFabs()/1024;
-
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::BeforeCall(),
-                    BLProfiler::NoTag());
-    MPI_Allgather(&thisbyte,
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  bytes.dataPtr(),
-                  1,
-                  ParallelDescriptor::Mpi_typemap<long>::type(),
-                  ParallelDescriptor::Communicator());
-    BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(long), BLProfiler::AfterCall(),
-                    BLProfiler::NoTag());
+    ParallelAllGather::AllGather(thisbyte, bytes.dataPtr(), ParallelContext::CommunicatorSub());

     std::vector<LIpair> LIpairV;
     std::vector<LIpair> LIworker;
@@ -397,6 +387,10 @@ DistributionMapping::RoundRobinDoIt (int                  nboxes,
                                      int                 /* nprocs */,
                                      std::vector<LIpair>* LIpairV)
 {
+    if (flag_verbose_mapper) {
+        Print() << "DM: RoundRobinDoIt called..." << std::endl;
+    }
+
     int nprocs = ParallelContext::NProcsSub();

     // If team is not use, we are going to treat it as a special case in which
@@ -435,6 +429,10 @@ DistributionMapping::RoundRobinDoIt (int                  nboxes,
 	    int wid = (w[tid]++) % nworkers;
 	    int rank = tid*nworkers + wrkerord[tid][wid];
 	    m_ref->m_pmap[(*LIpairV)[i].second] = ParallelContext::local_to_global_rank(rank);
+            if (flag_verbose_mapper) {
+                Print() << "  Mapping box " << (*LIpairV)[i].second << " of size "
+                        << (*LIpairV)[i].first << " to rank " << rank << std::endl;
+            }
 	}
     }
     else
@@ -445,6 +443,9 @@ DistributionMapping::RoundRobinDoIt (int                  nboxes,
 	    int wid = (w[tid]++) % nworkers;
 	    int rank = tid*nworkers + wrkerord[tid][wid];
 	    m_ref->m_pmap[i] = ParallelContext::local_to_global_rank(rank);
+            if (flag_verbose_mapper) {
+                Print() << "  Mapping box " << i << " to rank " << rank << std::endl;
+            }
 	}
     }
 }
@@ -721,6 +722,10 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,
                                    bool                     do_full_knapsack,
 				   int                      nmax)
 {
+    if (flag_verbose_mapper) {
+        Print() << "DM: KnapSackDoIt called..." << std::endl;
+    }
+
     BL_PROFILE("DistributionMapping::KnapSackDoIt()");

     int nprocs = ParallelContext::NProcsSub();
@@ -741,6 +746,15 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,

     knapsack(wgts,nteams,vec,efficiency,do_full_knapsack,nmax);

+    if (flag_verbose_mapper) {
+        for (int i = 0; i < vec.size(); ++i) {
+            Print() << "  Bucket " << i << " contains boxes:" << std::endl;
+            for (int j = 0; j < vec[i].size(); ++j) {
+                Print() << "    " << vec[i][j] << std::endl;
+            }
+        }
+    }
+
     BL_ASSERT(static_cast<int>(vec.size()) == nteams);

     std::vector<LIpair> LIpairV;
@@ -761,6 +775,12 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,

     Sort(LIpairV, true);

+    if (flag_verbose_mapper) {
+        for (const auto &p : LIpairV) {
+            Print() << "  Bucket " << p.second << " total weight: " << p.first << std::endl;
+        }
+    }
+
     Vector<int> ord;
     Vector<Vector<int> > wrkerord;

@@ -783,6 +803,10 @@ DistributionMapping::KnapSackDoIt (const std::vector<long>& wgts,
 	const std::vector<int>& vi = vec[idx];
 	const int N = vi.size();

+        if (flag_verbose_mapper) {
+            Print() << "  Mapping bucket " << idx << " to rank " << tid << std::endl;
+        }
+
 	if (nteams == nprocs) {
 	    for (int j = 0; j < N; ++j)
 	    {
@@ -917,6 +941,21 @@ Distribute (const std::vector<SFCToken>&     tokens,
             std::vector< std::vector<int> >& v)

 {
+    BL_PROFILE("DistributionMapping::Distribute()");
+
+    if (flag_verbose_mapper) {
+        Print() << "Distribute:" << std::endl;
+        Print() << "  volpercpu: " << volpercpu << std::endl;
+        Print() << "  Sorted SFC Tokens:" << std::endl;
+        int idx = 0;
+        for (const auto &t : tokens) {
+            Print() << "    " << idx++ << ": "
+                    << t.m_box << ": "
+                    << t.m_idx << ": "
+                    << t.m_vol << std::endl;
+        }
+    }
+
     BL_ASSERT(static_cast<int>(v.size()) == nprocs);

     int  K        = 0;
@@ -949,6 +988,26 @@ Distribute (const std::vector<SFCToken>&     tokens,
         }
     }

+    if (flag_verbose_mapper) {
+        Print() << "Distributed SFC Tokens:" << std::endl;
+        int idx = 0;
+        for (int i = 0; i < nprocs; ++i) {
+            Print() << "  Rank/Team " << i << ":" << std::endl;
+            Real rank_vol = 0;
+            for (const auto &box : v[i]) {
+                const auto &t = tokens[idx];
+                BL_ASSERT(box == t.m_box);
+                Print() << "    " << idx << ": "
+                        << t.m_box << ": "
+                        << t.m_idx << ": "
+                        << t.m_vol << std::endl;
+                rank_vol += t.m_vol;
+                idx++;
+            }
+            Print() << "    Total Rank Vol: " << rank_vol << std::endl;
+        }
+    }
+
 #ifdef AMREX_DEBUG
     int cnt = 0;
     for (int i = 0; i < nprocs; ++i) {
@@ -964,6 +1023,10 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
                                           int                   /*   nprocs */,
                                           bool                     sort)
 {
+    if (flag_verbose_mapper) {
+        Print() << "DM: SFCProcessorMapDoIt called..." << std::endl;
+    }
+
     BL_PROFILE("DistributionMapping::SFCProcessorMapDoIt()");

     int nprocs = ParallelContext::NProcsSub();
@@ -984,6 +1047,11 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
     }
 #endif

+    if (flag_verbose_mapper) {
+        Print() << "  (nprocs, nteams, nworkers) = ("
+                << nprocs << ", " << nteams << ", " << nworkers << ")\n";
+    }
+
     std::vector<SFCToken> tokens;

     const int N = boxes.size();
@@ -1048,6 +1116,12 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,

     if (sort) Sort(LIpairV, true);

+    if (flag_verbose_mapper) {
+        for (const auto &p : LIpairV) {
+            Print() << "  Bucket " << p.second << " contains " << p.first << std::endl;
+        }
+    }
+
     // LIpairV has a size of nteams and LIpairV[] is pair whose first is weight
     // and second is an index into vec.  LIpairV is sorted by weight such that
     // LIpairV is the heaviest.
@@ -1086,6 +1160,10 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
         const std::vector<int>& vi = vec[ivec];   // this vector contains boxes assigned to this team
 	const int Nbx = vi.size();                // # of boxes assigned to this team

+        if (flag_verbose_mapper) {
+            Print() << "Mapping bucket " << LIpairV[i].second << " to rank " << ord[i] << std::endl;
+        }
+
 	if (nteams == nprocs) { // In this case, team id is process id.
 	    for (int j = 0; j < Nbx; ++j)
 	    {
@@ -1136,7 +1214,7 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray&          boxes,
 	}
     }

-    if (verbose && ParallelDescriptor::IOProcessor())
+    if (verbose)
     {
         Real sum_wgt = 0, max_wgt = 0;
         for (int i = 0; i < nteams; ++i)
@@ -1398,7 +1476,7 @@ DistributionMapping::makeSFC (const MultiFab& weight, bool sort)
 }

 std::vector<std::vector<int> >
-DistributionMapping::makeSFC (const BoxArray& ba)
+DistributionMapping::makeSFC (const BoxArray& ba, bool use_box_vol)
 {
     std::vector<SFCToken> tokens;

@@ -1408,10 +1486,13 @@ DistributionMapping::makeSFC (const BoxArray& ba)

     int maxijk = 0;

+    Real vol_sum = 0;
     for (int i = 0; i < N; ++i)
     {
 	const Box& bx = ba[i];
-        tokens.push_back(SFCToken(i,bx.smallEnd(),1.0));
+        const auto & bx_vol = (use_box_vol ? bx.volume() : 1);
+        tokens.push_back(SFCToken(i,bx.smallEnd(),bx_vol));
+        vol_sum += bx_vol;

         const SFCToken& token = tokens.back();

@@ -1433,7 +1514,8 @@ DistributionMapping::makeSFC (const BoxArray& ba)
     std::sort(tokens.begin(), tokens.end(), SFCToken::Compare());

     const int nprocs = ParallelContext::NProcsSub();
-    Real volper = static_cast<Real>(N)/static_cast<Real>(nprocs);
+    Real volper;
+    volper = vol_sum / nprocs;

     std::vector< std::vector<int> > r(nprocs);
     Distribute(tokens, nprocs, volper, r);
diff --git a/Src/Base/AMReX_Machine.H b/Src/Base/AMReX_Machine.H
new file mode 100644
index 00000000000..b20817128fd
--- /dev/null
+++ b/Src/Base/AMReX_Machine.H
@@ -0,0 +1,18 @@
+#ifndef AMREX_MACHINE_H
+#define AMREX_MACHINE_H
+
+#include <AMReX_Vector.H>
+
+namespace amrex {
+namespace machine {
+
+void Initialize (); // called in amrex::Initialize()
+void Finalize ();
+
+// find the best topologically close neighborhood of ranks
+// returns a vector of global or local rank IDs based on flag_local_ranks
+Vector<int> find_best_nbh (int rank_n, bool flag_local_ranks = false);
+
+}}
+
+#endif
diff --git a/Src/Base/AMReX_Machine.cpp b/Src/Base/AMReX_Machine.cpp
new file mode 100644
index 00000000000..f2dfac9edc6
--- /dev/null
+++ b/Src/Base/AMReX_Machine.cpp
@@ -0,0 +1,586 @@
+#include <string>
+#include <vector>
+#include <array>
+#include <fstream>
+#include <cassert>
+#include <sstream>
+#include <map>
+#include <unordered_map>
+
+#include <AMReX_Print.H>
+#include <AMReX_ParmParse.H>
+#include <AMReX_ParallelReduce.H>
+#include <AMReX_Utility.H>
+#include <AMReX_Machine.H>
+
+using namespace amrex;
+
+namespace {
+
+struct DoubleInt {
+    double d;
+    int i;
+};
+
+using Coord = Array<int, 4>;
+
+// returns coordinate in an index space with no switches
+// for dragonfly network
+Coord read_df_node_coord (const std::string & name)
+{
+    int cabx, caby, cab_chas, slot, node;
+    {
+        std::ifstream ifs {"/proc/cray_xt/cname"};
+        if (!ifs) {
+            // not on a cray
+            return Coord {0,0,0,0}; // initializer_list
+        }
+        char t0, t1, t2, t3, t4;
+        ifs >> t0 >> cabx >> t1 >> caby >> t2 >> cab_chas >> t3 >> slot >> t4 >> node;
+        assert(t0 == 'c' && t1 == '-' && t2 == 'c' && t3 == 's' && t4 == 'n');
+    }
+
+    int group = 0;
+    if (name == "edison") {
+        group = cabx / 2 + caby * 4; // 2 cabinets per group, 4 groups per row
+        if (group > 12) { group--; } // nominal "group 12" is missing
+    } else if (name == "cori") {
+        group = cabx / 2 + caby * 6; // 2 cabinets per group, 6 groups per row
+    } else {
+        Print() << "Could not determine group!";
+        std::abort();
+    }
+    int chas = cab_chas + 3*(cabx & 1); // 2 cabinets per group (6 chassis per group)
+
+    return Coord {node, slot, chas, group};
+}
+
+std::string get_mpi_processor_name ()
+{
+    std::string result;
+#if BL_USE_MPI
+    int len;
+    char name[MPI_MAX_PROCESSOR_NAME];
+    MPI_Get_processor_name(name, &len);
+    result = std::string(name);
+#endif
+    return result;
+}
+
+// assumes groups are in 4x16x6 configuration
+int df_coord_to_id (const Coord & c)
+{
+    return c[0] + 4 * (c[1] + 16 * (c[2] + 6 * c[3]));
+}
+
+// assumes groups are in 4x16x6 configuration
+Coord df_id_to_coord (int id)
+{
+    int node = id % 4;  id /= 4;
+    int slot = id % 16; id /= 16;
+    int chas = id % 6;  id /= 6;
+    int group = id;
+    return Coord {node, slot, chas, group};
+}
+
+template <class T, size_t N>
+std::string to_str(const Array<T, N> & a)
+{
+    std::ostringstream oss;
+    oss << "(";
+    bool first = true;
+    for (int i = 0; i < N; ++i) {
+        if (!first) oss << ",";
+        oss << a[i];
+        first = false;
+    }
+    oss << ")";
+    return oss.str();
+}
+
+template <class T>
+std::string to_str(const Vector<T> & v)
+{
+    std::ostringstream oss;
+    oss << "(";
+    bool first = true;
+    for (int i = 0; i < v.size(); ++i) {
+        if (!first) oss << ",";
+        oss << v[i];
+        first = false;
+    }
+    oss << ")";
+    return oss.str();
+}
+
+Vector<int> get_subgroup_ranks ()
+{
+    int rank_n = ParallelContext::NProcsSub();
+    Vector<int> lranks(rank_n);
+    for (int i = 0; i < rank_n; ++i) {
+        lranks[i] = i;
+    }
+
+    Vector<int> granks(rank_n);
+    ParallelContext::local_to_global_rank(granks.data(), lranks.data(), rank_n);
+    return granks;
+}
+
+int pair_n (int x) {
+    return x*(x-1)/2;
+}
+
+int df_dist (const Coord & a, const Coord & b)
+{
+    if (a[3] != b[3]) {
+        // large penalty for traversing across groups
+        return 20;
+    } else {
+        // same group
+        int slot_diff = (a[1] != b[1] ? 1 : 0);
+        int chas_diff = (a[2] != b[2] ? 1 : 0);
+        if (slot_diff + chas_diff == 0 && a[0] == b[0]) {
+            // same node
+            return 0;
+        } else {
+            // add 2 for first and last node-to-switch hops
+            return 2 + slot_diff + chas_diff;
+        }
+    }
+}
+
+Coord id_to_coord (int id)
+{
+    // TODO: implement support for other types of networks
+    return df_id_to_coord(id);
+}
+
+int dist (const Coord & a, const Coord & b)
+{
+    // TODO: implement support for other types of networks
+    return df_dist(a, b);
+}
+
+struct Candidate
+{
+    int id;
+    Coord coord;
+    // how many ranks on this node
+    int rank_n = 0;
+    // sum of pairwise rank distances from the candidate node to already chosen nodes
+    int sum_dist = 0;
+
+    Candidate () = default;
+    Candidate (int i) : id(i), coord(id_to_coord(id)) {}
+};
+
+class NeighborhoodCache
+{
+  public:
+    void add (uint64_t key, Vector<int> val) {
+        AMREX_ASSERT(cache.count(key) == 0);
+        cache[key] = std::move(val);
+    }
+    bool get (uint64_t key, Vector<int> & val) {
+        bool result = cache.count(key) > 0;
+        if (result) {
+            val = cache.at(key);
+        }
+        return result;
+    }
+
+    // result is dependent on both the current set of ranks
+    // and the size of the neighborhood desired
+    uint64_t hash (const Vector<int> & cur_ranks, int nbh_rank_n) {
+        auto result = hash_vector(cur_ranks);
+        hash_combine(result, nbh_rank_n);
+        return result;
+    }
+
+  private:
+    std::unordered_map<uint64_t, Vector<int>> cache;
+};
+
+class Machine
+{
+  public:
+    Machine () {
+        get_params();
+        get_machine_envs();
+        node_ids = get_node_ids();
+    }
+
+    // find a compact neighborhood of size rank_n in the current ParallelContext subgroup
+    Vector<int> find_best_nbh (int nbh_rank_n, bool flag_local_ranks)
+    {
+#ifdef BL_USE_MPI
+        BL_PROFILE("Machine::find_best_nbh()");
+
+        auto sg_g_ranks = get_subgroup_ranks();
+        auto sg_rank_n = sg_g_ranks.size();
+        if (flag_verbose) {
+            Print() << "Machine::find_best_nbh(): called for " << nbh_rank_n
+                    << " of " << sg_rank_n << " ranks" << std::endl;
+        }
+
+        Vector<int> result;
+        auto key = nbh_cache.hash(sg_g_ranks, nbh_rank_n);
+        if (nbh_cache.get(key, result)) {
+            if (flag_verbose) {
+                Print() << "Machine::find_best_nbh(): found neighborhood in cache" << std::endl;
+            }
+        } else {
+            // get node IDs of current subgroup
+            Vector<int> sg_node_ids(sg_rank_n);
+            std::unordered_map<int, std::vector<int>> node_ranks;
+            for (int i = 0; i < sg_rank_n; ++i) {
+                AMREX_ASSERT(sg_g_ranks[i] >= 0 && sg_g_ranks[i] < node_ids.size());
+                sg_node_ids[i] = node_ids[sg_g_ranks[i]];
+                if (flag_local_ranks) {
+                    node_ranks[sg_node_ids[i]].push_back(i);
+                } else {
+                    node_ranks[sg_node_ids[i]].push_back(sg_g_ranks[i]);
+                }
+            }
+
+            if (flag_very_verbose) {
+                Print() << "SubRank: GloRank: Node ID: Node Coord:" << std::endl;
+                for (int i = 0; i < sg_rank_n; ++i) {
+                    Print() << "  " << i << ": " << sg_g_ranks[i] << ": " << sg_node_ids[i]
+                            << ": " << to_str(id_to_coord(sg_node_ids[i])) << std::endl;
+                }
+            }
+
+            Vector<int> local_nbh;
+            double score;
+            auto rank_me = ParallelContext::MyProcSub();
+            tie(local_nbh, score) = search_local_nbh(rank_me, sg_node_ids, nbh_rank_n);
+
+            if (flag_verbose) {
+                Vector<int> base_nbh;
+                double base_score;
+                tie(base_nbh, base_score) = baseline_score(sg_node_ids, nbh_rank_n);
+
+                Print() << "Baseline neighborhood: " << to_str(base_nbh) << ", score = " << base_score << std::endl;
+                Print() << "Rank 0's neighborhood: " << to_str(local_nbh) << ", score = " << score << std::endl;
+            }
+
+            // determine the best neighborhood among ranks
+            DoubleInt my_score_with_id {score, rank_me}, min_score_with_id;
+            MPI_Allreduce(&my_score_with_id, &min_score_with_id, 1, MPI_DOUBLE_INT, MPI_MINLOC, ParallelContext::CommunicatorSub());
+            double winner_score = min_score_with_id.d;
+            int    winner_rank  = min_score_with_id.i;
+
+            // broadcast the best hood from winner rank to everyone
+            int local_nbh_size = local_nbh.size();
+            MPI_Bcast(&local_nbh_size, 1, MPI_INT, winner_rank, ParallelContext::CommunicatorSub());
+            local_nbh.resize(local_nbh_size);
+            MPI_Bcast(local_nbh.data(), local_nbh.size(), MPI_INT, winner_rank, ParallelContext::CommunicatorSub());
+
+            std::sort(local_nbh.begin(), local_nbh.end());
+            if (flag_verbose) {
+                Print() << "Winning neighborhood: " << winner_rank << ": " << to_str(local_nbh)
+                        << ", score = " << winner_score << std::endl;
+            }
+
+            result.reserve(nbh_rank_n);
+            for (int i = 0; i < local_nbh.size(); ++i) {
+                for (auto rank : node_ranks.at(local_nbh[i])) {
+                    if (result.size() < nbh_rank_n) {
+                        result.push_back(rank);
+                    }
+                }
+            }
+            nbh_cache.add(key, result);
+        }
+
+        if (flag_very_verbose) {
+            Print() << "Ranks in neighborhood: " << to_str(result) << std::endl;
+        }
+
+        return result;
+#else
+        return Vector<int>(nbh_rank_n, 0);
+#endif
+    }
+
+  private:
+
+    std::string hostname;
+    std::string nersc_host;
+    std::string partition;
+    std::string node_list;
+    std::string topo_addr;
+
+    int flag_verbose = 0;
+    int flag_very_verbose = 0;
+    bool flag_nersc_df;
+    int my_node_id;
+    Vector<int> node_ids;
+
+    NeighborhoodCache nbh_cache;
+
+    void get_params ()
+    {
+        ParmParse pp("machine");
+        pp.query("verbose", flag_verbose);
+        pp.query("very_verbose", flag_very_verbose);
+    }
+
+    std::string get_env_str (std::string env_key)
+    {
+        std::string result;
+        auto val_c_str = std::getenv(env_key.c_str());
+        if (val_c_str) {
+            result = std::string(val_c_str);
+        }
+        return result;
+    }
+
+    void get_machine_envs ()
+    {
+        hostname   = get_env_str("HOSTNAME");
+        nersc_host = get_env_str("NERSC_HOST");
+        flag_nersc_df = (nersc_host == "edison" ||
+                         nersc_host == "cori" ||
+                         nersc_host == "saul");
+
+        if (flag_nersc_df) {
+            partition  = get_env_str("SLURM_JOB_PARTITION");
+            node_list  = get_env_str("SLURM_NODELIST");
+            topo_addr  = get_env_str("SLURM_TOPOLOGY_ADDR");
+
+            if (flag_verbose) {
+                Print() << "HOSTNAME = " << hostname << std::endl;
+                Print() << "NERSC_HOST = " << nersc_host << std::endl;
+                Print() << "SLURM_JOB_PARTITION = " << partition << std::endl;
+                Print() << "SLURM_NODELIST = " << node_list << std::endl;
+                Print() << "SLURM_TOPOLOGY_ADDR = " << topo_addr << std::endl;
+            }
+        }
+    }
+
+    // get this rank's machine node ID
+    int get_my_node_id ()
+    {
+        int result = -1;
+        if (flag_nersc_df) {
+            std::string tag = "nid";
+            auto pos = topo_addr.find(tag);
+            if (pos != std::string::npos) {
+                result = stoi(topo_addr.substr(pos + tag.size())); // assumes format ".*nid(\d+)"
+                if (flag_verbose) {
+                    Print() << "Got node ID from SLURM_TOPOLOGY_ADDR: " << result << std::endl;
+                }
+#if BL_USE_MPI
+            } else {
+                auto mpi_proc_name = get_mpi_processor_name();
+                Print() << "MPI_Get_processor_name: " << mpi_proc_name << std::endl;
+                pos = mpi_proc_name.find(tag);
+                if (pos != std::string::npos) {
+                    result = stoi(mpi_proc_name.substr(pos + tag.size())); // assumes format ".*nid(\d+)"
+                    if (flag_verbose) {
+                        Print() << "Got node ID from MPI_Get_processor_name(): " << result << std::endl;
+                    }
+                }
+#endif
+            }
+
+            // check result
+            AMREX_ALWAYS_ASSERT(result != -1);
+#ifndef NDEBUG
+            auto coord = read_df_node_coord(nersc_host);
+            int id_from_coord = df_coord_to_id(coord);
+            AMREX_ASSERT(id_from_coord == result);
+#endif
+        } else {
+            result = 0;
+        }
+
+        return result;
+    }
+
+    // get all node IDs in this job, indexed by job rank
+    // this is collective over ALL ranks in the job
+    Vector<int> get_node_ids ()
+    {
+        int node_id = -1;
+        Vector<int> ids(ParallelDescriptor::NProcs(), 0);
+#ifdef BL_USE_MPI
+        node_id = get_my_node_id();
+        ParallelAllGather::AllGather(node_id, ids.data(), ParallelContext::CommunicatorAll());
+#endif
+        if (flag_verbose) {
+            std::map<int, Vector<int>> node_ranks;
+            for (int i = 0; i < ids.size(); ++i) {
+                node_ranks[ids[i]].push_back(i);
+            }
+            Print() << "Node ID: Node Coord: Ranks:" << std::endl;
+            for (const auto & p : node_ranks) {
+                Print() << "  " << p.first << ": " << to_str(id_to_coord(p.first))
+                        << ": " << to_str(p.second) << std::endl;
+            }
+        }
+        return ids;
+    }
+
+    // do a local search starting at current node
+    std::pair<Vector<int>, double>
+    baseline_score(const Vector<int> & sg_node_ids, int nbh_rank_n)
+    {
+        AMREX_ASSERT(sg_node_ids.size() > 0 && nbh_rank_n > 0 &&
+                     nbh_rank_n <= sg_node_ids.size());
+
+        // construct map of node candidates to select
+        std::map<int, Candidate> cand_map;
+        for (int i = 0; i < nbh_rank_n; ++i) {
+            auto node_id = sg_node_ids[i];
+            if (cand_map.count(node_id) == 0) {
+                cand_map[node_id] = Candidate(node_id);
+            }
+            cand_map.at(node_id).rank_n++;
+        }
+
+        Vector<int> result(cand_map.size());
+        Vector<Candidate> candidates(cand_map.size());
+        int idx = 0;
+        for (auto & p : cand_map) {
+            result[idx] = p.second.id;
+            candidates[idx++] = std::move(p.second);
+        }
+
+        int sum_dist = 0;
+        for (int j = 1; j < candidates.size(); ++j) {
+            const auto & b = candidates[j];
+            for (int i = 0; i < j; ++i) {
+                const auto & a = candidates[i];
+                auto pair_dist = dist(a.coord, b.coord);
+                // multiply distance by number of rank pairs across the two nodes
+                sum_dist += pair_dist * (a.rank_n * b.rank_n);
+                if (flag_very_verbose) {
+                    Print() << "    Distance from " << a.id
+                            << " to " << b.id
+                            << ": " << pair_dist << std::endl;
+                }
+            }
+        }
+        double score = (nbh_rank_n > 1) ? (static_cast<double>(sum_dist) / pair_n(nbh_rank_n)) : 0;
+        return std::make_pair(std::move(result), score);
+    }
+
+    // do a local search starting at current node
+    std::pair<Vector<int>, double>
+    search_local_nbh(int rank_me, const Vector<int> & sg_node_ids, int nbh_rank_n)
+    {
+        BL_PROFILE("Machine::search_local_nbh()");
+
+        Print() << "Machine::search_local_nbh() called ..." << std::endl;
+
+        Vector<int> result;
+
+        // construct map of node candidates to select
+        std::map<int, Candidate> candidates;
+        for (auto node_id : sg_node_ids) {
+            if (candidates.count(node_id) == 0) {
+                candidates[node_id] = Candidate(node_id);
+            }
+            candidates.at(node_id).rank_n++;
+        }
+
+        if (flag_very_verbose) {
+            Print() << "  Candidates:" << std::endl;
+            for (const auto & p : candidates) {
+                const auto & cand = p.second;
+                Print() << "    " << cand.id << " : " << to_str(cand.coord)
+                        << ": " << cand.rank_n << " ranks" << std::endl;
+            }
+        }
+
+        AMREX_ASSERT(rank_me >= 0 && rank_me < sg_node_ids.size());
+        Candidate cur_node = std::move(candidates.at(sg_node_ids[rank_me]));
+        candidates.erase(cur_node.id);
+
+        // add source_node
+        result.push_back(cur_node.id);
+        int total_rank_n = cur_node.rank_n;
+        int total_pairs_dist = 0;
+        if (flag_verbose) {
+            Print() << "  Added " << cur_node.id
+                    << ": " << to_str(cur_node.coord)
+                    << ", ranks: " << cur_node.rank_n
+                    << ", total ranks: " << total_rank_n
+                    << ", avg dist: " << 0 << std::endl;
+        }
+        if (total_rank_n >= nbh_rank_n) {
+            return {std::move(result), 0};
+        }
+
+        double min_avg_dist;
+        while (total_rank_n < nbh_rank_n)
+        {
+            min_avg_dist = std::numeric_limits<double>::max();
+            Candidate * next_node = nullptr;
+            // update candidates with their pairwise rank distances to cur_node
+            for (auto & p : candidates) {
+                Candidate & cand_node = p.second;
+                auto cand_dist = dist(cand_node.coord, cur_node.coord);
+                // multiply distance by number of rank pairs across the two nodes
+                cand_node.sum_dist += cand_dist * (cand_node.rank_n * cur_node.rank_n);
+                double avg_dist = static_cast<double>(cand_node.sum_dist + total_pairs_dist) /
+                                  pair_n(cand_node.rank_n + total_rank_n);
+                if (flag_very_verbose) {
+                    Print() << "    Distance from " << cand_node.id
+                            << " to " << cur_node.id
+                            << ": " << cand_dist
+                            << ", candidate avg: " << avg_dist << std::endl;
+                }
+                // keep track of what should be the next node to add
+                if (avg_dist < min_avg_dist) {
+                    next_node = &cand_node;
+                    min_avg_dist = avg_dist;
+                }
+            }
+            cur_node = std::move(*next_node);
+            next_node = nullptr;
+            candidates.erase(cur_node.id);
+
+            // add cur_node to result
+            result.push_back(cur_node.id);
+            total_rank_n += cur_node.rank_n;
+            total_pairs_dist += cur_node.sum_dist;
+
+            if (flag_verbose) {
+                Print() << "  Added " << cur_node.id
+                        << ": " << to_str(cur_node.coord)
+                        << ", ranks: " << cur_node.rank_n
+                        << ", total ranks: " << total_rank_n
+                        << ", avg dist: " << min_avg_dist << std::endl;
+            }
+        }
+
+        return std::make_pair(std::move(result), min_avg_dist);
+    }
+};
+
+std::unique_ptr<Machine> the_machine;
+
+}
+
+namespace amrex {
+namespace machine {
+
+void Initialize () {
+    the_machine.reset(new Machine());
+    amrex::ExecOnFinalize(machine::Finalize);
+}
+
+void Finalize () {
+    the_machine.reset();
+}
+
+Vector<int> find_best_nbh (int rank_n, bool flag_local_ranks) {
+    AMREX_ASSERT(the_machine);
+    return the_machine->find_best_nbh(rank_n, flag_local_ranks);
+}
+
+}}
diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp
index 295a1bc0b2a..a11d65457f4 100644
--- a/Src/Base/AMReX_MultiFabUtil.cpp
+++ b/Src/Base/AMReX_MultiFabUtil.cpp
@@ -238,6 +238,7 @@ namespace amrex
 		       const Geometry& fgeom, const Geometry& cgeom,
                        int scomp, int ncomp, const IntVect& ratio)
     {
+        BL_PROFILE("amrex::average_down_w_geom");

         if (S_fine.is_nodal() || S_crse.is_nodal())
         {
@@ -337,6 +338,7 @@ namespace amrex
     void average_down (const MultiFab& S_fine, MultiFab& S_crse,
                        int scomp, int ncomp, const IntVect& ratio)
     {
+        BL_PROFILE("amrex::average_down");
         AMREX_ASSERT(S_crse.nComp() == S_fine.nComp());
         AMREX_ASSERT((S_crse.is_cell_centered() && S_fine.is_cell_centered()) ||
                      (S_crse.is_nodal()         && S_fine.is_nodal()));
diff --git a/Src/Base/AMReX_ParallelContext.H b/Src/Base/AMReX_ParallelContext.H
index c71fc0d82b6..c9f7f59de23 100644
--- a/Src/Base/AMReX_ParallelContext.H
+++ b/Src/Base/AMReX_ParallelContext.H
@@ -58,6 +58,8 @@ inline int NProcsAll () { return frames[0].NProcs(); }
 inline int MyProcAll () { return frames[0].MyProc(); }
 // IO rank in world communicator
 inline int IOProcessorNumberAll () { return frames[0].IOProc(); }
+// Am IO processor for world communicator?
+inline bool IOProcessorAll () { return MyProcAll() == IOProcessorNumberAll(); }
 // Pointer to ofstream
 inline std::ofstream * OFSPtrAll () { return frames[0].get_ofs_ptr(); }

@@ -71,6 +73,8 @@ inline int NProcsSub () { return frames.back().NProcs(); }
 inline int MyProcSub () { return frames.back().MyProc(); }
 // IO sub-rank in current frame
 inline int IOProcessorNumberSub () { return frames.back().IOProc(); }
+// Am IO processor for current frame?
+inline bool IOProcessorSub () { return MyProcSub() == IOProcessorNumberSub(); }
 // Pointer to ofstream
 inline std::ofstream * OFSPtrSub () { return frames.back().get_ofs_ptr(); }

diff --git a/Src/Base/AMReX_ParallelReduce.H b/Src/Base/AMReX_ParallelReduce.H
index 0776f5875a1..5ae621bf060 100644
--- a/Src/Base/AMReX_ParallelReduce.H
+++ b/Src/Base/AMReX_ParallelReduce.H
@@ -36,9 +36,11 @@ namespace detail {
         auto mpi_op = mpi_ops[static_cast<int>(op)];
         Vector<T> tmp(v, v+cnt);
         if (root == -1) {
+            // TODO: add BL_COMM_PROFILE commands
             MPI_Allreduce(tmp.data(), v, cnt, ParallelDescriptor::Mpi_typemap<T>::type(),
                           mpi_op, comm);
         } else {
+            // TODO: add BL_COMM_PROFILE commands
             MPI_Reduce(tmp.data(), v, cnt, ParallelDescriptor::Mpi_typemap<T>::type(),
                        mpi_op, root, comm);
         }
@@ -65,9 +67,15 @@ namespace detail {
     {
         auto mpi_type = ParallelDescriptor::Mpi_typemap<T>::type();
         if (root == -1) {
+            // TODO: check these BL_COMM_PROFILE commands
+            BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(T), BLProfiler::BeforeCall(),
+                            BLProfiler::NoTag());
             // const_cast for MPI-2
             MPI_Allgather(const_cast<T*>(v), cnt, mpi_type, vs, cnt, mpi_type, comm);
+            BL_COMM_PROFILE(BLProfiler::Allgather, sizeof(T), BLProfiler::AfterCall(),
+                            BLProfiler::NoTag());
         } else {
+            // TODO: add BL_COMM_PROFILE commands
             MPI_Gather(const_cast<T*>(v), cnt, mpi_type, vs, cnt, mpi_type, root, comm);
         }
     }
diff --git a/Src/Base/AMReX_Utility.H b/Src/Base/AMReX_Utility.H
index ba09a6cc17f..7d259f51159 100644
--- a/Src/Base/AMReX_Utility.H
+++ b/Src/Base/AMReX_Utility.H
@@ -318,6 +318,9 @@ namespace amrex
                                                std::chrono::steady_clock>::type;
     double second ();

+    template<typename T> void hash_combine (uint64_t & seed, const T & val);
+    template<typename T> uint64_t hash_vector (const Vector<T> & vec, uint64_t seed = 0xDEADBEEFDEADBEEF);
+
     template <class T>
     AMREX_GPU_HOST_DEVICE
     AMREX_INLINE const T& min (const T& a, const T& b)
@@ -447,4 +450,47 @@ extern "C" {
     void amrex_free (void* p);
 }

+// hash combiner borrowed from Boost
+/*
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+*/
+template<typename T>
+void
+amrex::hash_combine (uint64_t & seed, const T & val)
+{
+    seed ^= std::hash<T>()(val) + 0x9e3779b9 + (seed<<6) + (seed>>2);
+}
+
+template<typename T>
+uint64_t
+amrex::hash_vector (const Vector<T> & vec, uint64_t seed)
+{
+    for (const auto & x: vec) {
+        hash_combine(seed, x);
+    }
+    return seed;
+}
+
 #endif /*BL_UTILITY_H*/
diff --git a/Src/Base/CMakeLists.txt b/Src/Base/CMakeLists.txt
index 7a24d131f45..694bf0fa394 100644
--- a/Src/Base/CMakeLists.txt
+++ b/Src/Base/CMakeLists.txt
@@ -128,6 +128,8 @@ add_sources( AMReX_io_mod.F90 )

 add_sources( AMReX_BLProfiler.cpp AMReX_BLBackTrace.cpp )

+# Machine model
+add_sources ( AMReX_Machine.H AMReX_Machine.cpp )

 # LAZY mode not supported in CMake yet
 # ifeq ($(LAZY),TRUE)
diff --git a/Src/Base/Make.package b/Src/Base/Make.package
index 361adffbb65..cf512bb2a0e 100644
--- a/Src/Base/Make.package
+++ b/Src/Base/Make.package
@@ -211,6 +211,9 @@ ifeq ($(TINY_PROFILE),TRUE)
   C$(AMREX_BASE)_sources += AMReX_TinyProfiler.cpp
 endif

+# Machine model
+CEXE_headers += AMReX_Machine.H
+CEXE_sources += AMReX_Machine.cpp

 VPATH_LOCATIONS += $(AMREX_HOME)/Src/Base
 INCLUDE_LOCATIONS += $(AMREX_HOME)/Src/Base
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp
index ef74c5c1807..29b85ee7187 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp
@@ -80,7 +80,7 @@ MLCGSolver::solve_bicgstab (MultiFab&       sol,
                             Real            eps_rel,
                             Real            eps_abs)
 {
-    BL_PROFILE_REGION("MLCGSolver::bicgstab");
+    BL_PROFILE("MLCGSolver::bicgstab");

     const int nghost = sol.nGrow(), ncomp = sol.nComp();

@@ -192,7 +192,9 @@ MLCGSolver::solve_bicgstab (MultiFab&       sol,
         //
         Real tvals[2] = { dotxy(t,t,true), dotxy(t,s,true) };

+        BL_PROFILE_VAR("MLCGSolver::ParallelAllReduce", blp_par);
         ParallelAllReduce::Sum(tvals,2,Lp.BottomCommunicator());
+        BL_PROFILE_VAR_STOP(blp_par);

         if ( tvals[0] )
 	{
@@ -260,7 +262,7 @@ MLCGSolver::solve_cg (MultiFab&       sol,
                       Real            eps_rel,
                       Real            eps_abs)
 {
-    BL_PROFILE_REGION("MLCGSolver::cg");
+    BL_PROFILE("MLCGSolver::cg");

     const int nghost = sol.nGrow(), ncomp = sol.nComp();

@@ -390,7 +392,11 @@ MLCGSolver::solve_cg (MultiFab&       sol,
 Real
 MLCGSolver::dotxy (const MultiFab& r, const MultiFab& z, bool local)
 {
-    return Lp.xdoty(amrlev, mglev, r, z, local);
+    BL_PROFILE_VAR_NS("MLCGSolver::ParallelAllReduce", blp_par);
+    if (!local) { BL_PROFILE_VAR_START(blp_par); }
+    Real result = Lp.xdoty(amrlev, mglev, r, z, local);
+    if (!local) { BL_PROFILE_VAR_STOP(blp_par); }
+    return result;
 }

 Real
@@ -402,6 +408,7 @@ MLCGSolver::norm_inf (const MultiFab& res, bool local)
       result = std::max(result,res.norm0(n,0,true));

     if (!local) {
+        BL_PROFILE("MLCGSolver::ParallelAllReduce");
         ParallelAllReduce::Max(result, Lp.BottomCommunicator());
     }
     return result;
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
index 360aaf5a344..02824fe284a 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
+++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
@@ -34,6 +34,62 @@ public:
     }
     virtual void update () override;

+#ifdef AMREX_SOFT_PERF_COUNTERS
+    struct Counters
+    {
+        size_t smooth_n = 0;
+        size_t smooth_cell_n = 0;
+        size_t apply_n = 0;
+        size_t apply_cell_n = 0;
+        size_t restrict_n = 0;
+        size_t restrict_cell_n = 0;
+        size_t interp_n = 0;
+        size_t interp_cell_n = 0;
+
+        void reset () {
+            smooth_n = 0;
+            smooth_cell_n = 0;
+            apply_n = 0;
+            apply_cell_n = 0;
+            restrict_n = 0;
+            restrict_cell_n = 0;
+            interp_n = 0;
+            interp_cell_n = 0;
+        }
+
+        void smooth (const MultiFab & mf) {
+            smooth_n++;
+            smooth_cell_n += mf.boxArray().numPts();
+        }
+        void apply (const MultiFab & mf) {
+            apply_n++;
+            apply_cell_n += mf.boxArray().numPts();
+        }
+        void restrict (const MultiFab & mf) {
+            restrict_n++;
+            restrict_cell_n += mf.boxArray().numPts();
+        }
+        void interpolate (const MultiFab & mf) {
+            interp_n++;
+            interp_cell_n += mf.boxArray().numPts();
+        }
+
+        void print () {
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::smooth()        called " << smooth_n
+                    << " times on " << smooth_cell_n << " cells total" << std::endl;
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::apply()         called " << apply_n
+                    << " times on " << apply_cell_n << " cells total" << std::endl;
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::restriction()   called " << restrict_n
+                    << " times on " << restrict_cell_n << " cells total" << std::endl;
+            Print() << ParallelContext::MyProcSub() << ": perf: MLCellLinOp::interpolation() called " << interp_n
+                    << " times on " << interp_cell_n << " cells total" << std::endl;
+        }
+    };
+
+    // software counters for performance analysis
+    static Counters perf_counters;
+#endif
+
 protected:

     virtual bool isCrossStencil () const { return true; }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp
index d47e7de0870..385e032ef3f 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp
@@ -296,12 +296,18 @@ void
 MLCellLinOp::restriction (int, int, MultiFab& crse, MultiFab& fine) const
 {
     const int ncomp = getNComp();
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    perf_counters.restrict(crse);
+#endif
     amrex::average_down(fine, crse, 0, ncomp, 2);
 }

 void
 MLCellLinOp::interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const
 {
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    perf_counters.interpolate(fine);
+#endif
 #ifdef _OPENMP
 #pragma omp parallel
 #endif
@@ -336,6 +342,9 @@ MLCellLinOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode b
 {
     BL_PROFILE("MLCellLinOp::apply()");
     applyBC(amrlev, mglev, in, bc_mode, s_mode, bndry);
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    perf_counters.apply(out);
+#endif
     Fapply(amrlev, mglev, out, in);
 }

@@ -348,6 +357,9 @@ MLCellLinOp::smooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs,
     {
         applyBC(amrlev, mglev, sol, BCMode::Homogeneous, StateMode::Solution,
                 nullptr, skip_fillboundary);
+#ifdef AMREX_SOFT_PERF_COUNTERS
+        perf_counters.smooth(sol);
+#endif
         Fsmooth(amrlev, mglev, sol, rhs, redblack);
         skip_fillboundary = false;
     }
@@ -843,4 +855,9 @@ MLCellLinOp::update ()
     if (MLLinOp::needsUpdate()) MLLinOp::update();
 }

+#ifdef AMREX_SOFT_PERF_COUNTERS
+// perf_counters
+MLCellLinOp::Counters MLCellLinOp::perf_counters;
+#endif
+
 }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
index 32749de0505..52a87c675f5 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
+++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
@@ -56,6 +56,9 @@ public:

     enum struct Location { FaceCenter, FaceCentroid, CellCenter, CellCentroid };

+    static void Initialize ();
+    static void Finalize ();
+
     MLLinOp ();
     virtual ~MLLinOp ();

@@ -279,6 +282,7 @@ private:
     static void makeConsolidatedDMap (const Vector<BoxArray>& ba, Vector<DistributionMapping>& dm,
                                       int ratio, int strategy);
     MPI_Comm makeSubCommunicator (const DistributionMapping& dm);
+    void remapNeighborhoods (Vector<DistributionMapping> & dms);
 };

 }
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp
index 1a08ce6caef..131ef7b8b29 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp
@@ -1,8 +1,13 @@

 #include <cmath>
 #include <algorithm>
+#include <unordered_map>
+#include <set>
+#include <AMReX_Utility.H>
 #include <AMReX_MLLinOp.H>
+#include <AMReX_MLCellLinOp.H>
 #include <AMReX_ParmParse.H>
+#include <AMReX_Machine.H>

 #ifdef AMREX_USE_EB
 #include <AMReX_EB2.H>
@@ -21,9 +26,95 @@ constexpr int MLLinOp::mg_box_min_width;

 namespace {
     // experimental features
-    bool initialized = false;
+    bool initialized = false; // track initialization of static state
+    int consolidation_threshold = -1;
     int consolidation_ratio = 2;
     int consolidation_strategy = 3;
+
+    int flag_verbose_linop = 0;
+    int flag_comm_cache = 0;
+    int flag_use_mota = 0;
+    int remap_nbh_lb = 1;
+
+#ifdef BL_USE_MPI
+    class CommCache
+    {
+      public:
+        CommCache () = default;
+        CommCache (const CommCache&) = delete;
+        CommCache (CommCache&&) = delete;
+        void operator= (const CommCache&) = delete;
+        void operator= (CommCache&&) = delete;
+
+        ~CommCache () {
+            for (auto & p : cache) {
+                if (p.second != MPI_COMM_NULL) {
+                    MPI_Comm_free(&p.second);
+                }
+            }
+        }
+        void add (size_t key, MPI_Comm comm) {
+            AMREX_ASSERT(cache.count(key) == 0);
+            cache[key] = comm;
+        }
+        bool get (size_t key, MPI_Comm &comm) {
+            bool result = cache.count(key) > 0;
+            if (result) {
+                comm = cache.at(key);
+            }
+            return result;
+        }
+
+      private:
+        std::unordered_map<size_t, MPI_Comm> cache;
+    };
+
+    std::unique_ptr<CommCache> comm_cache;
+#endif
+
+    Vector<int> get_subgroup_ranks ()
+    {
+        int rank_n = ParallelContext::NProcsSub();
+        Vector<int> lranks(rank_n);
+        for (int i = 0; i < rank_n; ++i) {
+            lranks[i] = i;
+        }
+
+        Vector<int> granks(rank_n);
+        ParallelContext::local_to_global_rank(granks.data(), lranks.data(), rank_n);
+        return granks;
+    }
+}
+
+// static member function
+void MLLinOp::Initialize ()
+{
+    ParmParse pp("mg");
+    pp.query("consolidation_threshold", consolidation_threshold);
+    pp.query("consolidation_ratio", consolidation_ratio);
+    pp.query("consolidation_strategy", consolidation_strategy);
+    pp.query("verbose_linop", flag_verbose_linop);
+    pp.query("comm_cache", flag_comm_cache);
+    pp.query("mota", flag_use_mota);
+    pp.query("remap_nbh_lb", remap_nbh_lb);
+
+#ifdef BL_USE_MPI
+    comm_cache.reset(new CommCache());
+#endif
+    amrex::ExecOnFinalize(MLLinOp::Finalize);
+    initialized = true;
+}
+
+// static member function
+void MLLinOp::Finalize ()
+{
+    initialized = false;
+#ifdef BL_USE_MPI
+    comm_cache.reset();
+#endif
+#ifdef AMREX_SOFT_PERF_COUNTERS
+    MLCellLinOp::perf_counters.reset();
+#endif
 }

 MLLinOp::MLLinOp () {}
@@ -40,10 +131,7 @@ MLLinOp::define (const Vector<Geometry>& a_geom,
     BL_PROFILE("MLLinOp::define()");

     if (!initialized) {
-	ParmParse pp("mg");
-	pp.query("consolidation_ratio", consolidation_ratio);
-	pp.query("consolidation_strategy", consolidation_strategy);
-	initialized = true;
+        Initialize();
     }

     info = a_info;
@@ -158,6 +246,7 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,

     bool agged = false;
     bool coned = false;
+    int agg_lev, con_lev;

     if (info.do_agglomeration && aggable)
     {
@@ -237,6 +326,7 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,

                 m_dmap[0].push_back(DistributionMapping());
                 agged = true;
+                agg_lev = last_coarsenableto_lev;
             }

             m_num_mg_levels[0] = m_grids[0].size();
@@ -245,12 +335,14 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
     else
     {
         int rr = mg_coarsen_ratio;
-        Real avg_npts, threshold_npts;
+        Real avg_npts;
         if (info.do_consolidation) {
             avg_npts = static_cast<Real>(a_grids[0].d_numPts()) / static_cast<Real>(ParallelContext::NProcsSub());
-            threshold_npts = static_cast<Real>(AMREX_D_TERM(info.con_grid_size,
-                                                            *info.con_grid_size,
-                                                            *info.con_grid_size));
+            if (consolidation_threshold == -1) {
+                consolidation_threshold = static_cast<Real>(AMREX_D_TERM(info.con_grid_size,
+                                                                         *info.con_grid_size,
+                                                                         *info.con_grid_size));
+            }
         }
         while (m_num_mg_levels[0] < info.max_coarsening_level + 1
                and a_geom[0].Domain().coarsenable(rr)
@@ -263,9 +355,10 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,

             if (info.do_consolidation)
             {
-                if (avg_npts/(AMREX_D_TERM(rr,*rr,*rr)) < 0.999*threshold_npts)
+                if (avg_npts/(AMREX_D_TERM(rr,*rr,*rr)) < 0.999*consolidation_threshold)
                 {
                     coned = true;
+                    con_lev = m_dmap[0].size();
                     m_dmap[0].push_back(DistributionMapping());
                 }
                 else
@@ -292,6 +385,11 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
         makeConsolidatedDMap(m_grids[0], m_dmap[0], consolidation_ratio, consolidation_strategy);
     }

+    if (flag_use_mota && (agged || coned))
+    {
+        remapNeighborhoods(m_dmap[0]);
+    }
+
     if (info.do_agglomeration || info.do_consolidation)
     {
         m_bottom_comm = makeSubCommunicator(m_dmap[0].back());
@@ -304,6 +402,19 @@ MLLinOp::defineGrids (const Vector<Geometry>& a_geom,
     m_do_agglomeration = agged;
     m_do_consolidation = coned;

+    if (flag_verbose_linop) {
+        if (agged) {
+            Print() << "MLLinOp::defineGrids(): agglomerated AMR level 0 starting at MG level "
+                    << agg_lev << " of " << m_num_mg_levels[0] << std::endl;
+        } else if (coned) {
+            Print() << "MLLinOp::defineGrids(): consolidated AMR level 0 starting at MG level "
+                    << con_lev << " of " << m_num_mg_levels[0]
+                    << " (ratio = " << consolidation_ratio << ")" << std::endl;
+        } else {
+            Print() << "MLLinOp::defineGrids(): no agglomeration or consolidation of AMR level 0" << std::endl;
+        }
+    }
+
     for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev)
     {
         for (int mglev = 1; mglev < m_num_mg_levels[amrlev]; ++mglev)
@@ -377,30 +488,59 @@ MLLinOp::makeSubCommunicator (const DistributionMapping& dm)
     BL_PROFILE("MLLinOp::makeSubCommunicator()");

 #ifdef BL_USE_MPI
-    MPI_Comm newcomm;
-    MPI_Group defgrp, newgrp;
-
-    MPI_Comm_group(m_default_comm, &defgrp);

     Vector<int> newgrp_ranks = dm.ProcessorMap();
     std::sort(newgrp_ranks.begin(), newgrp_ranks.end());
     auto last = std::unique(newgrp_ranks.begin(), newgrp_ranks.end());
     newgrp_ranks.erase(last, newgrp_ranks.end());

-    if (ParallelContext::CommunicatorSub() == ParallelDescriptor::Communicator()) {
-        MPI_Group_incl(defgrp, newgrp_ranks.size(), newgrp_ranks.data(), &newgrp);
-    } else {
-        Vector<int> local_newgrp_ranks(newgrp_ranks.size());
-        ParallelContext::global_to_local_rank(local_newgrp_ranks.data(),
-                                              newgrp_ranks.data(), newgrp_ranks.size());
-        MPI_Group_incl(defgrp, local_newgrp_ranks.size(), local_newgrp_ranks.data(), &newgrp);
+    if (flag_verbose_linop) {
+        Print() << "MLLinOp::makeSubCommunicator(): called for " << newgrp_ranks.size() << " ranks" << std::endl;
     }

-    MPI_Comm_create(m_default_comm, newgrp, &newcomm);
-    m_raii_comm.reset(new CommContainer(newcomm));
+    MPI_Comm newcomm;
+    bool cache_hit = false;
+    uint64_t key = 0;
+    if (flag_comm_cache) {
+        AMREX_ASSERT(comm_cache);
+        key = hash_vector(newgrp_ranks, hash_vector(get_subgroup_ranks()));
+        cache_hit = comm_cache->get(key, newcomm);
+        if (cache_hit && flag_verbose_linop) {
+            Print() << "MLLinOp::makeSubCommunicator(): found subcomm in cache" << std::endl;
+        }
+    }
+
+    if (!flag_comm_cache || !cache_hit) {
+        MPI_Group defgrp, newgrp;
+        MPI_Comm_group(m_default_comm, &defgrp);
+        if (ParallelContext::CommunicatorSub() == ParallelDescriptor::Communicator()) {
+            MPI_Group_incl(defgrp, newgrp_ranks.size(), newgrp_ranks.data(), &newgrp);
+        } else {
+            Vector<int> local_newgrp_ranks(newgrp_ranks.size());
+            ParallelContext::global_to_local_rank(local_newgrp_ranks.data(),
+                                                  newgrp_ranks.data(), newgrp_ranks.size());
+            MPI_Group_incl(defgrp, local_newgrp_ranks.size(), local_newgrp_ranks.data(), &newgrp);
+        }
+
+        if (flag_verbose_linop) {
+            Print() << "MLLinOp::makeSubCommunicator(): MPI_Comm_create: (";
+            for (auto rank : newgrp_ranks) {
+                Print() << rank << ",";
+            }
+            Print() << "\b)" << std::endl;
+        }
+
+        MPI_Comm_create(m_default_comm, newgrp, &newcomm);
+
+        if (flag_comm_cache) {
+            comm_cache->add(key, newcomm);
+        } else {
+            m_raii_comm.reset(new CommContainer(newcomm));
+        }

-    MPI_Group_free(&defgrp);
-    MPI_Group_free(&newgrp);
+        MPI_Group_free(&defgrp);
+        MPI_Group_free(&newgrp);
+    }

     return newcomm;
 #else
@@ -490,6 +630,44 @@ MLLinOp::makeConsolidatedDMap (const Vector<BoxArray>& ba, Vector<DistributionMa
     }
 }

+void
+MLLinOp::remapNeighborhoods (Vector<DistributionMapping> & dms)
+{
+    BL_PROFILE("MLLinOp::remapNeighborhoods()");
+
+    if (flag_verbose_linop) {
+        Print() << "Remapping ranks to neighborhoods ..." << std::endl;
+    }
+
+    for (int j = 1; j < dms.size(); ++j)
+    {
+        const Vector<int> & pmap = dms[j].ProcessorMap();
+        std::set<int> g_ranks_set(pmap.begin(), pmap.end());
+        auto lev_rank_n = g_ranks_set.size();
+        if (lev_rank_n >= remap_nbh_lb && lev_rank_n < ParallelContext::NProcsSub())
+        {
+            // find best neighborhood with lev_rank_n ranks
+            auto nbh_g_ranks = machine::find_best_nbh(lev_rank_n);
+            AMREX_ASSERT(nbh_g_ranks.size() == lev_rank_n);
+
+            // construct mapping from original global rank to neighborhood global rank
+            int idx = 0;
+            std::unordered_map<int, int> rank_mapping;
+            for (auto orig_g_rank : g_ranks_set) {
+                AMREX_ASSERT(idx < nbh_g_ranks.size());
+                rank_mapping[orig_g_rank] = nbh_g_ranks[idx++];
+            }
+
+            // remap and create new DM
+            Vector<int> nbh_pmap(pmap.size());
+            for (int i = 0; i < pmap.size(); ++i) {
+                nbh_pmap[i] = rank_mapping.at(pmap[i]);
+            }
+            dms[j] = DistributionMapping(std::move(nbh_pmap));
+        }
+    }
+}
+
 #ifdef AMREX_USE_PETSC
 std::unique_ptr<PETScABecLap>
 MLLinOp::makePETSc () const
diff --git a/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp b/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp
index 55a791a650f..dfaff5d0e6b 100644
--- a/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp
+++ b/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp
@@ -53,7 +53,6 @@ Real
 MLMG::solve (const Vector<MultiFab*>& a_sol, const Vector<MultiFab const*>& a_rhs,
              Real a_tol_rel, Real a_tol_abs)
 {
-    BL_PROFILE_REGION("MLMG::solve()");
     BL_PROFILE("MLMG::solve()");

     if (bottom_solver == BottomSolver::hypre) {
@@ -357,21 +356,38 @@ MLMG::miniCycle (int amrlev)
     mgVcycle(amrlev, mglev);
 }

+namespace {
+
+void make_str_helper (std::ostringstream & oss) { }
+
+template <class T, class... Ts>
+void make_str_helper (std::ostringstream & oss, T x, Ts... xs) {
+    oss << x;
+    make_str_helper(oss, xs...);
+}
+
+template <class... Ts>
+std::string make_str (Ts... xs) {
+    std::ostringstream oss;
+    make_str_helper(oss, xs...);
+    return oss.str();
+}
+
+}
+
 // in   : Residual (res)
 // out  : Correction (cor) from bottom to this function's local top
 void
 MLMG::mgVcycle (int amrlev, int mglev_top)
 {
     BL_PROFILE("MLMG::mgVcycle()");
-    BL_PROFILE_VAR_NS("MLMG::mgVcycle_up", blp_up);
-    BL_PROFILE_VAR_NS("MLMG::mgVcycle_bottom", blp_bottom);
-    BL_PROFILE_VAR_NS("MLMG::mgVcycle_down", blp_down);

     const int mglev_bottom = linop.NMGLevels(amrlev) - 1;

-    BL_PROFILE_VAR_START(blp_down);
     for (int mglev = mglev_top; mglev < mglev_bottom; ++mglev)
     {
+        std::string blp_mgv_down_lev_str = make_str("MLMG::mgVcycle_down::", mglev);
+        BL_PROFILE_VAR(blp_mgv_down_lev_str, blp_mgv_down_lev);

         if (verbose >= 4)
         {
@@ -402,9 +418,8 @@ MLMG::mgVcycle (int amrlev, int mglev_top)
         linop.restriction(amrlev, mglev+1, res[amrlev][mglev+1], rescor[amrlev][mglev]);

     }
-    BL_PROFILE_VAR_STOP(blp_down);

-    BL_PROFILE_VAR_START(blp_bottom);
+    BL_PROFILE_VAR("MLMG::mgVcycle_bottom", blp_bottom);
     if (amrlev == 0)
     {
         if (verbose >= 4)
@@ -447,9 +462,10 @@ MLMG::mgVcycle (int amrlev, int mglev_top)
     }
     BL_PROFILE_VAR_STOP(blp_bottom);

-    BL_PROFILE_VAR_START(blp_up);
     for (int mglev = mglev_bottom-1; mglev >= mglev_top; --mglev)
     {
+        std::string blp_mgv_up_lev_str = make_str("MLMG::mgVcycle_up::", mglev);
+        BL_PROFILE_VAR(blp_mgv_up_lev_str, blp_mgv_up_lev);
         // cor_fine += I(cor_crse)
         addInterpCorrection(amrlev, mglev);
         if (verbose >= 4)
@@ -470,7 +486,6 @@ MLMG::mgVcycle (int amrlev, int mglev_top)
                            << "   UP: Norm after  smooth " << norm << "\n";
         }
     }
-    BL_PROFILE_VAR_STOP(blp_up);
 }

 // FMG cycle on the coarsest AMR level.
@@ -1461,7 +1476,6 @@ MLMG::computeVolInv ()

     if (linop.isCellCentered())
     {
-        Real temp1, temp2;
         volinv.resize(namrlevs);
         for (int amrlev = 0; amrlev < namrlevs; ++amrlev) {
             volinv[amrlev].resize(linop.NMGLevels(amrlev));
@@ -1491,6 +1505,7 @@ MLMG::computeVolInv ()
         f(0,mgbottom);

 #ifdef AMREX_USE_EB
+        Real temp1, temp2;
         if (rhs[0].hasEBFabFactory())
         {
             ParallelAllReduce::Sum<Real>({volinv[0][0], volinv[0][mgbottom]},
diff --git a/Tools/Backtrace/parse_bt.py b/Tools/Backtrace/parse_bt.py
index c1fb8a5cbd3..cdc64323820 100755
--- a/Tools/Backtrace/parse_bt.py
+++ b/Tools/Backtrace/parse_bt.py
@@ -16,10 +16,25 @@
   lines = f.readlines()

 for l in lines:
-  m = re.match("\s*(\d+): \[(0x[\dabcdef]+)\]", l)
+
+  matched = False
+
+  # gnu compiler
+  m = re.match("\s*(\d+): .*\(\+(0x[\dabcdef]+)\)", l)
   if m:
+    matched = True
     frame = m.group(1)
     addr = m.group(2)
+
+  # intel compiler
+  if not matched:
+    m = re.match("\s*(\d+): \[(0x[\dabcdef]+)\]", l)
+    if m:
+      matched = True
+      frame = m.group(1)
+      addr = m.group(2)
+
+  if matched:
     cmd = "addr2line -Cfie %s %s" % (exe_file, addr)
     info = subprocess.check_output(shlex.split(cmd)).decode("utf-8")
     print("%s: %s" % (frame, info))

diff --git a/Tests/LinearSolvers/MLMG/GNUmakefile b/Tests/LinearSolvers/MLMG/GNUmakefile
index 4a12a62bd49..eed6d56a155 100644
--- a/Tests/LinearSolvers/MLMG/GNUmakefile
+++ b/Tests/LinearSolvers/MLMG/GNUmakefile
@@ -1,19 +1,20 @@
 AMREX_HOME ?= ../../../

-DEBUG	= FALSE
-#DEBUG	= TRUE
+DEBUG	?= FALSE
+DIM	?= 3
+COMP    ?= gnu

-DIM	= 3
-
-COMP    = gnu
-
-USE_MPI   = TRUE
-#USE_OMP   = TRUE
-USE_OMP   = FALSE
+USE_MPI   ?= TRUE
+USE_OMP   ?= FALSE

 USE_HYPRE = FALSE

-TINY_PROFILE = TRUE
+TINY_PROFILE ?= TRUE
+PROFILE ?= FALSE
+COMM_PROFILE ?= FALSE
+TRACE_PROFILE ?= FALSE
+
+DEFINES += -DAMREX_SOFT_PERF_COUNTERS

 include $(AMREX_HOME)/Tools/GNUMake/Make.defs

diff --git a/Tests/LinearSolvers/MLMG/inputs b/Tests/LinearSolvers/MLMG/inputs
index a56526e83e6..3a69b9628bd 100644
--- a/Tests/LinearSolvers/MLMG/inputs
+++ b/Tests/LinearSolvers/MLMG/inputs
@@ -26,3 +26,10 @@ max_fmg_iter = 0     # # of F-cycles before switching to V.  To do pure V-cycle,
 linop_maxorder = 2
 agglomeration = 1    # Do agglomeration on AMR Level 0?
 consolidation = 1    # Do consolidation?
+
+mg.verbose_linop = 1
+mg.comm_cache = 1
+mg.consolidation_ratio = 2
+mg.mota = 0
+mg.remap_nbh_lb = 1
+machine.verbose = 1
diff --git a/Tests/LinearSolvers/MLMG/main.cpp b/Tests/LinearSolvers/MLMG/main.cpp
index bf4d8d6cbf5..e0a5e6f97de 100644
--- a/Tests/LinearSolvers/MLMG/main.cpp
+++ b/Tests/LinearSolvers/MLMG/main.cpp
@@ -2,6 +2,12 @@
 #include <AMReX.H>
 #include <AMReX_ParmParse.H>
 #include <AMReX_MultiFab.H>
+#include <AMReX_Machine.H>
+
+#ifdef AMREX_SOFT_PERF_COUNTERS
+// need for perf counters
+#include <AMReX_MLCellLinOp.H>
+#endif

 #include <prob_par.H>

@@ -70,6 +76,10 @@ int main (int argc, char* argv[])

         solve_with_mlmg (geom, ref_ratio, soln, alpha, beta, rhs, exact);

+#ifdef AMREX_SOFT_PERF_COUNTERS
+        MLCellLinOp::perf_counters.print();
+#endif
+
         write_plotfile (geom, ref_ratio, soln, exact, alpha, beta, rhs);
     }

diff --git a/Tests/LinearSolvers/MLMG/solve_with_mlmg.cpp b/Tests/LinearSolvers/MLMG/solve_with_mlmg.cpp
index 6aaf6b4564c..187c60237e4 100644
--- a/Tests/LinearSolvers/MLMG/solve_with_mlmg.cpp
+++ b/Tests/LinearSolvers/MLMG/solve_with_mlmg.cpp
@@ -28,6 +28,9 @@ void solve_with_mlmg(const Vector<Geometry>& geom, int ref_ratio,
                       Vector<MultiFab>& rhs, const Vector<MultiFab>& exact) {
   BL_PROFILE("solve_with_mlmg");

+  Real tol_rel = 1.e-10;
+  Real tol_abs = 0.0;
+
   {
     ParmParse pp;
     pp.query("composite_solve", composite_solve);
@@ -41,6 +44,8 @@ void solve_with_mlmg(const Vector<Geometry>& geom, int ref_ratio,
     pp.query("agglomeration", agglomeration);
     pp.query("consolidation", consolidation);
     pp.query("use_hypre", use_hypre);
+    pp.query("tol_rel", tol_rel);
+    pp.query("tol_abs", tol_abs);
   }

   LPInfo info;
@@ -48,9 +53,6 @@ void solve_with_mlmg(const Vector<Geometry>& geom, int ref_ratio,
   info.setConsolidation(consolidation);
   info.setMaxCoarseningLevel(max_coarsening_level);

-  const Real tol_rel = 1.e-10;
-  const Real tol_abs = 0.0;
-
   const int nlevels = geom.size();

   if (composite_solve) {


Fix ParmParse::query_enum_case_insensitive (AMReX-Codes/amrex#4144)

Since #4143, we can no longer cast the position of the found string in the vector of all names to obtain its enum value, because the enum values can now have duplicates.

AMREX_ENUM: Add more capabilites

Add support for `=`. For example,

```c++
    AMREX_ENUM(MyColor,
               red,
               chi=red,
               green,
               blue,
               Default = green);
```
The underlying values are then
```c++
    { red = 0,
      chi = 0,
      green = 1,
      blue = 2,
      Default = 1}
```
We can then have
```c++
    assert(amrex::getEnum<MyColor2>("red") == MyColor2::red);
    assert(amrex::getEnum<MyColor2>("chi") == MyColor2::chi);
    assert(amrex::getEnum<MyColor2>("green") == MyColor2::green);
    assert(amrex::getEnum<MyColor2>("blue") == MyColor2::blue);
    assert(amrex::getEnum<MyColor2>("Default") == MyColor2::Default);
```
Because of there are duplicated values, using value to get name string will return the first matched string. Currently, I don't see a way to fix it. But, since `MyColor::red == MyColor::chi` is true, it's also not outrageously wrong.
```c++
    assert(amrex::getEnumNameString(MyColor2::red) == "red");
    assert(amrex::getEnumNameString(MyColor2::chi) == "red");
    assert(amrex::getEnumNameString(MyColor2::green) == "green");
    assert(amrex::getEnumNameString(MyColor2::blue) == "blue");
    assert(amrex::getEnumNameString(MyColor2::Default) == "green");
```
P.S. Chi means red in Chinese. https://en.wikipedia.org/wiki/Radical_155

diff --git a/Src/Base/AMReX_Enum.H b/Src/Base/AMReX_Enum.H
index 2bb503d4113..b94e30d396e 100644
--- a/Src/Base/AMReX_Enum.H
+++ b/Src/Base/AMReX_Enum.H
@@ -67,6 +67,26 @@ namespace amrex
         }
     }

+    template <typename T, typename ET = amrex_enum_traits<T>,
+              std::enable_if_t<ET::value,int> = 0>
+    T getEnumCaseInsensitive (std::string_view const& s)
+    {
+        auto const& kv = getEnumNameValuePairs<T>();
+        std::string ls = amrex::toLower(std::string(s));
+        auto it = std::find_if(kv.begin(), kv.end(),
+                               [&] (auto const& x) -> bool
+                                   { return amrex::toLower(x.first) == ls; });
+        if (it != kv.end()) {
+            return it->second;
+        } else {
+            std::string error_msg("amrex::getEnumCaseInsensitive: Unknown enum: ");
+            error_msg.append(s).append(" in AMREX_ENUM(").append(ET::class_name)
+                .append(", ").append(ET::enum_names).append(").");
+            throw std::runtime_error(error_msg);
+            return T();
+        }
+    }
+
     template <typename T, typename ET = amrex_enum_traits<T>,
               std::enable_if_t<ET::value,int> = 0>
     std::string getEnumNameString (T const& v)
diff --git a/Src/Base/AMReX_ParmParse.H b/Src/Base/AMReX_ParmParse.H
index 63c0c45431a..42e62a05ab3 100644
--- a/Src/Base/AMReX_ParmParse.H
+++ b/Src/Base/AMReX_ParmParse.H
@@ -1329,19 +1329,14 @@ public:
         std::string s;
         int exist = this->query(name, s, ival);
         if (exist) {
-            s = amrex::toLower(s);
-            auto const& enum_names = amrex::getEnumNameStrings<T>();
-            auto found = std::find_if(enum_names.begin(), enum_names.end(),
-                                      [&] (std::string const& ename) {
-                                          return amrex::toLower(ename) == s;
-                                      });
-            if (found != enum_names.end()) {
-                ref = static_cast<T>(std::distance(enum_names.begin(), found));
-            } else {
-                std::string msg("query_enum_case_insensitive(\"");
-                msg.append(name).append("\",").append(amrex::getEnumClassName<T>())
-                    .append("&) failed.");
-                throw std::runtime_error(msg);
+            try {
+                ref = amrex::getEnumCaseInsensitive<T>(s);
+            } catch (...) {
+                if (amrex::Verbose() > 0) {
+                    amrex::Print() << "amrex::ParmParse::query_enum_case_insensitive (input name: "
+                                   << this->prefixedName(name) << "):\n";
+                }
+                throw;
             }
         }
         return exist;

diff --git a/Tests/Enum/main.cpp b/Tests/Enum/main.cpp
index 69ed91adc27..b0647836e49 100644
--- a/Tests/Enum/main.cpp
+++ b/Tests/Enum/main.cpp
@@ -129,6 +129,11 @@ int main (int argc, char* argv[])
         AMREX_ALWAYS_ASSERT(amrex::getEnum<MyColor2>("green") == MyColor2::green);
         AMREX_ALWAYS_ASSERT(amrex::getEnum<MyColor2>("blue") == MyColor2::blue);
         AMREX_ALWAYS_ASSERT(amrex::getEnum<MyColor2>("Default") == MyColor2::Default);
+
+        auto my_blue = MyColor2::Default;
+        ParmParse pp;
+        pp.query_enum_case_insensitive("color5", my_blue);
+        AMREX_ALWAYS_ASSERT(my_blue == MyColor2::blue);
     }

     amrex::Finalize();


Named SoA Support (AMReX-Codes/amrex#4163)

## Summary

Add support to optionally name SoA Real and Int components, both for compile-time and runtime components.

- [x] define and defaults
- [x] single tile `Get*Data(std::string)`
- [x] refactor out default name generation for reuse, finalize defaults in `AddReal/IntComp`
- [x] test

## Additional background

Close #3614

## Checklist

The proposed changes:
- [ ] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [ ] changes answers in the test suite to more than roundoff level
- [ ] are likely to significantly affect the results of downstream AMReX users
- [ ] include documentation in the code and/or rst files, if appropriate

Named Particle SoA Components

X-ref:
- https://github.com/AMReX-Codes/pyamrex/pull/220
- https://github.com/AMReX-Codes/pyamrex/pull/88

It would be nice if AMReX supported the functionality to name particle attributes (as strings). In WarpX, we added this as `NamedParticleContainer`, but as a mainstream implementation we could avoid deriving yet another class type (because if we do, we will again have additional compile-time to bind that custom type in pyAMReX), and instead could just extend the particle classes with these runtime details.

diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H
index 03a2254a10e..4b347d283b8 100644
--- a/Src/Particle/AMReX_ParticleContainer.H
+++ b/Src/Particle/AMReX_ParticleContainer.H
@@ -60,6 +60,7 @@
 #include <memory>
 #include <numeric>
 #include <random>
+#include <string>
 #include <tuple>
 #include <type_traits>
 #include <utility>
@@ -1144,7 +1145,8 @@ public:
      */
     ParticleTileType& DefineAndReturnParticleTile (int lev, int grid, int tile)
     {
-        m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps());
+        m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps(), &m_soa_rdata_names, &m_soa_idata_names);
+
         return ParticlesAt(lev, grid, tile);
     }

@@ -1247,10 +1249,10 @@ public:

     Long superParticleSize() const { return superparticle_size; }

-    template <typename T,
-              std::enable_if_t<std::is_same_v<T,bool>,int> = 0>
-    void AddRealComp (T communicate=true)
+    void AddRealComp (std::string const & name, int communicate=1)
     {
+        m_soa_rdata_names.push_back(name);
+
         m_runtime_comps_defined = true;
         m_num_runtime_real++;
         h_redistribute_real_comp.push_back(communicate);
@@ -1270,10 +1272,15 @@ public:
         }
     }

-    template <typename T,
-              std::enable_if_t<std::is_same_v<T,bool>,int> = 0>
-    void AddIntComp (T communicate=true)
+    void AddRealComp (int communicate=1)
     {
+        AddRealComp(getDefaultCompNameReal<ParticleType>(NArrayReal+m_num_runtime_real), communicate);
+    }
+
+    void AddIntComp (std::string const & name, int communicate=1)
+    {
+        m_soa_idata_names.push_back(name);
+
         m_runtime_comps_defined = true;
         m_num_runtime_int++;
         h_redistribute_int_comp.push_back(communicate);
@@ -1293,6 +1300,11 @@ public:
         }
     }

+    void AddIntComp (int communicate=1)
+    {
+        AddIntComp(getDefaultCompNameInt<ParticleType>(NArrayInt+m_num_runtime_int), communicate);
+    }
+
     int NumRuntimeRealComps () const { return m_num_runtime_real; }
     int NumRuntimeIntComps  () const { return m_num_runtime_int;  }

@@ -1403,6 +1415,15 @@ public:
 #include "AMReX_ParticlesHDF5.H"
 #endif

+    /** Overwrite the default names for the compile-time SoA components */
+    void SetSoACompileTimeNames (std::vector<std::string> const & rdata_name, std::vector<std::string> const & idata_name);
+
+    /** Get the names for the real SoA components **/
+    std::vector<std::string> GetRealSoANames () const {return m_soa_rdata_names;}
+
+    /** Get the names for the int SoA components **/
+    std::vector<std::string> GetIntSoANames () const {return m_soa_idata_names;}
+
 protected:

     template <class RTYPE>
@@ -1435,6 +1456,10 @@ private:
     size_t particle_size, superparticle_size;
     int num_real_comm_comps, num_int_comm_comps;
     Vector<ParticleLevel> m_particles;
+
+    // names of both compile-time and runtime Real and Int SoA data
+    std::vector<std::string> m_soa_rdata_names;
+    std::vector<std::string> m_soa_idata_names;
 };

 template <int T_NStructReal, int T_NStructInt, int T_NArrayReal, int T_NArrayInt, template<class> class Allocator, class CellAssignor>
diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H
index 74e65b792f0..718405f1e66 100644
--- a/Src/Particle/AMReX_ParticleContainerI.H
+++ b/Src/Particle/AMReX_ParticleContainerI.H
@@ -1,6 +1,10 @@
-#include <type_traits>
 #include <AMReX_MakeParticle.H>

+#include <string>
+#include <type_traits>
+#include <vector>
+
+
 template <typename ParticleType, int NArrayReal, int NArrayInt,
           template<class> class Allocator, class CellAssignor>
 void
@@ -60,10 +64,40 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
         pp.query("do_unlink", doUnlink);
         pp.queryAdd("do_mem_efficient_sort", memEfficientSort);

+        // add default names for SoA Real and Int compile-time arguments
+        for (int i=0; i<NArrayReal; ++i)
+        {
+            m_soa_rdata_names.push_back(getDefaultCompNameReal<ParticleType>(i));
+        }
+        for (int i=0; i<NArrayInt; ++i)
+        {
+            m_soa_idata_names.push_back(getDefaultCompNameInt<ParticleType>(i));
+        }
+
         initialized = true;
     }
 }

+template <typename ParticleType, int NArrayReal, int NArrayInt,
+        template<class> class Allocator, class CellAssignor>
+void
+ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> :: SetSoACompileTimeNames (
+    std::vector<std::string> const & rdata_name, std::vector<std::string> const & idata_name
+)
+{
+    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rdata_name.size() == NArrayReal, "rdata_name must be equal to NArrayReal");
+    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(idata_name.size() == NArrayInt, "idata_name must be equal to NArrayInt");
+
+    for (int i=0; i<NArrayReal; ++i)
+    {
+        m_soa_rdata_names.at(i) = rdata_name.at(i);
+    }
+    for (int i=0; i<NArrayInt; ++i)
+    {
+        m_soa_idata_names.at(i) = idata_name.at(i);
+    }
+}
+
 template <typename ParticleType, int NArrayReal, int NArrayInt,
           template<class> class Allocator, class CellAssignor>
 template <typename P, typename Assignor>
@@ -1161,7 +1195,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
         }
     } else {
         ParticleTileType ptile_tmp;
-        ptile_tmp.define(m_num_runtime_real, m_num_runtime_int);
+        ptile_tmp.define(m_num_runtime_real, m_num_runtime_int, &m_soa_rdata_names, &m_soa_idata_names);
         ptile_tmp.resize(np_total);
         // copy re-ordered particles
         gatherParticles(ptile_tmp, ptile, np, permutations);
@@ -1498,7 +1532,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
             tmp_local[lev][index].resize(num_threads);
             soa_local[lev][index].resize(num_threads);
             for (int t = 0; t < num_threads; ++t) {
-                soa_local[lev][index][t].define(m_num_runtime_real, m_num_runtime_int);
+                soa_local[lev][index][t].define(m_num_runtime_real, m_num_runtime_int, &m_soa_rdata_names, &m_soa_idata_names);
             }
         }
     }
diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H
index 01ab0ded869..601b7417d18 100644
--- a/Src/Particle/AMReX_ParticleIO.H
+++ b/Src/Particle/AMReX_ParticleIO.H
@@ -51,20 +51,18 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
 {
     Vector<int> write_real_comp;
     Vector<std::string> tmp_real_comp_names;
-    int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps();

-    for (int i = 0; i < nrc; ++i )
+    int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+    for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i )
     {
         write_real_comp.push_back(1);
         if (real_comp_names.empty())
         {
-            std::stringstream ss;
-            ss << "real_comp" << i;
-            tmp_real_comp_names.push_back(ss.str());
+            tmp_real_comp_names.push_back(getDefaultCompNameReal<ParticleType>(i));
         }
         else
         {
-            tmp_real_comp_names.push_back(real_comp_names[i]);
+            tmp_real_comp_names.push_back(real_comp_names[i-first_rcomp]);
         }
     }

@@ -75,9 +73,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
         write_int_comp.push_back(1);
         if (int_comp_names.empty())
         {
-            std::stringstream ss;
-            ss << "int_comp" << i;
-            tmp_int_comp_names.push_back(ss.str());
+            tmp_int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
         }
         else
         {
@@ -98,14 +94,12 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
 {
     Vector<int> write_real_comp;
     Vector<std::string> real_comp_names;
-    int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps();

-    for (int i = 0; i < nrc; ++i )
+    int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+    for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i )
     {
         write_real_comp.push_back(1);
-        std::stringstream ss;
-        ss << "real_comp" << i;
-        real_comp_names.push_back(ss.str());
+        real_comp_names.push_back(getDefaultCompNameReal<ParticleType>(i));
     }

     Vector<int> write_int_comp;
@@ -113,9 +107,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     for (int i = 0; i < NStructInt + NumIntComps(); ++i )
     {
         write_int_comp.push_back(1);
-        std::stringstream ss;
-        ss << "int_comp" << i;
-        int_comp_names.push_back(ss.str());
+        int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
     }

     WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp,
@@ -182,9 +174,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     Vector<std::string> int_comp_names;
     for (int i = 0; i < NStructInt + NumIntComps(); ++i )
     {
-        std::stringstream ss;
-        ss << "int_comp" << i;
-        int_comp_names.push_back(ss.str());
+        int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
     }

     WriteBinaryParticleData(dir, name,
@@ -211,20 +201,16 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     AMREX_ASSERT(write_int_comp.size()  == NStructInt  + NArrayInt );

     Vector<std::string> real_comp_names;
-    int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps();
-    for (int i = 0; i < nrc; ++i )
+    int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+    for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i )
     {
-        std::stringstream ss;
-        ss << "real_comp" << i;
-        real_comp_names.push_back(ss.str());
+        real_comp_names.push_back(getDefaultCompNameReal<ParticleType>(i));
     }

     Vector<std::string> int_comp_names;
     for (int i = 0; i < NStructInt + NumIntComps(); ++i )
     {
-        std::stringstream ss;
-        ss << "int_comp" << i;
-        int_comp_names.push_back(ss.str());
+        int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
     }

     WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp,
@@ -259,14 +245,12 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
 {
     Vector<int> write_real_comp;
     Vector<std::string> real_comp_names;
-    int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps();

-    for (int i = 0; i < nrc; ++i )
+    int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+    for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i )
     {
         write_real_comp.push_back(1);
-        std::stringstream ss;
-        ss << "real_comp" << i;
-        real_comp_names.push_back(ss.str());
+        real_comp_names.push_back(getDefaultCompNameReal<ParticleType>(i));
     }

     Vector<int> write_int_comp;
@@ -274,9 +258,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     for (int i = 0; i < NStructInt + NumIntComps(); ++i )
     {
         write_int_comp.push_back(1);
-        std::stringstream ss;
-        ss << "int_comp" << i;
-        int_comp_names.push_back(ss.str());
+        int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
     }

     WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp,
@@ -345,9 +327,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     Vector<std::string> int_comp_names;
     for (int i = 0; i < NStructInt + NumIntComps(); ++i )
     {
-        std::stringstream ss;
-        ss << "int_comp" << i;
-        int_comp_names.push_back(ss.str());
+        int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
     }

     WriteBinaryParticleData(dir, name,
@@ -374,20 +354,16 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
     AMREX_ASSERT(write_int_comp.size()  == NStructInt  + NumIntComps() );

     Vector<std::string> real_comp_names;
-    int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps();
-    for (int i = 0; i < nrc; ++i )
+    int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+    for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i )
     {
-        std::stringstream ss;
-        ss << "real_comp" << i;
-        real_comp_names.push_back(ss.str());
+        real_comp_names.push_back(getDefaultCompNameReal<ParticleType>(i));
     }

     Vector<std::string> int_comp_names;
     for (int i = 0; i < NStructInt + NumIntComps(); ++i )
     {
-        std::stringstream ss;
-        ss << "int_comp" << i;
-        int_comp_names.push_back(ss.str());
+        int_comp_names.push_back(getDefaultCompNameInt<ParticleType>(i));
     }

     WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp,
diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H
index a645330e04a..7546ff8a216 100644
--- a/Src/Particle/AMReX_ParticleTile.H
+++ b/Src/Particle/AMReX_ParticleTile.H
@@ -11,7 +11,10 @@
 #include <AMReX_RealVect.H>

 #include <array>
+#include <string>
 #include <type_traits>
+#include <vector>
+

 namespace amrex {

@@ -730,10 +733,15 @@ struct ParticleTile
     ParticleTile& operator= (ParticleTile &&) noexcept = default;
 #endif

-    void define (int a_num_runtime_real, int a_num_runtime_int)
+    void define (
+        int a_num_runtime_real,
+        int a_num_runtime_int,
+        std::vector<std::string>* soa_rdata_names=nullptr,
+        std::vector<std::string>* soa_idata_names=nullptr
+    )
     {
         m_defined = true;
-        GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int);
+        GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int, soa_rdata_names, soa_idata_names);
         m_runtime_r_ptrs.resize(a_num_runtime_real);
         m_runtime_i_ptrs.resize(a_num_runtime_int);
         m_runtime_r_cptrs.resize(a_num_runtime_real);
diff --git a/Src/Particle/AMReX_ParticleUtil.H b/Src/Particle/AMReX_ParticleUtil.H
index b09f1d3583f..f6f2506ec7f 100644
--- a/Src/Particle/AMReX_ParticleUtil.H
+++ b/Src/Particle/AMReX_ParticleUtil.H
@@ -883,6 +883,26 @@ void PermutationForDeposition (Gpu::DeviceVector<index_type>& perm, index_type n
         });
 }

+template <typename P>
+std::string getDefaultCompNameReal (const int i) {
+    int first_r_name = 0;
+    if constexpr (P::is_soa_particle) {
+        if (i < AMREX_SPACEDIM) {
+            constexpr int x_in_ascii = 120;
+            std::string const name{char(x_in_ascii+i)};
+            return name;
+        }
+        first_r_name = AMREX_SPACEDIM;
+    }
+    std::string const name{("real_comp" + std::to_string(i-first_r_name))};
+    return name;
+}
+
+template <typename P>
+std::string getDefaultCompNameInt (const int i) {
+    std::string const name{("int_comp" + std::to_string(i))};
+    return name;
+}

 #ifdef AMREX_USE_HDF5_ASYNC
 void async_vol_es_wait_particle();
diff --git a/Src/Particle/AMReX_StructOfArrays.H b/Src/Particle/AMReX_StructOfArrays.H
index 4de35e085ca..201cf340dc1 100644
--- a/Src/Particle/AMReX_StructOfArrays.H
+++ b/Src/Particle/AMReX_StructOfArrays.H
@@ -6,7 +6,11 @@
 #include <AMReX_Vector.H>
 #include <AMReX_GpuContainers.H>

+#include <algorithm>
 #include <array>
+#include <string>
+#include <vector>
+

 namespace amrex {

@@ -19,11 +23,18 @@ struct StructOfArrays {
     using RealVector = amrex::PODVector<ParticleReal, Allocator<ParticleReal> >;
     using IntVector = amrex::PODVector<int, Allocator<int> >;

-    void define (int a_num_runtime_real, int a_num_runtime_int)
+    void define (
+        int a_num_runtime_real,
+        int a_num_runtime_int,
+        std::vector<std::string>* soa_rdata_names=nullptr,
+        std::vector<std::string>* soa_idata_names=nullptr
+    )
     {
         m_defined = true;
         m_runtime_rdata.resize(a_num_runtime_real);
         m_runtime_idata.resize(a_num_runtime_int );
+        m_rdata_names = soa_rdata_names;
+        m_idata_names = soa_idata_names;
     }

     [[nodiscard]] int NumRealComps () const noexcept { return NReal + m_runtime_rdata.size(); }
@@ -79,6 +90,32 @@ struct StructOfArrays {
         }
     }

+    /** Get access to a particle Real component Array (compile-time and runtime component)
+     *
+     * @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments
+     */
+    [[nodiscard]] RealVector& GetRealData (std::string const & name) {
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined.");
+        auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name);
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components");
+
+        int const index = std::distance(m_rdata_names->begin(), pos);
+        return GetRealData(index);
+    }
+
+    /** Get access to a particle Real component Array (compile-time and runtime component)
+     *
+     * @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments
+     */
+    [[nodiscard]] const RealVector& GetRealData (std::string const & name) const {
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined.");
+        auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name);
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components");
+
+        int const index = std::distance(m_rdata_names->begin(), pos);
+        return GetRealData(index);
+    }
+
     /** Get access to a particle Int component Array (compile-time and runtime component)
      *
      * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments
@@ -118,6 +155,34 @@ struct StructOfArrays {
         }
     }

+    /** Get access to a particle Int component Array (compile-time and runtime component)
+     *
+     * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments
+     * @return
+     */
+    [[nodiscard]] IntVector& GetIntData (std::string const & name) {
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined.");
+        auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name);
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components");
+
+        int const index = std::distance(m_idata_names->begin(), pos);
+        return GetIntData(index);
+    }
+
+    /** Get access to a particle Int component Array (compile-time and runtime component)
+     *
+     * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments
+     * @return
+     */
+    [[nodiscard]] const IntVector& GetIntData (std::string const & name) const {
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined.");
+        auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name);
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components");
+
+        int const index = std::distance(m_idata_names->begin(), pos);
+        return GetIntData(index);
+    }
+
     /**
     * \brief Returns the total number of particles (real and neighbor)
     *
@@ -226,13 +291,20 @@ struct StructOfArrays {
     int m_num_neighbor_particles{0};

 private:
+    // compile-time data
     IdCPU m_idcpu;
     std::array<RealVector, NReal> m_rdata;
     std::array< IntVector,  NInt> m_idata;

+    // runtime data
     std::vector<RealVector> m_runtime_rdata;
     std::vector<IntVector > m_runtime_idata;

+    // names of both compile-time and runtime Real and Int data
+    std::vector<std::string>* m_rdata_names = nullptr;
+    std::vector<std::string>* m_idata_names = nullptr;
+
+    //! whether the runtime components are sized correctly
     bool m_defined{false};
 };

diff --git a/Tests/Particles/NamedSoAComponents/CMakeLists.txt b/Tests/Particles/NamedSoAComponents/CMakeLists.txt
new file mode 100644
index 00000000000..e14ddd68975
--- /dev/null
+++ b/Tests/Particles/NamedSoAComponents/CMakeLists.txt
@@ -0,0 +1,10 @@
+foreach(D IN LISTS AMReX_SPACEDIM)
+    set(_sources     main.cpp)
+    #set(_input_files)
+    #set(_input_files inputs)
+
+    setup_test(${D} _sources _input_files NTHREADS 2)
+
+    unset(_sources)
+    unset(_input_files)
+endforeach()
diff --git a/Tests/Particles/NamedSoAComponents/GNUmakefile b/Tests/Particles/NamedSoAComponents/GNUmakefile
new file mode 100644
index 00000000000..9f49d3ec029
--- /dev/null
+++ b/Tests/Particles/NamedSoAComponents/GNUmakefile
@@ -0,0 +1,22 @@
+AMREX_HOME = ../../../
+
+DEBUG	= FALSE
+
+DIM	= 3
+
+COMP    = gcc
+
+USE_MPI   = TRUE
+USE_OMP   = FALSE
+USE_CUDA  = FALSE
+
+#TINY_PROFILE = TRUE
+USE_PARTICLES = TRUE
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.defs
+
+include ./Make.package
+include $(AMREX_HOME)/Src/Base/Make.package
+include $(AMREX_HOME)/Src/Particle/Make.package
+
+include $(AMREX_HOME)/Tools/GNUMake/Make.rules
diff --git a/Tests/Particles/NamedSoAComponents/Make.package b/Tests/Particles/NamedSoAComponents/Make.package
new file mode 100644
index 00000000000..6b4b865e8fc
--- /dev/null
+++ b/Tests/Particles/NamedSoAComponents/Make.package
@@ -0,0 +1,1 @@
+CEXE_sources += main.cpp
diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp
new file mode 100644
index 00000000000..39b6a70ff52
--- /dev/null
+++ b/Tests/Particles/NamedSoAComponents/main.cpp
@@ -0,0 +1,139 @@
+#include <AMReX.H>
+#include <AMReX_Particle.H>
+#include <AMReX_ParticleContainer.H>
+#include <AMReX_ParticleTile.H>
+#include <AMReX_ParIter.H>
+#include <AMReX_REAL.H>
+#include <AMReX_Vector.H>
+#include <AMReX_GpuContainers.H>
+
+#include <array>
+
+using namespace amrex;
+
+void addParticles ()
+{
+    using PC = ParticleContainerPureSoA<AMREX_SPACEDIM+1, 2>;
+    int is_per[AMREX_SPACEDIM];
+    for (int & d : is_per) {
+        d = 1;
+    }
+
+    RealBox real_box;
+    for (int n = 0; n < AMREX_SPACEDIM; n++)
+    {
+        real_box.setLo(n, 0.0);
+        real_box.setHi(n, 100.0);
+    }
+
+    IntVect domain_lo(AMREX_D_DECL(0, 0, 0));
+    IntVect domain_hi(AMREX_D_DECL(127, 127, 127));
+    const Box base_domain(domain_lo, domain_hi);
+
+    Geometry geom(base_domain, &real_box, CoordSys::cartesian, is_per);
+    BoxArray ba(base_domain);
+    ba.maxSize(64);
+
+    DistributionMapping dm(ba);
+
+    PC pc(geom, dm, ba);
+
+    amrex::Print() << "Original Real SoA component names are: ";
+    for (auto& n : pc.GetRealSoANames()) {
+        amrex::Print() << n << ", ";
+    }
+    amrex::Print() << "\n";
+
+    amrex::Print() << "Original Int SoA component names are: ";
+    for (auto& n : pc.GetIntSoANames()) {
+        amrex::Print() << n << ", ";
+    }
+    amrex::Print() << "\n";
+
+    amrex::Print() << "Adding runtime comps. \n";
+    pc.AddRealComp("real_comp1");
+    pc.AddRealComp(); // without name - should be real_comp2
+    pc.AddIntComp(); // without name - should be int_comp0
+
+    amrex::Print() << "New Real SoA component names are: ";
+    for (auto& n : pc.GetRealSoANames()) {
+        amrex::Print() << n << ", ";
+    }
+    amrex::Print() << "\n";
+
+    amrex::Print() << "New Int SoA component names are: ";
+    for (auto& n : pc.GetIntSoANames()) {
+        amrex::Print() << n << ", ";
+    }
+    amrex::Print() << "\n";
+
+    amrex::Print() << "Reset compile-time SoA names \n";
+    pc.SetSoACompileTimeNames({AMREX_D_DECL("x", "y", "z"), "w"}, {"i1", "i2"});
+
+    amrex::Print() << "New Real SoA component names are: ";
+    for (auto& n : pc.GetRealSoANames()) {
+        amrex::Print() << n << ", ";
+    }
+    amrex::Print() << "\n";
+
+    amrex::Print() << "New Int SoA component names are: ";
+    for (auto& n : pc.GetIntSoANames()) {
+        amrex::Print() << n << ", ";
+    }
+    amrex::Print() << "\n";
+
+    int const NArrayReal = PC::NArrayReal;
+    int const NArrayInt = PC::NArrayInt;
+    using ParticleType = typename PC::ParticleType;
+
+    const int add_num_particles = 5;
+    auto& ptile1 = pc.DefineAndReturnParticleTile(0, 0, 0);
+    ptile1.resize(add_num_particles);
+
+    for (int i = 0; i < add_num_particles; ++i)
+    {
+        for (int d = 0; d < AMREX_SPACEDIM; d++) {
+            ptile1.pos(i, d) = 12.0;
+        }
+        ptile1.getParticleTileData().rdata(AMREX_SPACEDIM)[i] = 1.2;  // w
+
+        ptile1.push_back_int(0, int(ParticleType::NextID()));
+        ptile1.push_back_int(1, amrex::ParallelDescriptor::MyProc());
+    }
+
+    int lev=0;
+    using MyParIter = ParIter_impl<ParticleType, NArrayReal, NArrayInt>;
+    for (MyParIter pti(pc, lev); pti.isValid(); ++pti) {
+        auto& soa = pti.GetStructOfArrays();
+        AMREX_D_TERM(
+                     auto *xp = soa.GetRealData("x").data();,
+                     auto *yp = soa.GetRealData("y").data();,
+                     auto *zp = soa.GetRealData("z").data();
+                     );
+        auto *wp = soa.GetRealData("w").data();
+
+        const int np = pti.numParticles();
+        ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip)
+        {
+            AMREX_D_TERM(
+                         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(xp[ip] == 12_prt,
+                                                          "pos attribute expected to be 12");,
+                         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(yp[ip] == 12_prt,
+                                                          "pos attribute expected to be 12");,
+                         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(zp[ip] == 12_prt,
+                                                          "pos attribute expected to be 12");
+                         );
+            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(wp[ip] == 1.2_prt,
+                                             "pos attribute expected to be 1.2");
+        });
+    }
+}
+
+int main (int argc, char* argv[])
+ {
+    amrex::Initialize(argc,argv);
+    {
+        addParticles();
+    }
+    amrex::Finalize();
+ }