MpiDomainDecomposition.h

Go to the documentation of this file.

#ifndef MPI_DOMAIN_DECOMPOSITION_H
#define MPI_DOMAIN_DECOMPOSITION_H
 
#ifdef MPI_PRESENT
 
#include <mpi.h>
 
#include "SimulationClassBase.h"
 
template <typename T>
void domainDecompositionOneDim(SimulationClassBase<T>* sim)
{
    MPI_Comm_rank(MPI_COMM_WORLD, &sim->my_rank);
    MPI_Comm_size(MPI_COMM_WORLD, &sim->world_size);
    int number_of_facets{sim->m_simulation_config.getIntParameters("number_of_facets")};
    sim->m_simulation_config.forceValueOverwrite("global_number_of_facets",
                                                 std::to_string(number_of_facets));
 
    // The following distinction is needed for if the number of facets is not evenly divisable by
    // the number of processes.
    // Minimum number of of facets manages by each process (that's why its rounded off).
    sim->min_number_facets_per_proc = number_of_facets / sim->world_size;
    // Number of processes that need to manage min_number_facets_per_proc + 1 facet.
    sim->number_of_larger_procs =
        number_of_facets - sim->min_number_facets_per_proc * sim->world_size;
 
    // Check if no color split needs to be done, if number_of_facets is integer divisible by
    // world_size
    sim->other_color_exists = sim->number_of_larger_procs > 0 ? 1 : 0;
    // The first number_of_larger_procs processes get one extra facet, so that all facets are
    // accounted for in the end.
    if (sim->my_rank < sim->number_of_larger_procs)
    {
        sim->m_simulation_config.forceValueOverwrite(
            "number_of_facets", std::to_string(sim->min_number_facets_per_proc + 1));
        sim->m_simulation_config.forceValueOverwrite(
            "global_starting_facet_number",
            std::to_string(sim->my_rank * (sim->min_number_facets_per_proc + 1)));
        // The processes will be grouped along their length for collective communications along
        // subcomms.
        sim->my_color = 0;
    }
    else
    {
        sim->m_simulation_config.forceValueOverwrite(
            "number_of_facets", std::to_string(sim->min_number_facets_per_proc));
        sim->m_simulation_config.forceValueOverwrite(
            "global_starting_facet_number",
            std::to_string(sim->number_of_larger_procs * (sim->min_number_facets_per_proc + 1)
                           + (sim->my_rank - sim->number_of_larger_procs)
                                 * sim->min_number_facets_per_proc));
        // The processes will be grouped along their length for collective communications along
        // sub-comms.
        sim->my_color = 1;
    }
    // Skip creation of sub- and intercomm if not necessary
    if (sim->other_color_exists == 0)
    {
        sim->sub_size = -1;
        sim->my_sub_rank = -1;
        sim->remote_leader = -1;
        sim->my_inter_rank = -1;
        sim->sub_comm = MPI_COMM_NULL;
        sim->inter_comm = MPI_COMM_NULL;
        return;
    }
    // Creation of the sub-communicators
    MPI_Comm_split(MPI_COMM_WORLD, sim->my_color, 0, &sim->sub_comm);
    MPI_Comm_size(sim->sub_comm, &sim->sub_size);
    MPI_Comm_rank(sim->sub_comm, &sim->my_sub_rank);
 
    // Leader designation for the creation of the inter-communicator.
    // Leader is always process 0 in the other sub-comm.
    if (sim->my_color == 0)
    {
        sim->remote_leader = sim->sub_size;
    }
    else
    {
        sim->remote_leader = 0;
    }
 
    // Creation of inter-communicator
    if (sim->world_size > 1)
    {
        MPI_Intercomm_create(sim->sub_comm, 0, MPI_COMM_WORLD, sim->remote_leader, 0,
                             &sim->inter_comm);
        MPI_Comm_rank(sim->inter_comm, &sim->my_inter_rank);
    }
}
 
#endif  // MPI_PRESENT
 
#endif  // MPI_DOMAIN_DECOMPOSITION_H