SnapshotCreator.h

Go to the documentation of this file.

#include <algorithm>
#include <cstdint>
#include <fstream>
#include <map>
#include <memory>
#include <ostream>
#include <valarray>
#include <vector>
 
#include "GenericManagingModule.h"
#include "GenericSubmodule.h"
#include "SimulationClassDefault.h"
 
#ifdef MPI_PRESENT
#include <mpi.h>
 
#include "MpiCommunicationPatterns.h"
#endif
 
template <typename T>
class SnapshotCreator
{
    private:
        std::ofstream m_global_ini_file_content;
        std::vector<std::string> m_save_fields{"Temperature"};
        void gatherIniFileContent(
            std::vector<std::shared_ptr<GenericManagingModule<double>>>& all_modules,
            std::vector<std::shared_ptr<GenericSubmodule<double>>>& all_submodules);
        void writeFields(SimulationClassBase<T>* sim);
#ifdef MPI_PRESENT
        MPI_File file_handle;
        MPI_Datatype m_dtype;
        MPI_Offset displacement{0};
        MPI_Offset start{0};
        int count{0};
        int type_size, footer_type_size;
        int numerical_layers, local_number_of_facets, global_number_of_facets;
        int64_t data_sizes[2];
        std::string footer{};
        std::string field_file{};
        void _writeDataMpi(SimulationClassBase<T>* sim);
        void _getFieldFile(SimulationClassBase<T>* sim);
        MPI_Offset _calcDisplacement(const std::valarray<T>& field, SimulationClassBase<T>* sim);
        void _writeFooter(SimulationClassBase<T>* sim);
#endif
 
    public:
        SnapshotCreator(SimulationClassBase<T>* sim,
                        std::vector<std::shared_ptr<GenericManagingModule<double>>>& all_modules,
                        std::vector<std::shared_ptr<GenericSubmodule<double>>>& all_submodules);
};
 
template <typename T>
SnapshotCreator<T>::SnapshotCreator(
    SimulationClassBase<T>* sim,
    std::vector<std::shared_ptr<GenericManagingModule<double>>>& all_modules,
    std::vector<std::shared_ptr<GenericSubmodule<double>>>& all_submodules)
{
    try
    {
        if (sim->m_simulation_config.getStringParameters("snapshot_save_file") == "None")
        {
            // I wanted to use return here, but this is apparently disallowed by the standard
            throw BadInput("String is None.");
        }
        if (sim->my_rank == 0)
        {
            m_global_ini_file_content.open(
                sim->m_simulation_config.getStringParameters("snapshot_save_file"),
                std::ios::binary | std::ios::out | std::ios::trunc);
        }
        // Write the data from the specified fields (m_save_fields) into the snapshot.
#ifdef MPI_PRESENT
        if (sim->my_rank == 0)
        {
            m_global_ini_file_content << "INIFILEDATA:\n";
        }
        _getFieldFile(sim);
        // Get information necessary to calculate byte sizes of output
        numerical_layers = sim->m_simulation_config.getIntParameters("numerical_layers");
        local_number_of_facets = sim->m_simulation_config.getIntParameters("number_of_facets");
        global_number_of_facets =
            sim->m_simulation_config.getIntParameters("global_number_of_facets");
        // Set datatype of field to the one used across this simulation
        m_dtype = mpi_type_of<T>();
        MPI_Type_size(m_dtype, &type_size);
        MPI_Type_size(MPI_UNSIGNED_CHAR, &footer_type_size);
        MPI_File_open(MPI_COMM_WORLD, field_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE,
                      MPI_INFO_NULL, &file_handle);
        _writeDataMpi(sim);
        _writeFooter(sim);
        MPI_File_close(&file_handle);
#else
        writeFields(sim);
#endif
        // Get and write the data from the simulation config (default.ini + user supplied run ini).
        if (sim->my_rank == 0)
        {
            std::unordered_map<std::string, std::any> ini_file_content{
                sim->m_simulation_config.data()};
            for (const auto& [key, map_content] : ini_file_content)
            {
                m_global_ini_file_content << "SimulationConfig" << "$" << key << " = "
                                          << std::any_cast<std::string>(map_content) << "\n";
            }
            // Get an write the data from the module ini files.
            gatherIniFileContent(all_modules, all_submodules);
            m_global_ini_file_content.close();
            std::cout << "\033[32m[SnapshotCreator] Successfully created snapshot file!\033[0m\n";
        }
    }
    catch (const BadInput&)
    {
        // Do nothing here
    }
}
 
template <typename T>
void SnapshotCreator<T>::writeFields(SimulationClassBase<T>* sim)
{
    // I will probably add a little header with timestamp and maybe some more infos.
    m_global_ini_file_content << "FIELDDATA:\n";
    for (const auto& field_name : m_save_fields)
    {
        m_global_ini_file_content << field_name << "," << sim->elapsed_time << ",";
        const std::valarray<T>& field{sim->getField(field_name)};
        for (int i{0}; i < field.size() - 1; i++)
        {
            m_global_ini_file_content << field[i] << std::setprecision(15) << ",";
        }
        m_global_ini_file_content << field[field.size() - 1] << std::setprecision(15) << "\n";
    }
    m_global_ini_file_content << "INIFILEDATA:\n";
}
 
template <typename T>
void SnapshotCreator<T>::gatherIniFileContent(
    std::vector<std::shared_ptr<GenericManagingModule<double>>>& all_modules,
    std::vector<std::shared_ptr<GenericSubmodule<double>>>& all_submodules)
{
    for (auto& module : all_modules)
    {
        module->writeToGlobalIniContent(m_global_ini_file_content);
    }
 
    for (auto& submodule : all_submodules)
    {
        submodule->writeToGlobalIniContent(m_global_ini_file_content);
    }
}
 
#ifdef MPI_PRESENT
template <typename T>
void SnapshotCreator<T>::_writeDataMpi(SimulationClassBase<T>* sim)
{
    if (file_handle == MPI_FILE_NULL)
    {
        std::cerr << "Trying to access an unopened file. Please also use init and output chains "
                     "when using MPI I/O.\n";
        throw std::runtime_error(
            "Trying to access an unopened file. Please also use init and output chains when using "
            "MPI I/O.\n");
    }
    for (const auto& field_name : m_save_fields)
    {
        const std::valarray<T>& field{sim->getField(field_name)};
        displacement = _calcDisplacement(field, sim) + start;
        MPI_File_write_at_all(file_handle, displacement, &field[0], static_cast<int>(field.size()),
                              m_dtype, MPI_STATUS_IGNORE);
        // Update the starting position (in bytes) for the next write
        // Each process from the first k can write at position rank * size of its array.
        // Each process > k needs to account for the larger k processes to determine its pointer
        // start.
        if (field.size() == numerical_layers * local_number_of_facets)
        {
            start += type_size * numerical_layers * global_number_of_facets;
            footer += field_name + "," + std::to_string(sim->elapsed_time) + ","
                      + std::to_string(numerical_layers * global_number_of_facets) + ",";
        }
        else if (field.size() == local_number_of_facets)
        {
            start += type_size * global_number_of_facets;
            footer += field_name + "," + std::to_string(sim->elapsed_time) + ","
                      + std::to_string(global_number_of_facets) + ",";
        }
        else
        {
            start += type_size * field.size() * sim->world_size;
            footer += field_name + "," + std::to_string(sim->elapsed_time) + ","
                      + std::to_string(field.size() * sim->world_size) + ",";
        }
    }
}
 
template <typename T>
MPI_Offset SnapshotCreator<T>::_calcDisplacement(const std::valarray<T>& field,
                                                 SimulationClassBase<T>* sim)
{
    MPI_Offset proc_displacement;
    if (field.size() == numerical_layers * local_number_of_facets)
    {
        // For the block of same size processes, displacement is rank * local_field_size *
        // data_type_byte_size.
        if (sim->my_rank < sim->number_of_larger_procs)
        {
            proc_displacement = sim->my_rank * field.size() * type_size;
        }
        // For the second block, add the first block displacement and count how many second block
        // processes are between the targeted position and the first block.
        // local_field_size_large_proc = local_field_size_small_proc + numerical_layers.
        else
        {
            proc_displacement = (sim->number_of_larger_procs * (field.size() + numerical_layers)
                                 + (sim->my_rank - sim->number_of_larger_procs) * field.size())
                                * type_size;
        }
    }
    else if (field.size() == local_number_of_facets)
    {
        // For the block of same size processes, displacement is rank * local_field_size *
        // data_type_byte_size.
        if (sim->my_rank < sim->number_of_larger_procs)
        {
            proc_displacement = sim->my_rank * field.size() * type_size;
        }
        // For the second block, add the first block displacement and count how many second block
        // processes are between the targeted position and the first block.
        // local_field_size_large_proc = local_field_size_small_proc + 1.
        else
        {
            proc_displacement = (sim->number_of_larger_procs * (field.size() + 1)
                                 + (sim->my_rank - sim->number_of_larger_procs) * field.size())
                                * type_size;
        }
    }
    else
    {
        // Replace this section in the new MR with handling of "irregular" field sizes.
        std::cerr
            << "[SnapshotCreator]: Field in SnapshotCreator is of irregular size! Currently only "
               "supporting sizes of numerical_layers * numbers_of_facets or numbers_of_facets.\n";
        throw std::runtime_error(
            "[SnapshotCreator]: Field in SnapshotCreator is of irregular size! Currently only "
            "supporting sizes of numerical_layers * numbers_of_facets or numbers_of_facets.\n");
    }
    return proc_displacement;
}
 
template <typename T>
void SnapshotCreator<T>::_writeFooter(SimulationClassBase<T>* sim)
{
    MPI_Offset footer_displacement{start};
    MPI_Offset data_size_displacement{
        static_cast<MPI_Offset>(start + footer_type_size * footer.size())};
    if (sim->my_rank == 0)
    {
        MPI_File_write_at(file_handle, footer_displacement, footer.c_str(),
                          static_cast<int>(footer.size()), MPI_UNSIGNED_CHAR, MPI_STATUS_IGNORE);
        data_sizes[0] = start / type_size;
        data_sizes[1] = footer.size();
        MPI_File_write_at(file_handle, data_size_displacement, data_sizes, 2, MPI_INT64_T,
                          MPI_STATUS_IGNORE);
    }
}
 
template <typename T>
void SnapshotCreator<T>::_getFieldFile(SimulationClassBase<T>* sim)
{
    field_file = sim->m_simulation_config.getStringParameters("snapshot_save_file");
    const auto& file_type_start{std::find(field_file.begin(), field_file.end(), '.')};
    if (file_type_start == field_file.end())
    {
        std::cerr << "Could not find file type separator '.' in supplied snapshot file path. "
                     "Please specify the full path with file type.";
        throw std::runtime_error(
            "Could not find file type separator '.' in supplied snapshot file path. "
            "Please specify the full path with file type.");
    }
    std::string insertion{"_fields"};
    field_file.insert(file_type_start, insertion.begin(), insertion.end());
}
#endif