SnapshotLoader.h

Go to the documentation of this file.

#include <algorithm>
#include <cstdint>
#include <exception>
#include <memory>
#include <valarray>
#include <vector>
 
#include "CsvParser.h"
#include "GenericManagingModule.h"
#include "GenericSubmodule.h"
#include "IniParser.h"
#include "SimulationClassDefault.h"
 
#ifdef MPI_PRESENT
#include <mpi.h>
 
#include "MpiCommunicationPatterns.h"
#endif
 
template <typename T>
class SnapshotLoader
{
    private:
        std::unique_ptr<CsvParser> m_snapshot_field_data;
        std::unique_ptr<IniParserSnapshot> m_snapshot_ini_data;
        bool m_no_snapshot{false};
        bool m_keep_sim_config{false};
        bool m_keep_module_inis{false};
#ifdef MPI_PRESENT
        MPI_File file_handle;
        MPI_Datatype m_dtype;
        MPI_Offset full_file_size;
        MPI_Offset displacement{0};
        MPI_Offset start{0};
        int count{0};
        int type_size, footer_type_size;
        int numerical_layers, local_number_of_facets, global_number_of_facets;
        long data_sizes[2];
        void _parseFieldsMPI(SimulationClassBase<T>* sim);
        void _getFieldFile(SimulationClassBase<T>* sim);
        void _parseFooterString(const std::vector<unsigned char>& footer,
                                std::vector<std::string>& field_names, T& elapsed_time,
                                std::vector<MPI_Offset>& field_sizes);
        void _fillField(const std::string& field_name, const MPI_Offset field_size,
                        SimulationClassBase<T>* sim);
        MPI_Offset _calcDisplacement(const MPI_Offset field_size, SimulationClassBase<T>* sim);
        std::string field_file{};
#endif
 
    public:
        SnapshotLoader(SimulationClassBase<T>* sim);
 
        void overwriteSimFields(SimulationClassBase<T>* sim);
        void overwriteIniFiles(SimulationClassBase<T>* sim,
                               std::vector<std::shared_ptr<GenericManagingModule<T>>>& all_modules,
                               std::vector<std::shared_ptr<GenericSubmodule<T>>>& all_submodules);
        void overwriteSimConfig(SimulationClassBase<T>* sim);
};
 
template <typename T>
SnapshotLoader<T>::SnapshotLoader(SimulationClassBase<T>* sim)
{
    // Check if sim config or module ini file retention is selected.
    try
    {
        m_keep_sim_config = sim->m_simulation_config.getBoolParameters("_snapshot_keep_sim");
    }
    catch (const BadInput&)
    {
        // Do nothing here
    }
    try
    {
        m_keep_module_inis = sim->m_simulation_config.getBoolParameters("_snapshot_keep_module");
    }
    catch (const BadInput&)
    {
        // Do nothing here
    }
 
    // -----------------------------
    // Load snapshot if present
    // -----------------------------
    std::string snapshotStr;
    try
    {
        snapshotStr = sim->m_simulation_config.getStringParameters("snapshot");
    }
    catch (const BadInput&)
    {
        std::cout << "[SnapshotLoader] No snapshot key found in config. Skipping snapshot load.\n";
        m_no_snapshot = true;
        return;
    }
 
    if (snapshotStr.empty() || snapshotStr == "None")
    {
        std::cout << "[SnapshotLoader] Snapshot value is 'None'. Skipping snapshot load.\n";
        m_no_snapshot = true;
        return;
    }
 
    // Key exists and value is valid
    std::string filename = snapshotStr;
    std::string filetype{"snapshot"};
 
    try
    {
#ifdef MPI_PRESENT
        _getFieldFile(sim);
#else
        m_snapshot_field_data = std::make_unique<CsvParser>(filename, filetype);
#endif
        m_snapshot_ini_data = std::make_unique<IniParserSnapshot>(filename);
        if (sim->my_rank == 0)
        {
            std::cout << "\033[32m[SnapshotLoader] Successfully read snapshot file: " << filename
                      << "\033[0m\n";
        }
    }
    catch (const std::exception& e)
    {
        std::cerr << "[SnapshotLoader] Failed to read snapshot file '" << filename
                  << "': " << e.what() << "\n";
        m_no_snapshot = true;
    }
}
 
template <typename T>
void SnapshotLoader<T>::overwriteSimFields(SimulationClassBase<T>* sim)
{
    if (m_no_snapshot)
    {
        return;
    }
#ifdef MPI_PRESENT
    _parseFieldsMPI(sim);
#else
    std::vector<std::vector<T>> all_fields_content;
    std::vector<std::string> all_field_names;
    double elapsed_time{0.0};
 
    m_snapshot_field_data->getSnapshotContents(all_fields_content, all_field_names, elapsed_time);
 
    // For each field that is part of the snapshot
    for (int i{0}; i < all_field_names.size(); i++)
    {
        // Write the snapshot data into the field map. Can't just copy here due to valarray vector
        // mismatch.
        for (int j{0}; j < all_fields_content[i].size(); j++)
        {
            sim->m_field_map[all_field_names[i]][j] = all_fields_content[i][j];
        }
    }
 
    sim->elapsed_time = elapsed_time;
    sim->time_step = int(elapsed_time / sim->m_simulation_config.getDoubleParameters("time_delta"));
#endif
    if (sim->my_rank == 0)
    {
        std::cout
            << "\033[32m[SnapshotLoader] Successfully loaded snapshot file field data\033[0m\n";
    }
}
 
template <typename T>
void SnapshotLoader<T>::overwriteIniFiles(
    SimulationClassBase<T>* sim,
    std::vector<std::shared_ptr<GenericManagingModule<T>>>& all_modules,
    std::vector<std::shared_ptr<GenericSubmodule<T>>>& all_submodules)
{
    if (m_no_snapshot || m_keep_module_inis)
    {
        return;
    }
    std::vector<std::string> all_module_names;
    all_module_names = m_snapshot_ini_data->getModuleNames();
 
    bool module_placed{false};
    for (const auto& module_name : all_module_names)
    {
        if (module_name == "SimulationConfig")
        {
            module_placed = true;
        }
        for (const auto& loaded_module : all_modules)
        {
            if (module_placed)
            {
                break;
            }
            if (module_name == loaded_module->getNameLocal())
            {
                loaded_module->overwriteLocalIniContent(
                    m_snapshot_ini_data->getModuleContents(module_name));
                module_placed = true;
            }
        }
 
        for (const auto& loaded_submodule : all_submodules)
        {
            if (module_placed)
            {
                break;
            }
            if (module_name == loaded_submodule->getNameLocal())
            {
                loaded_submodule->overwriteLocalIniContent(
                    m_snapshot_ini_data->getModuleContents(module_name));
                module_placed = true;
            }
        }
        if (module_placed)
        {
            module_placed = false;
        }
        else
        {
            std::cerr << "[SnapshotLoader]: Couldn't find module: " << module_name
                      << " in all loaded modules. Please load this module to complete snapshot "
                         "loading.\n";
            throw std::runtime_error(
                static_cast<std::string>("[SnapshotLoader]: Couldn't find specified module from "
                                         "snapshot file in all loaded modules."));
        }
    }
    if (sim->my_rank == 0)
    {
        std::cout << "\033[32m[SnapshotLoader] Successfully loaded snapshot file module ini "
                     "files\033[0m\n";
    }
}
 
template <typename T>
void SnapshotLoader<T>::overwriteSimConfig(SimulationClassBase<T>* sim)
{
    if (m_no_snapshot || m_keep_sim_config)
    {
        return;
    }
    for (const auto& [key, map_content] :
         m_snapshot_ini_data->getModuleContents("SimulationConfig"))
    {
        // Do not write snapshot save file into the config/overwrite it, as it would lead to a loop
        // that always overwrites the current snapshot.
        if (key != "snapshot_save_file")
        {
            sim->m_simulation_config.forceValueOverwrite(key,
                                                         std::any_cast<std::string>(map_content));
        }
    }
    if (sim->my_rank == 0)
    {
        std::cout << "\033[32m[SnapshotLoader] Successfully loaded snapshot file simulation "
                     "configuration data\033[0m\n";
    }
}
 
#ifdef MPI_PRESENT
template <typename T>
void SnapshotLoader<T>::_parseFieldsMPI(SimulationClassBase<T>* sim)
{
    int64_t field_data_size{};
    int64_t footer_data_size{};
    T elapsed_time{};
    std::vector<std::string> field_names{};
    std::vector<MPI_Offset> field_sizes{};
 
    numerical_layers = sim->m_simulation_config.getIntParameters("numerical_layers");
    local_number_of_facets = sim->m_simulation_config.getIntParameters("number_of_facets");
    global_number_of_facets = sim->m_simulation_config.getIntParameters("global_number_of_facets");
    // Set datatype of field to the one used across this simulation
    m_dtype = mpi_type_of<T>();
    MPI_Type_size(m_dtype, &type_size);
    MPI_Type_size(MPI_UNSIGNED_CHAR, &footer_type_size);
 
    MPI_File_open(MPI_COMM_WORLD, field_file.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_handle);
    MPI_File_get_size(file_handle, &full_file_size);
    MPI_File_read_at_all(file_handle, full_file_size - sizeof(int64_t), &footer_data_size, 1,
                         MPI_INT64_T, MPI_STATUS_IGNORE);
    MPI_File_read_at_all(file_handle, full_file_size - 2 * sizeof(int64_t), &field_data_size, 1,
                         MPI_INT64_T, MPI_STATUS_IGNORE);
    std::vector<unsigned char> footer_content(footer_data_size);
    MPI_File_read_at_all(file_handle, field_data_size * sizeof(T), &footer_content[0],
                         footer_data_size, MPI_UNSIGNED_CHAR, MPI_STATUS_IGNORE);
    _parseFooterString(footer_content, field_names, elapsed_time, field_sizes);
    for (int i{0}; i < field_names.size(); i++)
    {
        _fillField(field_names[i], field_sizes[i], sim);
    }
    sim->elapsed_time = elapsed_time;
    sim->time_step = int(elapsed_time / sim->m_simulation_config.getDoubleParameters("time_delta"));
}
 
template <typename T>
void SnapshotLoader<T>::_parseFooterString(const std::vector<unsigned char>& footer,
                                           std::vector<std::string>& field_names, T& elapsed_time,
                                           std::vector<MPI_Offset>& field_sizes)
{
    std::string footer_string{footer.begin(), footer.end()};
    auto starting_it{footer_string.begin()};
    int current_entry{0};
    for (auto it{footer_string.begin()}; it != footer_string.end(); it++)
    {
        if (*it == ',')
        {
            std::string current_string{starting_it, it};
            if (current_entry % 3 == 0)
            {
                field_names.push_back(current_string);
            }
            else if (current_entry % 3 == 1)
            {
                elapsed_time = std::stod(current_string);
            }
            else if (current_entry % 3 == 2)
            {
                field_sizes.push_back(std::stoi(current_string));
            }
            starting_it = it + 1;
            current_entry++;
        }
    }
}
 
template <typename T>
void SnapshotLoader<T>::_getFieldFile(SimulationClassBase<T>* sim)
{
    field_file = sim->m_simulation_config.getStringParameters("snapshot");
    const auto& file_type_start{std::find(field_file.begin(), field_file.end(), '.')};
    if (file_type_start == field_file.end())
    {
        std::cerr << "Could not find file type separator '.' in supplied snapshot file path. "
                     "Please specify the full path with file type.";
        throw std::runtime_error(
            "Could not find file type separator '.' in supplied snapshot file path. "
            "Please specify the full path with file type.");
    }
    std::string insertion{"_fields"};
    field_file.insert(file_type_start, insertion.begin(), insertion.end());
}
 
template <typename T>
void SnapshotLoader<T>::_fillField(const std::string& field_name, const MPI_Offset field_size,
                                   SimulationClassBase<T>* sim)
{
    if (sim->m_field_map.find(field_name) == sim->m_field_map.end())
    {
        std::cerr << "Field: " << field_name
                  << " not found in m_field_map! Could not fully load snapshot - terminating.";
        throw std::runtime_error(
            "Field could not be found in m_field_map despite loading snapshot. Check "
            "your snapshot file and configuration!");
    }
 
    displacement = _calcDisplacement(field_size, sim) + start;
    MPI_File_read_at_all(file_handle, displacement, &(sim->m_field_map[field_name][0]),
                         static_cast<int>((sim->m_field_map[field_name]).size()), m_dtype,
                         MPI_STATUS_IGNORE);
    start += field_size * type_size;
}
 
template <typename T>
MPI_Offset SnapshotLoader<T>::_calcDisplacement(const MPI_Offset field_size,
                                                SimulationClassBase<T>* sim)
{
    MPI_Offset proc_displacement;
    if (field_size == numerical_layers * global_number_of_facets)
    {
        // For the block of same size processes, displacement is rank * local_number_of_facets *
        // data_type_byte_size.
        if (sim->my_rank < sim->number_of_larger_procs)
        {
            proc_displacement =
                sim->my_rank * local_number_of_facets * numerical_layers * type_size;
        }
        // For the second block, add the first block displacement and count how many second block
        // processes are between the targeted position and the first block.
        // local_number_of_facets_large_proc = local_number_of_facets_small_proc + numerical_layers.
        else
        {
            proc_displacement =
                (sim->number_of_larger_procs * (local_number_of_facets + 1)
                 + (sim->my_rank - sim->number_of_larger_procs) * local_number_of_facets)
                * numerical_layers * type_size;
        }
    }
    else if (field_size == global_number_of_facets)
    {
        // For the block of same size processes, displacement is rank * local_number_of_facets *
        // data_type_byte_size.
        if (sim->my_rank < sim->number_of_larger_procs)
        {
            proc_displacement = sim->my_rank * local_number_of_facets * type_size;
        }
        // For the second block, add the first block displacement and count how many second block
        // processes are between the targeted position and the first block.
        // local_number_of_facets_large_proc = local_number_of_facets_small_proc + 1.
        else
        {
            proc_displacement =
                (sim->number_of_larger_procs * (local_number_of_facets + 1)
                 + (sim->my_rank - sim->number_of_larger_procs) * local_number_of_facets)
                * type_size;
        }
    }
    else
    {
        // Replace this section in the new MR with handling of "irregular" field sizes.
        std::cerr
            << "[SnapshotLoader]: Field in SnapshotLoader is of irregular size! Currently only "
               "supporting sizes of numerical_layers * numbers_of_facets or numbers_of_facets.\n";
        throw std::runtime_error(
            "[SnapshotLoader]: Field in SnapshotLoader is of irregular size! Currently only "
            "supporting sizes of numerical_layers * numbers_of_facets or numbers_of_facets.\n");
    }
    return proc_displacement;
}
#endif