OutputCsvTool.h

Go to the documentation of this file.

#ifndef OUTPUT_CSV_TOOL_H
#define OUTPUT_CSV_TOOL_H
 
#define OUTPUT_CSV_TOOL_VERSION "9"
 
#include <fstream>
#include <iomanip>
#include <iostream>
#include <memory>
#include <ostream>
#include <string>
#include <valarray>
#include <vector>
 
#include "../../src/GenericManagingModule.h"
#include "../../src/ModuleFactory.h"
 
#ifdef MPI_PRESENT
#include <mpi.h>
 
#endif
 
template <typename T>
class OutputCsvTool : public GenericManagingModule<T>
{
    private:
        static bool m_registered;
        std::string m_ini_filepath{"tools/OutputCsvTool.ini"};
        std::string m_output_filepath{};
        std::string m_save_mode{};
        std::vector<std::string>
            m_save_fields;  // Right now this lives only in the local ini, but we make it live in
                            // the sim class, so that the other modules can write their save request
                            // to it.
        std::vector<int> m_save_steps{};
        int m_currentIndex{0};
        std::ofstream output_file;
 
        void _writeData();
        void _writeDataSurface(int period);
 
#ifdef MPI_PRESENT
        MPI_File file_handle;
        MPI_Datatype m_dtype;
        int displacement{0};
        int start{0};
        int count{0};
        int type_size, footer_type_size;
        int numerical_layers, local_number_of_facets, global_number_of_facets;
        long data_sizes[2];
        std::string footer{};
        void _writeDataMpi();
        void _writeDataSurfaceMpi(int period);
        int _calcDisplacement(const std::valarray<T>& field);
        void _writeFooter();
#endif
 
    public:
        OutputCsvTool(SimulationClassBase<T>* sim);
        bool setup(std::vector<std::shared_ptr<GenericSubmodule<T>>> all_submodules) override;
        bool exec(std::string_view param) override;  // main functionality of the module
 
        bool init() override;
        bool preTimeStep() override { return true; };
        bool postNonLinIter() override { return true; };
        bool postTimeStep() override;
        bool output() override;
        std::vector<std::string> getChainInsertion() const override
        {
            return this->ini_file_data.getStringVectorParameters("chain_insertion");
        };
 
        static std::shared_ptr<GenericManagingModule<T>> createMethode(SimulationClassBase<T>* sim)
        {
            return std::make_shared<OutputCsvTool<T>>(sim);
        };
        static std::string getName() { return "OutputCsvTool"; };
        std::string_view getNameLocal() const override { return "OutputCsvTool"; };
};
 
// ================= Implementation =================
 
template <typename T>
OutputCsvTool<T>::OutputCsvTool(SimulationClassBase<T>* sim) : GenericManagingModule<T>(sim)
{
    try
    {
        std::string ini_folder_path{
            this->sim->m_simulation_config.getStringParameters("ini_folder_path")};
        this->ini_file_data.loadUserInput(ini_folder_path + m_ini_filepath);
        m_output_filepath = this->ini_file_data.getStringParameters("file_path");
        m_save_mode = this->ini_file_data.getStringParameters("save_mode");
        m_save_fields = this->ini_file_data.getStringVectorParameters("save_fields");
        this->m_generic_submodules = this->ini_file_data.getStringVectorParameters("submodules");
    }
    catch (const BadInput& e)
    {
        std::cerr << "[OutputCsvTool]: " << e.what() << '\n';
        throw BadInput(static_cast<std::string>("[OutputCsvTool]: ")
                       + static_cast<std::string>(e.what()));
    }
}
 
template <typename T>
bool OutputCsvTool<T>::setup(std::vector<std::shared_ptr<GenericSubmodule<T>>> all_submodules)
{
#ifdef MPI_PRESENT
    // Get information necessary to calculate byte sizes of output
    numerical_layers = this->sim->m_simulation_config.getIntParameters("numerical_layers");
    local_number_of_facets = this->sim->m_simulation_config.getIntParameters("number_of_facets");
    global_number_of_facets =
        this->sim->m_simulation_config.getIntParameters("global_number_of_facets");
    // Set datatype of field to the one used across this simulation
    m_dtype = mpi_type_of<T>();
    MPI_Type_size(m_dtype, &type_size);
    MPI_Type_size(MPI_UNSIGNED_CHAR, &footer_type_size);
    if (this->sim->my_rank == 0)
    {
        // Flush the file
        std::ofstream file(m_output_filepath);
    }
#else
    // Flush the file
    std::ofstream file(m_output_filepath);
#endif
    if (m_save_mode == "specific_time_steps")
    {
        try
        {
            std::vector<double> save_steps{
                this->ini_file_data.getDoubleVectorParameters("save_time_steps")};
            for (int i{0}; i < save_steps.size(); i++)
            {
                m_save_steps.push_back(static_cast<int>(save_steps[i]));
            }
        }
        catch (const BadInput& e)
        {
            std::cerr << "[OutputCsvTool]: " << e.what() << '\n';
            throw BadInput(static_cast<std::string>("[OutputCsvTool]: ")
                           + static_cast<std::string>(e.what()));
        }
    }
    return true;
}
 
template <typename T>
bool OutputCsvTool<T>::exec(std::string_view param)
{
    if (param == "InitChain")
    {
        return init();
    }
    if (param == "PreTimeStepChain")
    {
        return preTimeStep();
    }
    if (param == "PostNonLinIterChain")
    {
        return postNonLinIter();
    }
    if (param == "PostTimeStepChain")
    {
        return postTimeStep();
    }
    if (param == "OutputChain")
    {
        return output();
    }
    return false;
}
 
template <typename T>
bool OutputCsvTool<T>::init()
{
#ifdef MPI_PRESENT
    MPI_File_open(MPI_COMM_WORLD, m_output_filepath.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE,
                  MPI_INFO_NULL, &file_handle);
#endif
    return true;
}
 
template <typename T>
bool OutputCsvTool<T>::postTimeStep()
{
    try
    {
        if ((m_save_mode == "specific_time_steps") && (m_currentIndex < m_save_steps.size())
            && (m_save_steps[m_currentIndex] == this->sim->time_step))
        {
#ifdef MPI_PRESENT
            _writeDataMpi();
#else
            _writeData();
#endif
            m_currentIndex++;
        }
        else if ((m_save_mode == "specific_time_steps_surface")
                 && (m_currentIndex < m_save_steps.size())
                 && (m_save_steps[m_currentIndex] == this->sim->time_step))
        {
#ifdef MPI_PRESENT
            _writeDataSurfaceMpi(
                this->sim->m_simulation_config.getIntParameters("numerical_layers"));
#else
            _writeDataSurface(this->sim->m_simulation_config.getIntParameters("numerical_layers"));
#endif
            m_currentIndex++;
        }
        else if ((m_save_mode == "interval")
                 && (this->sim->time_step % this->ini_file_data.getIntParameters("interval") == 0))
        {
#ifdef MPI_PRESENT
            _writeDataMpi();
#else
            _writeData();
#endif
        }
        else if ((m_save_mode == "interval_surface")
                 && (this->sim->time_step % this->ini_file_data.getIntParameters("interval") == 0))
        {
#ifdef MPI_PRESENT
            _writeDataSurfaceMpi(
                this->sim->m_simulation_config.getIntParameters("numerical_layers"));
#else
            _writeDataSurface(this->sim->m_simulation_config.getIntParameters("numerical_layers"));
#endif
        }
#ifdef MPI_PRESENT
        if (this->sim->elapsed_time + this->sim->m_simulation_config.getIntParameters("time_delta")
                > this->sim->m_simulation_config.getIntParameters("time_max")
            && !(footer.empty()))
        {
            // If this is the last time step, you can write the footer already.
            _writeFooter();
        }
#endif
    }
    catch (const BadInput& e)
    {
        std::cerr << "[OutputCsvTool]: " << e.what() << '\n';
        throw BadInput(static_cast<std::string>("[OutputCsvTool]: ")
                       + static_cast<std::string>(e.what()));
    }
    return true;
}
 
template <typename T>
bool OutputCsvTool<T>::output()
{
#ifdef MPI_PRESENT
    // Check to only write footer, if it hasn't already been written (in preTimeStep)
    if (!(footer.empty()))
    {
        _writeFooter();
    }
    MPI_File_close(&file_handle);
#else
    if (m_save_mode == "surface")
    {
        try
        {
            _writeDataSurface(this->sim->m_simulation_config.getIntParameters("numerical_layers"));
        }
        catch (const BadInput& e)
        {
            std::cerr << "[OutputCsvTool]: " << e.what() << '\n';
            throw BadInput(static_cast<std::string>("[OutputCsvTool]: ")
                           + static_cast<std::string>(e.what()));
        }
    }
    else
    {
        _writeData();
    }
#endif
    return true;
}
 
template <typename T>
void OutputCsvTool<T>::_writeData()
{
    output_file.open(m_output_filepath, std::ios::app);
 
    for (const auto& field_name : m_save_fields)
    {
        output_file << field_name << "," << this->sim->elapsed_time << ",";
        try
        {
            const std::valarray<T>& field{this->sim->getField(field_name)};
            for (int i{0}; i < field.size() - 1; i++)
            {
                output_file << field[i] << std::setprecision(15) << ",";
            }
            output_file << field[field.size() - 1] << std::setprecision(15) << "\n";
        }
        catch (const BadInput& e)
        {
            std::cerr << "[OutputCsvTool]: " << field_name
                      << " module not "
                         "loaded or accessable!\n "
                         "[OutputCsvTool]:"
                      << e.what() << '\n';
            throw BadInput(static_cast<std::string>("[OutputCsvTool]: ") + field_name
                           + static_cast<std::string>(" module "
                                                      "not loaded or "
                                                      "accessable!\n [OutputCsvTool]: ")
                           + static_cast<std::string>(e.what()));
        }
    }
 
    output_file.close();
}
 
#ifdef MPI_PRESENT
template <typename T>
void OutputCsvTool<T>::_writeDataMpi()
{
    if (file_handle == MPI_FILE_NULL)
    {
        std::cerr << "[OutputCsvTool]: Trying to access an unopened file. Please also use init and "
                     "output chains "
                     "when using MPI I/O.\n";
        throw std::runtime_error(
            "[OutputCsvTool]: Trying to access an unopened file. Please also use init and output "
            "chains when using "
            "MPI I/O.");
    }
    for (const auto& field_name : m_save_fields)
    {
        try
        {
            const std::valarray<T>& field{this->sim->getField(field_name)};
            displacement = _calcDisplacement(field) + start;
            MPI_File_write_at_all(file_handle, displacement, &field[0], field.size(), m_dtype,
                                  MPI_STATUS_IGNORE);
            // Update the starting position (in bytes) for the next write
            // Each process from the first k can write at position rank * size of its array.
            // Each process > k needs to account for the larger k processes to determine its pointer
            // start.
            if (field.size() == numerical_layers * local_number_of_facets)
            {
                start += type_size * numerical_layers * global_number_of_facets;
                footer += field_name + "," + std::to_string(this->sim->elapsed_time) + ","
                          + std::to_string(numerical_layers * global_number_of_facets) + ",";
            }
            else if (field.size() == local_number_of_facets)
            {
                start += type_size * global_number_of_facets;
                footer += field_name + "," + std::to_string(this->sim->elapsed_time) + ","
                          + std::to_string(global_number_of_facets) + ",";
            }
            else
            {
                start += type_size * field.size() * this->sim->world_size;
                footer += field_name + "," + std::to_string(this->sim->elapsed_time) + ","
                          + std::to_string(field.size() * this->sim->world_size) + ",";
            }
        }
        catch (const BadInput& e)
        {
            std::cerr << "[OutputCsvTool]: " << field_name
                      << " module not "
                         "loaded or accessable!\n "
                         "[OutputCsvTool]:"
                      << e.what() << '\n';
            throw BadInput(static_cast<std::string>("[OutputCsvTool]: ") + field_name
                           + static_cast<std::string>(" module "
                                                      "not loaded or "
                                                      "accessable!\n [OutputCsvTool]: ")
                           + static_cast<std::string>(e.what()));
        }
    }
}
#endif
 
template <typename T>
void OutputCsvTool<T>::_writeDataSurface(int period)
{
    output_file.open(m_output_filepath, std::ios::app);
 
    for (const auto& field_name : m_save_fields)
    {
        output_file << field_name << "," << this->sim->elapsed_time << ",";
        try
        {
            const std::valarray<T>& field{this->sim->getField(field_name)};
            for (int i{0}; i < static_cast<int>(field.size() / period) - 1; i++)
            {
                output_file << field[i * period] << std::setprecision(15) << ",";
            }
            output_file << field[field.size() - period] << std::setprecision(15) << "\n";
        }
        catch (const BadInput& e)
        {
            std::cerr << "[OutputCsvTool]: " << field_name
                      << " module not "
                         "loaded or accessable!\n "
                         "[OutputCsvTool]:"
                      << e.what() << '\n';
            throw BadInput(static_cast<std::string>("[OutputCsvTool]: ") + field_name
                           + static_cast<std::string>(" module "
                                                      "not loaded or "
                                                      "accessable!\n [OutputCsvTool]:")
                           + static_cast<std::string>(e.what()));
        }
    }
 
    output_file.close();
}
 
#ifdef MPI_PRESENT
template <typename T>
void OutputCsvTool<T>::_writeDataSurfaceMpi(int period)
{
    if (file_handle == MPI_FILE_NULL)
    {
        std::cerr << "[OutputCsvTool]: Trying to access an unopened file. Please also use init and "
                     "output chains "
                     "when using MPI I/O.\n";
        throw std::runtime_error(
            "[OutputCsvTool]: Trying to access an unopened file. Please also use init and output "
            "chains when using "
            "MPI I/O.");
    }
    for (const auto& field_name : m_save_fields)
    {
        // Get only the surface values of the field variable.
        try
        {
            const std::valarray<T>& field{this->sim->getField(field_name)};
            std::valarray<T> field_surface(field.size() / period);
            for (int i{0}; i < field_surface.size(); i++)
            {
                field_surface[i] = field[i * period];
            }
            // Update the starting position (in bytes) for the next write
            // Each process from the first k can write at position rank * size of its array.
            // Each process > k needs to account for the larger k processes to determine its pointer
            // start.
            displacement = _calcDisplacement(field) / period + start;
            MPI_File_write_at_all(file_handle, displacement, &field_surface[0],
                                  field_surface.size(), m_dtype, MPI_STATUS_IGNORE);
            // Update the starting position (in bytes) for the next write
            if (field.size() == numerical_layers * local_number_of_facets)
            {
                start += type_size * numerical_layers * global_number_of_facets / period;
                footer += field_name + "," + std::to_string(this->sim->elapsed_time) + ","
                          + std::to_string(numerical_layers * global_number_of_facets / period)
                          + ",";
            }
        }
        catch (const BadInput& e)
        {
            std::cerr << "[OutputCsvTool]: " << field_name
                      << " module not "
                         "loaded or accessable!\n "
                         "[OutputCsvTool]:"
                      << e.what() << '\n';
            throw BadInput(static_cast<std::string>("[OutputCsvTool]: ") + field_name
                           + static_cast<std::string>(" module "
                                                      "not loaded or "
                                                      "accessable!\n [OutputCsvTool]: ")
                           + static_cast<std::string>(e.what()));
        }
    }
}
 
template <typename T>
int OutputCsvTool<T>::_calcDisplacement(const std::valarray<T>& field)
{
    int proc_displacement;
    if (field.size() == numerical_layers * local_number_of_facets)
    {
        // For the block of same size processes, displacement is rank * local_field_size *
        // data_type_byte_size.
        if (this->sim->my_rank < this->sim->number_of_larger_procs)
        {
            proc_displacement = this->sim->my_rank * field.size() * type_size;
        }
        // For the second block, add the first block displacement and count how many second block
        // processes are between the targeted position and the first block.
        // local_field_size_large_proc = local_field_size_small_proc + numerical_layers.
        else
        {
            proc_displacement =
                (this->sim->number_of_larger_procs * (field.size() + numerical_layers)
                 + (this->sim->my_rank - this->sim->number_of_larger_procs) * field.size())
                * type_size;
        }
    }
    else if (field.size() == local_number_of_facets)
    {
        // For the block of same size processes, displacement is rank * local_field_size *
        // data_type_byte_size.
        if (this->sim->my_rank < this->sim->number_of_larger_procs)
        {
            proc_displacement = this->sim->my_rank * field.size() * type_size;
        }
        // For the second block, add the first block displacement and count how many second block
        // processes are between the targeted position and the first block.
        // local_field_size_large_proc = local_field_size_small_proc + 1.
        else
        {
            proc_displacement =
                (this->sim->number_of_larger_procs * (field.size() + 1)
                 + (this->sim->my_rank - this->sim->number_of_larger_procs) * field.size())
                * type_size;
        }
    }
    else
    {
        proc_displacement = type_size * field.size();
    }
    return proc_displacement;
}
 
template <typename T>
void OutputCsvTool<T>::_writeFooter()
{
    int footer_displacement{start};
    int data_size_displacement{static_cast<int>(start + footer_type_size * footer.size())};
    if (this->sim->my_rank == 0)
    {
        MPI_File_write_at(file_handle, footer_displacement, footer.c_str(), footer.size(),
                          MPI_UNSIGNED_CHAR, MPI_STATUS_IGNORE);
        data_sizes[0] = start / type_size;
        data_sizes[1] = footer.size();
        MPI_File_write_at(file_handle, data_size_displacement, data_sizes, 2, MPI_LONG,
                          MPI_STATUS_IGNORE);
    }
}
#endif
 
template <typename T>
bool OutputCsvTool<T>::m_registered = ModuleFactory<T>::registerModule(getName(), createMethode);
 
#endif  // OUTPUT_CSV_TOOL_H