Update ngen to generate catchment output variables in a combined netCDF file (NGWPC-9011)

CMakeLists.txt

@@ -355,6 +355,27 @@ if(NGEN_WITH_SQLITE)
     target_include_directories(partitionGenerator PUBLIC AFTER "${NGEN_INC_DIR}/geopackage")
 endif()
 
+if(NGEN_WITH_NETCDF)
+    add_library(NetCDFCreator STATIC "${NGEN_SRC_DIR}/NetCDFCreator.cpp")
+    target_include_directories(NetCDFCreator PUBLIC
+        ${NGEN_INC_DIR}
+        ${NGEN_INC_DIR}/core
+        ${NGEN_INC_DIR}/core/catchment
+        ${NGEN_INC_DIR}/realizations/catchment
+        ${NGEN_INC_DIR}/simulation_time
+        ${NGEN_INC_DIR}/utilities
+        ${NGEN_INC_DIR}/geojson
+        ${NGEN_INC_DIR}/forcing
+        ${NGEN_INC_DIR}/core/mediator
+        ${Python_INCLUDE_DIRS}
+        ${pybind11_INCLUDE_DIR}
+        ${MPI_CXX_INCLUDE_DIRS}
+    )
+    target_link_libraries(NetCDFCreator PUBLIC NGen::config_header)
+    target_link_libraries(NetCDFCreator PRIVATE NetCDF)
+    target_link_libraries(ngen PRIVATE NetCDFCreator)
+endif()
+
 target_link_libraries(partitionGenerator PUBLIC NGen::core NGen::geojson)
 if(NGEN_WITH_SQLITE)
     target_link_libraries(partitionGenerator PUBLIC NGen::geopackage)

include/core/Layer.hpp

@@ -9,6 +9,7 @@
 #include "State_Exception.hpp"
 #include "geojson/FeatureBuilder.hpp"
 #include <boost/core/span.hpp>
+#include <map>
 
 namespace hy_features
 {
@@ -110,6 +111,8 @@ namespace ngen
                                    std::unordered_map<std::string, int> &nexus_indexes,
                                    int current_step);
 
+        std::map<std::string, std::string> get_catchment_output_data_for_timestep();
+
         protected:
 
         const LayerDescription description;
@@ -121,6 +124,7 @@ namespace ngen
         //TODO is this really required at the top level? or can this be moved to SurfaceLayer?
         const geojson::GeoJSON catchment_data;
         long output_time_index;       
+        std::map<std::string, std::string> catchment_output_values;
 
     };
 }

include/core/NetCDFCreator.hpp

@@ -0,0 +1,47 @@
+#ifndef NGEN_NETCDF_CREATOR_HPP
+#define NGEN_NETCDF_CREATOR_HPP
+
+#include <NGenConfig.h>
+
+#if NGEN_WITH_NETCDF
+#include <vector>
+#include <Formulation_Manager.hpp>
+#include <Simulation_Time.hpp>
+#include <Catchment_Formulation.hpp>
+
+namespace netCDF {
+    class NcVar;
+    class NcFile;
+}
+class NetCDFCreator
+{
+public:
+    NetCDFCreator(std::shared_ptr<realization::Formulation_Manager> manager, 
+        const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs);
+    NetCDFCreator() = delete;
+    ~NetCDFCreator();
+
+    void write_simulations_response_from_formulation(size_t time_index, std::map<std::string, std::string> catchment_output_values);
+
+    netCDF::NcFile& get_ncfile();
+
+protected:
+    void add_output_variable_info_from_formulation(); 
+
+    void retrieve_output_variables_mpi();
+
+    std::vector<double> string_split(std::string str, char delimiter);
+
+    bool create_ncfile();
+
+    void close_ncfile();
+
+private:
+    std::shared_ptr<netCDF::NcFile> catchmentNcFile;
+    std::shared_ptr<realization::Formulation_Manager> manager_;
+    std::shared_ptr<Simulation_Time> sim_time_;
+    std::vector<std::string> catchments;
+    std::vector<netCDF::NcVar> nc_output_variables;
+};
+#endif // NGEN_WITH_NETCDF
+#endif // NGEN_NETCDF_CREATOR_HPP

include/core/NgenSimulation.hpp

@@ -4,8 +4,13 @@
 #include <NGenConfig.h>
 
 #include <Simulation_Time.hpp>
+#include <Formulation_Manager.hpp>
 #include <Layer.hpp>
 
+#if NGEN_WITH_NETCDF
+    #include <NetCDFCreator.hpp>
+#endif
+
 namespace hy_features
 {
     class HY_Features;
@@ -59,6 +64,8 @@ class NgenSimulation
     size_t get_num_output_times() const;
     std::string get_timestamp_for_step(int step) const;
 
+    void create_netcdf_writer(std::shared_ptr<realization::Formulation_Manager> manager, std::string nc_output_file_name, int mpi_rank, int mpi_num_procs);
+
 private:
     void advance_models_one_output_step();
 
@@ -81,6 +88,9 @@ class NgenSimulation
     // Serialization template will be defined and instantiated in the .cpp file
     template <class Archive>
     void serialize(Archive& ar);
+
+    //Pointer to netcdfcreator to write simulation output per timestep.
+    std::unique_ptr<NetCDFCreator> nc_writer_;
 };
 
 #endif

include/realizations/catchment/Bmi_Formulation.hpp

@@ -180,6 +180,15 @@ namespace realization {
             return REQUIRED_PARAMETERS;
         }
 
+        /**
+         * Get the units for the output variables organized as a vector of strings.
+         *
+         * @return The units for the output variables organized as a vector.
+         */
+        const std::vector<std::string> &get_output_variable_units() const {
+            return output_variable_units;
+        }
+
         virtual bool is_bmi_input_variable(const std::string &var_name) const = 0;
 
         /**
@@ -238,6 +247,10 @@ namespace realization {
             output_header_fields = output_headers;
         }
 
+        void set_output_variable_units(const std::vector<std::string> &output_units) {
+            output_variable_units = output_units;
+        }
+
         /**
          * Set the names of variables in formulation output.
          *
@@ -264,6 +277,11 @@ namespace realization {
          * the BMI module output variables accessible to the instance.
          */
         std::vector<std::string> output_variable_names;
+        /**
+         * Output units corresponding to the variables output by the realization, as defined in
+         * `output_variables`.
+         */
+        std::vector<std::string> output_variable_units;
         /** The degree of precision in output values when converting to text. */
         int output_precision;
 

include/realizations/catchment/Formulation_Manager.hpp

@@ -232,6 +232,10 @@ namespace realization {
                 return this->formulations.at(id);
             }
 
+            std::map<std::string, std::shared_ptr<Catchment_Formulation>> get_all_formulations() const {
+                return this->formulations;
+            }
+
             std::shared_ptr<Catchment_Formulation> get_domain_formulation(long id) const {
                 return this->domain_formulations.at(id);
             }

src/NGen.cpp

@@ -11,6 +11,10 @@
 #include <Catchment_Formulation.hpp>
 #include <HY_Features.hpp>
 
+#if NGEN_WITH_NETCDF
+#include <NetCDFCreator.hpp>
+#endif
+
 #if NGEN_WITH_SQLITE3
 #include <geopackage.hpp>
 #endif
@@ -712,12 +716,19 @@ int main(int argc, char* argv[]) {
                                                        std::move(nexus_indexes),
                                                        mpi_rank,
                                                        mpi_num_procs);
-
+    #if NGEN_WITH_NETCDF
+    #if NGEN_WITH_MPI
+        LOG("Under MPI mode, a NetCDF file for catchment output values is not created due to limitations in NetCDFCxx4 library.", LogLevel::INFO);
+    #else
+        simulation->create_netcdf_writer(manager, "catchment_output", mpi_rank, mpi_num_procs); //create a NetCDF file only for Non-MPI run.
+    #endif //NGEN_WITH_MPI
+    #endif //NGEN_WITH_NETCDF
     auto time_done_init                             = std::chrono::steady_clock::now();
     std::chrono::duration<double> time_elapsed_init = time_done_init - time_start;
     LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO);
-
+    
     simulation->run_catchments();
+
 
 #if NGEN_WITH_MPI
     MPI_Barrier(MPI_COMM_WORLD);