diff --git a/ChronoGrapher/CMakeLists.txt b/ChronoGrapher/CMakeLists.txt index 5c4ccc2a7..9d28dff48 100644 --- a/ChronoGrapher/CMakeLists.txt +++ b/ChronoGrapher/CMakeLists.txt @@ -11,6 +11,7 @@ add_executable(chrono_grapher) target_include_directories(chrono_grapher PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/chrono_common/include + ${HDF5_INCLUDE_DIRS} ) target_sources(chrono_grapher PRIVATE diff --git a/ChronoGrapher/include/CSVFileChunkExtractor.h b/ChronoGrapher/include/CSVFileChunkExtractor.h deleted file mode 100644 index 5966df683..000000000 --- a/ChronoGrapher/include/CSVFileChunkExtractor.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef CSV_FILE_CHUNK_EXTRACTOR_H -#define CSV_FILE_CHUNK_EXTRACTOR_H - -#include - -#include -#include -#include - - -namespace chronolog -{ - -class CSVFileStoryChunkExtractor: public StoryChunkExtractorBase -{ - -public: - CSVFileStoryChunkExtractor(std::string const& chrono_process_id_card, std::string const& csv_files_root_dir); - - ~CSVFileStoryChunkExtractor(); - - virtual int processStoryChunk(StoryChunk*); - -private: - std::string chrono_process_id; - std::string rootDirectory; -}; - - -} // namespace chronolog -#endif diff --git a/ChronoGrapher/include/ChronoGrapherConfiguration.h b/ChronoGrapher/include/ChronoGrapherConfiguration.h index 341a5e9d1..34456b1b0 100644 --- a/ChronoGrapher/include/ChronoGrapherConfiguration.h +++ b/ChronoGrapher/include/ChronoGrapherConfiguration.h @@ -7,6 +7,7 @@ #include #include +#include namespace chronolog { @@ -19,7 +20,7 @@ struct GrapherConfiguration RPCProviderConf VISOR_REGISTRY_SERVICE_CONF; LogConf LOG_CONF; DataStoreConf DATA_STORE_CONF{}; - ExtractorReaderConf EXTRACTOR_CONF; + ExtractionModuleConfiguration EXTRACTION_MODULE_CONF; GrapherConfiguration() { @@ -44,18 +45,22 @@ struct GrapherConfiguration DATA_STORE_CONF.acceptance_window_secs = 180; DATA_STORE_CONF.inactive_story_delay_secs = 300; - EXTRACTOR_CONF.story_files_dir = "/tmp/"; + EXTRACTION_MODULE_CONF.extraction_stream_count = 1; } int parseJsonConf(json_object*); [[nodiscard]] std::string to_String() const { - return "[CHRONO_GRAPHER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", EXTRACTOR_CONF: " + EXTRACTOR_CONF.to_String() + "]"; + std::string a_string = "[CHRONO_GRAPHER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + + ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + + ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + + ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + + ", LOG_CONF: " + LOG_CONF.to_String() + + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + ", "; + + EXTRACTION_MODULE_CONF.to_string(a_string); + a_string += "]"; + return a_string; } }; diff --git a/ChronoGrapher/include/GrapherExtractionChain.h b/ChronoGrapher/include/GrapherExtractionChain.h new file mode 100644 index 000000000..c764c0382 --- /dev/null +++ b/ChronoGrapher/include/GrapherExtractionChain.h @@ -0,0 +1,110 @@ +#ifndef CHRONO_GRAPHER_EXTRACTION_CHAIN +#define CHRONO_GRAPHER_EXTRACTION_CHAIN + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace tl = thallium; + +namespace chronolog +{ +using Extractor = std::variant; + +class ChronoGrapherExtractionChain +{ + std::vector theExtractors; + +public: + ChronoGrapherExtractionChain() {} + + ~ChronoGrapherExtractionChain() { theExtractors.clear(); } + + void add_extractor(Extractor e) { theExtractors.push_back(std::move(e)); } + + int process_chunk(StoryChunk* chunk) + { + int chain_result = CL_SUCCESS; + + // If extractor fails, mark the chain result as a failure, + // but keep going for the others. + for(auto& e: theExtractors) + { + int extractor_result = + std::visit([chunk](auto& extractor) -> int { return extractor.process_chunk(chunk); }, e); + + if(CL_SUCCESS != extractor_result) + { + chain_result = extractor_result; + } + } + return chain_result; + } + + bool is_active_chain() const + { + if(theExtractors.empty()) + { + return false; + } + + for(const auto& e: theExtractors) + { + bool active = std::visit([](const auto& extractor) -> bool { return extractor.is_active(); }, e); + + // if any single extractor is NOT active, the whole chain fails + if(!active) + { + return false; + } + } + + return true; + } + + int activate(ExtractionModuleConfiguration const& extraction_conf, ServiceId const& service_id) + { + int ret_value = CL_SUCCESS; + + for(auto iter = extraction_conf.extractors.begin(); iter != extraction_conf.extractors.end(); ++iter) + { + + if((*iter).first == "csv_extractor") + { + StoryChunkExtractorCSV csv_extractor(service_id); + ret_value = csv_extractor.reset((*iter).second); + if(CL_SUCCESS != ret_value) + { + break; + } + add_extractor(csv_extractor); + } + else if((*iter).first == "hdf5_extractor") + { + HDF5FileChunkExtractor hdf5_extractor; + ret_value = hdf5_extractor.reset((*iter).second); + if(CL_SUCCESS != ret_value) + { + break; + } + add_extractor(hdf5_extractor); + } + else if((*iter).first == "logging_extractor") + { + add_extractor(LoggingExtractor()); + } + } + + return ret_value; + } +}; + +} // namespace chronolog + +#endif diff --git a/ChronoGrapher/include/HDF5FileChunkExtractor.h b/ChronoGrapher/include/HDF5FileChunkExtractor.h index fe18ea45a..ea60960cd 100644 --- a/ChronoGrapher/include/HDF5FileChunkExtractor.h +++ b/ChronoGrapher/include/HDF5FileChunkExtractor.h @@ -1,9 +1,9 @@ #ifndef CHRONOLOG_HDF5_FILE_CHUNK_EXTRACTOR_H #define CHRONOLOG_HDF5_FILE_CHUNK_EXTRACTOR_H +#include #include - namespace chronolog { @@ -12,12 +12,17 @@ class StoryChunk; class HDF5FileChunkExtractor { public: - HDF5FileChunkExtractor(std::string const& hdf5_files_root_dir); + HDF5FileChunkExtractor(std::string const& hdf5_archive_dir = "/tmp"); ~HDF5FileChunkExtractor(); int process_chunk(StoryChunk*); + int reset(std::string const& hdf5_archive_dir); + int reset(json_object*); + + bool is_active() const { return (std::filesystem::exists(rootDirectory)); } + private: std::string rootDirectory; }; diff --git a/ChronoGrapher/src/CSVFileChunkExtractor.cpp b/ChronoGrapher/src/CSVFileChunkExtractor.cpp deleted file mode 100644 index 8d9962c84..000000000 --- a/ChronoGrapher/src/CSVFileChunkExtractor.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include -#include - -#include - -#include "CSVFileChunkExtractor.h" - -namespace tl = thallium; - -chronolog::CSVFileStoryChunkExtractor::CSVFileStoryChunkExtractor(std::string const& process_id_card, - std::string const& csv_files_root_dir) - : chrono_process_id(process_id_card) - , rootDirectory(csv_files_root_dir) -{} - -///////////// -chronolog::CSVFileStoryChunkExtractor::~CSVFileStoryChunkExtractor() -{ - LOG_INFO("[CSVFileStoryChunkExtractor] Destructor called. Cleaning up..."); -} - -///////////// -int chronolog::CSVFileStoryChunkExtractor::processStoryChunk(chronolog::StoryChunk* story_chunk) -{ - std::ofstream chunk_fstream; - std::string chunk_filename(rootDirectory); - chunk_filename += "/" + story_chunk->getChronicleName() + "." + story_chunk->getStoryName() + "." + - std::to_string(story_chunk->getStartTime() / 1000000000) + "." + - std::to_string(story_chunk->getEndTime() / 1000000000) + ".csv"; - - tl::xstream es = tl::xstream::self(); - LOG_DEBUG("[CSVFileStoryChunkExtractor] Processing StoryChunk: ES={}, ULT={}, StoryID={}, StartTime={}", - es.get_rank(), - tl::thread::self_id(), - story_chunk->getStoryId(), - story_chunk->getStartTime()); - // current thread if the only one that has this storyChunk and the only one that's writing to this chunk csv file - // thus no additional locking is needed ... - chunk_fstream.open(chunk_filename, std::ofstream::out | std::ofstream::app); - for(auto event_iter = story_chunk->begin(); event_iter != story_chunk->end(); ++event_iter) - { - chronolog::LogEvent const& event = (*event_iter).second; - chunk_fstream << event << std::endl; - } - chunk_fstream.close(); - LOG_DEBUG("[CSVFileStoryChunkExtractor] Finished processing StoryChunk. File={}", chunk_filename); - return chronolog::CL_SUCCESS; -} diff --git a/ChronoGrapher/src/ChronoGrapher.cpp b/ChronoGrapher/src/ChronoGrapher.cpp index 2f71fcb0f..0ca4eab6a 100644 --- a/ChronoGrapher/src/ChronoGrapher.cpp +++ b/ChronoGrapher/src/ChronoGrapher.cpp @@ -18,11 +18,9 @@ #include #include #include -#include -#include -#include #include #include +#include namespace chl = chronolog; namespace tl = thallium; @@ -158,18 +156,30 @@ int main(int argc, char** argv) LOG_INFO("[ChronoGrapher] GrapherIdCard: {}", chl::to_string(processIdCard)); - // Instantiate MemoryDataStore & ExtractionModule - chronolog::ChunkIngestionQueue ingestionQueue; + // Instantiate StoryChunkExtractionModule + + std::string log_string; + GRAPHER_CONF.EXTRACTION_MODULE_CONF.to_string(log_string); + LOG_INFO("[ChronoGrapherInstance] Initializing StoryChunkExtractionModule with {}", log_string); + + chronolog::StoryChunkExtractionModule theExtractionModule; - std::string archive_directory = GRAPHER_CONF.EXTRACTOR_CONF.story_files_dir; + theExtractionModule.getExtractionChain().activate(GRAPHER_CONF.EXTRACTION_MODULE_CONF, + processIdCard.getRecordingServiceId()); - chl::LoggingExtractor logging_extractor; - chl::HDF5FileChunkExtractor hdf5_extractor(archive_directory); + theExtractionModule.initialize(GRAPHER_CONF.EXTRACTION_MODULE_CONF.extraction_stream_count); - chronolog::StoryChunkExtractionModule extractionModule(logging_extractor, hdf5_extractor); + if(!theExtractionModule.is_initialized()) + { + LOG_ERROR("[ChronoGrapher] StoryChunkExtractionModule failed to initialize, exiting"); + return (-1); + } + + // Instantiate MemoryDataStore + chronolog::ChunkIngestionQueue ingestionQueue; chronolog::GrapherDataStore theDataStore(ingestionQueue, - extractionModule.getExtractionQueue(), + theExtractionModule.getExtractionQueue(), GRAPHER_CONF.DATA_STORE_CONF.max_story_chunk_size, GRAPHER_CONF.DATA_STORE_CONF.story_chunk_duration_secs, GRAPHER_CONF.DATA_STORE_CONF.acceptance_window_secs, @@ -291,7 +301,7 @@ int main(int argc, char** argv) tl::abt scope; theDataStore.startDataCollection(3); // start extraction streams & threads - extractionModule.startExtraction(2); + theExtractionModule.startExtraction(); /// Main loop for sending stats message until receiving SIGTERM ____________________________________________________ // now we are ready to ingest records coming from the storyteller clients .... @@ -318,7 +328,7 @@ int main(int argc, char** argv) theDataStore.shutdownDataCollection(); // Shutdown extraction module // drain extractionQueue and stop extraction xStreams - extractionModule.shutdownExtraction(); + theExtractionModule.shutdownExtraction(); // these are not probably needed as thallium handles the engine finalization... // recordingEngine.finalize(); // collectionEngine.finalize(); diff --git a/ChronoGrapher/src/ChronoGrapherConfiguration.cpp b/ChronoGrapher/src/ChronoGrapherConfiguration.cpp index 90950b76b..e345eead0 100644 --- a/ChronoGrapher/src/ChronoGrapherConfiguration.cpp +++ b/ChronoGrapher/src/ChronoGrapherConfiguration.cpp @@ -1,7 +1,7 @@ #include #include #include - +#include namespace chl = chronolog; @@ -113,31 +113,22 @@ int chronolog::GrapherConfiguration::parseJsonConf(json_object* json_conf) if(DATA_STORE_CONF.parseJsonConf(data_store_conf) != chl::CL_SUCCESS) return chl::CL_ERR_INVALID_CONF; } - else if(strcmp(key, "Extractors") == 0) + else if(strcmp(key, "ExtractionModule") == 0) { if(!json_object_is_type(val, json_type_object)) { - std::cerr << "[GrapherConfiguration] Invalid 'Extractors': expected object" << std::endl; + std::cerr << "[GrapherConfiguration] Invalid 'ExtractionModule' segment: expected json object" + << std::endl; return chl::CL_ERR_INVALID_CONF; } - json_object* extractors = json_object_object_get(json_conf, "Extractors"); - json_object_object_foreach(extractors, key, val) + json_object* extraction_module_json_object = json_object_object_get(json_conf, "ExtractionModule"); + if(EXTRACTION_MODULE_CONF.parse_json_object(extraction_module_json_object) != chl::CL_SUCCESS) { - if(strcmp(key, "story_files_dir") == 0) - { - if(!json_object_is_type(val, json_type_string)) - { - std::cerr << "[GrapherConfiguration] Invalid 'story_files_dir': expected string" << std::endl; - return chl::CL_ERR_INVALID_CONF; - } - EXTRACTOR_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown Extractors configuration " << key << std::endl; - } + return chl::CL_ERR_INVALID_CONF; + std::cerr << "[GrapherConfiguration] Error parsing ExtractionModule configuration: " << std::endl; } } + else { std::cerr << "[GrapherConfiguration] Unknown Grapher configuration " << key << std::endl; diff --git a/ChronoGrapher/src/HDF5FileChunkExtractor.cpp b/ChronoGrapher/src/HDF5FileChunkExtractor.cpp index 599df2467..e75d2d075 100644 --- a/ChronoGrapher/src/HDF5FileChunkExtractor.cpp +++ b/ChronoGrapher/src/HDF5FileChunkExtractor.cpp @@ -1,6 +1,9 @@ +#include +#include #include #include +#include #include #include #include @@ -12,13 +15,68 @@ namespace chl = chronolog; chronolog::HDF5FileChunkExtractor::HDF5FileChunkExtractor(const std::string& hdf5_files_root_dir) : rootDirectory(hdf5_files_root_dir) { - LOG_DEBUG("[HDF5FileChunkExtractor] Destructor called. Cleaning up..."); + LOG_TRACE("[HDF5FileChunkExtractor] Destructor called. Cleaning up..."); } chronolog::HDF5FileChunkExtractor::~HDF5FileChunkExtractor() { - LOG_DEBUG("[HDF5FileChunkExtractor] Destructor called. Cleaning up..."); + LOG_TRACE("[HDF5FileChunkExtractor] Destructor called. Cleaning up..."); } +////// + +int chronolog::HDF5FileChunkExtractor::reset(std::string const& new_archive_dir) +{ + rootDirectory = new_archive_dir; + LOG_INFO("HDF5FileChunkExtractor] Reset success: using directory :", rootDirectory); + return chl::CL_SUCCESS; +} + +// json block for HDF5 File Chunk Extractor looks like this +// +// "extractor_name": { +// "type": "hdf5_extractor", +// "hdf5_archive_dir": "/tmp/hdf5_archive" +// } +// +////////// + +int chronolog::HDF5FileChunkExtractor::reset(json_object* json_block) +{ + if((json_block == nullptr) || !json_object_is_type(json_block, json_type_object) || + (json_object_object_get(json_block, "type") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "type"), json_type_string) || + (std::string("hdf5_extractor").compare(json_object_get_string(json_object_object_get(json_block, "type"))) != 0)) + { + rootDirectory = "/tmp"; + LOG_ERROR("HDF5FileChunkExtractor] Reset failure: invalid json_conf; using {}", rootDirectory); + return chl::CL_ERR_INVALID_CONF; + } + + if((json_object_object_get(json_block, "hdf5_archive_dir") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "hdf5_archive_dir"), json_type_string)) + { + rootDirectory = "/tmp"; + LOG_ERROR("HDF5FileChunkExtractor] Reset failure: invalid json_conf; using {} ", rootDirectory); + return chl::CL_ERR_INVALID_CONF; + } + + rootDirectory = json_object_get_string(json_object_object_get(json_block, "hdf5_archive_dir")); + + // check if archive directory exists and is writable by the extractor process + if(!std::filesystem::exists(rootDirectory)) + { + rootDirectory = "/tmp"; + LOG_ERROR("HDF5FileChunkExtractor] Reset failure: hdf5_archive_dir doesn't exist or not writable; using {} ", + rootDirectory); + return chl::CL_ERR_INVALID_CONF; + } + + LOG_INFO("HDF5FileChunkExtractor] Reset success: using {}", rootDirectory); + return chl::CL_SUCCESS; +} + +////// + int chronolog::HDF5FileChunkExtractor::process_chunk(chl::StoryChunk* story_chunk) { diff --git a/ChronoKeeper/CMakeLists.txt b/ChronoKeeper/CMakeLists.txt index 9f97ccdce..b960b1b19 100644 --- a/ChronoKeeper/CMakeLists.txt +++ b/ChronoKeeper/CMakeLists.txt @@ -17,6 +17,7 @@ target_sources(chrono_keeper PRIVATE src/ChronoKeeperConfiguration.cpp src/KeeperStoryPipeline.cpp src/KeeperDataStore.cpp + src/DualEndpointChunkExtractorRDMA.cpp ) target_link_libraries(chrono_keeper diff --git a/ChronoKeeper/include/ChronoKeeperConfiguration.h b/ChronoKeeper/include/ChronoKeeperConfiguration.h index 64a799833..38e16e0f3 100644 --- a/ChronoKeeper/include/ChronoKeeperConfiguration.h +++ b/ChronoKeeper/include/ChronoKeeperConfiguration.h @@ -4,16 +4,20 @@ #include #include #include -#include -#include + +#include #include #include "chronolog_errcode.h" #include +#include namespace chronolog { + +/////////////////// + struct KeeperConfiguration { uint32_t RECORDING_GROUP; @@ -22,9 +26,10 @@ struct KeeperConfiguration RPCProviderConf VISOR_REGISTRY_SERVICE_CONF; RPCProviderConf KEEPER_GRAPHER_DRAIN_SERVICE_CONF; DataStoreConf DATA_STORE_CONF{}; - ExtractorReaderConf EXTRACTOR_CONF; LogConf LOG_CONF; + ExtractionModuleConfiguration EXTRACTION_MODULE_CONF; + KeeperConfiguration() { RECORDING_GROUP = 0; @@ -48,18 +53,23 @@ struct KeeperConfiguration DATA_STORE_CONF.acceptance_window_secs = 10; DATA_STORE_CONF.inactive_story_delay_secs = 180; - EXTRACTOR_CONF.story_files_dir = "/tmp/"; + EXTRACTION_MODULE_CONF.extraction_stream_count = 1; } int parseJsonConf(json_object*); + [[nodiscard]] std::string to_String() const { - return "[CHRONO_GRAPHER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", EXTRACTOR_CONF: " + EXTRACTOR_CONF.to_String() + "]"; + std::string a_string = "[CHRONO_GRAPHER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + + ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + + ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + + ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + + ", LOG_CONF: " + LOG_CONF.to_String() + + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + ", "; + + EXTRACTION_MODULE_CONF.to_string(a_string); + a_string += "]"; + return a_string; } }; diff --git a/ChronoKeeper/include/DualEndpointChunkExtractorRDMA.h b/ChronoKeeper/include/DualEndpointChunkExtractorRDMA.h new file mode 100644 index 000000000..b51fa1943 --- /dev/null +++ b/ChronoKeeper/include/DualEndpointChunkExtractorRDMA.h @@ -0,0 +1,52 @@ +#ifndef DUAL_CHUNK_EXTRACTOR_RDMA_H +#define DUAL_CHUNK_EXTRACTOR_RDMA_H + +#include +#include + +namespace tl = thallium; + +namespace chronolog +{ + +class StoryChunk; +class RDMATransferAgent; + +class DualEndpointChunkExtractorRDMA +{ + +public: + DualEndpointChunkExtractorRDMA(tl::engine& tl_engine, + ServiceId const& receiver_player = ServiceId(), + ServiceId const& receiver_grapher = ServiceId()); + DualEndpointChunkExtractorRDMA(DualEndpointChunkExtractorRDMA const& other); + DualEndpointChunkExtractorRDMA& operator=(DualEndpointChunkExtractorRDMA const& other); + + ~DualEndpointChunkExtractorRDMA(); + + tl::engine& get_sender_engine() const { return sender_tl_engine; } + ServiceId const& get_player_receiver_id() const { return player_receiver_service_id; } + ServiceId const& get_grapher_receiver_id() const { return grapher_receiver_service_id; } + + int process_chunk(StoryChunk*); + + int reset_player_endpoint(ServiceId const& receiver_player); + int reset_grapher_endpoint(ServiceId const& receiver_grapher); + int reset(json_object*); + + bool is_active() const { return (nullptr != rdma_sender_for_grapher) && (nullptr != rdma_sender_for_player); } + +private: + tl::engine& sender_tl_engine; // local tl::engine + ServiceId player_receiver_service_id; // ChronoGrapher receiving ServiceId + ServiceId grapher_receiver_service_id; // ChronoPlayer receiving ServiceId + RDMATransferAgent* rdma_sender_for_player; + RDMATransferAgent* rdma_sender_for_grapher; + + void restart_rdma_sender_for_grapher(ServiceId const&); + void restart_rdma_sender_for_player(ServiceId const&); +}; + + +} // namespace chronolog +#endif diff --git a/ChronoKeeper/include/KeeperExtractionChain.h b/ChronoKeeper/include/KeeperExtractionChain.h new file mode 100644 index 000000000..606fe0e44 --- /dev/null +++ b/ChronoKeeper/include/KeeperExtractionChain.h @@ -0,0 +1,125 @@ +#ifndef CHRONO_KEEPER_EXTRACTION_CHAIN +#define CHRONO_KEEPER_EXTRACTION_CHAIN + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace tl = thallium; + +namespace chronolog +{ +using Extractor = + std::variant; + +class ChronoKeeperExtractionChain +{ + std::vector theExtractors; + +public: + ChronoKeeperExtractionChain() {} + + ~ChronoKeeperExtractionChain() { theExtractors.clear(); } + + void add_extractor(Extractor e) { theExtractors.push_back(std::move(e)); } + + int process_chunk(StoryChunk* chunk) + { + int chain_result = CL_SUCCESS; + + // If extractor fails, mark the chain result as a failure, + // but keep going for the others. + for(auto& e: theExtractors) + { + int extractor_result = + std::visit([chunk](auto& extractor) -> int { return extractor.process_chunk(chunk); }, e); + + if(CL_SUCCESS != extractor_result) + { + chain_result = extractor_result; + } + } + return chain_result; + } + + bool is_active_chain() const + { + if(theExtractors.empty()) + { + return false; + } + + for(const auto& e: theExtractors) + { + bool active = std::visit([](const auto& extractor) -> bool { return extractor.is_active(); }, e); + + // if any single extractor is NOT active, the whole chain fails + if(!active) + { + return false; + } + } + + return true; + } + + int activate(tl::engine& extraction_engine, + ExtractionModuleConfiguration const& extraction_conf, + ServiceId const& service_id) + { + + int ret_value = CL_SUCCESS; + + for(auto iter = extraction_conf.extractors.begin(); iter != extraction_conf.extractors.end(); ++iter) + { + + if((*iter).first == "csv_extractor") + { + StoryChunkExtractorCSV csv_extractor(service_id); + ret_value = csv_extractor.reset((*iter).second); + if(CL_SUCCESS != ret_value) + { + break; + } + add_extractor(csv_extractor); + } + else if((*iter).first == "single_endpoint_rdma_extractor") + { + StoryChunkExtractorRDMA single_endpoint_rdma_extractor(extraction_engine); + ret_value = single_endpoint_rdma_extractor.reset((*iter).second); + if(CL_SUCCESS != ret_value) + { + break; + } + add_extractor(single_endpoint_rdma_extractor); + } + else if((*iter).first == "dual_endpoint_rdma_extractor") + { + DualEndpointChunkExtractorRDMA dual_endpoint_rdma_extractor(extraction_engine); + ret_value = dual_endpoint_rdma_extractor.reset((*iter).second); + if(CL_SUCCESS != ret_value) + { + break; + } + add_extractor(dual_endpoint_rdma_extractor); + } + else if((*iter).first == "logging_extractor") + { + add_extractor(LoggingExtractor()); + } + } + + return ret_value; + } +}; + +} // namespace chronolog + +#endif diff --git a/ChronoKeeper/src/ChronoKeeperConfiguration.cpp b/ChronoKeeper/src/ChronoKeeperConfiguration.cpp index aa99fb353..c834c2a47 100644 --- a/ChronoKeeper/src/ChronoKeeperConfiguration.cpp +++ b/ChronoKeeper/src/ChronoKeeperConfiguration.cpp @@ -139,35 +139,25 @@ int chronolog::KeeperConfiguration::parseJsonConf(json_object* json_conf) if(DATA_STORE_CONF.parseJsonConf(data_store_conf) != chl::CL_SUCCESS) return chl::CL_ERR_INVALID_CONF; } - else if(strcmp(key, "Extractors") == 0) + else if(strcmp(key, "ExtractionModule") == 0) { if(!json_object_is_type(val, json_type_object)) { - std::cerr << "[KeeperConfiguration] Invalid 'Extractors': expected object" << std::endl; + std::cerr << "[KeeperConfiguration] Invalid 'ExtractionModule' segment: expected json object" + << std::endl; return chl::CL_ERR_INVALID_CONF; } - json_object* extractors = json_object_object_get(json_conf, "Extractors"); - json_object_object_foreach(extractors, key, val) + json_object* extraction_module_json_object = json_object_object_get(json_conf, "ExtractionModule"); + if(EXTRACTION_MODULE_CONF.parse_json_object(extraction_module_json_object) != chl::CL_SUCCESS) { - if(strcmp(key, "story_files_dir") == 0) - { - if(!json_object_is_type(val, json_type_string)) - { - std::cerr << "[KeeperConfiguration] Invalid 'story_files_dir': expected string" << std::endl; - return chl::CL_ERR_INVALID_CONF; - } - EXTRACTOR_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Extractors configuration " << key - << std::endl; - } + return chl::CL_ERR_INVALID_CONF; + std::cerr << "[KeeperConfiguration] Error parsing ExtractionModule configuration: " << std::endl; } } else { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Keeper configuration: " << key << std::endl; + std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Keeper configuration block: " << key + << std::endl; } } return chronolog::CL_SUCCESS; diff --git a/ChronoKeeper/src/ChronoKeeperInstance.cpp b/ChronoKeeper/src/ChronoKeeperInstance.cpp index d9063e554..1a6a2ff6a 100644 --- a/ChronoKeeper/src/ChronoKeeperInstance.cpp +++ b/ChronoKeeper/src/ChronoKeeperInstance.cpp @@ -16,9 +16,7 @@ #include #include #include -#include -#include - +#include #include #include @@ -97,6 +95,7 @@ int main(int argc, char** argv) } LOG_INFO("[ChronoKeeper] Running ChronoKeeper Server."); + LOG_INFO("[ChronoKeeper] Configuration {}", KEEPER_CONF.to_String()); // Instantiate ChronoKeeper MemoryDataStore @@ -159,15 +158,21 @@ int main(int argc, char** argv) LOG_INFO("[ChronoKeeperInstance] KeeperIdCard: {}", chronolog::to_string(keeperIdCard)); - // Instantiate ChronoKeeper MemoryDataStore & ExtractionModule - chronolog::IngestionQueue ingestionQueue; - std::string keeper_csv_files_directory = KEEPER_CONF.EXTRACTOR_CONF.story_files_dir; - // Instantiate KeeperGrapherDrainService + + // Instantiate ChronoKeeper ExtractionModule + tl::engine* extractionEngine = nullptr; + // default extraction engine fabric is "ofi+sockets" + std::string extraction_engine_protocol = KEEPER_CONF.EXTRACTION_MODULE_CONF.extraction_protocol; + + + // if single_endpoint_rdma_extractor or dual_endpoint_rdma_extractor + // are configured then extraction engine needs to be instantiated with + // the protocol of the grapher receiving endpoint + try { - extractionEngine = - new tl::engine(KEEPER_CONF.KEEPER_GRAPHER_DRAIN_SERVICE_CONF.PROTO_CONF, THALLIUM_CLIENT_MODE); + extractionEngine = new tl::engine(extraction_engine_protocol, THALLIUM_CLIENT_MODE); std::stringstream s1; s1 << extractionEngine->self(); @@ -181,16 +186,30 @@ int main(int argc, char** argv) return (-1); } - chl::ServiceId grapherReceivingServiceId(KEEPER_CONF.KEEPER_GRAPHER_DRAIN_SERVICE_CONF.PROTO_CONF, - KEEPER_CONF.KEEPER_GRAPHER_DRAIN_SERVICE_CONF.IP, - KEEPER_CONF.KEEPER_GRAPHER_DRAIN_SERVICE_CONF.BASE_PORT, - KEEPER_CONF.KEEPER_GRAPHER_DRAIN_SERVICE_CONF.SERVICE_PROVIDER_ID); + std::string log_string; + + LOG_INFO("[ChronoKeeperInstance] Initializing StoryChunkExtractionModule with {}", + KEEPER_CONF.EXTRACTION_MODULE_CONF.to_string(log_string)); - chl::StoryChunkExtractorRDMA single_endpoint_rdma_extractor(*extractionEngine, grapherReceivingServiceId); - chronolog::StoryChunkExtractionModule extractionModule(chl::LoggingExtractor(), single_endpoint_rdma_extractor); + chl::StoryChunkExtractionModule theExtractionModule( + KEEPER_CONF.EXTRACTION_MODULE_CONF.extraction_stream_count); + theExtractionModule.getExtractionChain().activate(*extractionEngine, + KEEPER_CONF.EXTRACTION_MODULE_CONF, + keeperIdCard.getRecordingServiceId()); + + theExtractionModule.initialize(KEEPER_CONF.EXTRACTION_MODULE_CONF.extraction_stream_count); + + if(!theExtractionModule.is_initialized()) + { + LOG_ERROR("[ChronoKeeperInstance] StoryChunkExtractionModule failed to initialize, exiting"); + return (-1); + } + + // Instantiate KeeperDataStore + chronolog::IngestionQueue ingestionQueue; chronolog::KeeperDataStore theDataStore(ingestionQueue, - extractionModule.getExtractionQueue(), + theExtractionModule.getExtractionQueue(), KEEPER_CONF.DATA_STORE_CONF.max_story_chunk_size, KEEPER_CONF.DATA_STORE_CONF.story_chunk_duration_secs, KEEPER_CONF.DATA_STORE_CONF.acceptance_window_secs, @@ -316,8 +335,7 @@ int main(int argc, char** argv) tl::abt scope; theDataStore.startDataCollection(3); // start extraction streams & threads - //storyExtractor.startExtractionThreads(2); - extractionModule.startExtraction(2); + theExtractionModule.startExtraction(); /// Main loop for sending stats message until receiving SIGTERM ____________________________________________________ @@ -345,7 +363,8 @@ int main(int argc, char** argv) theDataStore.shutdownDataCollection(); // Shutdown extraction module // drain extractionQueue and stop extraction xStreams - extractionModule.shutdownExtraction(); + theExtractionModule.shutdownExtraction(); + // these are not probably needed as thallium handles the engine finalization... // recordingEngine.finalize(); // collectionEngine.finalize(); diff --git a/ChronoKeeper/src/DualEndpointChunkExtractorRDMA.cpp b/ChronoKeeper/src/DualEndpointChunkExtractorRDMA.cpp new file mode 100644 index 000000000..d4edc6ee5 --- /dev/null +++ b/ChronoKeeper/src/DualEndpointChunkExtractorRDMA.cpp @@ -0,0 +1,431 @@ +#include +#include +//#include +#include + +#include +#include +#include +#include +#include +#include + +namespace tl = thallium; +namespace chl = chronolog; + +chronolog::DualEndpointChunkExtractorRDMA::DualEndpointChunkExtractorRDMA( + tl::engine& tl_engine, + chronolog::ServiceId const& player_receiving_service_id, + chronolog::ServiceId const& grapher_receiving_service_id) + : sender_tl_engine(tl_engine) + , player_receiver_service_id(player_receiving_service_id) + , grapher_receiver_service_id(grapher_receiving_service_id) + , rdma_sender_for_player(nullptr) + , rdma_sender_for_grapher(nullptr) +{ + restart_rdma_sender_for_grapher(grapher_receiver_service_id); + + restart_rdma_sender_for_player(player_receiver_service_id); +} + +///////////// + +void chronolog::DualEndpointChunkExtractorRDMA::restart_rdma_sender_for_grapher( + chl::ServiceId const& receiver_service_id) +{ + if(rdma_sender_for_grapher != nullptr) + { + LOG_TRACE("[DualEndpointExtractorRDMA] assingment : deleting receiver {} ", + chl::to_string(grapher_receiver_service_id)); + delete rdma_sender_for_grapher; + } + + grapher_receiver_service_id = receiver_service_id; + + if(!grapher_receiver_service_id.is_valid()) + { + return; + } + + try + { + rdma_sender_for_grapher = + RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, grapher_receiver_service_id); + LOG_TRACE("[DualEndpointChunkExtractor] created rdma_sender for grpaher_receiver {} ", + chl::to_string(grapher_receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[DualEndpointChunkExtractor] failed to create rdma_sender for grapher_receiver {} ", + chl::to_string(grapher_receiver_service_id)); + rdma_sender_for_grapher = nullptr; + } +} + +///////////// + +void chronolog::DualEndpointChunkExtractorRDMA::restart_rdma_sender_for_player( + chl::ServiceId const& receiver_service_id) +{ + if(rdma_sender_for_player != nullptr) + { + LOG_TRACE("[DualEndpointExtractorRDMA] assingment : deleting receiver {} ", + chl::to_string(player_receiver_service_id)); + delete rdma_sender_for_player; + } + + player_receiver_service_id = receiver_service_id; + + if(!player_receiver_service_id.is_valid()) + { + return; + } + + try + { + rdma_sender_for_player = + RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, player_receiver_service_id); + LOG_TRACE("[DualEndpointChunkExtractor] Constructor created rdma_sender for player_receiver {} ", + chl::to_string(player_receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[DualEndpointChunkExtractor] Constructor : failed to create rdma_sender for player_receiver {} ", + chl::to_string(player_receiver_service_id)); + rdma_sender_for_player = nullptr; + } +} + + +chronolog::DualEndpointChunkExtractorRDMA::DualEndpointChunkExtractorRDMA(DualEndpointChunkExtractorRDMA const& other) + : sender_tl_engine(other.get_sender_engine()) + , player_receiver_service_id(other.get_player_receiver_id()) + , grapher_receiver_service_id(other.get_grapher_receiver_id()) + , rdma_sender_for_player(nullptr) + , rdma_sender_for_grapher(nullptr) +{ + try + { + rdma_sender_for_player = + RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, player_receiver_service_id); + LOG_TRACE("[DualEndpointExtractorRDMA] Copy Constructor created rdma_sender for player_receiver {} ", + chl::to_string(player_receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[DualEndpointExtractorRDMA] Constructor : failed to create rdma_sender for player_receiver {} ", + chl::to_string(player_receiver_service_id)); + rdma_sender_for_player = nullptr; + } + + try + { + rdma_sender_for_grapher = + RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, grapher_receiver_service_id); + LOG_TRACE("Dual[DualEndpointExtractorRDMA] Copy Constructor created rdma_sender for grpaher_receiver {} ", + chl::to_string(grapher_receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[DualEndpointExtractorRDMA] Constructor : failed to create rdma_sender for grapher_receiver {} ", + chl::to_string(grapher_receiver_service_id)); + rdma_sender_for_grapher = nullptr; + } +} + +chl::DualEndpointChunkExtractorRDMA& +chronolog::DualEndpointChunkExtractorRDMA::operator=(DualEndpointChunkExtractorRDMA const& other) +{ + if(this == &other) + { + return *this; + } + + if(rdma_sender_for_player != nullptr) + { + LOG_TRACE("[DualEndpointExtractorRDMA] assingment : deleting receiver {} ", + chl::to_string(player_receiver_service_id)); + delete rdma_sender_for_player; + } + if(rdma_sender_for_grapher != nullptr) + { + LOG_TRACE("[DualEndpointExtractorRDMA] assingment : deleting receiver {} ", + chl::to_string(grapher_receiver_service_id)); + delete rdma_sender_for_grapher; + } + + sender_tl_engine = other.get_sender_engine(); + player_receiver_service_id = other.get_player_receiver_id(); + grapher_receiver_service_id = other.get_grapher_receiver_id(); + + try + { + rdma_sender_for_player = + RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, player_receiver_service_id); + LOG_TRACE("[DualEndpointExtractorRDMA] assingment: created rdma_sender for receiver {} ", + chl::to_string(player_receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[DualEndpointExtractorRDMA] assignment: failed to create rdma_sender for receiver {} ", + chl::to_string(player_receiver_service_id)); + rdma_sender_for_player = nullptr; + } + + try + { + rdma_sender_for_grapher = + RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, grapher_receiver_service_id); + LOG_TRACE("[DualEndpointExtractorRDMA] assingment: created rdma_sender for receiver {} ", + chl::to_string(grapher_receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[DualEndpointExtractorRDMA] assignment: failed to create rdma_sender for receiver {} ", + chl::to_string(grapher_receiver_service_id)); + rdma_sender_for_grapher = nullptr; + } + + return *this; +} + + +chronolog::DualEndpointChunkExtractorRDMA::~DualEndpointChunkExtractorRDMA() +{ + if(rdma_sender_for_player != nullptr) + { + LOG_TRACE("[DualEndpointExtractorRDMA] Destructor: deleting receiver {} ", + chl::to_string(player_receiver_service_id)); + delete rdma_sender_for_player; + } + if(rdma_sender_for_grapher != nullptr) + { + LOG_TRACE("[DualEndpointExtractorRDMA] Destructor: deleting receiver {} ", + chl::to_string(grapher_receiver_service_id)); + delete rdma_sender_for_grapher; + } +} + +//// + +int chronolog::DualEndpointChunkExtractorRDMA::process_chunk(chronolog::StoryChunk* story_chunk) +{ + + // Note: a failure to transfer a story_chunk to the ChronoPlayer alone is not considered a critical enough error + // to exit this function and try again later. In this case the error would be logged but the function would + // proceed to sending the story_chunk to the ChronoGrapher + // Failure to transfer the story_chunk to the ChronoGrapher would trigger the exit from the function, + // return the StoryChunk back on the extraction_queue, and subsequent retry + + int transfer_return = chl::CL_ERR_UNKNOWN; + + try + { + LOG_DEBUG("[DualEndpointChunkExtractor] tl::thread_id={} processing chunk StoryId={} {}-{} {}-{} eventCount {}", + thallium::thread::self_id(), + story_chunk->getStoryId(), + story_chunk->getChronicleName(), + story_chunk->getStoryName(), + story_chunk->getStartTime(), + story_chunk->getEndTime(), + story_chunk->getEventCount()); + + if(rdma_sender_for_grapher == nullptr) + { + LOG_ERROR("[DualEndpointChunkExtractor] Failed to transfer StoryChunk StoryId={} StartTime={}, no valid " + "rdma_sender_for_grapher", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + return chl::CL_ERR_STORY_CHUNK_EXTRACTION; + } + + std::ostringstream oss(std::ios::binary); + try + { + cereal::BinaryOutputArchive oarchive(oss); + oarchive(*story_chunk); + } + catch(cereal::Exception const& ex) + { + LOG_ERROR("[DualEndpointChunkExtractor] Cereal exception while serializing StoryChunk StoryId={} " + "StartTime={} ex {}", + story_chunk->getStoryId(), + story_chunk->getStartTime(), + ex.what()); + } + + + std::string serialized_story_chunk = oss.str(); + + if(rdma_sender_for_player != nullptr) + { + try + { + transfer_return = rdma_sender_for_player->transfer_serialized_story_chunk(serialized_story_chunk); + if(transfer_return == chl::CL_SUCCESS) + { + LOG_INFO("[DualEndpointChunkExtractor] Transfered to Player StoryChunk StoryId={} StartTime={}", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + } + else + { + LOG_ERROR("[DualEndpointChunkExtractor] Failed to transfer to Player StoryChunk StoryId={} " + "StartTime={}", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + } + } + catch(std::exception const& ex) + { + LOG_ERROR("[DualEndpointChunkExtractor] Standard exception while serializing StoryChunk StoryId={} " + "StartTime={} ex {}", + story_chunk->getStoryId(), + story_chunk->getStartTime(), + ex.what()); + } + } + + transfer_return = rdma_sender_for_grapher->transfer_serialized_story_chunk(serialized_story_chunk); + + if(transfer_return == chl::CL_SUCCESS) + { + LOG_INFO("[DualEndpointChunkExtractor] Transfered to Grapher StoryChunk StoryId={} StartTime={}", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + } + else + { + LOG_ERROR("[DualEndpointChunkExtractor] Failed to transfer to Grapher StoryChunk StoryId={} StartTime={}", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + } + + return transfer_return; + } + catch(std::exception const& ex) + { + LOG_ERROR("[DualEndpointChunkExtractor] Standard exception while serializing StoryChunk StoryId={} " + "StartTime={} ex {}", + story_chunk->getStoryId(), + story_chunk->getStartTime(), + ex.what()); + } + catch(...) + { + LOG_ERROR("[DualEndpointChunkExtractor] Exception while transferring StoryChunkiStoryId={} StartTime={}", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + } + + return chl::CL_ERR_STORY_CHUNK_EXTRACTION; +} + + +/* + JSON Block for Dual Endpoint RDMA Extractor + + "extractor_name": { + "type": "dual_endpoint_rdma_extractor", + "player_receiving_endpoint": { + "protocol_conf": "ofi+sockets", + "service_ip": "127.0.0.1", + "service_base_port": 2230, + "service_provider_id": 30 + }, + "grapher_receiving_endpoint": { + "protocol_conf": "ofi+sockets", + "service_ip": "127.0.0.1", + "service_base_port": 3333, + "service_provider_id": 33 + } + +*/ +int chronolog::DualEndpointChunkExtractorRDMA::reset(json_object* json_block) +{ + if((json_block == nullptr) || !json_object_is_type(json_block, json_type_object) || + (json_object_object_get(json_block, "type") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "type"), json_type_string) || + (std::string("dual_endpoint_rdma_extractor") + .compare(json_object_get_string(json_object_object_get(json_block, "type"))) != 0)) + { + LOG_ERROR("[DualEndpointExtractorRDMA] Reset failure: invalid json config"); + return chl::CL_ERR_INVALID_CONF; + } + + if((json_object_object_get(json_block, "grapher_receiving_endpoint") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "grapher_receiving_endpoint"), json_type_object)) + { + LOG_ERROR("[DualEndpointChunkExtractorRDMA] Reset failure: invalid json config"); + return chl::CL_ERR_INVALID_CONF; + } + + json_object* json_rpc_block = json_object_object_get(json_block, "grapher_receiving_endpoint"); + chl::RPCProviderConf grapher_receiving_endpoint_conf; + if(grapher_receiving_endpoint_conf.parseJsonConf(json_rpc_block) != chl::CL_SUCCESS) + { + LOG_ERROR("[ChunkExtractorRDMA] Reset failure: invalid grapher_endpoint config"); + return chl::CL_ERR_INVALID_CONF; + } + + chl::ServiceId grapher_receiver_id(grapher_receiving_endpoint_conf.PROTO_CONF, + grapher_receiving_endpoint_conf.IP, + grapher_receiving_endpoint_conf.BASE_PORT, + grapher_receiving_endpoint_conf.SERVICE_PROVIDER_ID); + + LOG_DEBUG("[ChunkExtractorRDMA] reset: about to create grapher_rdma_sender for receiver_service {} ", + chl::to_string(grapher_receiver_id)); + + if(!grapher_receiver_id.is_valid()) + { + LOG_ERROR("[DualEndpointChunkExtractorRDMA] Reset failure: invalid grapher endpoint config"); + return chl::CL_ERR_INVALID_CONF; + } + + restart_rdma_sender_for_grapher(grapher_receiver_id); + + if((json_object_object_get(json_block, "player_receiving_endpoint") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "player_receiving_endpoint"), json_type_object)) + { + LOG_ERROR("[DualEndpointChunkExtractorRDMA] Reset failure: invalid json config"); + return chl::CL_ERR_INVALID_CONF; + } + + json_object* player_rpc_block = json_object_object_get(json_block, "player_receiving_endpoint"); + chl::RPCProviderConf player_receiving_endpoint_conf; + if(player_receiving_endpoint_conf.parseJsonConf(player_rpc_block) != chl::CL_SUCCESS) + { + LOG_ERROR("[DualEndpointChunkExtractorRDMA] Reset failure: invalid player endpoint config"); + return chl::CL_ERR_INVALID_CONF; + } + + chl::ServiceId player_receiver_id(player_receiving_endpoint_conf.PROTO_CONF, + player_receiving_endpoint_conf.IP, + player_receiving_endpoint_conf.BASE_PORT, + player_receiving_endpoint_conf.SERVICE_PROVIDER_ID); + + if(!player_receiver_id.is_valid()) + { + LOG_ERROR("[ChunkExtractorRDMA] Reset failure: invalid player endpoint config"); + return chl::CL_ERR_INVALID_CONF; + } + + LOG_DEBUG("[ChunkExtractorRDMA] reset: about to create player_rdma_sender for receiver_service {} ", + chl::to_string(player_receiver_id)); + + restart_rdma_sender_for_player(player_receiver_id); + + return chl::CL_SUCCESS; +} + +int chronolog::DualEndpointChunkExtractorRDMA::reset_player_endpoint(chl::ServiceId const& receiver_player) +{ + + return chl::CL_SUCCESS; +} + +int chronolog::DualEndpointChunkExtractorRDMA::reset_grapher_endpoint(chl::ServiceId const& receiver_grapher) +{ + + return chl::CL_SUCCESS; +} diff --git a/ChronoPlayer/CMakeLists.txt b/ChronoPlayer/CMakeLists.txt index 0f348b176..443393423 100644 --- a/ChronoPlayer/CMakeLists.txt +++ b/ChronoPlayer/CMakeLists.txt @@ -20,6 +20,7 @@ target_sources(chrono_player PRIVATE src/ArchiveReadingAgent.cpp src/HDF5ArchiveReadingAgent.cpp src/PlaybackService.cpp + src/StoryChunkExtractor.cpp src/StoryChunkTransferAgent.cpp ) diff --git a/chrono_common/include/StoryChunkExtractor.h b/ChronoPlayer/include/StoryChunkExtractor.h similarity index 100% rename from chrono_common/include/StoryChunkExtractor.h rename to ChronoPlayer/include/StoryChunkExtractor.h diff --git a/chrono_common/src/StoryChunkExtractor.cpp b/ChronoPlayer/src/StoryChunkExtractor.cpp similarity index 100% rename from chrono_common/src/StoryChunkExtractor.cpp rename to ChronoPlayer/src/StoryChunkExtractor.cpp diff --git a/ChronoVisor/include/ChronoVisorConfiguration.h b/ChronoVisor/include/ChronoVisorConfiguration.h index 6c778de12..34302e432 100644 --- a/ChronoVisor/include/ChronoVisorConfiguration.h +++ b/ChronoVisor/include/ChronoVisorConfiguration.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include diff --git a/chrono_common/CMakeLists.txt b/chrono_common/CMakeLists.txt index 631fe0421..10e3dbda2 100644 --- a/chrono_common/CMakeLists.txt +++ b/chrono_common/CMakeLists.txt @@ -1,7 +1,6 @@ cmake_minimum_required(VERSION 3.25) find_package(Thallium REQUIRED) -find_package(HDF5 REQUIRED COMPONENTS C CXX) #------------------------------------------------------------------------------ # chrono_common Library @@ -10,12 +9,12 @@ add_library(chrono_common STATIC src/city.cpp src/ConfigurationBlocks.cpp src/StoryChunk.cpp - src/StoryChunkExtractor.cpp src/StoryChunkWriter.cpp src/StoryPipeline.cpp src/ChunkExtractorCSV.cpp src/ChunkExtractorRDMA.cpp src/RDMATransferAgent.cpp + src/ExtractionModuleConfiguration.cpp ) target_include_directories(chrono_common @@ -29,7 +28,6 @@ target_include_directories(chrono_common target_link_libraries(chrono_common PRIVATE thallium - ${HDF5_LIBRARIES} ) # Get all header files from include directory diff --git a/chrono_common/include/ChunkExtractorCSV.h b/chrono_common/include/ChunkExtractorCSV.h index 117ce6d5f..b4414941e 100644 --- a/chrono_common/include/ChunkExtractorCSV.h +++ b/chrono_common/include/ChunkExtractorCSV.h @@ -1,6 +1,9 @@ #ifndef CHUNK_EXTRACTOR_CSV_H #define CHUNK_EXTRACTOR_CSV_H +#include +#include + #include #include @@ -14,15 +17,21 @@ class StoryChunkExtractorCSV { public: - StoryChunkExtractorCSV(ServiceId const& service_id, std::string const& csv_files_dir); + StoryChunkExtractorCSV(ServiceId const& service_id, std::string const& csv_archive_dir = "/tmp"); + + int reset(std::string const& csv_archive_dir); + int reset(json_object*); StoryChunkExtractorCSV() = default; int process_chunk(StoryChunk*); + bool is_active() const { return (std::filesystem::exists(outputDirectory)); } + private: ServiceId serviceId; std::string outputDirectory; + std::string service_id_string; }; diff --git a/chrono_common/include/ChunkExtractorRDMA.h b/chrono_common/include/ChunkExtractorRDMA.h index 27b44abcd..16dc85963 100644 --- a/chrono_common/include/ChunkExtractorRDMA.h +++ b/chrono_common/include/ChunkExtractorRDMA.h @@ -1,6 +1,9 @@ #ifndef CHUNK_EXTRACTOR_RDMA_H #define CHUNK_EXTRACTOR_RDMA_H +#include +#include + #include #include @@ -16,7 +19,7 @@ class StoryChunkExtractorRDMA { public: - StoryChunkExtractorRDMA(tl::engine& tl_engine, ServiceId const& service_id); + StoryChunkExtractorRDMA(tl::engine& tl_engine, ServiceId const& service_id = ServiceId()); StoryChunkExtractorRDMA(StoryChunkExtractorRDMA const& other); StoryChunkExtractorRDMA& operator=(StoryChunkExtractorRDMA const& other); @@ -26,10 +29,17 @@ class StoryChunkExtractorRDMA ServiceId const& get_receiver_service_id() const { return receiver_service_id; } int process_chunk(StoryChunk*); + int reset(ServiceId const&); + int reset(json_object*); + + bool is_active() const { return (nullptr != rdma_sender); } + private: tl::engine& sender_tl_engine; // local tl::engine ServiceId receiver_service_id; // receiving ServiceId RDMATransferAgent* rdma_sender; + + void restart_rdma_sender(ServiceId const&); }; diff --git a/chrono_common/include/ChunkLoggingExtractor.h b/chrono_common/include/ChunkLoggingExtractor.h index 776b311b6..7e54849ee 100644 --- a/chrono_common/include/ChunkLoggingExtractor.h +++ b/chrono_common/include/ChunkLoggingExtractor.h @@ -27,6 +27,8 @@ class LoggingExtractor return chronolog::CL_SUCCESS; } + + bool is_active() const { return true; } }; } // namespace chronolog diff --git a/chrono_common/include/ConfigurationBlocks.h b/chrono_common/include/ConfigurationBlocks.h index be9538f96..3a3012ded 100644 --- a/chrono_common/include/ConfigurationBlocks.h +++ b/chrono_common/include/ConfigurationBlocks.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include diff --git a/chrono_common/include/ExtractionModuleConfiguration.h b/chrono_common/include/ExtractionModuleConfiguration.h new file mode 100644 index 000000000..cf5d1d532 --- /dev/null +++ b/chrono_common/include/ExtractionModuleConfiguration.h @@ -0,0 +1,43 @@ +#ifndef CHRONO_EXTRACTION_MODULE_CONFIGURATION +#define CHRONO_EXTRACTION_MODULE_CONFIGURATION + + +#include +#include +#include + +#include + +namespace chronolog +{ + +struct ExtractionModuleConfiguration +{ + int extraction_stream_count = 1; + std::string extraction_protocol = "ofi+sockets"; + std::map extractors; + + std::string& to_string(std::string& a_string) const + { + a_string += "\n[EXTRACTION_MODULE_CONF: { extraction_stream_count: " + std::to_string(extraction_stream_count) + + ", extraction_protocol: " + extraction_protocol + "} extractors {"; + if(!extractors.empty()) + { + for(auto iter = extractors.begin(); iter != extractors.end(); ++iter) + { + a_string += " {" + (*iter).first + ":" + json_object_get_string((*iter).second) + "}"; + } + } + a_string += "} ]"; + + return a_string; + } + + int parse_json_object(json_object* json_object); + + ~ExtractionModuleConfiguration(); +}; + +} // namespace chronolog + +#endif diff --git a/chrono_common/include/StoryChunkExtractionModule.h b/chrono_common/include/StoryChunkExtractionModule.h index 85fe02199..08669b3d5 100644 --- a/chrono_common/include/StoryChunkExtractionModule.h +++ b/chrono_common/include/StoryChunkExtractionModule.h @@ -1,5 +1,5 @@ -#ifndef STORY_CHUNK_EXTRACTION_CHAIN_H -#define STORY_CHUNK_EXTRACTION_CHAIN_H +#ifndef STORY_CHUNK_EXTRACTION_MODULE_H +#define STORY_CHUNK_EXTRACTION_MODULE_H #include #include @@ -12,78 +12,74 @@ #include #include #include +#include namespace tl = thallium; + namespace chronolog { - -// Recursive ExtractorChain Definition -template -class ExtractorChain +template +class StoryChunkExtractionModule { + enum State + { + UNKNOWN = 0, + INITIALIZED = 1, // ExtractionChain initialized and ready + RUNNING = 2, // active extraction threads + SHUTTING_DOWN = 3 // Shutting down extraction threads + }; + public: - ExtractorChain(const T& extractor, const Args&... rest) - : the_extractor(extractor) - , the_rest(rest...) + StoryChunkExtractionModule(int extraction_stream_count = 2) + : state(UNKNOWN) + , stream_count(extraction_stream_count) {} - constexpr size_t size() const { return 1 + the_rest.size(); } - - const T& extractor() const { return the_extractor; } - - const ExtractorChain& get_rest() const { return the_rest; } - - int process_chunk(StoryChunk* some_chunk) + StoryChunkExtractionModule(ServiceId const& recording_service_id, + ExtractionModuleConfiguration const& configuration) + : state(UNKNOWN) + , stream_count(2) { - the_extractor.process_chunk(some_chunk); - return the_rest.process_chunk(some_chunk); + initialize(recording_service_id, configuration); } -private: - T the_extractor; - ExtractorChain the_rest; -}; + int initialize(ServiceId const& recording_service_id, ExtractionModuleConfiguration const& configuration) + { + //TODO: move extraction engine instantiation here + // then move Extraction Chain instantiation here as well + stream_count = configuration.extraction_stream_count; -// Base Case ExtractorChain -template -class ExtractorChain -{ -public: - ExtractorChain(const T& an_extractor) - : the_extractor(an_extractor) - {} + //TODO: theExtractionChain.activate(recording_service_id, configuration); - constexpr size_t size() const { return 1; } + if(!theExtractionChain.is_active_chain()) + { + return CL_ERR_INVALID_CONF; + } - const T& extractor() const { return the_extractor; } + state = INITIALIZED; + return CL_SUCCESS; + } - int process_chunk(StoryChunk* some_chunk) { return the_extractor.process_chunk(some_chunk); } + int initialize(int extraction_stream_count) + { + stream_count = extraction_stream_count; -private: - T the_extractor; -}; + if(!theExtractionChain.is_active_chain()) + { + return CL_ERR_INVALID_CONF; + } -/////////////////////////// + state = INITIALIZED; + return CL_SUCCESS; + } -template -class StoryChunkExtractionModule -{ - enum State - { - UNKNOWN = 0, - RUNNING = 1, // active extraction threads - SHUTTING_DOWN = 2 // Shutting down extraction threads - }; + StoryChunkExtractionQueue& getExtractionQueue() { return chunkExtractionQueue; } -public: - StoryChunkExtractionModule(const Args&... extractors) - : state(UNKNOWN) - , extractorChain(extractors...) - {} + T& getExtractionChain() { return theExtractionChain; } - StoryChunkExtractionQueue& getExtractionQueue() { return chunkExtractionQueue; } + bool is_initialized() const { return (state == INITIALIZED); } bool is_running() const { return (state == RUNNING); } @@ -101,6 +97,11 @@ class StoryChunkExtractionModule thallium::thread::self_id(), chunkExtractionQueue.size()); + // while state== RUNNING keep draining the queue + // or waiting if the queue is empty + + int extraction_result; + while(state == RUNNING) { if(!chunkExtractionQueue.empty()) @@ -122,11 +123,20 @@ class StoryChunkExtractionModule story_chunk->getEndTime(), story_chunk->getEventCount()); - // each extractor in the chain would handle its own intermittent failure appropriately - extractorChain.process_chunk(story_chunk); - - // free the memory or reset the chunk to the original state and return it to the pool of prealocated chunks - delete story_chunk; + extraction_result = theExtractionChain.process_chunk(story_chunk); + // INNA: commented out lines are temporary to get the existing PR + // past the CI deployment check + // more nuanced handling of intermitent communication outtage + // will be addressed in the follow-up issue #635 + //if(CL_SUCCESS == extraction_result) + { // free the story_chunk memory or + // return it to the pool of prealocated chunks + delete story_chunk; + } + // else + // { //return the story_chunk to the extractionQueue and try again later + // chunkExtractionQueue.stashStoryChunk(story_chunk); + // } } } else @@ -136,7 +146,7 @@ class StoryChunkExtractionModule } } - void startExtraction(int stream_count) + void startExtraction() { if(state == RUNNING) @@ -145,6 +155,13 @@ class StoryChunkExtractionModule return; } + if(state != INITIALIZED) + { + LOG_INFO("[StoryChunkExtractionModule] Can't start extraction: ExtractionModule is either not initialized " + "or is shutting down"); + return; + } + state = RUNNING; for(int i = 0; i < stream_count; ++i) @@ -174,11 +191,62 @@ class StoryChunkExtractionModule state = SHUTTING_DOWN; // make sure extractionQueue is drained before the extraction streams are shut down - LOG_DEBUG("[StoryChunkExtractionModule] Initiating shutdown: extractionQueue size {}", - chunkExtractionQueue.size()); + LOG_INFO("[StoryChunkExtractionModule] Initiating shutdown: extractionQueue size {}", + chunkExtractionQueue.size()); - drainExtractionQueue(); + // make the best to drain the extraction queue before exiting + while(!chunkExtractionQueue.empty()) + { + // chunkExtractionQueue has internal mutex protecting its integrity + StoryChunk* story_chunk = chunkExtractionQueue.ejectStoryChunk(); + // the queue might have been drained by another thread before the current thread acquired extractionQueue mutex + // in this case the nullptr is returned.. + if(story_chunk == nullptr) + { + continue; + } + + LOG_DEBUG("[StoryChunkExtractionModule] tl::thread_id={} processing chunk StoryId={} {}-{} {}-{} " + "eventCount {}", + thallium::thread::self_id(), + story_chunk->getStoryId(), + story_chunk->getChronicleName(), + story_chunk->getStoryName(), + story_chunk->getStartTime(), + story_chunk->getEndTime(), + story_chunk->getEventCount()); + + //try to extract the chunk 5 times before giving up + int extraction_result = CL_ERR_UNKNOWN; + int tries = 0; + while(CL_SUCCESS != extraction_result && (tries < 5)) + { + tries++; + extraction_result = theExtractionChain.process_chunk(story_chunk); + } + + if(CL_SUCCESS == extraction_result) + { + LOG_INFO("[StoryChunkExtractionModule] extracted chunk StoryId={} {}-{} {}-{}", + story_chunk->getStoryId(), + story_chunk->getChronicleName(), + story_chunk->getStoryName(), + story_chunk->getStartTime(), + story_chunk->getEndTime()); + } + else + { + LOG_ERROR("[StoryChunkExtractionModule] failed to extract chunk StoryId={} {}-{} {}-{}", + story_chunk->getStoryId(), + story_chunk->getChronicleName(), + story_chunk->getStoryName(), + story_chunk->getStartTime(), + story_chunk->getEndTime()); + } + + delete story_chunk; + } // join and stop threads & executionstreams for(auto& eth: extractionThreads) { eth->join(); } LOG_DEBUG("[StoryChunkExtractionModule] Extraction threads have been successfully shut down."); @@ -205,14 +273,14 @@ class StoryChunkExtractionModule StoryChunkExtractionModule& operator=(StoryChunkExtractionModule const&) = delete; - std::atomic state; StoryChunkExtractionQueue chunkExtractionQueue; - ExtractorChain extractorChain; - + int stream_count; std::vector> extractionStreams; std::vector> extractionThreads; + + T theExtractionChain; }; diff --git a/chrono_common/include/StoryChunkWriter.h b/chrono_common/include/StoryChunkWriter.h index 93d3c1e19..ff7d4788b 100644 --- a/chrono_common/include/StoryChunkWriter.h +++ b/chrono_common/include/StoryChunkWriter.h @@ -15,20 +15,19 @@ namespace chronolog class StoryChunkWriter { public: - StoryChunkWriter(std::string const &root_dir, std::string const &group_name, std::string const &dset_name) - : rootDirectory(root_dir), groupName(group_name), dsetName(dset_name), numDims(1) - {}; + StoryChunkWriter(std::string const& root_dir, std::string const& group_name, std::string const& dset_name) + : rootDirectory(root_dir) + , groupName(group_name) + , dsetName(dset_name) + , numDims(1){}; - ~StoryChunkWriter() - { - LOG_INFO("[StoryChunkWriter] Destructor called. Cleaning up..."); - } + ~StoryChunkWriter() { LOG_DEBUG("[StoryChunkWriter] Destructor called. Cleaning up..."); } - hsize_t writeStoryChunk(StoryChunkHVL &story_chunk); + hsize_t writeStoryChunk(StoryChunkHVL& story_chunk); - hsize_t writeStoryChunk(StoryChunk &story_chunk); + hsize_t writeStoryChunk(StoryChunk& story_chunk); - hsize_t writeEvents(std::unique_ptr &file, std::vector &data); + hsize_t writeEvents(std::unique_ptr& file, std::vector& data); static H5::CompType createEventCompoundType() { @@ -37,12 +36,14 @@ class StoryChunkWriter data_type.insertMember("eventTime", HOFFSET(LogEventHVL, eventTime), H5::PredType::NATIVE_UINT64); data_type.insertMember("clientId", HOFFSET(LogEventHVL, clientId), H5::PredType::NATIVE_UINT32); data_type.insertMember("eventIndex", HOFFSET(LogEventHVL, eventIndex), H5::PredType::NATIVE_UINT32); - data_type.insertMember("logRecord", HOFFSET(LogEventHVL, logRecord), H5::VarLenType(H5::PredType::NATIVE_UINT8)); + data_type.insertMember("logRecord", + HOFFSET(LogEventHVL, logRecord), + H5::VarLenType(H5::PredType::NATIVE_UINT8)); return data_type; } // base_file_name should be in the format of chronicleName.storyName.startTime.vlen.h5, not including the path - static std::string getStoryChunkFileName(std::string const &root_dir, std::string const &base_file_name); + static std::string getStoryChunkFileName(std::string const& root_dir, std::string const& base_file_name); private: std::string rootDirectory; @@ -50,6 +51,6 @@ class StoryChunkWriter std::string dsetName; int numDims; }; -} // chronolog +} // namespace chronolog #endif //CHRONOLOG_STORY_CHUNK_WRITER_H diff --git a/chrono_common/include/deprecated_enum.h b/chrono_common/include/deprecated_enum.h deleted file mode 100644 index a9a0df1b0..000000000 --- a/chrono_common/include/deprecated_enum.h +++ /dev/null @@ -1,77 +0,0 @@ -// -// Created by kfeng on 4/1/22. -// - -#ifndef CHRONOLOG_ENUM_H -#define CHRONOLOG_ENUM_H - -typedef enum ChronoLogRPCImplementation -{ - CHRONOLOG_THALLIUM_SOCKETS = 0, - CHRONOLOG_THALLIUM_TCP = 1, - CHRONOLOG_THALLIUM_ROCE = 2 -} ChronoLogRPCImplementation; - -inline const char* getRPCImplString(ChronoLogRPCImplementation impl) -{ - switch(impl) - { - case CHRONOLOG_THALLIUM_SOCKETS: - return "CHRONOLOG_THALLIUM_SOCKETS"; - case CHRONOLOG_THALLIUM_TCP: - return "CHRONOLOG_THALLIUM_TCP"; - case CHRONOLOG_THALLIUM_ROCE: - return "CHRONOLOG_THALLIUM_ROCE"; - default: - return "UNKNOWN"; - } -} - -typedef enum ChronoLogServiceRole -{ - CHRONOLOG_UNKNOWN = 0, - CHRONOLOG_VISOR = 1, - CHRONOLOG_CLIENT = 2, - CHRONOLOG_KEEPER = 3 -} ChronoLogServiceRole; - -inline const char* getServiceRoleString(ChronoLogServiceRole role) -{ - switch(role) - { - case CHRONOLOG_UNKNOWN: - return "CHRONOLOG_UNKNOWN"; - case CHRONOLOG_VISOR: - return "CHRONOLOG_VISOR"; - case CHRONOLOG_CLIENT: - return "CHRONOLOG_CLIENT"; - case CHRONOLOG_KEEPER: - return "CHRONOLOG_KEEPER"; - default: - return "CHRONOLOG_UNKNOWN"; - } -} - -enum ClocksourceType -{ - C_STYLE = 0, - CPP_STYLE = 1, - TSC = 2 -}; - -inline const char* getClocksourceTypeString(ClocksourceType type) -{ - switch(type) - { - case C_STYLE: - return "C_STYLE"; - case CPP_STYLE: - return "CPP_STYLE"; - case TSC: - return "TSC"; - default: - return "UNKNOWN"; - } -} - -#endif //CHRONOLOG_ENUM_H diff --git a/chrono_common/src/ChunkExtractorCSV.cpp b/chrono_common/src/ChunkExtractorCSV.cpp index 89ab173a6..829e4356c 100644 --- a/chrono_common/src/ChunkExtractorCSV.cpp +++ b/chrono_common/src/ChunkExtractorCSV.cpp @@ -1,5 +1,6 @@ -#include +#include #include +#include #include #include @@ -11,12 +12,66 @@ namespace tl = thallium; namespace chl = chronolog; chronolog::StoryChunkExtractorCSV::StoryChunkExtractorCSV(chronolog::ServiceId const& service_id, - std::string const& csv_files_dir) + std::string const& csv_archive_dir) : serviceId(service_id) - , outputDirectory(csv_files_dir) -{} + , outputDirectory(csv_archive_dir) +{ + serviceId.get_ip_as_dotted_string(service_id_string); +} + +int chronolog::StoryChunkExtractorCSV::reset(std::string const& new_archive_dir) +{ + outputDirectory = new_archive_dir; + LOG_INFO("StoryChunkExtractorCSV] Reset success: using csv directory :", outputDirectory); + return chl::CL_SUCCESS; +} +// json block for CSV Extractor looks like this +// +// "extractor": { +// "type": "csv_extractor", +// "csv_archive_dir": "/tmp/csv_archive" +// } +// ////////// + +int chronolog::StoryChunkExtractorCSV::reset(json_object* json_block) +{ + if((json_block == nullptr) || !json_object_is_type(json_block, json_type_object) || + (json_object_object_get(json_block, "type") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "type"), json_type_string) || + (std::string("csv_extractor").compare(json_object_get_string(json_object_object_get(json_block, "type"))) != 0)) + { + outputDirectory = "/tmp"; + LOG_ERROR("StoryChunkExtractorCSV] Reset failure, using csv directory {}", outputDirectory); + return chl::CL_ERR_INVALID_CONF; + } + + if((json_object_object_get(json_block, "csv_archive_dir") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "csv_archive_dir"), json_type_string)) + { + outputDirectory = "/tmp"; + LOG_ERROR("StoryChunkExtractorCSV] Reset failure, using csv directory {}", outputDirectory); + return chl::CL_ERR_INVALID_CONF; + } + + outputDirectory = json_object_get_string(json_object_object_get(json_block, "csv_archive_dir")); + + // check if archive directory exists and is writable by the extractor process + if(!std::filesystem::exists(outputDirectory)) + { + outputDirectory = "/tmp"; + LOG_ERROR("StoryChunkExtractorCSV] Reset failure: csv_archive_dir doesn't exist or not writable; using {}", + outputDirectory); + return chl::CL_ERR_INVALID_CONF; + } + + LOG_INFO("StoryChunkExtractorCSV] Reset success: using csv directory {}", outputDirectory); + return chl::CL_SUCCESS; +} + +/////////////////// + int chronolog::StoryChunkExtractorCSV::process_chunk(StoryChunk* story_chunk) { diff --git a/chrono_common/src/ChunkExtractorRDMA.cpp b/chrono_common/src/ChunkExtractorRDMA.cpp index cffdb466b..125271aaa 100644 --- a/chrono_common/src/ChunkExtractorRDMA.cpp +++ b/chrono_common/src/ChunkExtractorRDMA.cpp @@ -1,9 +1,10 @@ -#include #include - +#include +#include #include #include #include +#include #include #include @@ -16,17 +17,20 @@ chronolog::StoryChunkExtractorRDMA::StoryChunkExtractorRDMA(tl::engine& tl_engin , receiver_service_id(receiving_service_id) , rdma_sender(nullptr) { - try - { - rdma_sender = RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, receiver_service_id); - LOG_TRACE("[ChunkExtractorRDMA] Constructor created rdma_sender for receiver_service {} ", - chl::to_string(receiver_service_id)); - } - catch(...) + if(receiver_service_id.is_valid()) { - LOG_ERROR("[ChunkExtractorRDMA] Constructor : failed to create rdma_sender for receiver_service {} ", - chl::to_string(receiver_service_id)); - rdma_sender = nullptr; + try + { + rdma_sender = RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, receiver_service_id); + LOG_TRACE("[ChunkExtractorRDMA] Constructor created rdma_sender for receiver_service {} ", + chl::to_string(receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[ChunkExtractorRDMA] Constructor : failed to create rdma_sender for receiver_service {} ", + chl::to_string(receiver_service_id)); + rdma_sender = nullptr; + } } } @@ -35,35 +39,44 @@ chronolog::StoryChunkExtractorRDMA::StoryChunkExtractorRDMA(StoryChunkExtractorR , receiver_service_id(other.get_receiver_service_id()) , rdma_sender(nullptr) { - try - { - rdma_sender = RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, receiver_service_id); - LOG_TRACE("[ChunkExtractorRDMA] Constructor copy: created rdma_sender for receiver_service {} ", - chl::to_string(receiver_service_id)); - } - catch(...) + if(receiver_service_id.is_valid()) { - LOG_ERROR("[ChunkExtractorRDMA] Constructor: failed to create rdma_sender for receiver_service {} ", - chl::to_string(receiver_service_id)); - rdma_sender = nullptr; + try + { + rdma_sender = RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, receiver_service_id); + LOG_TRACE("[ChunkExtractorRDMA] Constructor copy: created rdma_sender for receiver_service {} ", + chl::to_string(receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[ChunkExtractorRDMA] Constructor: failed to create rdma_sender for receiver_service {} ", + chl::to_string(receiver_service_id)); + rdma_sender = nullptr; + } } } chl::StoryChunkExtractorRDMA& chronolog::StoryChunkExtractorRDMA::operator=(StoryChunkExtractorRDMA const& other) { - if(this != &other) + if(this == &other) { - if(rdma_sender != nullptr) - { - LOG_TRACE("[ChunkExtractorRDMA] assingment : deleting receiver_service {} ", - chl::to_string(receiver_service_id)); + return *this; + } - delete rdma_sender; - } - sender_tl_engine = other.get_sender_engine(); - receiver_service_id = other.get_receiver_service_id(); + if(rdma_sender != nullptr) + { + LOG_TRACE("[ChunkExtractorRDMA] assingment : deleting receiver_service {} ", + chl::to_string(receiver_service_id)); + delete rdma_sender; + } + + sender_tl_engine = other.get_sender_engine(); + receiver_service_id = other.get_receiver_service_id(); + + if(receiver_service_id.is_valid()) + { try { rdma_sender = RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, receiver_service_id); @@ -81,6 +94,7 @@ chl::StoryChunkExtractorRDMA& chronolog::StoryChunkExtractorRDMA::operator=(Stor return *this; } +//// chronolog::StoryChunkExtractorRDMA::~StoryChunkExtractorRDMA() { @@ -92,13 +106,139 @@ chronolog::StoryChunkExtractorRDMA::~StoryChunkExtractorRDMA() } } +////////////// + +void chronolog::StoryChunkExtractorRDMA::restart_rdma_sender(chl::ServiceId const& new_receiver_service_id) +{ + LOG_DEBUG("[ChunkExtractorRDMA] restart_rdma_sender for new receiver_service {} ", + chl::to_string(new_receiver_service_id)); + + if(rdma_sender != nullptr) + { + LOG_TRACE("[ChunkExtractorRDMA] assingment : deleting receiver_service {} ", + chl::to_string(receiver_service_id)); + + delete rdma_sender; + } + + receiver_service_id = new_receiver_service_id; + + if(!receiver_service_id.is_valid()) + { + return; + } + + try + { + rdma_sender = RDMATransferAgent::CreateRDMATransferAgent(sender_tl_engine, receiver_service_id); + LOG_TRACE("[ChunkExtractorRDMA] assingment: created rdma_sender for receiver_service {} ", + chl::to_string(receiver_service_id)); + } + catch(...) + { + LOG_ERROR("[ChunkExtractorRDMA] assignment: failed to create rdma_sender for receiver_service {} ", + chl::to_string(receiver_service_id)); + rdma_sender = nullptr; + } +} + +//////////// + +int chronolog::StoryChunkExtractorRDMA::reset(chl::ServiceId const& new_receiver_id) +{ + if(!new_receiver_id.is_valid()) + { + return chl::CL_ERR_INVALID_CONF; + } + + LOG_DEBUG("[ChunkExtractorRDMA] reset: about to create rdma_sender for receiver_service {} ", + chl::to_string(new_receiver_id)); + + restart_rdma_sender(new_receiver_id); + + if(is_active()) + { + return chl::CL_SUCCESS; + } + else + { + return chl::CL_ERR_STORY_CHUNK_EXTRACTION; + } +} + +//// +// JSON configuration for single endpoint RDMA Chunk Extractor RDMA +// looks like this: +// +// "extractor_name": { +// "type": "single_endpoint_rdma_extractor", +// "receiving_endpoint": { +// "protocol_conf": "ofi+sockets", +// "service_ip": "127.0.0.1", +// "service_base_port": 2230, +// "service_provider_id": 30 +// } +// } + +int chronolog::StoryChunkExtractorRDMA::reset(json_object* json_block) +{ + if((json_block == nullptr) || !json_object_is_type(json_block, json_type_object) || + (json_object_object_get(json_block, "type") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "type"), json_type_string) || + (std::string("single_endpoint_rdma_extractor") + .compare(json_object_get_string(json_object_object_get(json_block, "type"))) != 0)) + { + LOG_ERROR("[ChunkExtractorRDMA] Reset failure: invalid json config"); + return chl::CL_ERR_INVALID_CONF; + } + + if((json_object_object_get(json_block, "receiving_endpoint") == nullptr) || + !json_object_is_type(json_object_object_get(json_block, "receiving_endpoint"), json_type_object)) + { + LOG_ERROR("[ChunkExtractorRDMA] Reset failure: invalid json config"); + return chl::CL_ERR_INVALID_CONF; + } + + json_object* json_rpc_block = json_object_object_get(json_block, "receiving_endpoint"); + chl::RPCProviderConf receiving_endpoint_conf; + if(receiving_endpoint_conf.parseJsonConf(json_rpc_block) != chl::CL_SUCCESS) + { + LOG_ERROR("[ChunkExtractorRDMA] Reset failure: invalid endpoint config"); + return chl::CL_ERR_INVALID_CONF; + } + + chl::ServiceId new_receiver_id(receiving_endpoint_conf.PROTO_CONF, + receiving_endpoint_conf.IP, + receiving_endpoint_conf.BASE_PORT, + receiving_endpoint_conf.SERVICE_PROVIDER_ID); + + if(!new_receiver_id.is_valid()) + { + return chl::CL_ERR_INVALID_CONF; + } + + LOG_DEBUG("[ChunkExtractorRDMA] reset: about to create rdma_sender for receiver_service {} ", + chl::to_string(new_receiver_id)); + + restart_rdma_sender(new_receiver_id); + + if(is_active()) + { + return chl::CL_SUCCESS; + } + else + { + return chl::CL_ERR_STORY_CHUNK_EXTRACTION; + } +} + //// int chronolog::StoryChunkExtractorRDMA::process_chunk(chronolog::StoryChunk* story_chunk) { try { - LOG_DEBUG("[ExtractorRDMA] tl::thread_id={} processing chunk StoryId={} {}-{} {}-{} eventCount {}", + LOG_DEBUG("[ChunkExtractorRDMA] tl::thread_id={} processing chunk StoryId={} {}-{} {}-{} eventCount {}", thallium::thread::self_id(), story_chunk->getStoryId(), story_chunk->getChronicleName(), @@ -109,10 +249,11 @@ int chronolog::StoryChunkExtractorRDMA::process_chunk(chronolog::StoryChunk* sto if(rdma_sender == nullptr) { - LOG_ERROR("[ChunkExtractorRDMA] Failed to transfer StoryChunk StoryId={} StartTime={}", - story_chunk->getStoryId(), - story_chunk->getStartTime()); - return chl::CL_ERR_UNKNOWN; + LOG_ERROR( + "[ChunkExtractorRDMA] Failed to transfer StoryChunk StoryId={} StartTime={}, no valid rdma_sender", + story_chunk->getStoryId(), + story_chunk->getStartTime()); + return chl::CL_ERR_STORY_CHUNK_EXTRACTION; } std::ostringstream oss(std::ios::binary); diff --git a/chrono_common/src/ConfigurationManager.cpp.mv b/chrono_common/src/ConfigurationManager.cpp.mv deleted file mode 100644 index 023f89a14..000000000 --- a/chrono_common/src/ConfigurationManager.cpp.mv +++ /dev/null @@ -1,817 +0,0 @@ -#include "ConfigurationManager.h" - -#include "ExtractionModuleConfiguration.h" - -namespace chl = chronolog; - -int chronolog::ClockConf::parseJsonConf(json_object* clock_conf) -{ - json_object_object_foreach(clock_conf, key, val) - { - if(strcmp(key, "clocksource_type") == 0) - { - if(json_object_is_type(val, json_type_string)) - { - const char* clocksource_type = json_object_get_string(val); - if(strcmp(clocksource_type, "C_STYLE") == 0) - CLOCKSOURCE_TYPE = ClocksourceType::C_STYLE; - else if(strcmp(clocksource_type, "CPP_STYLE") == 0) - CLOCKSOURCE_TYPE = ClocksourceType::CPP_STYLE; - else if(strcmp(clocksource_type, "TSC") == 0) - CLOCKSOURCE_TYPE = ClocksourceType::TSC; - else - std::cout << "[ClockConfiguration] Unknown clocksource type: " << clocksource_type << std::endl; - } - else - { - std::cerr << "[ClockConfiguration] Failed to parse configuration file: clocksource_type is not a string" - << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - } - else if(strcmp(key, "drift_cal_sleep_sec") == 0) - { - if(json_object_is_type(val, json_type_int)) - { - DRIFT_CAL_SLEEP_SEC = json_object_get_int(val); - } - else - { - std::cerr << "[ClockConfiguration] Failed to parse configuration file: drift_cal_sleep_sec is not an " - "integer" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - else if(strcmp(key, "drift_cal_sleep_nsec") == 0) - { - if(json_object_is_type(val, json_type_int)) - { - DRIFT_CAL_SLEEP_NSEC = json_object_get_int(val); - } - else - { - std::cerr << "[ConfigurationManager] Failed to parse configuration file: drift_cal_sleep_nsec is not " - "an integer" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - } - return 1; -} - -int chronolog::AuthConf::parseJsonConf(json_object* auth_conf) -{ - if(auth_conf == nullptr || !json_object_is_type(auth_conf, json_type_object)) - { - std::cerr << "[AuthConfiguration] Error while parsing configuration file. Authentication configuration is not " - "found or is not an object." - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - json_object_object_foreach(auth_conf, key, val) - { - if(strcmp(key, "auth_type") == 0) - { - if(json_object_is_type(val, json_type_string)) - { - AUTH_TYPE = json_object_get_string(val); - } - else - { - std::cerr << "[AuthConfiguration] Failed to parse configuration file: auth_type is not a string" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - else if(strcmp(key, "module_location") == 0) - { - if(json_object_is_type(val, json_type_string)) - { - MODULE_PATH = json_object_get_string(val); - } - else - { - std::cerr << "[AuthConfiguration] Failed to parse configuration file: module_location is not a string" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - } - return 1; -} - -int chronolog::RPCProviderConf::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "protocol_conf") == 0) - { - assert(json_object_is_type(val, json_type_string)); - PROTO_CONF = json_object_get_string(val); - } - else if(strcmp(key, "service_ip") == 0) - { - assert(json_object_is_type(val, json_type_string)); - IP = json_object_get_string(val); - } - else if(strcmp(key, "service_base_port") == 0) - { - assert(json_object_is_type(val, json_type_int)); - BASE_PORT = json_object_get_int(val); - } - else if(strcmp(key, "service_provider_id") == 0) - { - assert(json_object_is_type(val, json_type_int)); - SERVICE_PROVIDER_ID = json_object_get_int(val); - } - else - { - std::cerr << "[RPCProviderConf] Unknown client end configuration: " << key << std::endl; - } - } - return 1; -} - -int chronolog::LogConf::parseJsonConf(json_object* json_conf) -{ - - if(!json_object_is_type(json_conf, json_type_object)) - { - std::cerr << "[LogConf] Logger configuration is not found or is not an object." << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - json_object_object_foreach(json_conf, key, json_val) - { - if(strcmp(key, "monitor") != 0) - { - std::cerr << "[LogConf] Unknown Log configuration key : " << key << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - - json_object_object_foreach(json_val, key, val) - { - if(strcmp(key, "type") == 0) - { - assert(json_object_is_type(val, json_type_string)); - LOGTYPE = json_object_get_string(val); - } - else if(strcmp(key, "file") == 0) - { - assert(json_object_is_type(val, json_type_string)); - LOGFILE = json_object_get_string(val); - } - else if(strcmp(key, "level") == 0) - { - assert(json_object_is_type(val, json_type_string)); - parselogLevelConf(val, LOGLEVEL); - } - else if(strcmp(key, "name") == 0) - { - assert(json_object_is_type(val, json_type_string)); - LOGNAME = json_object_get_string(val); - } - else if(strcmp(key, "filesize") == 0) - { - assert(json_object_is_type(val, json_type_int)); - LOGFILESIZE = json_object_get_int(val); - } - else if(strcmp(key, "filenum") == 0) - { - assert(json_object_is_type(val, json_type_int)); - LOGFILENUM = json_object_get_int(val); - } - else if(strcmp(key, "flushlevel") == 0) - { - assert(json_object_is_type(val, json_type_string)); - parseFlushLevelConf(val, FLUSHLEVEL); - } - else - { - std::cerr << "[LogConf] Unknown log configuration: " << key << std::endl; - } - } - } - return 1; -} - -int chronolog::VisorConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "VisorClientPortalService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_client_portal_service_conf = - json_object_object_get(json_conf, "VisorClientPortalService"); - json_object_object_foreach(visor_client_portal_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_CLIENT_PORTAL_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[VisorConfiguration] [chrono_visor] Unknown VisorClientPortalService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "VisorKeeperRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = - json_object_object_get(json_conf, "VisorKeeperRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_KEEPER_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[VisorConfiguration] Unknown VisorKeeperRegistryService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* chronovisor_log = json_object_object_get(json_conf, "Monitoring"); - VISOR_LOG_CONF.parseJsonConf(chronovisor_log); - } - else if(strcmp(key, "delayed_data_admin_exit_in_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int delayed_exit_value = json_object_get_int(val); - DELAYED_DATA_ADMIN_EXIT_IN_SECS = - ((0 < delayed_exit_value && delayed_exit_value < 60) ? delayed_exit_value : 5); - } - else - { - std::cerr << "[VisorConfiguration] Unknown Visor configuration: " << key << std::endl; - } - } - return 1; -} -////////////// - -chl::KeeperConfiguration::~KeeperConfiguration() -{ - if(extraction_module_conf != nullptr) { delete extraction_module_conf; } -} -////////////// - -int chronolog::KeeperConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "RecordingGroup") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int value = json_object_get_int(val); - RECORDING_GROUP = (value >= 0 ? value : 0); - } - else if(strcmp(key, "KeeperRecordingService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_recording_service_conf = json_object_object_get(json_conf, "KeeperRecordingService"); - json_object_object_foreach(keeper_recording_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - KEEPER_RECORDING_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown KeeperRecordingService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "KeeperDataStoreAdminService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_data_store_admin_service_conf = - json_object_object_get(json_conf, "KeeperDataStoreAdminService"); - json_object_object_foreach(keeper_data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - DATA_STORE_ADMIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown KeeperDataStoreAdminService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "VisorKeeperRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = - json_object_object_get(json_conf, "VisorKeeperRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown VisorKeeperRegistryService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "KeeperGrapherDrainService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_grapher_drain_service_conf = - json_object_object_get(json_conf, "KeeperGrapherDrainService"); - json_object_object_foreach(keeper_grapher_drain_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - KEEPER_GRAPHER_DRAIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown KeeperGrapherDrainService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_monitor = json_object_object_get(json_conf, "Monitoring"); - LOG_CONF.parseJsonConf(keeper_monitor); - /* json_object_object_foreach(chronokeeper_log, key, val) - { - if(strcmp(key, "monitor") == 0) - { - LOG_CONF.parseJsonConf(chronokeeper_log); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Monitoring configuration: " - << key << std::endl; - } - }*/ - } - else if(strcmp(key, "DataStoreInternals") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_conf = json_object_object_get(json_conf, "DataStoreInternals"); - DATA_STORE_CONF.parseJsonConf(data_store_conf); - } - else if(strcmp(key,"ExtractionModule") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extraction_conf_json_object = json_object_object_get(json_conf, "ExtractionModule"); - if(extraction_module_conf != nullptr) { delete extraction_module_conf; } - extraction_module_conf = new chl::ExtractionModuleConfiguration(extraction_conf_json_object); - } - else if(strcmp(key, "Extractors") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extractors = json_object_object_get(json_conf, "Extractors"); - json_object_object_foreach(extractors, key, val) - { - if(strcmp(key, "story_files_dir") == 0) - { - assert(json_object_is_type(val, json_type_string)); - EXTRACTOR_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Extractors configuration " << key - << std::endl; - } - } - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Keeper configuration: " << key << std::endl; - } - } - return 1; -} - -////////////// - -chl::GrapherConfiguration::~GrapherConfiguration() -{ - if(extraction_module_conf != nullptr) { delete extraction_module_conf; } -} - -////////////// - -int chronolog::GrapherConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "RecordingGroup") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int value = json_object_get_int(val); - RECORDING_GROUP = (value >= 0 ? value : 0); - } - else if(strcmp(key, "KeeperGrapherDrainService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_grapher_drain_service_conf = - json_object_object_get(json_conf, "KeeperGrapherDrainService"); - json_object_object_foreach(keeper_grapher_drain_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - KEEPER_GRAPHER_DRAIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown KeeperGrapherDrainService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "DataStoreAdminService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_admin_service_conf = json_object_object_get(json_conf, "DataStoreAdminService"); - json_object_object_foreach(data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - DATA_STORE_ADMIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown DataStoreAdminService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "VisorRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = json_object_object_get(json_conf, "VisorRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown VisorRegistryService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* grapher_monitor = json_object_object_get(json_conf, "Monitoring"); - LOG_CONF.parseJsonConf(grapher_monitor); - /* - json_object_object_foreach(chrono_logging, key, val) - { - if(strcmp(key, "monitor") == 0) - { - LOG_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConf] Unknown Monitoring configuration: " << key - << std::endl; - } - }*/ - } - else if(strcmp(key, "DataStoreInternals") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_conf = json_object_object_get(json_conf, "DataStoreInternals"); - DATA_STORE_CONF.parseJsonConf(data_store_conf); - } - else if(strcmp(key,"ExtractionModule") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extraction_conf_json_object = json_object_object_get(json_conf, "ExtractionModule"); - if(extraction_module_conf != nullptr) { delete extraction_module_conf; } - extraction_module_conf = new chl::ExtractionModuleConfiguration(extraction_conf_json_object); - } - else if(strcmp(key, "Extractors") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extractors = json_object_object_get(json_conf, "Extractors"); - json_object_object_foreach(extractors, key, val) - { - if(strcmp(key, "story_files_dir") == 0) - { - assert(json_object_is_type(val, json_type_string)); - EXTRACTOR_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown Extractors configuration " << key << std::endl; - } - } - } - else - { - std::cerr << "[GrapherConfiguration] Unknown Grapher configuration " << key << std::endl; - } - } - return 1; -} - -int chronolog::PlayerConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "RecordingGroup") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int value = json_object_get_int(val); - RECORDING_GROUP = (value >= 0 ? value : 0); - } - else if(strcmp(key, "PlayerStoreAdminService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_admin_service_conf = json_object_object_get(json_conf, "PlayerStoreAdminService"); - json_object_object_foreach(data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - DATA_STORE_ADMIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr - << "[ConfigurationManager] [chrono_player] Unknown PlayerStoreAdminService configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "PlaybackQueryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_admin_service_conf = json_object_object_get(json_conf, "PlaybackQueryService"); - json_object_object_foreach(data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - PLAYBACK_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown PlaybackQueryService configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "VisorRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = json_object_object_get(json_conf, "VisorRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown VisorRegistryService configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* player_monitor = json_object_object_get(json_conf, "Monitoring"); - LOG_CONF.parseJsonConf(player_monitor); - /*json_object_object_foreach(chrono_logging, key, val) - { - if(strcmp(key, "monitor") == 0) - { - LOG_CONF.parseJsonConf(palyer_monitor); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown Monitoring configuration: " << key - << std::endl; - } - }*/ - } - else if(strcmp(key, "DataStoreInternals") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_conf = json_object_object_get(json_conf, "DataStoreInternals"); - DATA_STORE_CONF.parseJsonConf(data_store_conf); //, key, val) - } - else if(strcmp(key, "ArchiveReaders") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* archive_readers = json_object_object_get(json_conf, "ArchiveReaders"); - json_object_object_foreach(archive_readers, key, val) - { - if(strcmp(key, "story_files_dir") == 0) - { - assert(json_object_is_type(val, json_type_string)); - READER_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown ArchiveReaders configuration " << key - << std::endl; - } - } - } - else - { - std::cerr << "[ConfigurationManager][chrono_player] Unknown Player configuration " << key << std::endl; - } - } - return 1; -} - -int chronolog::DataStoreConf::parseJsonConf(json_object* data_store_json_conf) -{ - json_object_object_foreach(data_store_json_conf, key, val) - { - if(strcmp(key, "max_story_chunk_size") == 0) - { - assert(json_object_is_type(val, json_type_int)); - max_story_chunk_size = json_object_get_int(val); - } - else if(strcmp(key, "story_chunk_duration_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - story_chunk_duration_secs = json_object_get_int(val); - } - else if(strcmp(key, "acceptance_window_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - acceptance_window_secs = json_object_get_int(val); - } - else if(strcmp(key, "inactive_story_delay_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - inactive_story_delay_secs = json_object_get_int(val); - } - else - { - std::cerr << "[DataStoreConf] Unknown DataStoreInternals configuration: " << key << std::endl; - } - } - - return 1; -} - -//////////////// - -std::string & chl::KeeperConfiguration::to_string( std::string & a_string) const -{ - a_string += "[CHRONO_KEEPER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", " + (extraction_module_conf != nullptr ? extraction_module_conf->to_string(a_string) : std::string("ExtractionModuleConf { }")) + - "]"; - - return a_string; -} - -//////////////// - -std::string & chl::GrapherConfiguration::to_string( std::string & a_string) const - { - a_string += "[CHRONO_GRAPHER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", EXTRACTOR_CONF: " + EXTRACTOR_CONF.to_String() + "]"; - ", " + (extraction_module_conf != nullptr ? extraction_module_conf->to_string(a_string) : std::string("ExtractionModuleConf { }")) + - "]"; - - return a_string; -} - -//////////////// - -std::string & chl::PlayerConfiguration::to_string(std::string & a_string) const -{ - a_string +="[CHRONO_PLAYER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", PLAYBACK_SERVICE_CONF: " + PLAYBACK_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", READER_CONF: " + READER_CONF.to_String() + "]"; - - return a_string; -} - - -//////////////// - -void chl::ConfigurationManager::LoadConfFromJSONFile(const std::string& conf_file_path) -{ - json_object* root = json_object_from_file(conf_file_path.c_str()); - if(root == nullptr) - { - std::cerr << "[ConfigurationManager] Failed to open configuration file at path: " << conf_file_path.c_str() - << ". Exiting..." << std::endl; - exit(chronolog::CL_ERR_UNKNOWN); - } - - json_object_object_foreach(root, key, val) - { - if(strcmp(key, "clock") == 0) - { - json_object* clock_conf = json_object_object_get(root, "clock"); - if(clock_conf == nullptr || !json_object_is_type(clock_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " - << conf_file_path.c_str() << ". Clock configuration is not found or is not an object." - << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - CLOCK_CONF.parseJsonConf(clock_conf); - } - else if(strcmp(key, "authentication") == 0) - { - json_object* auth_conf = json_object_object_get(root, "authentication"); - if(auth_conf == nullptr || !json_object_is_type(auth_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " - << conf_file_path.c_str() - << ". Authentication configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - AUTH_CONF.parseJsonConf(auth_conf); - } - else if(strcmp(key, "chrono_visor") == 0) - { - json_object* chrono_visor_conf = json_object_object_get(root, "chrono_visor"); - if(chrono_visor_conf == nullptr || !json_object_is_type(chrono_visor_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " - << conf_file_path.c_str() - << ". ChronoVisor configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - VISOR_CONF.parseJsonConf(chrono_visor_conf); - } - else if(strcmp(key, "chrono_keeper") == 0) - { - json_object* chrono_keeper_conf = json_object_object_get(root, "chrono_keeper"); - if(chrono_keeper_conf == nullptr || !json_object_is_type(chrono_keeper_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " - << conf_file_path.c_str() - << ". ChronoKeeper configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - KEEPER_CONF.parseJsonConf(chrono_keeper_conf); - } - else if(strcmp(key, "chrono_grapher") == 0) - { - json_object* chrono_grapher_conf = json_object_object_get(root, "chrono_grapher"); - if(chrono_grapher_conf == nullptr || !json_object_is_type(chrono_grapher_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " - << conf_file_path.c_str() - << ". ChronoGrapher configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - GRAPHER_CONF.parseJsonConf(chrono_grapher_conf); - } - else if(strcmp(key, "chrono_player") == 0) - { - json_object* chrono_player_conf = json_object_object_get(root, "chrono_player"); - if(chrono_player_conf == nullptr || !json_object_is_type(chrono_player_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " - << conf_file_path.c_str() - << ". ChronoPlayer configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - PLAYER_CONF.parseJsonConf(chrono_player_conf); - } - } - json_object_put(root); - } - diff --git a/chrono_common/src/ExtractionModuleConfiguration.cpp b/chrono_common/src/ExtractionModuleConfiguration.cpp new file mode 100644 index 000000000..a7211c8b0 --- /dev/null +++ b/chrono_common/src/ExtractionModuleConfiguration.cpp @@ -0,0 +1,101 @@ + +#include +#include + +#include + + +namespace chl = chronolog; + +chronolog::ExtractionModuleConfiguration::~ExtractionModuleConfiguration() +{ + //clear old extractors map + for(auto iter = extractors.begin(); iter != extractors.end(); ++iter) { json_object_put((*iter).second); } + + extractors.clear(); +} + +/////// + +int chronolog::ExtractionModuleConfiguration::parse_json_object(json_object* json_block) +{ + + if(json_block == nullptr || !json_object_is_type(json_block, json_type_object)) + { + return chl::CL_ERR_INVALID_CONF; + } + + //clear old extractors map + + for(auto iter = extractors.begin(); iter != extractors.end(); ++iter) { json_object_put((*iter).second); } + + extractors.clear(); + + json_object_object_foreach(json_block, key, json_val) + { + + if(strcmp(key, "extraction_stream_count") == 0) + { + if(!json_object_is_type(json_val, json_type_int)) + { + std::cerr << "[ExtractionConfiguration] Invalid 'extraction_stream_count': expected json_object" + << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + int value = json_object_get_int(json_val); + extraction_stream_count = (value >= 0 ? value : 0); + if(extraction_stream_count == 0) + { + std::cerr << "[ExtractionModuleConfiguration] Invalid 'extraction_stream_count': expected integer " + << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + } + else if(strcmp(key, "extraction_protocol") == 0) + { + if(!json_object_is_type(json_val, json_type_string)) + { + std::cerr << "[ExtractionConfiguration] Invalid 'extraction_protocol': expected json_object" + << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + extraction_protocol = json_object_get_string(json_val); + } + else if(strcmp(key, "extractors") == 0) + { + if((json_val == nullptr) || !json_object_is_type(json_val, json_type_object)) + { + std::cerr << "[ExtractionModuleConfiguration] Invalid 'extractors': expected json_object" << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + + json_object_object_foreach(json_val, key, val) + { + if(!json_object_is_type(val, json_type_object)) + { + std::cerr << "[ExtractionConfiguration] Invalid 'extractor': expected json_object" << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + else if(!json_object_is_type(json_object_object_get(val, "type"), json_type_string)) + { + std::cerr << "[ExtractionConfiguration] Invalid 'extractor_type' " << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + else + { + std::string extractor_type = json_object_get_string(json_object_object_get(val, "type")); + //retrieve_extractor json_object and increment its reference count + json_object* extractor_json_object = json_object_object_get(json_val, key); + extractors[extractor_type] = json_object_get(extractor_json_object); + } + } + } + else + { + std::cerr << "[ExtractionModuleConfiguration] Unknown configuration block: " << key << std::endl; + return chl::CL_ERR_INVALID_CONF; + } + } + + return chronolog::CL_SUCCESS; +} diff --git a/chrono_common/src/deprecated_ConfigurationManager.cpp b/chrono_common/src/deprecated_ConfigurationManager.cpp deleted file mode 100644 index faf79a19b..000000000 --- a/chrono_common/src/deprecated_ConfigurationManager.cpp +++ /dev/null @@ -1,825 +0,0 @@ -#include "ConfigurationManager.h" - -#include "ExtractionModuleConfiguration.h" - -namespace chl = chronolog; - -int chronolog::ClockConf::parseJsonConf(json_object* clock_conf) -{ - json_object_object_foreach(clock_conf, key, val) - { - if(strcmp(key, "clocksource_type") == 0) - { - if(json_object_is_type(val, json_type_string)) - { - const char* clocksource_type = json_object_get_string(val); - if(strcmp(clocksource_type, "C_STYLE") == 0) - CLOCKSOURCE_TYPE = ClocksourceType::C_STYLE; - else if(strcmp(clocksource_type, "CPP_STYLE") == 0) - CLOCKSOURCE_TYPE = ClocksourceType::CPP_STYLE; - else if(strcmp(clocksource_type, "TSC") == 0) - CLOCKSOURCE_TYPE = ClocksourceType::TSC; - else - std::cout << "[ClockConfiguration] Unknown clocksource type: " << clocksource_type << std::endl; - } - else - { - std::cerr << "[ClockConfiguration] Failed to parse configuration file: clocksource_type is not a string" - << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - } - else if(strcmp(key, "drift_cal_sleep_sec") == 0) - { - if(json_object_is_type(val, json_type_int)) - { - DRIFT_CAL_SLEEP_SEC = json_object_get_int(val); - } - else - { - std::cerr << "[ClockConfiguration] Failed to parse configuration file: drift_cal_sleep_sec is not an " - "integer" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - else if(strcmp(key, "drift_cal_sleep_nsec") == 0) - { - if(json_object_is_type(val, json_type_int)) - { - DRIFT_CAL_SLEEP_NSEC = json_object_get_int(val); - } - else - { - std::cerr << "[ConfigurationManager] Failed to parse configuration file: drift_cal_sleep_nsec is not " - "an integer" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - } - return 1; -} - -int chronolog::AuthConf::parseJsonConf(json_object* auth_conf) -{ - if(auth_conf == nullptr || !json_object_is_type(auth_conf, json_type_object)) - { - std::cerr << "[AuthConfiguration] Error while parsing configuration file. Authentication configuration is not " - "found or is not an object." - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - json_object_object_foreach(auth_conf, key, val) - { - if(strcmp(key, "auth_type") == 0) - { - if(json_object_is_type(val, json_type_string)) - { - AUTH_TYPE = json_object_get_string(val); - } - else - { - std::cerr << "[AuthConfiguration] Failed to parse configuration file: auth_type is not a string" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - else if(strcmp(key, "module_location") == 0) - { - if(json_object_is_type(val, json_type_string)) - { - MODULE_PATH = json_object_get_string(val); - } - else - { - std::cerr << "[AuthConfiguration] Failed to parse configuration file: module_location is not a string" - << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - } - } - return 1; -} - -int chronolog::RPCProviderConf::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "protocol_conf") == 0) - { - assert(json_object_is_type(val, json_type_string)); - PROTO_CONF = json_object_get_string(val); - } - else if(strcmp(key, "service_ip") == 0) - { - assert(json_object_is_type(val, json_type_string)); - IP = json_object_get_string(val); - } - else if(strcmp(key, "service_base_port") == 0) - { - assert(json_object_is_type(val, json_type_int)); - BASE_PORT = json_object_get_int(val); - } - else if(strcmp(key, "service_provider_id") == 0) - { - assert(json_object_is_type(val, json_type_int)); - SERVICE_PROVIDER_ID = json_object_get_int(val); - } - else - { - std::cerr << "[RPCProviderConf] Unknown client end configuration: " << key << std::endl; - } - } - return 1; -} - -int chronolog::LogConf::parseJsonConf(json_object* json_conf) -{ - - if(!json_object_is_type(json_conf, json_type_object)) - { - std::cerr << "[LogConf] Logger configuration is not found or is not an object." << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - json_object_object_foreach(json_conf, key, json_val) - { - if(strcmp(key, "monitor") != 0) - { - std::cerr << "[LogConf] Unknown Log configuration key : " << key << std::endl; - return (chronolog::CL_ERR_INVALID_CONF); - } - - json_object_object_foreach(json_val, key, val) - { - if(strcmp(key, "type") == 0) - { - assert(json_object_is_type(val, json_type_string)); - LOGTYPE = json_object_get_string(val); - } - else if(strcmp(key, "file") == 0) - { - assert(json_object_is_type(val, json_type_string)); - LOGFILE = json_object_get_string(val); - } - else if(strcmp(key, "level") == 0) - { - assert(json_object_is_type(val, json_type_string)); - parselogLevelConf(val, LOGLEVEL); - } - else if(strcmp(key, "name") == 0) - { - assert(json_object_is_type(val, json_type_string)); - LOGNAME = json_object_get_string(val); - } - else if(strcmp(key, "filesize") == 0) - { - assert(json_object_is_type(val, json_type_int)); - LOGFILESIZE = json_object_get_int(val); - } - else if(strcmp(key, "filenum") == 0) - { - assert(json_object_is_type(val, json_type_int)); - LOGFILENUM = json_object_get_int(val); - } - else if(strcmp(key, "flushlevel") == 0) - { - assert(json_object_is_type(val, json_type_string)); - parseFlushLevelConf(val, FLUSHLEVEL); - } - else - { - std::cerr << "[LogConf] Unknown log configuration: " << key << std::endl; - } - } - } - return 1; -} - -int chronolog::VisorConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "VisorClientPortalService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_client_portal_service_conf = - json_object_object_get(json_conf, "VisorClientPortalService"); - json_object_object_foreach(visor_client_portal_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_CLIENT_PORTAL_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[VisorConfiguration] [chrono_visor] Unknown VisorClientPortalService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "VisorKeeperRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = - json_object_object_get(json_conf, "VisorKeeperRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_KEEPER_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[VisorConfiguration] Unknown VisorKeeperRegistryService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* chronovisor_log = json_object_object_get(json_conf, "Monitoring"); - VISOR_LOG_CONF.parseJsonConf(chronovisor_log); - } - else if(strcmp(key, "delayed_data_admin_exit_in_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int delayed_exit_value = json_object_get_int(val); - DELAYED_DATA_ADMIN_EXIT_IN_SECS = - ((0 < delayed_exit_value && delayed_exit_value < 60) ? delayed_exit_value : 5); - } - else - { - std::cerr << "[VisorConfiguration] Unknown Visor configuration: " << key << std::endl; - } - } - return 1; -} -////////////// - -chl::KeeperConfiguration::~KeeperConfiguration() -{ - if(extraction_module_conf != nullptr) - { - delete extraction_module_conf; - } -} -////////////// - -int chronolog::KeeperConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "RecordingGroup") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int value = json_object_get_int(val); - RECORDING_GROUP = (value >= 0 ? value : 0); - } - else if(strcmp(key, "KeeperRecordingService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_recording_service_conf = json_object_object_get(json_conf, "KeeperRecordingService"); - json_object_object_foreach(keeper_recording_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - KEEPER_RECORDING_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown KeeperRecordingService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "KeeperDataStoreAdminService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_data_store_admin_service_conf = - json_object_object_get(json_conf, "KeeperDataStoreAdminService"); - json_object_object_foreach(keeper_data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - DATA_STORE_ADMIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown KeeperDataStoreAdminService " - "configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "VisorKeeperRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = - json_object_object_get(json_conf, "VisorKeeperRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown VisorKeeperRegistryService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "KeeperGrapherDrainService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_grapher_drain_service_conf = - json_object_object_get(json_conf, "KeeperGrapherDrainService"); - json_object_object_foreach(keeper_grapher_drain_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - KEEPER_GRAPHER_DRAIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[KeeperConfiguration] Unknown KeeperGrapherDrainService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_monitor = json_object_object_get(json_conf, "Monitoring"); - LOG_CONF.parseJsonConf(keeper_monitor); - /* json_object_object_foreach(chronokeeper_log, key, val) - { - if(strcmp(key, "monitor") == 0) - { - LOG_CONF.parseJsonConf(chronokeeper_log); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Monitoring configuration: " - << key << std::endl; - } - }*/ - } - else if(strcmp(key, "DataStoreInternals") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_conf = json_object_object_get(json_conf, "DataStoreInternals"); - DATA_STORE_CONF.parseJsonConf(data_store_conf); - } - else if(strcmp(key, "ExtractionModule") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extraction_conf_json_object = json_object_object_get(json_conf, "ExtractionModule"); - if(extraction_module_conf != nullptr) - { - delete extraction_module_conf; - } - extraction_module_conf = new chl::ExtractionModuleConfiguration(extraction_conf_json_object); - } - else if(strcmp(key, "Extractors") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extractors = json_object_object_get(json_conf, "Extractors"); - json_object_object_foreach(extractors, key, val) - { - if(strcmp(key, "story_files_dir") == 0) - { - assert(json_object_is_type(val, json_type_string)); - EXTRACTOR_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Extractors configuration " << key - << std::endl; - } - } - } - else - { - std::cerr << "[ConfigurationManager] [chrono_keeper] Unknown Keeper configuration: " << key << std::endl; - } - } - return 1; -} - -////////////// - -chl::GrapherConfiguration::~GrapherConfiguration() -{ - if(extraction_module_conf != nullptr) - { - delete extraction_module_conf; - } -} - -////////////// - -int chronolog::GrapherConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "RecordingGroup") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int value = json_object_get_int(val); - RECORDING_GROUP = (value >= 0 ? value : 0); - } - else if(strcmp(key, "KeeperGrapherDrainService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* keeper_grapher_drain_service_conf = - json_object_object_get(json_conf, "KeeperGrapherDrainService"); - json_object_object_foreach(keeper_grapher_drain_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - KEEPER_GRAPHER_DRAIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown KeeperGrapherDrainService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "DataStoreAdminService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_admin_service_conf = json_object_object_get(json_conf, "DataStoreAdminService"); - json_object_object_foreach(data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - DATA_STORE_ADMIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown DataStoreAdminService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "VisorRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = json_object_object_get(json_conf, "VisorRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown VisorRegistryService configuration: " << key - << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* grapher_monitor = json_object_object_get(json_conf, "Monitoring"); - LOG_CONF.parseJsonConf(grapher_monitor); - /* - json_object_object_foreach(chrono_logging, key, val) - { - if(strcmp(key, "monitor") == 0) - { - LOG_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[GrapherConf] Unknown Monitoring configuration: " << key - << std::endl; - } - }*/ - } - else if(strcmp(key, "DataStoreInternals") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_conf = json_object_object_get(json_conf, "DataStoreInternals"); - DATA_STORE_CONF.parseJsonConf(data_store_conf); - } - else if(strcmp(key, "ExtractionModule") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extraction_conf_json_object = json_object_object_get(json_conf, "ExtractionModule"); - if(extraction_module_conf != nullptr) - { - delete extraction_module_conf; - } - extraction_module_conf = new chl::ExtractionModuleConfiguration(extraction_conf_json_object); - } - else if(strcmp(key, "Extractors") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* extractors = json_object_object_get(json_conf, "Extractors"); - json_object_object_foreach(extractors, key, val) - { - if(strcmp(key, "story_files_dir") == 0) - { - assert(json_object_is_type(val, json_type_string)); - EXTRACTOR_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[GrapherConfiguration] Unknown Extractors configuration " << key << std::endl; - } - } - } - else - { - std::cerr << "[GrapherConfiguration] Unknown Grapher configuration " << key << std::endl; - } - } - return 1; -} - -int chronolog::PlayerConfiguration::parseJsonConf(json_object* json_conf) -{ - json_object_object_foreach(json_conf, key, val) - { - if(strcmp(key, "RecordingGroup") == 0) - { - assert(json_object_is_type(val, json_type_int)); - int value = json_object_get_int(val); - RECORDING_GROUP = (value >= 0 ? value : 0); - } - else if(strcmp(key, "PlayerStoreAdminService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_admin_service_conf = json_object_object_get(json_conf, "PlayerStoreAdminService"); - json_object_object_foreach(data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - DATA_STORE_ADMIN_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr - << "[ConfigurationManager] [chrono_player] Unknown PlayerStoreAdminService configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "PlaybackQueryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_admin_service_conf = json_object_object_get(json_conf, "PlaybackQueryService"); - json_object_object_foreach(data_store_admin_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - PLAYBACK_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown PlaybackQueryService configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "VisorRegistryService") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* visor_keeper_registry_service_conf = json_object_object_get(json_conf, "VisorRegistryService"); - json_object_object_foreach(visor_keeper_registry_service_conf, key, val) - { - if(strcmp(key, "rpc") == 0) - { - VISOR_REGISTRY_SERVICE_CONF.parseJsonConf(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown VisorRegistryService configuration: " - << key << std::endl; - } - } - } - else if(strcmp(key, "Monitoring") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* player_monitor = json_object_object_get(json_conf, "Monitoring"); - LOG_CONF.parseJsonConf(player_monitor); - /*json_object_object_foreach(chrono_logging, key, val) - { - if(strcmp(key, "monitor") == 0) - { - LOG_CONF.parseJsonConf(palyer_monitor); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown Monitoring configuration: " << key - << std::endl; - } - }*/ - } - else if(strcmp(key, "DataStoreInternals") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* data_store_conf = json_object_object_get(json_conf, "DataStoreInternals"); - DATA_STORE_CONF.parseJsonConf(data_store_conf); //, key, val) - } - else if(strcmp(key, "ArchiveReaders") == 0) - { - assert(json_object_is_type(val, json_type_object)); - json_object* archive_readers = json_object_object_get(json_conf, "ArchiveReaders"); - json_object_object_foreach(archive_readers, key, val) - { - if(strcmp(key, "story_files_dir") == 0) - { - assert(json_object_is_type(val, json_type_string)); - READER_CONF.story_files_dir = json_object_get_string(val); - } - else - { - std::cerr << "[ConfigurationManager] [chrono_player] Unknown ArchiveReaders configuration " << key - << std::endl; - } - } - } - else - { - std::cerr << "[ConfigurationManager][chrono_player] Unknown Player configuration " << key << std::endl; - } - } - return 1; -} - -int chronolog::DataStoreConf::parseJsonConf(json_object* data_store_json_conf) -{ - json_object_object_foreach(data_store_json_conf, key, val) - { - if(strcmp(key, "max_story_chunk_size") == 0) - { - assert(json_object_is_type(val, json_type_int)); - max_story_chunk_size = json_object_get_int(val); - } - else if(strcmp(key, "story_chunk_duration_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - story_chunk_duration_secs = json_object_get_int(val); - } - else if(strcmp(key, "acceptance_window_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - acceptance_window_secs = json_object_get_int(val); - } - else if(strcmp(key, "inactive_story_delay_secs") == 0) - { - assert(json_object_is_type(val, json_type_int)); - inactive_story_delay_secs = json_object_get_int(val); - } - else - { - std::cerr << "[DataStoreConf] Unknown DataStoreInternals configuration: " << key << std::endl; - } - } - - return 1; -} - -//////////////// - -std::string& chl::KeeperConfiguration::to_string(std::string& a_string) const -{ - a_string += "[CHRONO_KEEPER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + ", " + - (extraction_module_conf != nullptr ? extraction_module_conf->to_string(a_string) - : std::string("ExtractionModuleConf { }")) + - "]"; - - return a_string; -} - -//////////////// - -std::string& chl::GrapherConfiguration::to_string(std::string& a_string) const -{ - a_string += "[CHRONO_GRAPHER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", KEEPER_GRAPHER_DRAIN_SERVICE_CONF: " + KEEPER_GRAPHER_DRAIN_SERVICE_CONF.to_String() + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", EXTRACTOR_CONF: " + EXTRACTOR_CONF.to_String() + "]"; - ", " + - (extraction_module_conf != nullptr ? extraction_module_conf->to_string(a_string) - : std::string("ExtractionModuleConf { }")) + - "]"; - - return a_string; -} - -//////////////// - -std::string& chl::PlayerConfiguration::to_string(std::string& a_string) const -{ - a_string += "[CHRONO_PLAYER_CONFIGURATION: RECORDING_GROUP: " + std::to_string(RECORDING_GROUP) + - ", DATA_STORE_ADMIN_SERVICE_CONF: " + DATA_STORE_ADMIN_SERVICE_CONF.to_String() + - ", PLAYBACK_SERVICE_CONF: " + PLAYBACK_SERVICE_CONF.to_String() + - ", VISOR_REGISTRY_SERVICE_CONF: " + VISOR_REGISTRY_SERVICE_CONF.to_String() + - ", LOG_CONF: " + LOG_CONF.to_String() + ", DATA_STORE_CONF: " + DATA_STORE_CONF.to_String() + - ", READER_CONF: " + READER_CONF.to_String() + "]"; - - return a_string; -} - - -//////////////// - -void chl::ConfigurationManager::LoadConfFromJSONFile(const std::string& conf_file_path) -{ - json_object* root = json_object_from_file(conf_file_path.c_str()); - if(root == nullptr) - { - std::cerr << "[ConfigurationManager] Failed to open configuration file at path: " << conf_file_path.c_str() - << ". Exiting..." << std::endl; - exit(chronolog::CL_ERR_UNKNOWN); - } - - json_object_object_foreach(root, key, val) - { - if(strcmp(key, "clock") == 0) - { - json_object* clock_conf = json_object_object_get(root, "clock"); - if(clock_conf == nullptr || !json_object_is_type(clock_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " << conf_file_path.c_str() - << ". Clock configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - CLOCK_CONF.parseJsonConf(clock_conf); - } - else if(strcmp(key, "authentication") == 0) - { - json_object* auth_conf = json_object_object_get(root, "authentication"); - if(auth_conf == nullptr || !json_object_is_type(auth_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " << conf_file_path.c_str() - << ". Authentication configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - AUTH_CONF.parseJsonConf(auth_conf); - } - else if(strcmp(key, "chrono_visor") == 0) - { - json_object* chrono_visor_conf = json_object_object_get(root, "chrono_visor"); - if(chrono_visor_conf == nullptr || !json_object_is_type(chrono_visor_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " << conf_file_path.c_str() - << ". ChronoVisor configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - VISOR_CONF.parseJsonConf(chrono_visor_conf); - } - else if(strcmp(key, "chrono_keeper") == 0) - { - json_object* chrono_keeper_conf = json_object_object_get(root, "chrono_keeper"); - if(chrono_keeper_conf == nullptr || !json_object_is_type(chrono_keeper_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " << conf_file_path.c_str() - << ". ChronoKeeper configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - KEEPER_CONF.parseJsonConf(chrono_keeper_conf); - } - else if(strcmp(key, "chrono_grapher") == 0) - { - json_object* chrono_grapher_conf = json_object_object_get(root, "chrono_grapher"); - if(chrono_grapher_conf == nullptr || !json_object_is_type(chrono_grapher_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " << conf_file_path.c_str() - << ". ChronoGrapher configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - GRAPHER_CONF.parseJsonConf(chrono_grapher_conf); - } - else if(strcmp(key, "chrono_player") == 0) - { - json_object* chrono_player_conf = json_object_object_get(root, "chrono_player"); - if(chrono_player_conf == nullptr || !json_object_is_type(chrono_player_conf, json_type_object)) - { - std::cerr << "[ConfigurationManager] Error while parsing configuration file " << conf_file_path.c_str() - << ". ChronoPlayer configuration is not found or is not an object." << std::endl; - exit(chronolog::CL_ERR_INVALID_CONF); - } - PLAYER_CONF.parseJsonConf(chrono_player_conf); - } - } - json_object_put(root); -} diff --git a/default_conf.json.in b/default_conf.json.in index 2a6f9e5f5..1b46917b2 100644 --- a/default_conf.json.in +++ b/default_conf.json.in @@ -89,8 +89,24 @@ "acceptance_window_secs": 15, "inactive_story_delay_secs": 120 }, - "Extractors": { - "story_files_dir": "/tmp" + "ExtractionModule": { + "extraction_stream_count":2, + "extraction_protocol": "ofi+sockets", + "extractors": { + "csv_tier_extractor": { + "type": "csv_extractor", + "csv_archive_dir": "/tmp/csv_archive" + }, + "extractor_to_grapher": { + "type": "single_endpoint_rdma_extractor", + "receiving_endpoint": { + "protocol_conf": "ofi+sockets", + "service_ip": "127.0.0.1", + "service_base_port": 3333, + "service_provider_id": 33 + } + } + } } }, "chrono_grapher": { @@ -136,8 +152,15 @@ "acceptance_window_secs": 180, "inactive_story_delay_secs": 300 }, - "Extractors": { - "story_files_dir": "/tmp" + "ExtractionModule": { + "extraction_stream_count":2, + "extraction_protocol": "ofi+sockets", + "extractors": { + "hdf5_archive_extractor": { + "type": "hdf5_extractor", + "hdf5_archive_dir": "/tmp" + } + } } }, "chrono_player": { @@ -186,5 +209,5 @@ "ArchiveReaders": { "story_files_dir": "/tmp" } - } +} } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4933b7e64..ffd93486e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.25) add_subdirectory(unit) +add_subdirectory(functional) add_subdirectory(integration) add_subdirectory(communication) add_subdirectory(end-to-end) diff --git a/test/functional/CMakeLists.txt b/test/functional/CMakeLists.txt new file mode 100644 index 000000000..5aab60dbc --- /dev/null +++ b/test/functional/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.25) + +find_package(GTest REQUIRED) +include(GoogleTest) + +#------------------------------------------------------------------------------ +# Unit test subdirectories (one component per folder) +#------------------------------------------------------------------------------ +add_subdirectory(chrono-keeper) +add_subdirectory(chrono-grapher) diff --git a/test/functional/chrono-grapher/CMakeLists.txt b/test/functional/chrono-grapher/CMakeLists.txt new file mode 100644 index 000000000..39204eb60 --- /dev/null +++ b/test/functional/chrono-grapher/CMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 3.25) + +find_package(GTest REQUIRED) +find_package(Thallium REQUIRED) +include(GoogleTest) + +find_package(HDF5 REQUIRED COMPONENTS C CXX) + +#------------------------------------------------------------------------------ +# chrono_grapher standalone tests (Thallium/Margo) +#------------------------------------------------------------------------------ + +add_executable(chrono_grapher_extraction_chain_test + chrono_grapher_extraction_chain_test.cpp + ${CMAKE_SOURCE_DIR}/ChronoGrapher/src/HDF5FileChunkExtractor.cpp +) + +target_include_directories(chrono_grapher_extraction_chain_test PRIVATE + ${CMAKE_SOURCE_DIR}/chrono_common/include + ${CMAKE_SOURCE_DIR}/Client/cpp/include + ${CMAKE_SOURCE_DIR}/ChronoGrapher/include + ${HDF5_INCLUDE_DIRS} +) + +target_link_libraries(chrono_grapher_extraction_chain_test PRIVATE + chrono_common + chronolog_client + thallium + ${HDF5_LIBRARIES} +) + +set_target_properties(chrono_grapher_extraction_chain_test PROPERTIES + OUTPUT_NAME chronolog-test-grapher-extraction-chain + BUILD_WITH_INSTALL_RPATH TRUE + SKIP_BUILD_RPATH TRUE +) + +add_test(NAME Functional_ChronoGrapher_ExtractionChain COMMAND chrono_grapher_extraction_chain_test) +set_tests_properties(Functional_ChronoGrapher_ExtractionChain PROPERTIES DISABLED TRUE) +if(CHRONOLOG_INSTALL_TESTS) + chronolog_install_target(chrono_grapher_extraction_chain_test DESTINATION tests) +endif() diff --git a/test/functional/chrono-grapher/chrono_grapher_extraction_chain_test.cpp b/test/functional/chrono-grapher/chrono_grapher_extraction_chain_test.cpp new file mode 100644 index 000000000..43973f64e --- /dev/null +++ b/test/functional/chrono-grapher/chrono_grapher_extraction_chain_test.cpp @@ -0,0 +1,146 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace tl = thallium; +namespace chl = chronolog; + +bool keep_running = true; +void sigterm_handler(int) +{ + std::cout << "Received SIGTERM signal. Initiating shutdown procedure." << std::endl; + keep_running = false; + return; +} + +static constexpr uint64_t NS = 1000000000ULL; + +void chunk_contributor_thread(chl::StoryChunkExtractionQueue* extractionQueue, uint32_t thread_id) +{ + LOG_INFO("[GrapherExtractionChainTest] starting contributing thread {} ", thread_id); + + for(unsigned int k = 0; k < 10; ++k) + { + auto time_now = std::chrono::high_resolution_clock::now(); + uint64_t chunk_starttime = time_now.time_since_epoch().count(); + uint64_t chunk_endtime = (time_now + std::chrono::seconds(5)).time_since_epoch().count(); + + chl::StoryChunk* story_chunk = + new chl::StoryChunk("Chronicle", "Story" + std::to_string(k), k, chunk_starttime, chunk_endtime); + + for(unsigned int i = 0; i < 10; ++i) + { + story_chunk->insertEvent( + chl::LogEvent{k, + chunk_starttime + i, + thread_id, + 1, + "thread " + std::to_string(thread_id) + " line " + std::to_string(i)}); + } + + extractionQueue->stashStoryChunk(story_chunk); + } + + + LOG_INFO("[GrapherExtractionChainTest] exiting contributing thread {} ", thread_id); +} + + +std::string extraction_module_json_string = + std::string("{ \"ExtractionModule\": ") + "{ \"extraction_stream_count\":2," + "\"extractors\": { " + + "\"test_csv_extractor\": { \"type\": \"csv_extractor\", \"csv_archive_dir\": \"/tmp/csv_archive\" }" + "," + + "\"test_hdf5_extractor\": { \"type\": \"hdf5_extractor\", \"hdf5_archive_dir\": \"/tmp/hdf5_archive\" }" + "}" + + "}" + "}"; + +int main() +{ + int contributor_threads = 5; + int extraction_threads = 2; + + signal(SIGTERM, sigterm_handler); + + int result = chronolog::chrono_monitor::initialize( + "file" //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGTYPE + , + "/tmp/grapher_extraction_test.log" //, confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILE + , + chronolog::LogLevel::debug // confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGLEVEL + , + "ExtractionModuleTest" //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGNAME + , + 100000000 //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILESIZE + , + 2 //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILENUM + , + chronolog::LogLevel::debug //confManager.CLIENT_CONF.CLIENT_LOG_CONF.FLUSHLEVEL); + ); + + + chl::ServiceId localServiceId("ofi+sockets", "127.0.0.1", 2225, 25); + + chronolog::LoggingExtractor logging_extractor; + chronolog::StoryChunkExtractorCSV csv_extractor(localServiceId, "/tmp/"); + chronolog::HDF5FileChunkExtractor hdf5_extractor("/tmp/"); + + // 2. Test chained ExtractionModule instantiation with ChronoGrapherExtractionChain + chronolog::StoryChunkExtractionModule extractionModule; + + extractionModule.getExtractionChain().add_extractor(csv_extractor); + extractionModule.getExtractionChain().add_extractor(hdf5_extractor); + + extractionModule.initialize(extraction_threads); + + if(!extractionModule.is_initialized()) + { + + LOG_ERROR("[GrapherExtractionChainTest] ExtractionModule failed to initialize "); + return (-1); + } + // 3. Start extraction threads + chl::StoryChunkExtractionQueue& extractionQueue = extractionModule.getExtractionQueue(); + + extractionModule.startExtraction(); + + // 4. create chunk contributing threads + std::thread contributors[contributor_threads]; + + static uint32_t thread_id = 0; + while(true == keep_running) + { + LOG_INFO("[GrapherExtractionChainTest] ExtractionChainTest is running"); + + // now the main thread will start a few contributor threads, that would stash a few StoryChunks on the extractionQueue, + // than fall asleep until it's time to wake up and start some more contributor threads... + + // the StoryChunkExtraction module is expected to run in the background and take care of the extruction duties + + for(short int i = 0; i < contributor_threads; ++i, ++thread_id) + { + std::thread t{chunk_contributor_thread, &extractionQueue, thread_id}; + contributors[i] = std::move(t); + } + for(int i = 0; i < contributor_threads; ++i) { contributors[i].join(); } + + std::this_thread::sleep_for(std::chrono::seconds(20)); + } + + + // 5. Test ExtractionModule shutdown + LOG_INFO("[GrapherExtractionChainTest] Shutting down StoryChunkExtractionModule for {}", + chl::to_string(localServiceId)); + + extractionModule.shutdownExtraction(); + + return 1; +} diff --git a/test/functional/chrono-keeper/CMakeLists.txt b/test/functional/chrono-keeper/CMakeLists.txt new file mode 100644 index 000000000..e9928f2bf --- /dev/null +++ b/test/functional/chrono-keeper/CMakeLists.txt @@ -0,0 +1,40 @@ +cmake_minimum_required(VERSION 3.25) + +find_package(GTest REQUIRED) +find_package(Thallium REQUIRED) +include(GoogleTest) + +find_package(HDF5 REQUIRED COMPONENTS C CXX) + +#------------------------------------------------------------------------------ +# chrono_keeper standalone tests (Thallium/Margo) +#------------------------------------------------------------------------------ + +add_executable(chrono_keeper_extraction_chain_test + chrono_keeper_extraction_chain_test.cpp + ${CMAKE_SOURCE_DIR}/ChronoKeeper/src/DualEndpointChunkExtractorRDMA.cpp +) + +target_include_directories(chrono_keeper_extraction_chain_test PRIVATE + ${CMAKE_SOURCE_DIR}/chrono_common/include + ${CMAKE_SOURCE_DIR}/Client/cpp/include + ${CMAKE_SOURCE_DIR}/ChronoKeeper/include +) + +target_link_libraries(chrono_keeper_extraction_chain_test PRIVATE + chrono_common + chronolog_client + thallium +) + +set_target_properties(chrono_keeper_extraction_chain_test PROPERTIES + OUTPUT_NAME chronolog-test-keeper-extraction-chain + BUILD_WITH_INSTALL_RPATH TRUE + SKIP_BUILD_RPATH TRUE +) + +add_test(NAME Functional_ChronoKeeper_ExtractionChain COMMAND chrono_keeper_extraction_chain_test) +set_tests_properties(Functional_ChronoKeeper_ExtractionChain PROPERTIES DISABLED TRUE) +if(CHRONOLOG_INSTALL_TESTS) + chronolog_install_target(chrono_keeper_extraction_chain_test DESTINATION tests) +endif() diff --git a/test/functional/chrono-keeper/chrono_keeper_extraction_chain_test.cpp b/test/functional/chrono-keeper/chrono_keeper_extraction_chain_test.cpp new file mode 100644 index 000000000..793f2ff48 --- /dev/null +++ b/test/functional/chrono-keeper/chrono_keeper_extraction_chain_test.cpp @@ -0,0 +1,210 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tl = thallium; +namespace chl = chronolog; + +bool keep_running = true; +void sigterm_handler(int) +{ + std::cout << "Received SIGTERM signal. Initiating shutdown procedure." << std::endl; + keep_running = false; + return; +} + +static constexpr uint64_t NS = 1000000000ULL; + +void chunk_contributor_thread(chl::StoryChunkExtractionQueue* extractionQueue, uint32_t thread_id) +{ + LOG_INFO("[ExtractionModuleTest] starting contributing thread {} ", thread_id); + + for(unsigned int k = 0; k < 10; ++k) + { + auto time_now = std::chrono::high_resolution_clock::now(); + uint64_t chunk_starttime = time_now.time_since_epoch().count(); + uint64_t chunk_endtime = (time_now + std::chrono::seconds(5)).time_since_epoch().count(); + + chl::StoryChunk* story_chunk = + new chl::StoryChunk("Chronicle", "Story" + std::to_string(k), k, chunk_starttime, chunk_endtime); + + for(unsigned int i = 0; i < 10; ++i) + { + story_chunk->insertEvent( + chl::LogEvent{k, + chunk_starttime + i, + thread_id, + 1, + "thread " + std::to_string(thread_id) + " line " + std::to_string(i)}); + } + + extractionQueue->stashStoryChunk(story_chunk); + } + + + LOG_INFO("[ExtractionModuleTest] exiting contributing thread {} ", thread_id); +} + +//// + +std::string extraction_module_json_1 = + std::string("{ \"ExtractionModule\": ") + "{ \"extraction_stream_count\":2," + "\"extractors\": { " + + "\"test_single_rdma_extractor\": {" + "\"type\": \"single_endpoint_rdma_extractor\"," + + "\"receiving_endpoint\": {" + "\"protocol_conf\": \"ofi+sockets\"," + "\"service_ip\": \"127.0.0.1\"," + + "\"service_base_port\": 2230," + "\"service_provider_id\": 30" + " }" + "}" + "}" + "}" + "}"; + +std::string extraction_module_json_2 = + std::string("{ \"ExtractionModule\": ") + "{ \"extraction_stream_count\":2," + "\"extractors\": { " + + "\"test_dual_rdma_extractor\": {" + "\"type\": \"dual_endpoint_rdma_extractor\"," + + "\"player_receiving_endpoint\": {" + "\"protocol_conf\": \"ofi+sockets\"," + "\"service_ip\": \"127.0.0.1\"," + + "\"service_base_port\": 2232," + "\"service_provider_id\": 32" + " }," + "\"grapher_receiving_endpoint\": {" + + "\"protocol_conf\": \"ofi+sockets\"," + "\"service_ip\": \"127.0.0.1\"," + "\"service_base_port\": 2233," + + "\"service_provider_id\": 33" + "}" + "}" + "}" + "}" + "}"; + +/// + + +int main() +{ + int contributor_threads = 5; + int extraction_threads = 2; + + signal(SIGTERM, sigterm_handler); + + int result = chronolog::chrono_monitor::initialize( + "file" //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGTYPE + , + "/tmp/keeper_extraction_test.log" //, confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILE + , + chronolog::LogLevel::debug // confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGLEVEL + , + "ExtractionModuleTest" //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGNAME + , + 100000000 //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILESIZE + , + 2 //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILENUM + , + chronolog::LogLevel::debug //confManager.CLIENT_CONF.CLIENT_LOG_CONF.FLUSHLEVEL); + ); + + + chl::ServiceId localServiceId("ofi+sockets", "127.0.0.1", 2225, 25); + + std::string LOCAL_SERVICE_NA_STRING; + localServiceId.get_service_as_string(LOCAL_SERVICE_NA_STRING); + + tl::engine* localEngine = nullptr; + + chronolog::LoggingExtractor logging_extractor; + chronolog::StoryChunkExtractorCSV csv_extractor(localServiceId, "/tmp/"); + + try + { + margo_instance_id margo_id = margo_init(LOCAL_SERVICE_NA_STRING.c_str(), MARGO_SERVER_MODE, 1, 1); + localEngine = new tl::engine(margo_id); + } + catch(tl::exception const&) + { + return (-1); + } + + // 1. Test single endpoint RDMA extractor instantiation + chl::ServiceId receiving_service_id("ofi+sockets", "127.0.0.1", 3333, 33); + chl::StoryChunkExtractorRDMA rdma_extractor(*localEngine, receiving_service_id); + + // 2. Test chained ExtractionModule instantiation with chained logging extractor & csv extractor + chronolog::StoryChunkExtractionModule extractionModule; + + extractionModule.getExtractionChain().add_extractor(logging_extractor); + extractionModule.getExtractionChain().add_extractor(csv_extractor); + extractionModule.getExtractionChain().add_extractor(rdma_extractor); + + extractionModule.initialize(extraction_threads); + + // 3. Test ExtractionModule instantiation using json configuration object + + chronolog::StoryChunkExtractionModule new_extractionModule; + + json_object* parsed_json = json_tokener_parse(extraction_module_json_1.c_str()); + + if(!json_object_is_type(json_object_object_get(parsed_json, "ExtractionModule"), json_type_object)) + { + std::cerr << "\n [ExtractionModuleConfiguration] Test parsing stage:" + << " failed : json_block is not a json object" << std::endl; + return -1; + } + + json_object* extraction_module_block = json_object_object_get(parsed_json, "ExtractionModule"); + chronolog::ExtractionModuleConfiguration extraction_config; + int return_value = extraction_config.parse_json_object(extraction_module_block); + if(return_value != chl::CL_SUCCESS) + { + std::cerr << "\n [ExtractionModuleConfiguration] Test parsing stage:" << " failed " << std::endl; + return -1; + } + + std::string log_string; + extraction_config.to_string(log_string); + + std::cout << "\n [ExtractionModuleConfiguration] Test parsing stage:" << " passed " << log_string << std::endl; + + json_object_put(parsed_json); + + extractionModule.getExtractionChain().activate(*localEngine, extraction_config, localServiceId); + + new_extractionModule.initialize(extraction_config.extraction_stream_count); + + if(!new_extractionModule.is_initialized()) + { + std::cerr << "\n [ExtractionModuleConfiguration] ExtractionModule failed to initialize " << std::endl; + return -1; + } + + chl::StoryChunkExtractionQueue& extractionQueue = new_extractionModule.getExtractionQueue(); + + new_extractionModule.startExtraction(); + + // 4. create chunk contributing threads + std::thread contributors[contributor_threads]; + + static uint32_t thread_id = 0; + while(true == keep_running) + { + LOG_INFO("[ExtractionModuleTest] ExtractionChainTest is running"); + + // now the main thread will start a few contributor threads, that would stash a few StoryChunks on the extractionQueue, + // than fall asleep until it's time to wake up and start some more contributor threads... + + // the StoryChunkExtraction module is expected to run in the background and take care of the extraction duties + + for(short int i = 0; i < contributor_threads; ++i, ++thread_id) + { + std::thread t{chunk_contributor_thread, &extractionQueue, thread_id}; + contributors[i] = std::move(t); + } + for(int i = 0; i < contributor_threads; ++i) { contributors[i].join(); } + + std::this_thread::sleep_for(std::chrono::seconds(20)); + } + + + // 5. Test ExtractionModule shutdown + LOG_INFO("[ExtractionModuleTest] Shutting down StoryChunkExtractionModule for {}", chl::to_string(localServiceId)); + + extractionModule.shutdownExtraction(); + + delete localEngine; + return 1; +} diff --git a/test/unit/chrono-common/CMakeLists.txt b/test/unit/chrono-common/CMakeLists.txt index b1dd773c4..e35a564e4 100644 --- a/test/unit/chrono-common/CMakeLists.txt +++ b/test/unit/chrono-common/CMakeLists.txt @@ -52,35 +52,57 @@ if(CHRONOLOG_INSTALL_TESTS) chronolog_install_target(story_pipeline_test DESTINATION tests) endif() +# StoryChunk (GTest) +add_executable(extraction_chain_config_test + chrono_common_extraction_chain_config_test.cpp +) + +target_include_directories(extraction_chain_config_test PRIVATE + ${CMAKE_SOURCE_DIR}/chrono_common/include + ${CMAKE_SOURCE_DIR}/Client/cpp/include +) + +target_link_libraries(extraction_chain_config_test PRIVATE + GTest::gtest_main + chrono_common + chronolog_client +) + +set_target_properties(extraction_chain_config_test PROPERTIES OUTPUT_NAME chronolog-test-common-extraction-chain-config) +gtest_discover_tests(extraction_chain_config_test TEST_PREFIX "Unit_ChronoCommon_") +if(CHRONOLOG_INSTALL_TESTS) + chronolog_install_target(extraction_chain_config_test DESTINATION tests) +endif() + #------------------------------------------------------------------------------ # chrono_common standalone tests (Thallium/Margo) #------------------------------------------------------------------------------ -add_executable(extraction_chain_test - chrono_common_extraction_chain_test.cpp +add_executable(extraction_module_test + chrono_common_extraction_module_test.cpp ) -target_include_directories(extraction_chain_test PRIVATE +target_include_directories(extraction_module_test PRIVATE ${CMAKE_SOURCE_DIR}/chrono_common/include ${CMAKE_SOURCE_DIR}/Client/cpp/include ) -target_link_libraries(extraction_chain_test PRIVATE +target_link_libraries(extraction_module_test PRIVATE chrono_common chronolog_client thallium ) -set_target_properties(extraction_chain_test PROPERTIES - OUTPUT_NAME chronolog-test-common-extraction-chain +set_target_properties(extraction_module_test PROPERTIES + OUTPUT_NAME chronolog-test-common-extraction-module BUILD_WITH_INSTALL_RPATH TRUE SKIP_BUILD_RPATH TRUE ) -add_test(NAME Unit_ChronoCommon_ExtractionChain COMMAND extraction_chain_test) -set_tests_properties(Unit_ChronoCommon_ExtractionChain PROPERTIES DISABLED TRUE) +add_test(NAME Unit_ChronoCommon_ExtractionModule COMMAND extraction_module_test) +set_tests_properties(Unit_ChronoCommon_ExtractionModule PROPERTIES DISABLED TRUE) if(CHRONOLOG_INSTALL_TESTS) - chronolog_install_target(extraction_chain_test DESTINATION tests) + chronolog_install_target(extraction_module_test DESTINATION tests) endif() #------------------------------------------------------------------------------ diff --git a/test/unit/chrono-common/chrono_common_extraction_chain_config_test.cpp b/test/unit/chrono-common/chrono_common_extraction_chain_config_test.cpp new file mode 100644 index 000000000..eb55dbfaf --- /dev/null +++ b/test/unit/chrono-common/chrono_common_extraction_chain_config_test.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +namespace chl = chronolog; + +const char* logging_extractor_json_str = "{ \"extractror\" : {} }"; + +/* +JSON block for ExtractionModule Configuration is expected to look like this + + "ExtractionModule": { + "extraction_stream_count":2, + "extraction_protocol": "ofi+sockets", + "extractors": { + "test_csv_extractor": { + "type": "csv_extractor", + "csv_archive_dir": "/tmp/csv_archive" + }, + "test_rdma_extractor": { + "type": "single_endpoint_rdma_extractor", + "grapher_receiving_endpoint": { + "protocol_conf": "ofi+sockets", + "service_ip": "127.0.0.1", + "service_base_port": 3333, + "service_provider_id": 33 + } + }, + "test_dual_rdma_extractor": { + "type": "dual_endpoint_rdma_extractor", + "player_receiving_endpoint": { + "protocol_conf": "ofi+sockets", + "service_ip": "127.0.0.1", + "service_base_port": 2230, + "service_provider_id": 30 + }, + "grapher_receiving_endpoint": { + "protocol_conf": "ofi+sockets", + "service_ip": "127.0.0.1", + "service_base_port": 3333, + "service_provider_id": 33 + } + } + } + } +*/ + + +std::string rdma_extractor_json_string = std::string("{") + "\"test_rdma_extractor\": { " + + "\"type\": \"single_endpoint_rdma_extractor\", " + + "\"receiving_endpoint\": { \"protocol_conf\": \"ofi+sockets\",\"service_ip\": " + "\"127.0.0.1\", \"service_base_port\": 3333,\"service_provider_id\": 33 } } " + + "}"; + +std::string extraction_module_json_string = + std::string("{ \"ExtractionModule\": { \"extraction_stream_count\":2,") + "\"extractors\": { " + + "\"test_csv_extractor\": { \"type\": \"csv_extractor\", \"csv_archive_dir\": \"/tmp/csv_archive\" }" + + + "," + "\"test_rdma_extractor\": { " + "\"type\": \"single_endpoint_rdma_extractor\", " + + "\"receiving_endpoint\": { \"protocol_conf\": \"ofi+sockets\",\"service_ip\": \"127.0.0.1\", " + "\"service_base_port\": 3333,\"service_provider_id\": 33 } } " + + "} }" + "}"; + + +///////// +int test_csv_extractor_config() +{ + std::string csv_extractor_json_string = + std::string("{") + + " \"test_csv_extractor\": { \"type\": \"csv_extractor\", \"csv_archive_dir\": \"/tmp\" }" + "}"; + + json_object* parsed_json = json_tokener_parse(csv_extractor_json_string.c_str()); + + json_object* extractor_block = json_object_object_get(parsed_json, "test_csv_extractor"); + + if(!json_object_is_type(extractor_block, json_type_object)) + { + std::cout << "\n test_csv_extractor_config " << " failed : json_block is not a json object" << std::endl; + return -1; + } + + chl::StoryChunkExtractorCSV csv_extractor(chl::ServiceId{}); + + int return_value = csv_extractor.reset(extractor_block); + + json_object_put(parsed_json); + + return return_value; +} + +int test_rdma_extractor_config(tl::engine& tl_engine, std::string const& json_string) +{ + json_object* parsed_json = json_tokener_parse(json_string.c_str()); + + json_object* extractor_block = json_object_object_get(parsed_json, "test_rdma_extractor"); + + if(!json_object_is_type(extractor_block, json_type_object)) + { + std::cout << "\n test_rdma_extractor_config " << " failed : json_block is not a json object" << std::endl; + return -1; + } + + chl::StoryChunkExtractorRDMA single_endpoint_rdma_extractor(tl_engine, chl::ServiceId()); + + int return_value = single_endpoint_rdma_extractor.reset(extractor_block); + + json_object_put(parsed_json); + + return return_value; +} + +int main() +{ + + int result = chronolog::chrono_monitor::initialize( + "file" //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGTYPE + , + "/tmp/extraction_test.log" //, confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILE + , + chronolog::LogLevel::trace // confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGLEVEL + , + "ExtractionModuleTest" //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGNAME + , + 100000000 //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILESIZE + , + 2 //confManager.CLIENT_CONF.CLIENT_LOG_CONF.LOGFILENUM + , + chronolog::LogLevel::trace //confManager.CLIENT_CONF.CLIENT_LOG_CONF.FLUSHLEVEL); + ); + + // Test 1 : csv_extractor configuration + + int test_ret_value = test_csv_extractor_config(); + if(test_ret_value == chl::CL_SUCCESS) + { + std::cout << "\n [ExtractionModuleConfiguration] Test 1 csv_extractor_config:" << " passed " << std::endl; + } + else + { + std::cout << "\n [ExtractionModuleConfiguration] Test 1 csv_extractor_config:" << " failed " << std::endl; + return (-1); + } + + // Test 2 : single endpoint rdma extractor + // reset receiver configuration + chl::ServiceId localServiceId("ofi+sockets", "127.0.0.1", 2225, 25); + + std::string LOCAL_SERVICE_NA_STRING; + localServiceId.get_service_as_string(LOCAL_SERVICE_NA_STRING); + + tl::engine* localEngine = nullptr; + + try + { + margo_instance_id margo_id = margo_init(LOCAL_SERVICE_NA_STRING.c_str(), MARGO_SERVER_MODE, 1, 1); + localEngine = new tl::engine(margo_id); + } + catch(tl::exception const&) + { + return (-1); + } + + + test_ret_value = test_rdma_extractor_config(*localEngine, rdma_extractor_json_string); + if(test_ret_value == chl::CL_SUCCESS) + { + std::cout << "\n [ExtractionModuleConfiguration] Test 2 rdma_extractor_config:" << " passed " << std::endl; + } + else + { + std::cout << "\n [ExtractionModuleConfiguration] Test 2 rdma_extractor_config:" << " failed " << std::endl; + return (-1); + } + + // Test 3 ExtractionModule Configuration + + struct json_object *json_block, *parsed_json, *tags_array, *temp_item; + + // Parse the full JSON string + json_block = json_tokener_parse(extraction_module_json_string.c_str()); + + chronolog::ExtractionModuleConfiguration module_config; + + + if(!json_object_is_type(json_object_object_get(json_block, "ExtractionModule"), json_type_object)) + { + std::cerr << "\n [ExtractionModuleConfiguration] Test parsing stage:" + << " failed : json_block is not a json object" << std::endl; + return -1; + } + + json_object* extraction_module_block = json_object_object_get(json_block, "ExtractionModule"); + int return_value = module_config.parse_json_object(extraction_module_block); + + std::string log_string; + module_config.to_string(log_string); + + std::cout << "\n [ExtractionModuleConfiguration] Test parsing stage:" << " passed " << log_string << std::endl; + + json_object_put(json_block); + + // confirm that borrowed extractor_json_objects in ExtractionModuleConfiguration instance are retained + // after the original json_block object is released + + for(auto iter = module_config.extractors.begin(); iter != module_config.extractors.end(); ++iter) + { + if(!json_object_is_type((*iter).second, json_type_object)) + { + std::cerr << "\n [ExtractionModuleConfiguration] Test borrowed reference stage:" + << " failed to retain extractor_json_object for" << (*iter).first << std::endl; + return (-1); + } + } + + log_string.clear(); + + std::cout << "\n [ExtractionModuleConfiguration] Test borowed reference stage:" << " passed :" + << module_config.to_string(log_string) << std::endl; + + + return return_value; +} diff --git a/test/unit/chrono-common/chrono_common_extraction_chain_test.cpp b/test/unit/chrono-common/chrono_common_extraction_module_test.cpp similarity index 68% rename from test/unit/chrono-common/chrono_common_extraction_chain_test.cpp rename to test/unit/chrono-common/chrono_common_extraction_module_test.cpp index 33f52427e..c1d1dc1a7 100644 --- a/test/unit/chrono-common/chrono_common_extraction_chain_test.cpp +++ b/test/unit/chrono-common/chrono_common_extraction_module_test.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include @@ -54,6 +56,60 @@ void chunk_contributor_thread(chl::StoryChunkExtractionQueue* extractionQueue, u LOG_INFO("[ExtractionModuleTest] exiting contributing thread {} ", thread_id); } +using Extractor = std::variant; + +class TestExtractionChain +{ + std::vector theExtractors; + +public: + TestExtractionChain() {} + + ~TestExtractionChain() { theExtractors.clear(); } + + void add_extractor(Extractor e) { theExtractors.push_back(std::move(e)); } + + int process_chunk(chl::StoryChunk* chunk) + { + int chain_result = chl::CL_SUCCESS; + + // If extractor fails, mark the chain result as a failure, + // but keep going for the others. + for(auto& e: theExtractors) + { + int extractor_result = + std::visit([chunk](auto& extractor) -> int { return extractor.process_chunk(chunk); }, e); + + if(chl::CL_SUCCESS != extractor_result) + { + chain_result = extractor_result; + } + } + return chain_result; + } + + bool is_active_chain() const + { + if(theExtractors.empty()) + { + return false; + } + + for(const auto& e: theExtractors) + { + bool active = std::visit([](const auto& extractor) -> bool { return extractor.is_active(); }, e); + + // if any single extractor is NOT active, the whole chain fails + if(!active) + { + return false; + } + } + + return true; + } +}; + int main() { int contributor_threads = 5; @@ -88,10 +144,6 @@ int main() chronolog::LoggingExtractor logging_extractor; chronolog::StoryChunkExtractorCSV csv_extractor(localServiceId, "/tmp/"); - chl::ExtractorChain extractorChain1(csv_extractor); - chl::ExtractorChain extractorChain2(logging_extractor, csv_extractor); - - try { margo_instance_id margo_id = margo_init(LOCAL_SERVICE_NA_STRING.c_str(), MARGO_SERVER_MODE, 1, 1); @@ -102,18 +154,25 @@ int main() return (-1); } + // 1. Test single endpoint RDMA extractor instantiation chl::ServiceId receiving_service_id("ofi+sockets", "127.0.0.1", 3333, 33); chl::StoryChunkExtractorRDMA rdma_extractor(*localEngine, receiving_service_id); + // 2. Test chained ExtractionModule instantiation with chained logging extractor & csv extractor + chronolog::StoryChunkExtractionModule extractionModule; + + extractionModule.getExtractionChain().add_extractor(logging_extractor); + extractionModule.getExtractionChain().add_extractor(csv_extractor); + extractionModule.getExtractionChain().add_extractor(rdma_extractor); - chronolog::StoryChunkExtractionModule extractionModule(logging_extractor, rdma_extractor); - //Start extraction threads + extractionModule.initialize(extraction_threads); + // 3. Start extraction threads chl::StoryChunkExtractionQueue& extractionQueue = extractionModule.getExtractionQueue(); - extractionModule.startExtraction(extraction_threads); + extractionModule.startExtraction(); - //Start chunk contributing threads + // 4. create chunk contributing threads std::thread contributors[contributor_threads]; static uint32_t thread_id = 0; @@ -137,6 +196,7 @@ int main() } + // 5. Test ExtractionModule shutdown LOG_INFO("[ExtractionModuleTest] Shutting down StoryChunkExtractionModule for {}", chl::to_string(localServiceId)); extractionModule.shutdownExtraction(); diff --git a/test/unit/chrono-player/CMakeLists.txt b/test/unit/chrono-player/CMakeLists.txt index 7e470246c..7e5bc7835 100644 --- a/test/unit/chrono-player/CMakeLists.txt +++ b/test/unit/chrono-player/CMakeLists.txt @@ -9,6 +9,7 @@ find_package(HDF5 REQUIRED COMPONENTS C CXX) add_executable(transfer_agent_test chrono_player_transfer_agent_test.cpp + ${CMAKE_SOURCE_DIR}/ChronoPlayer/src/StoryChunkExtractor.cpp ${CMAKE_SOURCE_DIR}/ChronoPlayer/src/StoryChunkTransferAgent.cpp ) @@ -41,6 +42,7 @@ endif() add_executable(playback_service_test chrono_player_playback_service_test.cpp ${CMAKE_SOURCE_DIR}/ChronoPlayer/src/PlaybackService.cpp + ${CMAKE_SOURCE_DIR}/ChronoPlayer/src/StoryChunkExtractor.cpp ${CMAKE_SOURCE_DIR}/ChronoPlayer/src/StoryChunkTransferAgent.cpp ) diff --git a/tools/deploy/ChronoLog/deploy_cluster.sh b/tools/deploy/ChronoLog/deploy_cluster.sh index 8237d58b3..cd8c3d40d 100755 --- a/tools/deploy/ChronoLog/deploy_cluster.sh +++ b/tools/deploy/ChronoLog/deploy_cluster.sh @@ -388,10 +388,10 @@ generate_conf_for_each_recording_group() { jq ".chrono_keeper.RecordingGroup = ${i}" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" jq ".chrono_grapher.RecordingGroup = ${i}" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" jq ".chrono_player.RecordingGroup = ${i}" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" - jq ".chrono_keeper.KeeperGrapherDrainService.rpc.service_ip = \"${grapher_ip}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" + jq ".chrono_keeper.ExtractionModule.extractors.extractor_to_grapher.receiving_endpoint.service_ip= \"${grapher_ip}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" jq ".chrono_grapher.KeeperGrapherDrainService.rpc.service_ip = \"${grapher_ip}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" - jq ".chrono_keeper.Extractors.story_files_dir = \"${OUTPUT_DIR}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" - jq ".chrono_grapher.Extractors.story_files_dir = \"${OUTPUT_DIR}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" + jq ".chrono_keeper.ExtractionModule.extractors.csv_tier_extractor.csv_archive_dir = \"${OUTPUT_DIR}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" + jq ".chrono_grapher.ExtractionModule.extractors.hdf5_archive_extractor.hdf5_archive_dir = \"${OUTPUT_DIR}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" jq ".chrono_player.PlayerStoreAdminService.rpc.service_ip = \"${player_ip}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" jq ".chrono_player.PlaybackQueryService.rpc.service_ip = \"${player_ip}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" jq ".chrono_player.ArchiveReaders.story_files_dir = \"${OUTPUT_DIR}\"" "${CONF_FILE}.${i}" >${CONF_DIR}/temp.json && mv ${CONF_DIR}/temp.json "${CONF_FILE}.${i}" diff --git a/tools/deploy/ChronoLog/deploy_local.sh b/tools/deploy/ChronoLog/deploy_local.sh index e9d8942a9..4874fbd4e 100755 --- a/tools/deploy/ChronoLog/deploy_local.sh +++ b/tools/deploy/ChronoLog/deploy_local.sh @@ -142,6 +142,7 @@ generate_config_files() { echo "Generating grapher configuration files ..." mkdir -p "${output_dir}" + for (( i=0; i "$grapher_output_file" + .chrono_grapher.ExtractionModule.extractors.hdf5_archive_extractor.hdf5_archive_dir = ($output_dir + "/")' "$default_conf" > "$grapher_output_file" echo "Generated $grapher_output_file with ports $new_port_grapher_drain and $new_port_grapher_datastore" done @@ -203,9 +204,9 @@ generate_config_files() { --argjson grapher_index "$grapher_index" \ --arg keeper_index "$keeper_index" \ '.chrono_keeper.KeeperRecordingService.rpc.service_base_port = $new_port_keeper_record | - .chrono_keeper.KeeperGrapherDrainService.rpc.service_base_port = $new_port_keeper_drain | .chrono_keeper.KeeperDataStoreAdminService.rpc.service_base_port = $new_port_keeper_datastore | - .chrono_keeper.Extractors.story_files_dir = ($output_dir + "/") | + .chrono_keeper.ExtractionModule.extractors.csv_tier_extractor.csv_archive_dir = ($output_dir + "/") | + .chrono_keeper.ExtractionModule.extractors.extractor_to_grapher.receiving_endpoint.service_base_port = $new_port_keeper_drain | .chrono_keeper.RecordingGroup = $grapher_index | .chrono_keeper.Monitoring.monitor.file = ($monitor_dir + "/chrono-keeper-" + ($keeper_index | tostring) + ".log")' "$default_conf" > "$keeper_output_file" echo "Generated $keeper_output_file with ports $new_port_keeper_record, $new_port_keeper_datastore, and $new_port_keeper_drain"