diff --git a/JournalParserApp/src/JournalParser.cpp b/JournalParserApp/src/JournalParser.cpp index 8b007d7..659811d 100644 --- a/JournalParserApp/src/JournalParser.cpp +++ b/JournalParserApp/src/JournalParser.cpp @@ -43,12 +43,12 @@ int main(int argc, char* argv[]) return -1; } std::string file_prefix = argv[1]; // could be inst name or inst short name - std::string run_number = argv[2]; // 5 or 8 digit with leading zeros + std::string run_number = argv[2]; // 5 or 8 digit with leading zeros, or * to do all in cycle file std::string isis_cycle = argv[3]; // e.g. cycle_14_2 std::string journal_dir = argv[4]; // e.g. c:\data\export only std::string computer_name = argv[5]; // e.g. NDXGEM int stat = parseJournal(file_prefix, run_number, isis_cycle, journal_dir, computer_name); time(&time2); - std::cerr << "JournalParser: took " << difftime(time2, time1) << " seconds" << std::endl; + std::cout << "JournalParser: took " << difftime(time2, time1) << " seconds" << std::endl; return stat; } diff --git a/JournalParserApp/src/JournalParserSup.cpp b/JournalParserApp/src/JournalParserSup.cpp index 7195b48..f5a3cb2 100644 --- a/JournalParserApp/src/JournalParserSup.cpp +++ b/JournalParserApp/src/JournalParserSup.cpp @@ -196,73 +196,75 @@ static void sendSlackAndTeamsMessage(std::string inst_name, std::string slack_me } } + + +// List of items to extract from XML. +// Should be in the same order and same amount of things as the columns in the database. +const char* xml_names[][2] = { + { "run_number", "0" }, + {"title", "" }, + {"start_time", "" }, + {"duration", "0" }, + {"proton_charge", "0.0" }, + {"experiment_identifier", "" }, + {"user_name", "" }, + {"simulation_mode", "" }, + {"local_contact", "" }, + {"user_institute", "" }, + {"instrument_name", "" }, + {"sample_id", "" }, + {"measurement_first_run", "0" }, + {"measurement_id", "" }, + {"measurement_label", "" }, + {"measurement_type", "" }, + {"measurement_subid", "" }, + {"end_time", "" }, + {"raw_frames", "0" }, + {"good_frames", "0" }, + {"number_periods", "0" }, + {"number_spectra", "0" }, + {"number_detectors", "0" }, + {"number_time_regimes", "0" }, + {"frame_sync", "" }, + {"icp_version", "" }, + {"detector_table_file", "" }, + {"spectra_table_file", "" }, + {"wiring_table_file", "" }, + {"monitor_spectrum", "0" }, + {"monitor_sum", "0" }, + {"total_mevents", "0.0" }, + {"comment", "" }, + {"field_label", "" }, + {"instrument_geometry", "" }, + {"script_name", "" }, + {"sample_name", "" }, + {"sample_orientation", "" }, + {"temperature_label", "" }, + {"npratio_average", "0.0" }, + {"isis_cycle", "" }, + {"seci_config", "" }, + {"event_mode", "0.0" } +}; + +const int number_of_xml_elements = sizeof(xml_names) / (2 * sizeof(const char*)); + /** * Writes to the database based on the contents of an XML node. * * Args: * entry: The XML node representing the entry to be written to the database. + * prep_stmt: prepared mysql statement to use * * Returns: * 0 if successful, non-zero if unsuccessful. */ -int writeToDatabase(pugi::xml_node& entry) -{ - - // List of items to extract from XML. - // Should be in the same order and same amount of things as the columns in the database. - const char* xml_names[][2] = { - { "run_number", "0" }, - {"title", "" }, - {"start_time", "" }, - {"duration", "0" }, - {"proton_charge", "0.0" }, - {"experiment_identifier", "" }, - {"user_name", "" }, - {"simulation_mode", "" }, - {"local_contact", "" }, - {"user_institute", "" }, - {"instrument_name", "" }, - {"sample_id", "" }, - {"measurement_first_run", "0" }, - {"measurement_id", "" }, - {"measurement_label", "" }, - {"measurement_type", "" }, - {"measurement_subid", "" }, - {"end_time", "" }, - {"raw_frames", "0" }, - {"good_frames", "0" }, - {"number_periods", "0" }, - {"number_spectra", "0" }, - {"number_detectors", "0" }, - {"number_time_regimes", "0" }, - {"frame_sync", "" }, - {"icp_version", "" }, - {"detector_table_file", "" }, - {"spectra_table_file", "" }, - {"wiring_table_file", "" }, - {"monitor_spectrum", "0" }, - {"monitor_sum", "0" }, - {"total_mevents", "0.0" }, - {"comment", "" }, - {"field_label", "" }, - {"instrument_geometry", "" }, - {"script_name", "" }, - {"sample_name", "" }, - {"sample_orientation", "" }, - {"temperature_label", "" }, - {"npratio_average", "0.0" }, - {"isis_cycle", "" }, - {"seci_config", "" }, - {"event_mode", "0.0" } - }; - - const int number_of_elements = sizeof(xml_names) / (2 * sizeof(const char*)); - - std::string data[number_of_elements]; +static int writeToDatabase(pugi::xml_node& entry, sql::PreparedStatement *prep_stmt) +{ + std::string data[number_of_xml_elements]; // Get value of XML node and trim whitespace. int i; - for (i=0; i con(mysql_driver->connect("localhost", "journal", "$journal")); - std::auto_ptr stmt(con->createStatement()); - con->setAutoCommit(0); - con->setSchema("journal"); - sql::PreparedStatement *prep_stmt; - - std::string query ("INSERT INTO journal_entries VALUES ("); - // Loop to number_of_elements-1 because last "?" should not have a trailing comma. - for(i=0; iprepareStatement(query); - - for (i=0; isetString(i+1, data[i]); - } - + } prep_stmt->execute(); - con->commit(); } catch (sql::SQLException &e) { - errlogSevPrintf(errlogMinor, "JournalParser: MySQL ERR: %s (MySQL error code: %d, SQLState: %s)\n", e.what(), e.getErrorCode(), e.getSQLStateCStr()); + fprintf(stderr, "JournalParser: MySQL ERR: %s (MySQL error code: %d, SQLState: %s)\n", e.what(), e.getErrorCode(), e.getSQLStateCStr()); return -1; } catch (std::runtime_error &e) { - errlogSevPrintf(errlogMinor, "JournalParser: MySQL ERR: %s\n", e.what()); + fprintf(stderr, "JournalParser: MySQL ERR: %s\n", e.what()); return -1; } catch(...) { - errlogSevPrintf(errlogMinor, "JournalParser: MySQL ERR: FAILED TRYING TO WRITE TO THE ISIS PV DB\n"); + fprintf(stderr, "JournalParser: MySQL ERR: FAILED TRYING TO WRITE TO THE ISIS PV DB\n"); return -1; } return 0; } + +static void writeSlackEntry(pugi::xml_node& entry, const std::string& inst_name) +{ + + std::ostringstream slack_mess, teams_mess, summ_mess; + // we need to have title in a ``` so if it contains markdown like + // characters they are not interpreted + const char* collect_mode_slack = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "*event* mode" : "*histogram* mode"); + const char* collect_mode_teams = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "**event** mode" : "**histogram** mode"); + time_t now; + time(&now); + char tbuffer[64]; + strftime(tbuffer, sizeof(tbuffer), "%a %d %b %H:%M", localtime(&now)); + // << getJV(entry, "monitor_sum") << "* monitor spectrum " << getJV(entry, "monitor_spectrum") << " sum, *" + slack_mess << tbuffer << " Run *" << getJV(entry, "run_number") << "* finished (*" << getJV(entry, "proton_charge") << "* uAh, *" << getJV(entry, "good_frames") << "* frames, *" << getJV(entry, "duration") << "* seconds, *" << getJV(entry, "number_spectra") << "* spectra, *" << getJV(entry, "number_periods") << "* periods, " << collect_mode_slack << ", *" << getJV(entry, "total_mevents") << "* total DAE MEvents) ```" << getJV(entry, "title") << "```"; + teams_mess << tbuffer << " Run **" << getJV(entry, "run_number") << "** finished (**" << getJV(entry, "proton_charge") << "** uAh, **" << getJV(entry, "good_frames") << "** frames, **" << getJV(entry, "duration") << "** seconds, **" << getJV(entry, "number_spectra") << "** spectra, **" << getJV(entry, "number_periods") << "** periods, " << collect_mode_teams << ", **" << getJV(entry, "total_mevents") << "** total DAE MEvents) ```" << getJV(entry, "title") << "```"; + std::cout << slack_mess.str() << std::endl; + + summ_mess << getJV(entry, "run_number") << ": " << getJV(entry, "title"); + + //sendSlackAndTeamsMessage(inst_name, slack_mess.str(), teams_mess.str(), summ_mess.str()); + +} + +static int writeEntries(pugi::xpath_node_set& entries, const std::string& inst_name) +{ + int stat = 0, count = 0; + try { + sql::Driver * mysql_driver = sql::mysql::get_driver_instance(); + std::auto_ptr< sql::Connection > con(mysql_driver->connect("localhost", "journal", "$journal")); + con->setAutoCommit(0); + con->setSchema("journal"); + + std::auto_ptr stmt(con->createStatement()); + sql::PreparedStatement *prep_stmt; + + std::string query ("INSERT INTO journal_entries VALUES ("); + // Loop to number_of_elements-1 because last "?" should not have a trailing comma. + for(int i=0; iprepareStatement(query); + + for (pugi::xpath_node_set::const_iterator it = entries.begin(); it != entries.end(); ++it) + { + pugi::xpath_node node = *it; + pugi::xml_node entry = node.node(); + writeSlackEntry(entry, inst_name); + stat |= writeToDatabase(entry, prep_stmt); + ++count; + } + con->commit(); + } + catch (sql::SQLException &e) + { + fprintf(stderr, "JournalParser: MySQL ERR: %s (MySQL error code: %d, SQLState: %s)\n", e.what(), e.getErrorCode(), e.getSQLStateCStr()); + return -1; + } + catch (std::runtime_error &e) + { + fprintf(stderr, "JournalParser: MySQL ERR: %s\n", e.what()); + return -1; + } + catch(...) + { + fprintf(stderr, "JournalParser: MySQL ERR: FAILED TRYING TO WRITE TO THE ISIS PV DB\n"); + return -1; + } + if (count == 0) { + std::cerr << "JournalParser: Cannot find entry in journal file" << std::endl; + return -1; + } else if (stat == -1) { + std::cerr << "JournalParser: Error writing one of more entries to DB" << std::endl; + } + return stat; +} + int createJournalFile(const std::string& file_prefix, const std::string& run_number, const std::string& isis_cycle, const std::string& journal_dir, const std::string& computer_name, const std::string inst_name) { size_t pos = isis_cycle.find("_"); @@ -360,30 +422,15 @@ int createJournalFile(const std::string& file_prefix, const std::string& run_num return -1; } char main_entry_xpath[128]; - sprintf(main_entry_xpath, "/NXroot/NXentry[@name='%s%08d']", inst_name.c_str(), atoi(run_number.c_str())); - pugi::xpath_node main_entry = doc.select_single_node(main_entry_xpath); - std::ostringstream slack_mess, teams_mess, summ_mess; - pugi::xml_node entry = main_entry.node(); - // we need to have title in a ``` so if it contains markdown like - // characters they are not interpreted - const char* collect_mode_slack = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "*event* mode" : "*histogram* mode"); - const char* collect_mode_teams = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "**event** mode" : "**histogram** mode"); - time_t now; - time(&now); - char tbuffer[64]; - strftime(tbuffer, sizeof(tbuffer), "%a %d %b %H:%M", localtime(&now)); - // << getJV(entry, "monitor_sum") << "* monitor spectrum " << getJV(entry, "monitor_spectrum") << " sum, *" - slack_mess << tbuffer << " Run *" << getJV(entry, "run_number") << "* finished (*" << getJV(entry, "proton_charge") << "* uAh, *" << getJV(entry, "good_frames") << "* frames, *" << getJV(entry, "duration") << "* seconds, *" << getJV(entry, "number_spectra") << "* spectra, *" << getJV(entry, "number_periods") << "* periods, " << collect_mode_slack << ", *" << getJV(entry, "total_mevents") << "* total DAE MEvents) ```" << getJV(entry, "title") << "```"; - teams_mess << tbuffer << " Run **" << getJV(entry, "run_number") << "** finished (**" << getJV(entry, "proton_charge") << "** uAh, **" << getJV(entry, "good_frames") << "** frames, **" << getJV(entry, "duration") << "** seconds, **" << getJV(entry, "number_spectra") << "** spectra, **" << getJV(entry, "number_periods") << "** periods, " << collect_mode_teams << ", **" << getJV(entry, "total_mevents") << "** total DAE MEvents) ```" << getJV(entry, "title") << "```"; - std::cerr << slack_mess.str() << std::endl; - - summ_mess << getJV(entry, "run_number") << ": " << getJV(entry, "title"); - - sendSlackAndTeamsMessage(inst_name, slack_mess.str(), teams_mess.str(), summ_mess.str()); - - return writeToDatabase(entry); + if (run_number == "*") { + sprintf(main_entry_xpath, "/NXroot/NXentry[starts-with(@name, '%s')]", inst_name.c_str()); + } else { + sprintf(main_entry_xpath, "/NXroot/NXentry[@name='%s%08d']", inst_name.c_str(), atoi(run_number.c_str())); + } + pugi::xpath_node_set entries = doc.select_nodes(main_entry_xpath); + return writeEntries(entries, inst_name); } - + /* * Parses a journal file. * diff --git a/JournalParserApp/src/Makefile b/JournalParserApp/src/Makefile index 52e43d7..2c9d0b4 100644 --- a/JournalParserApp/src/Makefile +++ b/JournalParserApp/src/Makefile @@ -37,6 +37,16 @@ JournalParser_SYS_LIBS_WIN32 += wldap32 crypt32 Normaliz JournalParser_SYS_LIBS_Linux += curl JournalParser_LIBS += $(EPICS_BASE_IOC_LIBS) +# add some DLLs so easier to run standalone +BIN_INSTALLS_WIN32 += $(wildcard $(MYSQL)/bin/$(EPICS_HOST_ARCH)/*.dll) +BIN_INSTALLS_WIN32 += $(wildcard $(PUGIXML)/bin/$(EPICS_HOST_ARCH)/*.dll) + +# ifeq ($(SHARED_LIBRARIES),YES) +# USR_CPPFLAGS_WIN32 += -DSQLITE_API=__declspec(dllimport) +# else +# #USR_CXXFLAGS += -DCPPCONN_LIB_BUILD +# endif + #=========================== include $(TOP)/configure/RULES diff --git a/ingest/add_journal_entries.bat b/ingest/add_journal_entries.bat index 61cc02a..8f30ddb 100644 --- a/ingest/add_journal_entries.bat +++ b/ingest/add_journal_entries.bat @@ -1,6 +1,6 @@ call %~dp0..\..\..\..\config_env.bat set "PATH=C:\Instrument\Apps\EPICS\support\pugixml\master\bin\windows-x64;C:\Instrument\Apps\EPICS\support\curl\master\bin\windows-x64;C:\Instrument\Apps\EPICS\support\MySQL\master\bin\windows-x64;%PATH%" -"%PYTHON3%" %~dp0add_journal_entries.py %* +"%PYTHON3%" -u %~dp0add_journal_entries.py %* if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/ingest/add_journal_entries.py b/ingest/add_journal_entries.py index 994f628..9f04d9e 100644 --- a/ingest/add_journal_entries.py +++ b/ingest/add_journal_entries.py @@ -2,6 +2,7 @@ import argparse import os +import glob import subprocess import xml.etree.ElementTree as ET from contextlib import contextmanager @@ -17,15 +18,20 @@ """ INGEST_DIR = os.path.dirname(os.path.abspath(__file__)) +DATA_DIR = INGEST_DIR JOURNAL_PARSER_DIR = os.path.abspath(os.path.join(INGEST_DIR, "..", "bin", "windows-x64")) JOURNAL_PARSER_CONFIG_FILE = os.path.join(JOURNAL_PARSER_DIR, "JournalParser.conf") JOURNAL_PARSER = os.path.join(JOURNAL_PARSER_DIR, "JournalParser.exe") JOURNAL_PREFIX = "journal_" +JOURNAL_SUFFIX = ".xml" +JOURNAL_GLOB = JOURNAL_PREFIX + "[0-9]*_[0-9]*" + JOURNAL_SUFFIX def run_journal_parser(*args): with open(os.devnull, "w") as devnull: - subprocess.check_call(" ".join([JOURNAL_PARSER] + list(args)), stderr=devnull, stdout=devnull) + #subprocess.check_call(" ".join([JOURNAL_PARSER] + list(args)), stderr=devnull, stdout=devnull) + #subprocess.check_call(" ".join([JOURNAL_PARSER] + list(args))) + subprocess.call(" ".join([JOURNAL_PARSER] + list(args)), stdout=devnull) @contextmanager @@ -49,30 +55,39 @@ def temporarily_rename_config_file(): parser.add_argument('-i', '--instrument', help="Specify the instrument to run on, e.g. ENGINX") parser.add_argument('-host', '--hostname', help="Specify the instrument hostname, e.g. NDXENGINX") parser.add_argument('-f', '--files', help="Specify a list of files to add", nargs="+", default=None) + parser.add_argument('-d', '--dir', help="Directory to ingest", default=None) + arguments = parser.parse_args() instrument_name = arguments.instrument computer_name = arguments.hostname + if arguments.dir is not None: + DATA_DIR = arguments.dir if arguments.files is None: - files = [f for f in os.listdir(INGEST_DIR) if f.startswith(JOURNAL_PREFIX)] + files = glob.glob(JOURNAL_GLOB, root_dir=DATA_DIR) else: files = arguments.files with temporarily_rename_config_file(): for filename in files: - year_and_cycle = filename[len(JOURNAL_PREFIX):-len(".xml")] + year_and_cycle = filename[len(JOURNAL_PREFIX):-len(JOURNAL_SUFFIX)] try: - print("\n\n-----\nParsing {}\n-----\n\n".format(filename)) - - tree = ET.parse(filename) - for run in tree.getroot(): - print(".", end="") - run_number = int(run.attrib['name'][len(instrument_name):]) - - run_journal_parser(instrument_name, "{:08d}".format(run_number), "cycle_{}".format(year_and_cycle), - '"{}"'.format(INGEST_DIR), computer_name) + print("\n\n-----\nParsing {} from {}\n-----\n\n".format(filename, DATA_DIR)) + tree = ET.parse(os.path.join(DATA_DIR, filename)) + except Exception as e: + print("Malformed data from '{}': {} {}".format(filename, e.__class__.__name__, e)) + try: +# tree = ET.parse(os.path.join(DATA_DIR, filename)) +# for run in tree.getroot(): +# print(".", end="") +# run_number = int(run.attrib['name'][len(instrument_name):]) + +# run_journal_parser(instrument_name, "{:08d}".format(run_number), "cycle_{}".format(year_and_cycle), +# '"{}"'.format(DATA_DIR), computer_name) + run_journal_parser(instrument_name, "*", "cycle_{}".format(year_and_cycle), + '"{}"'.format(DATA_DIR), computer_name) except Exception as e: print("Couldn't load data from '{}': {} {}".format(filename, e.__class__.__name__, e)) raise