ISISComputingGroup · FreddieAkeroyd · Aug 27, 2024 · Aug 27, 2024 · Oct 17, 2024 · Jan 10, 2025
diff --git a/JournalParserApp/src/JournalParser.cpp b/JournalParserApp/src/JournalParser.cpp
@@ -43,12 +43,12 @@ int main(int argc, char* argv[])
 		return -1;
 	}
 	std::string file_prefix = argv[1]; // could be inst name or inst short name
-	std::string run_number = argv[2];  // 5 or 8 digit with leading zeros
+	std::string run_number = argv[2];  // 5 or 8 digit with leading zeros, or * to do all in cycle file
 	std::string isis_cycle = argv[3];  // e.g. cycle_14_2
 	std::string journal_dir = argv[4];  // e.g. c:\data\export only
 	std::string computer_name = argv[5];  // e.g. NDXGEM
     int stat = parseJournal(file_prefix, run_number, isis_cycle, journal_dir, computer_name);
 	time(&time2);
-	std::cerr << "JournalParser: took " << difftime(time2, time1) << " seconds" << std::endl;
+	std::cout << "JournalParser: took " << difftime(time2, time1) << " seconds" << std::endl;
 	return stat;
 }
diff --git a/JournalParserApp/src/JournalParserSup.cpp b/JournalParserApp/src/JournalParserSup.cpp
@@ -196,73 +196,75 @@ static void sendSlackAndTeamsMessage(std::string inst_name, std::string slack_me
     }
 }
 
+
+
+// List of items to extract from XML.
+// Should be in the same order and same amount of things as the columns in the database.
+const char* xml_names[][2] = {
+    { "run_number",             "0" },
+    {"title",                   "" },
+    {"start_time",              "" },
+    {"duration",                "0" },
+    {"proton_charge",           "0.0" },
+    {"experiment_identifier",   "" },
+    {"user_name",               "" },
+    {"simulation_mode",         "" },
+    {"local_contact",           "" },
+    {"user_institute",          "" },
+    {"instrument_name",         "" },
+    {"sample_id",               "" },
+    {"measurement_first_run",   "0" },
+    {"measurement_id",          "" },
+    {"measurement_label",       "" },
+    {"measurement_type",        "" },
+    {"measurement_subid",       "" },
+    {"end_time",                "" },
+    {"raw_frames",              "0" },
+    {"good_frames",             "0" },
+    {"number_periods",          "0" },
+    {"number_spectra",          "0" },
+    {"number_detectors",        "0" },
+    {"number_time_regimes",     "0" },
+    {"frame_sync",              "" },
+    {"icp_version",             "" },
+    {"detector_table_file",     "" },
+    {"spectra_table_file",      "" },
+    {"wiring_table_file",       "" },
+    {"monitor_spectrum",        "0" },
+    {"monitor_sum",             "0" },
+    {"total_mevents",           "0.0" },
+    {"comment",                 "" },
+    {"field_label",             "" },
+    {"instrument_geometry",     "" },
+    {"script_name",             "" },
+    {"sample_name",             "" },
+    {"sample_orientation",      "" },
+    {"temperature_label",       "" },
+    {"npratio_average",         "0.0" },
+    {"isis_cycle",              "" },
+    {"seci_config",             "" },
+    {"event_mode",              "0.0" }
+};
+
+const int number_of_xml_elements = sizeof(xml_names) / (2 * sizeof(const char*)); 
+
 /**
  * Writes to the database based on the contents of an XML node.
  *
  * Args: 
  *     entry: The XML node representing the entry to be written to the database.
+ *     prep_stmt: prepared mysql statement to use
  *
  * Returns: 
  *     0 if successful, non-zero if unsuccessful.
  */
-int writeToDatabase(pugi::xml_node& entry)
-{
-
-	// List of items to extract from XML.
-	// Should be in the same order and same amount of things as the columns in the database.
-	const char* xml_names[][2] = {
-		{ "run_number",             "0" },
-		{"title",                   "" },
-		{"start_time",              "" },
-		{"duration",                "0" },
-		{"proton_charge",           "0.0" },
-		{"experiment_identifier",   "" },
-		{"user_name",               "" },
-		{"simulation_mode",         "" },
-		{"local_contact",           "" },
-		{"user_institute",          "" },
-		{"instrument_name",         "" },
-		{"sample_id",               "" },
-		{"measurement_first_run",   "0" },
-		{"measurement_id",          "" },
-		{"measurement_label",       "" },
-		{"measurement_type",        "" },
-		{"measurement_subid",       "" },
-		{"end_time",                "" },
-		{"raw_frames",              "0" },
-		{"good_frames",             "0" },
-		{"number_periods",          "0" },
-		{"number_spectra",          "0" },
-		{"number_detectors",        "0" },
-		{"number_time_regimes",     "0" },
-		{"frame_sync",              "" },
-		{"icp_version",             "" },
-        {"detector_table_file",     "" },
-        {"spectra_table_file",      "" },
-        {"wiring_table_file",       "" },
-        {"monitor_spectrum",        "0" },
-        {"monitor_sum",             "0" },
-        {"total_mevents",           "0.0" },
-        {"comment",                 "" },
-        {"field_label",             "" },
-        {"instrument_geometry",     "" },
-        {"script_name",             "" },
-        {"sample_name",             "" },
-        {"sample_orientation",      "" },
-        {"temperature_label",       "" },
-        {"npratio_average",         "0.0" },
-        {"isis_cycle",              "" },
-        {"seci_config",             "" },
-		{"event_mode",              "0.0" }
-	};
-
-	const int number_of_elements = sizeof(xml_names) / (2 * sizeof(const char*)); 
-
-	std::string data[number_of_elements];
+static int writeToDatabase(pugi::xml_node& entry, sql::PreparedStatement *prep_stmt)
+{	
+	std::string data[number_of_xml_elements];
 
 	// Get value of XML node and trim whitespace.
 	int i;
-	for (i=0; i<number_of_elements; i++)
+	for (i=0; i<number_of_xml_elements; i++)
 	{
 		data[i] = trimXmlNode(entry, xml_names[i][0]);
 		if (data[i] == "")
@@ -273,49 +275,109 @@ int writeToDatabase(pugi::xml_node& entry)
 
 	try
 	{
-		sql::Driver * mysql_driver = sql::mysql::get_driver_instance();
-		std::auto_ptr< sql::Connection > con(mysql_driver->connect("localhost", "journal", "$journal"));
-		std::auto_ptr<sql::Statement> stmt(con->createStatement());
-		con->setAutoCommit(0);
-		con->setSchema("journal");
-		sql::PreparedStatement *prep_stmt;
-
-		std::string query ("INSERT INTO journal_entries VALUES (");
-		// Loop to number_of_elements-1 because last "?" should not have a trailing comma.
-		for(i=0; i<number_of_elements-1; i++)
-		{
-			query.append("?, ");
-		}
-		query.append("?)");
-
-		prep_stmt = con->prepareStatement(query);
-
-		for (i=0; i<number_of_elements; i++) 
+		for (i=0; i<number_of_xml_elements; i++) 
 		{
 			prep_stmt->setString(i+1, data[i]);
-		}
-
+		}	
 		prep_stmt->execute();
-		con->commit();
 	}
 	catch (sql::SQLException &e)
 	{
-        errlogSevPrintf(errlogMinor, "JournalParser: MySQL ERR: %s (MySQL error code: %d, SQLState: %s)\n", e.what(), e.getErrorCode(), e.getSQLStateCStr());
+        fprintf(stderr, "JournalParser: MySQL ERR: %s (MySQL error code: %d, SQLState: %s)\n", e.what(), e.getErrorCode(), e.getSQLStateCStr());
         return -1;
 	}
 	catch (std::runtime_error &e)
 	{
-        errlogSevPrintf(errlogMinor, "JournalParser: MySQL ERR: %s\n", e.what());
+        fprintf(stderr, "JournalParser: MySQL ERR: %s\n", e.what());
         return -1;
 	}
     catch(...)
     {
-        errlogSevPrintf(errlogMinor, "JournalParser: MySQL ERR: FAILED TRYING TO WRITE TO THE ISIS PV DB\n");
+        fprintf(stderr, "JournalParser: MySQL ERR: FAILED TRYING TO WRITE TO THE ISIS PV DB\n");
         return -1;
     }
 	return 0;
 }
 
+
+static void writeSlackEntry(pugi::xml_node& entry, const std::string& inst_name)
+{
+
+	std::ostringstream slack_mess, teams_mess, summ_mess;
+	// we need to have title in a ``` so if it contains markdown like
+	// characters they are not interpreted
+	const char* collect_mode_slack = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "*event* mode" : "*histogram* mode");
+	const char* collect_mode_teams = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "**event** mode" : "**histogram** mode");
+	time_t now;
+	time(&now);
+	char tbuffer[64];
+	strftime(tbuffer, sizeof(tbuffer), "%a %d %b %H:%M", localtime(&now));
+	// << getJV(entry, "monitor_sum") << "* monitor spectrum " << getJV(entry, "monitor_spectrum") << " sum, *"
+	slack_mess << tbuffer << " Run *" << getJV(entry, "run_number") << "* finished (*" << getJV(entry, "proton_charge") << "* uAh, *" << getJV(entry, "good_frames") << "* frames, *" << getJV(entry, "duration") << "* seconds, *" << getJV(entry, "number_spectra") << "* spectra, *" << getJV(entry, "number_periods") << "* periods, " << collect_mode_slack << ", *" << getJV(entry, "total_mevents") << "* total DAE MEvents) ```" << getJV(entry, "title") << "```";
+	teams_mess << tbuffer << " Run **" << getJV(entry, "run_number") << "** finished (**" << getJV(entry, "proton_charge") << "** uAh, **" << getJV(entry, "good_frames") << "** frames, **" << getJV(entry, "duration") << "** seconds, **" << getJV(entry, "number_spectra") << "** spectra, **" << getJV(entry, "number_periods") << "** periods, " << collect_mode_teams << ", **" << getJV(entry, "total_mevents") << "** total DAE MEvents) ```" << getJV(entry, "title") << "```";
+	std::cout << slack_mess.str() << std::endl;
+
+    summ_mess << getJV(entry, "run_number") << ": " << getJV(entry, "title");
+
+	//sendSlackAndTeamsMessage(inst_name, slack_mess.str(), teams_mess.str(), summ_mess.str());
+
+}
+
+static int writeEntries(pugi::xpath_node_set& entries, const std::string& inst_name)
+{
+    int stat = 0, count = 0;
+    try {
+        sql::Driver * mysql_driver = sql::mysql::get_driver_instance();
+        std::auto_ptr< sql::Connection > con(mysql_driver->connect("localhost", "journal", "$journal"));
+	    con->setAutoCommit(0);
+	    con->setSchema("journal");
+
+        std::auto_ptr<sql::Statement> stmt(con->createStatement());
+	    sql::PreparedStatement *prep_stmt;
+
+	    std::string query ("INSERT INTO journal_entries VALUES (");
+	    // Loop to number_of_elements-1 because last "?" should not have a trailing comma.
+	    for(int i=0; i<number_of_xml_elements-1; i++)
+	    {
+		    query.append("?, ");
+	    }
+	    query.append("?)");
+	    prep_stmt = con->prepareStatement(query);
+
+        for (pugi::xpath_node_set::const_iterator it = entries.begin(); it != entries.end(); ++it)
+        {
+            pugi::xpath_node node = *it;
+            pugi::xml_node entry = node.node();
+            writeSlackEntry(entry, inst_name);
+            stat |= writeToDatabase(entry, prep_stmt);
+            ++count;
+        }
+    	con->commit();
+    }
+	catch (sql::SQLException &e)
+	{
+        fprintf(stderr, "JournalParser: MySQL ERR: %s (MySQL error code: %d, SQLState: %s)\n", e.what(), e.getErrorCode(), e.getSQLStateCStr());
+        return -1;
+	}
+	catch (std::runtime_error &e)
+	{
+        fprintf(stderr, "JournalParser: MySQL ERR: %s\n", e.what());
+        return -1;
+	}
+    catch(...)
+    {
+        fprintf(stderr, "JournalParser: MySQL ERR: FAILED TRYING TO WRITE TO THE ISIS PV DB\n");
+        return -1;
+    }
+    if (count == 0) {
+		std::cerr << "JournalParser: Cannot find entry in journal file"  << std::endl;
+        return -1;
+    } else if (stat == -1) {
+        std::cerr << "JournalParser: Error writing one of more entries to DB" << std::endl;
+    }
+    return stat;
+}
+
 int createJournalFile(const std::string& file_prefix, const std::string& run_number, const std::string& isis_cycle, const std::string& journal_dir, const std::string& computer_name, const std::string inst_name)
 {
 	size_t pos = isis_cycle.find("_");
@@ -360,30 +422,15 @@ int createJournalFile(const std::string& file_prefix, const std::string& run_num
 		return -1;
     }
 	char main_entry_xpath[128];
-	sprintf(main_entry_xpath, "/NXroot/NXentry[@name='%s%08d']", inst_name.c_str(), atoi(run_number.c_str()));
-    pugi::xpath_node main_entry = doc.select_single_node(main_entry_xpath);
-	std::ostringstream slack_mess, teams_mess, summ_mess;
-	pugi::xml_node entry = main_entry.node();
-	// we need to have title in a ``` so if it contains markdown like
-	// characters they are not interpreted
-	const char* collect_mode_slack = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "*event* mode" : "*histogram* mode");
-	const char* collect_mode_teams = (atof(getJV(entry, "event_mode").c_str()) > 0.0 ? "**event** mode" : "**histogram** mode");
-	time_t now;
-	time(&now);
-	char tbuffer[64];
-	strftime(tbuffer, sizeof(tbuffer), "%a %d %b %H:%M", localtime(&now));
-	// << getJV(entry, "monitor_sum") << "* monitor spectrum " << getJV(entry, "monitor_spectrum") << " sum, *"
-	slack_mess << tbuffer << " Run *" << getJV(entry, "run_number") << "* finished (*" << getJV(entry, "proton_charge") << "* uAh, *" << getJV(entry, "good_frames") << "* frames, *" << getJV(entry, "duration") << "* seconds, *" << getJV(entry, "number_spectra") << "* spectra, *" << getJV(entry, "number_periods") << "* periods, " << collect_mode_slack << ", *" << getJV(entry, "total_mevents") << "* total DAE MEvents) ```" << getJV(entry, "title") << "```";
-	teams_mess << tbuffer << " Run **" << getJV(entry, "run_number") << "** finished (**" << getJV(entry, "proton_charge") << "** uAh, **" << getJV(entry, "good_frames") << "** frames, **" << getJV(entry, "duration") << "** seconds, **" << getJV(entry, "number_spectra") << "** spectra, **" << getJV(entry, "number_periods") << "** periods, " << collect_mode_teams << ", **" << getJV(entry, "total_mevents") << "** total DAE MEvents) ```" << getJV(entry, "title") << "```";
-	std::cerr << slack_mess.str() << std::endl;
-
-    summ_mess << getJV(entry, "run_number") << ": " << getJV(entry, "title");
-
-	sendSlackAndTeamsMessage(inst_name, slack_mess.str(), teams_mess.str(), summ_mess.str());
-
-	return writeToDatabase(entry);
+    if (run_number == "*") {
+	    sprintf(main_entry_xpath, "/NXroot/NXentry[starts-with(@name, '%s')]", inst_name.c_str());
+    } else {
+        sprintf(main_entry_xpath, "/NXroot/NXentry[@name='%s%08d']", inst_name.c_str(), atoi(run_number.c_str()));
+    }
+    pugi::xpath_node_set entries = doc.select_nodes(main_entry_xpath);
+    return writeEntries(entries, inst_name);
 }
-
+    
 /*
  * Parses a journal file.
  *

diff --git a/JournalParserApp/src/Makefile b/JournalParserApp/src/Makefile
@@ -37,6 +37,16 @@ JournalParser_SYS_LIBS_WIN32 += wldap32 crypt32 Normaliz
 JournalParser_SYS_LIBS_Linux += curl
 JournalParser_LIBS += $(EPICS_BASE_IOC_LIBS)
 
+# add some DLLs so easier to run standalone
+BIN_INSTALLS_WIN32 += $(wildcard $(MYSQL)/bin/$(EPICS_HOST_ARCH)/*.dll)
+BIN_INSTALLS_WIN32 += $(wildcard $(PUGIXML)/bin/$(EPICS_HOST_ARCH)/*.dll)
+
+# ifeq ($(SHARED_LIBRARIES),YES)
+# USR_CPPFLAGS_WIN32 += -DSQLITE_API=__declspec(dllimport)
+# else
+# #USR_CXXFLAGS += -DCPPCONN_LIB_BUILD
+# endif
+
 #===========================
 
 include $(TOP)/configure/RULES

diff --git a/ingest/add_journal_entries.bat b/ingest/add_journal_entries.bat
@@ -1,6 +1,6 @@
 call %~dp0..\..\..\..\config_env.bat
 set "PATH=C:\Instrument\Apps\EPICS\support\pugixml\master\bin\windows-x64;C:\Instrument\Apps\EPICS\support\curl\master\bin\windows-x64;C:\Instrument\Apps\EPICS\support\MySQL\master\bin\windows-x64;%PATH%"
 
-"%PYTHON3%" %~dp0add_journal_entries.py %*
+"%PYTHON3%" -u %~dp0add_journal_entries.py %*
 
 if %errorlevel% neq 0 exit /b %errorlevel%