started unifying tracing

.github/workflows/cmake.yml

@@ -73,7 +73,8 @@ jobs:
           6.2)
             wget https://repo.radeon.com/amdgpu-install/6.2/ubuntu/jammy/amdgpu-install_6.2.60200-1_all.deb
             sudo apt install ./amdgpu-install_6.2.60200-1_all.deb
-            sudo amdgpu-install -y --usecase=hiplibsdk,rocm --no-dkms
+            sudo amdgpu-install -y --usecase=hiplibsdk --no-dkms
+            rm ./amdgpu-install_6.2.60200-1_all.deb
             ;;
         esac
 

CMakeLists.txt

@@ -57,6 +57,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 	endif()
 endif()
 
+set(FIRESTARTER_TRACING "None" CACHE STRING "FIRESTARTER_TRACING can be any of None, or External. This feature is currently under development and static linking might not work.")
+set_property(CACHE FIRESTARTER_TRACING PROPERTY STRINGS None External)
+
+set(FIRESTARTER_TRACING_CXX_FLAGS "" CACHE STRING "FIRESTARTER_TRACING_CXX_FLAGS should add include paths for the selected tracing infrastructure.")
+set(FIRESTARTER_TRACING_LD_FLAGS "" CACHE STRING "FIRESTARTER_TRACING_LD_FLAGS should set linker flags for the selected tracing infrastructure.")
+set(FIRESTARTER_TRACING_ADDITIONAL_FILES "" CACHE STRING "FIRESTARTER_TRACING_ADDITIONAL_FILES can be used to add files to the compilation process that are needed for tracing.")
+
+
 set(FIRESTARTER_BUILD_TYPE "FIRESTARTER" CACHE STRING "FIRESTARTER_BUILD_TYPE can be any of FIRESTARTER, FIRESTARTER_CUDA, FIRESTARTER_ONEAPI, or FIRESTARTER_HIP.")
 set_property(CACHE FIRESTARTER_BUILD_TYPE PROPERTY STRINGS FIRESTARTER FIRESTARTER_CUDA FIRESTARTER_ONEAPI)
 if (${FIRESTARTER_BUILD_TYPE} STREQUAL "FIRESTARTER")

examples/tracing/README.md

@@ -0,0 +1,22 @@
+# In this folder, there are examples on how one could include tracing with an external library
+
+Since these will be mostly shared libraries, we probably will only be able to use them with a dynamic build of FIRESTARTER
+
+You should be able to add the external libraries also to a static build by using the flags `FIRESTARTER_TRACING_*` for `cmake`. There you probably would include an archive. But this was not tested yet.
+
+## Example 1: Score-P
+
+- Needs Score-P: https://www.vi-hps.org/projects/score-p
+- File: `scorep.c`
+- Compilation:
+  - 1. create the adapter configuration: `scorep-config --adapter-init --dynamic --user --nokokkos --nocompiler --thread=pthread > .scorep_init.c`
+  - 2. compile the adapter and the tracing library: `scorep --user --nocompiler --dynamic --nokokkos --noopenmp --thread=pthread gcc -fPIC -c -DSCOREP_USER_ENABLE scorep.c .scorep_init.c`
+  - 3. link the tracing library: `scorep --no-as-needed --dynamic --user --nokokkos --nocompiler --noopenmp --thread=pthread gcc -shared -o libfirestarter_scorep.so scorep.o .scorep_init.o`
+  - 4. cmake FIRESTARTER: `cmake -DFIRESTARTER_TRACING=External -DFIRESTARTER_TRACING_LD_FLAGS="-L/home/rschoene/git/FIRESTARTER/examples/tracing -lfirestarter_scorep" -DFIRESTARTER_LINK_STATIC=OFF ..`
+  - 5. make FIRESTARTER: `make -j`
+
+- Running:
+  - Make sure that the library `libfirestarter_scorep.so` can be found in the `LD_LIBRARY_PATH`
+  - Run `FIRESTARTER` as usual
+  - Running `FIRESTARTER` should create a profile in your current directory starting with `scorep...`
+  - You can change environment variables to tune Score-P for your purposes. Have a look at: https://perftools.pages.jsc.fz-juelich.de/cicd/scorep/tags/latest/pdf/scorep.pdf

examples/tracing/scorep.c

@@ -0,0 +1,35 @@
+/******************************************************************************
+ * FIRESTARTER - A Processor Stress Test Utility
+ * Copyright (C) 2024 TU Dresden, Center for Information Services and High
+ * Performance Computing
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/\>.
+ *
+ * Contact: [email protected]
+ *****************************************************************************/
+
+#include "../../include/firestarter/Tracing/FIRESTARTER_Tracing.h"
+#include <scorep/SCOREP_User.h>
+
+void firestarter_tracing_initialize(int argc, const char **argv) {
+}
+
+void firestarter_tracing_region_begin(char const* region_name) {
+    SCOREP_USER_REGION_BY_NAME_BEGIN(region_name,
+                                     SCOREP_USER_REGION_TYPE_COMMON);
+}
+
+void firestarter_tracing_region_end(char const* region_name) {
+    SCOREP_USER_REGION_BY_NAME_END(region_name);
+}

include/firestarter/Tracing/FIRESTARTER_Tracing.h

@@ -0,0 +1,32 @@
+/******************************************************************************
+ * FIRESTARTER - A Processor Stress Test Utility
+ * Copyright (C) 2024 TU Dresden, Center for Information Services and High
+ * Performance Computing
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/\>.
+ *
+ * Contact: [email protected]
+ *****************************************************************************/
+
+#pragma once
+
+#ifdef FIRESTARTER_TRACING
+void firestarter_tracing_initialize(int argc, const char **argv);
+void firestarter_tracing_region_begin(char const* region_name);
+void firestarter_tracing_region_end(char const* region_name);
+#else
+#define firestarter_tracing_initialize(argc, argv) {}
+#define firestarter_tracing_region_begin(region_name) {}
+#define firestarter_tracing_region_end(region_name) {}
+#endif

src/CMakeLists.txt

@@ -41,8 +41,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 		)
 endif()
 
-SET(FIRESTARTER_LINK_LIBRARIES
+if(NOT "${FIRESTARTER_TRACING}" STREQUAL "None")
+	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFIRESTARTER_TRACING")
+	SET(FIRESTARTER_FILES
+		${FIRESTARTER_FILES}
+		${FIRESTARTER_TRACING_ADDITIONAL_FILES}
 	)
+	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FIRESTARTER_TRACING_CXXFLAGS}")
+endif()
 
 if (FIRESTARTER_THREAD_AFFINITY)
 	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFIRESTARTER_THREAD_AFFINITY")
@@ -231,3 +237,10 @@ elseif(${FIRESTARTER_BUILD_TYPE} STREQUAL "FIRESTARTER")
 		nlohmann_json::nlohmann_json
 		)
 endif()
+
+if(NOT "${FIRESTARTER_TRACING}" STREQUAL "None")
+	string(STRIP "${FIRESTARTER_TRACING_LD_FLAGS}" FIRESTARTER_TRACING_LD_FLAGS)
+	target_link_libraries("${FIRESTARTER_BUILD_TYPE}"
+		"${FIRESTARTER_TRACING_LD_FLAGS}"
+		)
+endif()

src/firestarter/Firestarter.cpp

@@ -21,6 +21,11 @@
 
 #include <firestarter/Firestarter.hpp>
 #include <firestarter/Logging/Log.hpp>
+
+extern "C" {
+#include <firestarter/Tracing/FIRESTARTER_Tracing.h>
+}
+
 #if defined(linux) || defined(__linux__)
 #include <firestarter/Optimizer/Algorithm/NSGA2.hpp>
 #include <firestarter/Optimizer/History.hpp>
@@ -325,6 +330,9 @@ Firestarter::~Firestarter() {
 }
 
 void Firestarter::mainThread() {
+
+  firestarter_tracing_region_begin("Main-Thread");
+
   this->environment().printThreadSummary();
 
 #if defined(FIRESTARTER_BUILD_CUDA) || defined(FIRESTARTER_BUILD_HIP)
@@ -352,6 +360,7 @@ void Firestarter::mainThread() {
     int returnCode;
     if (EXIT_SUCCESS != (returnCode = this->initDumpRegisterWorker(
                              _dumpRegistersTimeDelta, _dumpRegistersOutpath))) {
+      firestarter_tracing_region_end("Main-Thread");
       std::exit(returnCode);
     }
   }
@@ -363,6 +372,7 @@ void Firestarter::mainThread() {
 #if defined(linux) || defined(__linux__)
   // check if optimization is selected
   if (_optimize) {
+    firestarter_tracing_region_begin("Main-Thread-Optimize");
     auto startTime = optimizer::History::getTime();
 
     Firestarter::_optimizer = std::make_unique<optimizer::OptimizerWorker>(
@@ -381,6 +391,10 @@ void Firestarter::mainThread() {
     firestarter::optimizer::History::printBest(_optimizationMetrics,
                                                payloadItems);
 
+
+    firestarter_tracing_region_end("Main-Thread-Optimize");
+    firestarter_tracing_region_end("Main-Thread");
+
     // stop all the load threads
     std::raise(SIGTERM);
   }
@@ -415,6 +429,8 @@ void Firestarter::mainThread() {
   if (_errorDetection) {
     this->printThreadErrorReport();
   }
+
+  firestarter_tracing_region_end("Main-Thread");
 }
 
 void Firestarter::setLoad(unsigned long long value) {
@@ -437,6 +453,11 @@ void Firestarter::sigalrmHandler(int signum) { (void)signum; }
 void Firestarter::sigtermHandler(int signum) {
   (void)signum;
 
+    if (Firestarter::loadVar == LOAD_LOW)
+      firestarter_tracing_region_end("WD_LOW");
+    if (Firestarter::loadVar == LOAD_HIGH)
+      firestarter_tracing_region_end("WD_HIGH");
+
   Firestarter::setLoad(LOAD_STOP);
   // exit loop
   // used in case of 0 < load < 100

src/firestarter/LoadWorker.cpp

@@ -23,18 +23,16 @@
 #include <firestarter/Firestarter.hpp>
 #include <firestarter/Logging/Log.hpp>
 
+extern "C" {
+#include <firestarter/Tracing/FIRESTARTER_Tracing.h>
+}
+
 #if defined(linux) || defined(__linux__)
 extern "C" {
 #include <firestarter/Measurement/Metric/IPCEstimate.h>
 }
 #endif
 
-#ifdef ENABLE_VTRACING
-#include <vt_user.h>
-#endif
-#ifdef ENABLE_SCOREP
-#include <SCOREP_User.h>
-#endif
 
 #include <cmath>
 #include <cstdlib>
@@ -350,33 +348,20 @@ void Firestarter::loadThreadWorker(std::shared_ptr<LoadWorkerData> td) {
       // will be terminated by watchdog
       for (;;) {
         // call high load function
-#ifdef ENABLE_VTRACING
-        VT_USER_START("HIGH_LOAD_FUNC");
-#endif
-#ifdef ENABLE_SCOREP
-        SCOREP_USER_REGION_BY_NAME_BEGIN("HIGH",
-                                         SCOREP_USER_REGION_TYPE_COMMON);
-#endif
+
+        firestarter_tracing_region_begin("High");
+
         td->iterations = td->config().payload().highLoadFunction(
             td->addrMem, td->addrHigh, td->iterations);
 
+        firestarter_tracing_region_end("High");
+        firestarter_tracing_region_begin("Low");
+
         // call low load function
-#ifdef ENABLE_VTRACING
-        VT_USER_END("HIGH_LOAD_FUNC");
-        VT_USER_START("LOW_LOAD_FUNC");
-#endif
-#ifdef ENABLE_SCOREP
-        SCOREP_USER_REGION_BY_NAME_END("HIGH");
-        SCOREP_USER_REGION_BY_NAME_BEGIN("LOW", SCOREP_USER_REGION_TYPE_COMMON);
-#endif
         td->config().payload().lowLoadFunction(td->addrHigh, td->period);
-#ifdef ENABLE_VTRACING
-        VT_USER_END("LOW_LOAD_FUNC");
-#endif
-#ifdef ENABLE_SCOREP
-        SCOREP_USER_REGION_BY_NAME_END("LOW");
-#endif
 
+        firestarter_tracing_region_end("Low");
+
         // terminate if master signals end of run and record stop timestamp
         if (*td->addrHigh == LOAD_STOP) {
           td->stopTsc = td->environment().topology().timestamp();

src/firestarter/Main.cpp

@@ -22,6 +22,10 @@
 #include <firestarter/Firestarter.hpp>
 #include <firestarter/Logging/Log.hpp>
 
+extern "C" {
+#include <firestarter/Tracing/FIRESTARTER_Tracing.h>
+}
+
 #include <cxxopts.hpp>
 
 #include <string>
@@ -468,6 +472,8 @@ int main(int argc, const char **argv) {
          "Computing"
       << "\n";
 
+    firestarter_tracing_initialize(argc, argv);
+
   Config cfg{argc, argv};
 
   try {

src/firestarter/WatchdogWorker.cpp

@@ -21,13 +21,13 @@
 
 #include <firestarter/Firestarter.hpp>
 
+extern "C" {
+#include <firestarter/Tracing/FIRESTARTER_Tracing.h>
+}
+
 #include <cerrno>
 #include <csignal>
 
-#ifdef ENABLE_SCOREP
-#include <SCOREP_User.h>
-#endif
-
 using namespace firestarter;
 
 int Firestarter::watchdogWorker(std::chrono::microseconds period,
@@ -75,13 +75,8 @@ int Firestarter::watchdogWorker(std::chrono::microseconds period,
       nsec load_nsec = load - load_reduction;
 
       // wait for time to be ellapsed with high load
-#ifdef ENABLE_VTRACING
-      VT_USER_START("WD_HIGH");
-#endif
-#ifdef ENABLE_SCOREP
-      SCOREP_USER_REGION_BY_NAME_BEGIN("WD_HIGH",
-                                       SCOREP_USER_REGION_TYPE_COMMON);
-#endif
+      firestarter_tracing_region_begin("WD_HIGH");
+
       {
         std::unique_lock<std::mutex> lk(this->_watchdogTerminateMutex);
         // abort waiting if we get the interrupt signal
@@ -92,12 +87,7 @@ int Firestarter::watchdogWorker(std::chrono::microseconds period,
           return EXIT_SUCCESS;
         }
       }
-#ifdef ENABLE_VTRACING
-      VT_USER_END("WD_HIGH");
-#endif
-#ifdef ENABLE_SCOREP
-      SCOREP_USER_REGION_BY_NAME_END("WD_HIGH");
-#endif
+      firestarter_tracing_region_end("WD_HIGH");
 
       // signal low load
       this->setLoad(LOAD_LOW);
@@ -106,13 +96,8 @@ int Firestarter::watchdogWorker(std::chrono::microseconds period,
       nsec idle_nsec = idle - idle_reduction;
 
       // wait for time to be ellapsed with low load
-#ifdef ENABLE_VTRACING
-      VT_USER_START("WD_LOW");
-#endif
-#ifdef ENABLE_SCOREP
-      SCOREP_USER_REGION_BY_NAME_BEGIN("WD_LOW",
-                                       SCOREP_USER_REGION_TYPE_COMMON);
-#endif
+
+      firestarter_tracing_region_begin("WD_LOW");
       {
         std::unique_lock<std::mutex> lk(this->_watchdogTerminateMutex);
         // abort waiting if we get the interrupt signal
@@ -123,13 +108,7 @@ int Firestarter::watchdogWorker(std::chrono::microseconds period,
           return EXIT_SUCCESS;
         }
       }
-#ifdef ENABLE_VTRACING
-      VT_USER_END("WD_LOW");
-#endif
-#ifdef ENABLE_SCOREP
-      SCOREP_USER_REGION_BY_NAME_END("WD_LOW");
-#endif
-
+      firestarter_tracing_region_end("WD_LOW");
       // increment elapsed time
       time += period;
 
@@ -155,6 +134,10 @@ int Firestarter::watchdogWorker(std::chrono::microseconds period,
       Firestarter::_watchdogTerminateAlert.wait_for(
           lk, timeout, []() { return Firestarter::_watchdog_terminate; });
     }
+    if (Firestarter::loadVar == LOAD_LOW)
+      firestarter_tracing_region_end("WD_LOW");
+    if (Firestarter::loadVar == LOAD_HIGH)
+      firestarter_tracing_region_end("WD_HIGH");
 
     this->setLoad(LOAD_STOP);