HPCC-31020 Fix postmortem files overwriting one another.

Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Dec 14, 2023 · bb2486b · bb2486b
1 parent 597fd83
commit bb2486b
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/initfiles/bin/check_executes b/initfiles/bin/check_executes
@@ -99,8 +99,8 @@ if [ $PMD_ALWAYS = true ] || [ $retVal -ne 0 ]; then
     cp $f $POST_MORTEM_DIR/$f
     echo "Copied $f to $POST_MORTEM_DIR/$f" | tee -a $POST_MORTEM_DIR/info.log
   done
-  cp `ls -rt /tmp/postmortem.log.*` $POST_MORTEM_DIR
-  rm /tmp/postmortem.log.*
+  cp `ls -rt /tmp/postmortem.$progPid.log.*` $POST_MORTEM_DIR
+  rm /tmp/postmortem.$progPid.log.*
   if [ -f core ]; then
     echo "Generating info from core file to $POST_MORTEM_DIR/info.log" | tee -a $POST_MORTEM_DIR/info.log
     gdb -batch -ix /opt/HPCCSystems/bin/.gdbinit -x /opt/HPCCSystems/bin/post-mortem-gdb ${PMD_PROGNAME} core 2>$POST_MORTEM_DIR/info.err >>$POST_MORTEM_DIR/info.log

diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp
@@ -2480,7 +2480,12 @@ void setupContainerizedLogMsgHandler()
         unsigned postMortemLines = logConfig->getPropInt(capturePostMortemAtt, 0);
         if (postMortemLines)
         {
-            ILogMsgHandler *fileMsgHandler = getPostMortemLogMsgHandler("/tmp/postmortem.log", postMortemLines, MSGFIELD_STANDARD);
+            // augment postmortem files with <pid> to avoid clashes where multiple processes are running within
+            // same process space, e.g. hthor processes running in same k8s container
+            unsigned pid = GetCurrentProcessId();
+            VStringBuffer portMortemFileBase("/tmp/postmortem.%u.log", pid);
+
+            ILogMsgHandler *fileMsgHandler = getPostMortemLogMsgHandler(portMortemFileBase, postMortemLines, MSGFIELD_STANDARD);
             queryLogMsgManager()->addMonitorOwn(fileMsgHandler, getCategoryLogMsgFilter(MSGAUD_all, MSGCLS_all, TopDetail));
         }
     }