Merge remote-tracking branch 'origin/candidate-9.8.x'

Signed-off-by: Gordon Smith <[email protected]>

dockerfiles/platform-build/README.md

@@ -18,7 +18,7 @@ on the build behavior remains exactly the same.
 Default of signing_secret is empty. Populate this variable within your Github
 Secrets for the repository with an exported armored secret key.  It is hidden by
 Github Actions from view and is hidden from the docker image layers by using 
-BuildKit and [secret mounting](https://docs.docker.com.xy2401.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information).
+BuildKit and [secret mounting](https://docs.docker.com/build/building/secrets/).
 This ensures that the secret key does not leak into the final docker image layers.
 
 > signing_keyid: ${{ secrets.SIGNING_KEYID }}

docs/EN_US/ECLStandardLibraryReference/SLR-Mods/WorkunitTimeStamps.xml

@@ -4,21 +4,17 @@
 <sect1 id="WorkunitTimeStamps">
   <title>WorkunitTimeStamps</title>
 
-  <para><emphasis
-  role="bold">STD.System.Workunit.WorkunitTimeStamps    <indexterm>
+  <para><emphasis role="bold">STD.System.Workunit.WorkunitTimeStamps
+  <indexterm>
       <primary>STD.System.Workunit.WorkunitTimeStamps</primary>
-    </indexterm>
-    <indexterm>
+    </indexterm> <indexterm>
       <primary>System.Workunit.WorkunitTimeStamps</primary>
-    </indexterm>
-    <indexterm>
+    </indexterm> <indexterm>
       <primary>Workunit.WorkunitTimeStamps</primary>
-    </indexterm>
-    <indexterm>
+    </indexterm> <indexterm>
       <primary>WorkunitTimeStamps</primary>
-    </indexterm>
-(</emphasis> <emphasis>
-  wuid </emphasis> <emphasis role="bold">)</emphasis></para>
+    </indexterm> (</emphasis> <emphasis> wuid </emphasis> <emphasis
+  role="bold">)</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">
@@ -46,7 +42,7 @@
   <para>The <emphasis role="bold">WorkunitTimeStamps </emphasis>function
   returns a DATASET with this format:</para>
 
-  <programlisting>EXPORT WsTimeStamp := RECORD
+  <programlisting>EXPORT TimeStampRecord := RECORD
   STRING32 application;
   STRING16 id;
   STRING20 time;
@@ -58,14 +54,15 @@ END;</programlisting>
 
   <para>Example:</para>
 
-  <programlisting format="linespecific">OUTPUT(STD.System.Workunit.WorkunitTimeStamps('W20070308-164946'));
+  <programlisting format="linespecific">OUTPUT(STD.System.Workunit.WorkunitTimeStamps('W20240801-122755'));
+
 /* produces output like this:
-'workunit     ','Created ','2008-02-13T18:28:20Z','              '
-'workunit     ','Modified','2008-02-13T18:32:47Z','              '
-'EclServer    ','Compiled','2008-02-13T18:28:20Z','10.173.9.2:0  '
-'EclAgent     ','Started ','2008-02-13T18:32:35Z','training009003'
-'Thor - graph1','Finished','2008-02-13T18:32:47Z','training009004'
-'Thor - graph1','Started ','2008-02-13T18:32:13Z','training009004'
-'EclAgent     ','Finished','2008-02-13T18:33:09Z','training009003'
+'workunit     ','Created ','2024-08-01T16:28:20Z','              '
+'workunit     ','Modified','2024-08-01T16:32:47Z','              '
+'EclServer    ','Compiled','2024-08-01T16:28:20Z','172.31.4.17'
+'EclAgent     ','Started ','2024-08-01T16:32:35Z','172.31.4.17'
+'Thor - graph1','Finished','2024-08-01T16:32:47Z','172.31.4.17'
+'Thor - graph1','Started ','2024-08-01T16:32:13Z','172.31.4.17'
+'EclAgent     ','Finished','2024-08-01T16:33:09Z','172.31.4.17'
 */</programlisting>
 </sect1>

ecl/agentexec/agentexec.cpp

@@ -261,6 +261,7 @@ class WaitThread : public CInterfaceOf<IPooledThread>
     virtual void threadmain() override
     {
         Owned<IException> exception;
+        bool sharedK8sJob = false;
         try
         {
             StringAttr jobSpecName(apptype);
@@ -276,6 +277,7 @@ class WaitThread : public CInterfaceOf<IPooledThread>
             bool useChildProcesses = compConfig->getPropBool("@useChildProcesses");
             if (isContainerized() && !useChildProcesses)
             {
+                sharedK8sJob = true;
                 constexpr unsigned queueWaitingTimeoutMs = 10000;
                 constexpr unsigned queueWaitingCheckPeriodMs = 1000;
                 if (!owner.lingerQueue || !queueJobIfQueueWaiting(owner.lingerQueue, item, queueWaitingCheckPeriodMs, queueWaitingCheckPeriodMs))
@@ -337,13 +339,26 @@ class WaitThread : public CInterfaceOf<IPooledThread>
         {
             EXCLOG(exception);
             Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
-            Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
-            if (workunit)
+            Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
+            if (cw)
             {
-                workunit->setState(WUStateFailed);
-                StringBuffer eStr;
-                addExceptionToWorkunit(workunit, SeverityError, "agentexec", exception->errorCode(), exception->errorMessage(eStr).str(), nullptr, 0, 0, 0);
-                workunit->commit();
+                // if either a) NOT a thoragent with useChildProcesses=false (default in k8s config) or b) is still in an executing state
+                if (!sharedK8sJob || (cw->getState() == WUStateRunning) || (cw->getState() == WUStateBlocked) || (cw->getState() == WUStateWait))
+                {
+                    // For a shared k8s job, i.e. where this agent is thoragent launching shared (multiJobLinger) k8s jobs
+                    // the job agent should handle the job state.
+                    // In that scenario, this is a fallback that should only come into effect if the job workflow instance has failed to handle the exception
+                    // e.g. because it abruptly disappeared.
+                    Owned<IWorkUnit> workunit = &cw->lock();
+                    // recheck now locked
+                    if ((workunit->getState() == WUStateRunning) || (workunit->getState() == WUStateBlocked) || (workunit->getState() == WUStateWait))
+                    {
+                        workunit->setState(WUStateFailed);
+                        StringBuffer eStr;
+                        addExceptionToWorkunit(workunit, SeverityError, "agentexec", exception->errorCode(), exception->errorMessage(eStr).str(), nullptr, 0, 0, 0);
+                        workunit->commit();
+                    }
+                }
             }
         }
     }

ecl/hqlcpp/hqlcatom.cpp

@@ -533,14 +533,17 @@ IIdAtom * regexNewStrFindId;
 IIdAtom * regexNewStrFoundId;
 IIdAtom * regexNewStrFoundXId;
 IIdAtom * regexNewStrReplaceXId;
+IIdAtom * regexNewStrReplaceFixedId;
 IIdAtom * regexNewUStrFindId;
 IIdAtom * regexNewU8StrFindId;
 IIdAtom * regexNewUStrFoundId;
 IIdAtom * regexNewU8StrFoundId;
 IIdAtom * regexNewUStrFoundXId;
 IIdAtom * regexNewU8StrFoundXId;
 IIdAtom * regexNewUStrReplaceXId;
+IIdAtom * regexNewUStrReplaceFixedId;
 IIdAtom * regexNewU8StrReplaceXId;
+IIdAtom * regexNewU8StrReplaceFixedId;
 IIdAtom * regexMatchSetId;
 IIdAtom * regexUStrMatchSetId;
 IIdAtom * regexU8StrMatchSetId;
@@ -1218,14 +1221,17 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEID(regexNewStrFound);
     MAKEID(regexNewStrFoundX);
     MAKEID(regexNewStrReplaceX);
+    MAKEID(regexNewStrReplaceFixed);
     MAKEID(regexNewUStrFind);
     MAKEID(regexNewU8StrFind);
     MAKEID(regexNewUStrFound);
     MAKEID(regexNewU8StrFound);
     MAKEID(regexNewUStrFoundX);
     MAKEID(regexNewU8StrFoundX);
     MAKEID(regexNewUStrReplaceX);
+    MAKEID(regexNewUStrReplaceFixed);
     MAKEID(regexNewU8StrReplaceX);
+    MAKEID(regexNewU8StrReplaceFixed);
     MAKEID(regexMatchSet);
     MAKEID(regexUStrMatchSet);
     MAKEID(regexU8StrMatchSet);

ecl/hqlcpp/hqlcatom.hpp

@@ -531,14 +531,17 @@ extern IIdAtom * regexNewStrFindId;
 extern IIdAtom * regexNewStrFoundId;
 extern IIdAtom * regexNewStrFoundXId;
 extern IIdAtom * regexNewStrReplaceXId;
+extern IIdAtom * regexNewStrReplaceFixedId;
 extern IIdAtom * regexNewUStrFindId;
 extern IIdAtom * regexNewU8StrFindId;
 extern IIdAtom * regexNewUStrFoundId;
 extern IIdAtom * regexNewU8StrFoundId;
 extern IIdAtom * regexNewUStrFoundXId;
 extern IIdAtom * regexNewU8StrFoundXId;
 extern IIdAtom * regexNewUStrReplaceXId;
+extern IIdAtom * regexNewUStrReplaceFixedId;
 extern IIdAtom * regexNewU8StrReplaceXId;
+extern IIdAtom * regexNewU8StrReplaceFixedId;
 extern IIdAtom * regexMatchSetId;
 extern IIdAtom * regexUStrMatchSetId;
 extern IIdAtom * regexU8StrMatchSetId;

ecl/hqlcpp/hqlcpp.cpp

@@ -2226,8 +2226,28 @@ void HqlCppTranslator::buildFunctionCall(BuildCtx & ctx, IIdAtom * name, HqlExpr
 void HqlCppTranslator::callProcedure(BuildCtx & ctx, IIdAtom * name, HqlExprArray & args)
 {
     OwnedHqlExpr call = bindTranslatedFunctionCall(name, args);
-    assertex(call->queryExternalDefinition());
+    IHqlExpression * funcdef = call->queryExternalDefinition();
+
+    assertex(funcdef);
+
+    CHqlBoundExpr boundTimer, boundStart;
+    IHqlExpression * external = funcdef->queryChild(0);
+    bool isTimed = external->hasAttribute(timeAtom);
+    if (isTimed)
+    {
+        StringBuffer nameTemp;
+        const char * name = str(external->queryId());
+        if (getStringValue(nameTemp, queryAttributeChild(external, timeAtom, 0)).length())
+            name = nameTemp;
+        buildStartTimer(ctx, boundTimer, boundStart, name);
+    }
+
     ctx.addExpr(call);
+
+    if (isTimed)
+    {
+        buildStopTimer(ctx, boundTimer, boundStart);
+    }
 }
 
 bool HqlCppTranslator::getDebugFlag(const char * name, bool defValue)
@@ -6697,6 +6717,10 @@ void HqlCppTranslator::doBuildAssignCast(BuildCtx & ctx, const CHqlBoundTarget &
                     //don't do this if the target type is unicode at the moment
                     ignoreStretched = isStringType(targetType);
                     break;
+                case no_regex_replace:
+                    // replacing into a fixed-sized target should not require a temp
+                    useTemp = false;
+                    break;
                 }
 
                 if (ignoreStretched)

ecl/hqlcpp/hqlcppsys.ecl

@@ -558,20 +558,23 @@ const char * cppSystemText[]  = {
     "   boolean regexNewStrFound() : method,pure,entrypoint='found';"
     "   string regexNewStrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';"
     "   string regexNewStrReplaceX(const string _search, const string _replace) : method,pure,entrypoint='replace',time('REGEXREPLACE');"
+    "   regexNewStrReplaceFixed(noconst string _tgt, const string _search, const string _replace) : method,pure,entrypoint='replaceFixed',time('REGEXREPLACE');"
     "   set of string regexMatchSet(const string _search) : method,pure,entrypoint='getMatchSet',time('REGEXFINDSET');"
 
     "   regexNewSetUStrPattern(const varunicode _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPattern',time('CompileUnicodeRegex');"
     "   regexNewUStrFind(boolean _compiled, const unicode _search) : omethod,entrypoint='find',time('REGEXFIND');"
     "   boolean regexNewUStrFound() : method,pure,entrypoint='found';"
     "   unicode regexNewUStrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';"
     "   unicode regexNewUStrReplaceX(const unicode _search, const unicode _replace) : method,pure,entrypoint='replace',time('REGEXREPLACE');"
+    "   regexNewUStrReplaceFixed(noconst unicode _tgt, const unicode _search, const unicode _replace) : method,pure,entrypoint='replaceFixed',time('REGEXREPLACE');"
     "   set of unicode regexUStrMatchSet(const unicode _search) : method,pure,entrypoint='getMatchSet',time('REGEXFINDSET');"
 
     "   regexNewSetU8StrPattern(const utf8 _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPattern',time('CompileUTF8Regex');"
     "   regexNewU8StrFind(boolean _compiled, const utf8 _search, boolean _cloneSearch) : omethod,entrypoint='find',time('REGEXFIND');"
     "   boolean regexNewU8StrFound() : method,pure,entrypoint='found';"
     "   utf8 regexNewU8StrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';"
     "   utf8 regexNewU8StrReplaceX(const utf8 _search, const utf8 _replace) : method,pure,entrypoint='replace',time('REGEXREPLACE');"
+    "   regexNewU8StrReplaceFixed(noconst utf8 _tgt, const utf8 _search, const utf8 _replace) : method,pure,entrypoint='replaceFixed',time('REGEXREPLACE');"
     "   set of utf8 regexU8StrMatchSet(const utf8 _search) : method,pure,entrypoint='getMatchSet',time('REGEXFINDSET');"
 
     //clibrary functions that are called from the code generation

ecl/hqlcpp/hqlhtcpp.cpp

@@ -18659,20 +18659,54 @@ void HqlCppTranslator::doBuildNewRegexFindReplace(BuildCtx & ctx, const CHqlBoun
     // as long as the find instance.  Only exception could be if call created a temporary class instance.
     if (expr->getOperator() == no_regex_replace)
     {
-        HqlExprArray args;
-        args.append(*LINK(compiled));
-        args.append(*LINK(search));
-        args.append(*LINK(expr->queryChild(2)));
-        IIdAtom * func = nullptr;
-        if (isUTF8Type(searchStringType))
-            func = regexNewU8StrReplaceXId;
-        else if (isUnicodeType(searchStringType))
-            func = regexNewUStrReplaceXId;
+        // If the target is a preallocated fixed-length buffer and the
+        // datatype matches the result expression datatype, we can call an optimized replace function
+        if (target && target->isFixedSize() && target->queryType()->getTypeCode() == expr->queryType()->getTypeCode())
+        {
+            // We need to build our arguments manually because we need to
+            // pass the size of the output buffer (the target) as an argument
+            IHqlExpression * targetVar = target->expr;
+            unsigned targetSize = target->queryType()->getStringLen();
+
+            CHqlBoundExpr searchExpr, replaceExpr;
+            buildCachedExpr(ctx, search, searchExpr);
+            buildCachedExpr(ctx, expr->queryChild(2), replaceExpr);
+
+            HqlExprArray args;
+            args.append(*LINK(compiled)); // instance on which method is called
+            args.append(*getSizetConstant(targetSize)); // size of the output buffer in code units
+            args.append(*getElementPointer(targetVar)); // pointer to the output buffer
+            args.append(*getBoundLength(searchExpr)); // length of regex expression, in characters
+            args.append(*LINK(searchExpr.expr)); // pointer to regex expression
+            args.append(*getBoundLength(replaceExpr)); // length of replacement expression, in characters
+            args.append(*LINK(replaceExpr.expr)); // pointer to replacement expression
+
+            IIdAtom * func = nullptr;
+            if (isUTF8Type(searchStringType))
+                func = regexNewU8StrReplaceFixedId;
+            else if (isUnicodeType(searchStringType))
+                func = regexNewUStrReplaceFixedId;
+            else
+                func = regexNewStrReplaceFixedId;
+            callProcedure(ctx, func, args);
+        }
         else
-            func = regexNewStrReplaceXId;
-        OwnedHqlExpr call = bindFunctionCall(func, args);
-        //Need to associate???
-        buildExprOrAssign(ctx, target, call, bound);
+        {
+            HqlExprArray args;
+            args.append(*LINK(compiled));
+            args.append(*LINK(search));
+            args.append(*LINK(expr->queryChild(2)));
+            IIdAtom * func = nullptr;
+            if (isUTF8Type(searchStringType))
+                func = regexNewU8StrReplaceXId;
+            else if (isUnicodeType(searchStringType))
+                func = regexNewUStrReplaceXId;
+            else
+                func = regexNewStrReplaceXId;
+            OwnedHqlExpr call = bindFunctionCall(func, args);
+            //Need to associate???
+            buildExprOrAssign(ctx, target, call, bound);
+        }
     }
     else
     {