From b74d7fd6f69d7138fc045339ac1d9a5d5446d9ce Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Mon, 1 Apr 2024 13:00:24 -0400 Subject: [PATCH 01/12] HPCC-31549 Dafilesrv throttle arg order Signed-off-by: Mark Kelly --- fs/dafsserver/dafsserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index e2f2c3a9cad..47279455f7c 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -3361,7 +3361,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface */ unsigned ms = timer.elapsedMs(); totalThrottleDelay += ms; - PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %u milliseconds, proceeding as cpu(%u) Date: Fri, 29 Mar 2024 10:09:05 -0400 Subject: [PATCH 02/12] HPCC-31211 Document new planename parameter to STD.File LZ file methods Signed-off-by: Jim DeFabia --- .../SLR-Mods/CreateExternalDirectory.xml | 23 ++++++++++--- .../SLR-Mods/DeleteExternalFile.xml | 23 ++++++++++--- .../SLR-Mods/MoveExternalFile.xml | 27 +++++++++++---- .../SLR-Mods/RemoteDirectory.xml | 34 +++++++++++++------ 4 files changed, 80 insertions(+), 27 deletions(-) diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml index 482a9ad8ebe..55b62e7c113 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml @@ -10,7 +10,9 @@ File.CreateExternalDirectory CreateExternalDirectory - ( location, path ( location, path [ , planename ] ) @@ -24,7 +26,8 @@ location A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planename is + provided. @@ -33,6 +36,14 @@ A null-terminated string containing the directory path to create. + + + planename + + A null-terminated string containing name of the data plane + containing the file. Optional if location is + provided, but planename is preferred. + @@ -44,8 +55,10 @@ Example: - IP := '10.150.254.6'; -path := '/c$/training/import/NewDir'; -STD.File.CreateExternalDirectory(IP,path); + IMPORT STD; +IP := ''; +path := '/var/lib/HPCCSystems/dropzone/advancedtraining/'; +planename := 'mydropzone'; +STD.File.CreateExternalDirectory(IP,path,planename); diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml index 211aedd6bb2..15194c9d374 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml @@ -10,7 +10,9 @@ File.DeleteExternalFile DeleteExternalFile - ( location, path ( location, path [ , planename ] ) @@ -24,7 +26,8 @@ location A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planename is + provided. @@ -33,6 +36,14 @@ A null-terminated string containing the path/name of the file to remove. + + + planename + + A null-terminated string containing name of the data plane + containing the file. Optional if location is + provided, but planename is preferred. + @@ -44,8 +55,10 @@ Example: - IP := '10.150.254.6'; -infile := '/c$/training/import/AdvancedECL/people'; -STD.File.DeleteExternalFile(IP,infile); + IMPORT STD; +IP := ''; +infile := '/var/lib/HPCCSystems/dropzone/originalperson'; +planename := 'mydropzone'; +STD.File.DeleteExternalFile(IP,infile,planename); diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml index 2983b611365..a812219cee0 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml @@ -10,8 +10,10 @@ File.MoveExternalFile MoveExternalFile - ( location, frompath, topath - ) + ( location, frompath, topath [ , planename ] ) @@ -24,7 +26,8 @@ location A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planename is + provided. @@ -40,6 +43,14 @@ A null-terminated string containing the path/name of the target file. + + + planeName + + A null-terminated string containing name of the data plane + containing the file. Optional if location is + provided, but planename is preferred. + @@ -53,8 +64,10 @@ Example: - IP := '10.150.254.6'; -infile := '/c$/training/import/AdvancedECL/people'; -outfile := '/c$/training/import/DFUtest/people'; -STD.File.MoveExternalFile(IP,infile,outfile); + IMPORT STD; +IP := ''; +infile := '/var/lib/HPCCSystems/dropzone/originalperson'; +outfile := '/var/lib/HPCCSystems/dropzone/originalperson_bak'; +planename := 'mydropzone'; +STD.File.MoveExternalFile(IP,infile,outfile,planename); diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml index 69e390bd2df..add90d0bf22 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml @@ -4,17 +4,19 @@ RemoteDirectory - STD.File.RemoteDirectory + STD.File.RemoteDirectory( + machineIP, dir [ , + mask ][ , recurse + ][ , planeName ] ) + STD.File.RemoteDirectory File.RemoteDirectory RemoteDirectory - ( machineIP, dir [ , mask ][ , recurse ] ) + @@ -27,7 +29,8 @@ machineIP A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planeName is + provided. @@ -54,6 +57,14 @@ omitted, the default is FALSE. + + planeName + + A null-terminated string containing name of the data plane + containing the file. Optional if machineIP is + provided, but planeName is preferred. + + Return: @@ -84,7 +95,10 @@ END; Example: - OUTPUT(STD.File.RemoteDirectory('edata12','\in','*.d00')); -OUTPUT(STD.File.RemoteDirectory('10.150.254.6', - '/var/lib/HPCCSystems/hpcc-data/thor/','acc*',TRUE)); + IMPORT STD; +machineIP := ''; +dir := '/var/lib/HPCCSystems/dropzone/training'; +recurse:= FALSE; +planeName := 'mydropzone'; +OUTPUT(STD.File.RemoteDirectory(machineIP,dir,'*.csv',recurse,planeName)); From fd6a9c6d78c60c9b179c92545ac0d2936bf7a392 Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 26 Mar 2024 09:54:37 +0000 Subject: [PATCH 03/12] HPCC-31338 Return cost from cost optimizer as separate field Signed-off-by: Shamser Ahmed --- common/workunit/workunit.cpp | 23 +++++++++++++++++++ common/workunit/workunit.hpp | 2 ++ common/workunit/workunit.ipp | 2 ++ common/wuanalysis/anacommon.cpp | 4 ++-- esp/scm/ws_workunits_struct.ecm | 1 + .../ws_workunits/ws_workunitsHelpers.cpp | 2 ++ 6 files changed, 32 insertions(+), 2 deletions(-) diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index 7faa1dffcf5..805b9a6f0de 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -4862,6 +4862,7 @@ class CLocalWUException : implements IWUException, public CInterface virtual unsigned getSequence() const override; virtual const char * queryScope() const override; virtual unsigned getPriority() const override; + virtual double getCost() const override; virtual void setExceptionSource(const char *str) override; virtual void setExceptionMessage(const char *str) override; virtual void setExceptionCode(unsigned code) override; @@ -4873,6 +4874,7 @@ class CLocalWUException : implements IWUException, public CInterface virtual void setActivityId(unsigned _id) override; virtual void setScope(const char * _scope) override; virtual void setPriority(unsigned _priority) override; + virtual void setCost(double cost) override; }; //========================================================================================== @@ -7645,6 +7647,18 @@ WUPriorityClass CLocalWorkUnit::getPriority() const return (WUPriorityClass) getEnum(p, "@priorityClass", priorityClasses); } +void CLocalWorkUnit::setCost(double cost) +{ + CriticalBlock block(crit); + p->setPropReal("@cost", cost); +} + +double CLocalWorkUnit::getCost() const +{ + CriticalBlock block(crit); + return p->getPropReal("@cost", 0); +} + const char *CLocalWorkUnit::queryPriorityDesc() const { return getEnumText(getPriority(), priorityClasses); @@ -11924,6 +11938,11 @@ unsigned CLocalWUException::getPriority() const return p->getPropInt("@prio", 0); } +double CLocalWUException::getCost() const +{ + return p->getPropReal("@cost", 0); +} + void CLocalWUException::setExceptionSource(const char *str) { p->setProp("@source", str); @@ -11979,6 +11998,10 @@ void CLocalWUException::setPriority(unsigned _priority) p->setPropInt("@prio", _priority); } +void CLocalWUException::setCost(double _cost) +{ + p->setPropReal("@cost", _cost); +} //========================================================================================== CLocalWUAppValue::CLocalWUAppValue(const IPropertyTree *_owner, const IPropertyTree *_props) : owner(_owner), props(_props) diff --git a/common/workunit/workunit.hpp b/common/workunit/workunit.hpp index 3e08f2d16eb..d53b98b5e9d 100644 --- a/common/workunit/workunit.hpp +++ b/common/workunit/workunit.hpp @@ -520,6 +520,7 @@ interface IConstWUException : extends IInterface virtual unsigned getActivityId() const = 0; virtual const char * queryScope() const = 0; virtual unsigned getPriority() const = 0; // For ordering within a severity - e.g. warnings about inefficiency + virtual double getCost() const = 0; // cost optimizer cost saving estimate. }; @@ -536,6 +537,7 @@ interface IWUException : extends IConstWUException virtual void setActivityId(unsigned _id) = 0; virtual void setScope(const char * _scope) = 0; virtual void setPriority(unsigned _priority) = 0; + virtual void setCost(double cost) = 0; // cost optimizer cost saving estimate. }; diff --git a/common/workunit/workunit.ipp b/common/workunit/workunit.ipp index d2d7156258d..85f0287b460 100644 --- a/common/workunit/workunit.ipp +++ b/common/workunit/workunit.ipp @@ -241,6 +241,7 @@ public: virtual IConstWUPluginIterator & getPlugins() const; virtual IConstWULibraryIterator & getLibraries() const; virtual WUPriorityClass getPriority() const; + virtual double getCost() const; virtual const char *queryPriorityDesc() const; virtual int getPriorityLevel() const; virtual int getPriorityValue() const; @@ -328,6 +329,7 @@ public: void setDebugValueInt(const char * propname, int value, bool overwrite); void setJobName(const char * value); void setPriority(WUPriorityClass cls); + void setCost(double cost); void setPriorityLevel(int level); void setRescheduleFlag(bool value); void setResultLimit(unsigned value); diff --git a/common/wuanalysis/anacommon.cpp b/common/wuanalysis/anacommon.cpp index 89d451607f6..be92d006405 100644 --- a/common/wuanalysis/anacommon.cpp +++ b/common/wuanalysis/anacommon.cpp @@ -65,12 +65,12 @@ void PerformanceIssue::createException(IWorkUnit * wu, double costRate) we->setExceptionFileName(filename); StringBuffer s(comment); // Append scope to comment as scope column is not visible in ECLWatch s.appendf(" (%s)", scope.str()); + we->setExceptionMessage(s.str()); if (costRate!=0.0) { double timePenaltyPerHour = (double)statUnits2seconds(timePenalty) / 3600; - s.appendf(" cost %.2f", timePenaltyPerHour*costRate); + we->setCost(timePenaltyPerHour*costRate); } - we->setExceptionMessage(s.str()); we->setExceptionSource(CostOptimizerName); } diff --git a/esp/scm/ws_workunits_struct.ecm b/esp/scm/ws_workunits_struct.ecm index b187ff88283..cab333fe846 100644 --- a/esp/scm/ws_workunits_struct.ecm +++ b/esp/scm/ws_workunits_struct.ecm @@ -143,6 +143,7 @@ ESPStruct [nil_remove] ECLException [min_ver("1.63")] int Activity; [min_ver("1.69")] string Scope; [min_ver("1.69")] int Priority; + [min_ver("1.70")] double Cost; }; // =========================================================================== ESPStruct [nil_remove] ECLSchemaItem diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 8598752f888..1a2de1ce067 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -202,6 +202,8 @@ WsWUExceptions::WsWUExceptions(IConstWorkUnit& wu): numerr(0), numwrn(0), numinf e->setActivity(cur.getActivityId()); if (cur.getPriority()) e->setPriority(cur.getPriority()); + if (cur.getCost()) + e->setCost(cur.getCost()); e->setScope(cur.queryScope()); const char * label = ""; From 704918ad4507632dd39a81e85046af4d1a488d54 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 4 Apr 2024 17:00:48 +0100 Subject: [PATCH 04/12] Split off 9.2.76 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index bc9ab25f676..f776941e750 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.75-closedown0 +version: 9.2.77-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.75-closedown0 +appVersion: 9.2.77-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 8c47215266c..e32dd10fa9f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index d499c484b3c..4580c3c729d 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 4aa41ec2e96..4efae3345b5 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index d62be94906e..2865b66058a 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index a4e164c3d2f..6ef7ef369fa 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index def1e2123dc..e33bda147b6 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 0c5c4577769..023a1151cbd 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 5959ace8c7e..ed1a8c4a645 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index e70a8c7ffdc..917ebb021b7 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 318c94d0eba..4e6f9e897c3 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index e6828835700..00818cb2417 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 202842992ee..b5633f7341d 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.75-closedown0 + helmVersion: 9.2.77-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index e19c0fb38e1..d5ef71ab5b7 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 75 ) +set ( HPCC_POINT 77 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-03-28T17:52:14Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-04-04T16:00:48Z" ) ### From 79b8a869a7d21d5e3dcb43f20812290fb18f57d0 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 4 Apr 2024 17:04:06 +0100 Subject: [PATCH 05/12] Split off 9.0.98 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 71e8531f1bc..4c4c7466d7b 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.97-closedown0 +version: 9.0.99-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.97-closedown0 +appVersion: 9.0.99-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 01b80f95f98..54ba8764e52 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 9ea69a8a013..57dcff36d0d 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 4bddee3a3f4..5af6af90856 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 5140c98ecb6..80e21d47aa4 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 0c3797ad9e9..146737fd16e 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 65f13323f26..16528ada2d2 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index e1b75508d27..dda550233f8 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index a173d1bf691..58bdab28518 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 6e212461e4f..77130e0f12e 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 4cecf6852c3..acedd9ef4f0 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 2592982e235..8a6939dee83 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 36020ecf8b4..6e33342b41b 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.97-closedown0 + helmVersion: 9.0.99-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index d27dbd8b0e8..8ab60f4286b 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 97 ) +set ( HPCC_POINT 99 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-03-28T17:51:30Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-04-04T16:04:06Z" ) ### From 34f88b35eb575c10861e0c65d22fea060c19e566 Mon Sep 17 00:00:00 2001 From: g-pan Date: Thu, 4 Apr 2024 16:05:09 -0400 Subject: [PATCH 06/12] HPCC-27242 Update introduction of containerized doc Signed-off-by: g-pan --- .../ContainerizedHPCCSystemsPlatform.xml | 43 ++++++------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml index d1efc80c613..a6cf61bccfb 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml @@ -37,7 +37,7 @@ similarity to actual persons, living or dead, is purely coincidental. - + Containerized HPCC Overview - Starting with version 8.0, the HPCC - Systems® Platform is focusing on containerized - deployments. This is useful for cloud-based deployments (large or small) - or local testing/development deployments. + Since version 8.0, the HPCC Systems® + Platform started focusing significantly on containerized deployments. This + is useful for cloud-based deployments (large or small) or local + testing/development deployments. Docker containers managed by Kubernetes (K8s) is a new target operating environment, alongside continued support for traditional @@ -81,10 +81,10 @@ to run, but a significant change in how components are configured, how and when they start up, and where they store their data. - This book focuses on containerized deployments. The first section is - about using Docker containers and Helm charts locally. Docker and Helm do - a lot of the work for you. The second part uses the same techniques in the - cloud. + This book focuses on these containerized deployments. The first + section is about using Docker containers and Helm charts locally. Docker + and Helm do a lot of the work for you. The second part uses the same + techniques in the cloud. For local small deployments (for development and testing), we suggest using Docker Desktop and Helm. This is useful for learning, @@ -92,8 +92,7 @@ For Cloud deployments, you can use any flavor of Cloud services, if it supports Docker, Kubernetes, and Helm. This book, however, will focus - on Microsoft Azure for Cloud Services. Future versions may include - specifics for other Cloud providers. + on Microsoft Azure for Cloud Services. If you want to manually manage your local or Cloud deployment, you can still use the traditional installers and Configuration Manager, but @@ -106,28 +105,12 @@ be easy for someone familiar with Kubernetes and Helm to install and manage the HPCC Systems platform. - - - Note: - - - The traditional bare-metal version of the HPCC Systems - platform is mature and has been heavily used in commercial - applications for almost two decades and is fully intended for - production use. The containerized version is new and is not yet 100% - ready for production. In addition, aspects of that version could - change without notice. We encourage you to use it and provide - feedback so we can make this version as robust as a bare-metal - installation. - - - - Bare-metal vs Containers - If you are familiar with the HPCC Systems platform, there are a - few fundamental changes to note. + If you are familiar with the traditional bare-metal HPCC Systems + platform deployments, there are a few fundamental changes to + note. Processes and Pods, not Machines From 485cda0fc5b1194ced496ba347a868faad49d7ca Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 5 Apr 2024 04:14:27 +0100 Subject: [PATCH 07/12] HPCC-31568 Switch xz mirror for liblzma tukaani-project/xz is currently suspended Signed-off-by: Gordon Smith --- vcpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcpkg b/vcpkg index 761c26eeb51..229f836da66 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 761c26eeb51bcf9a100e34a2b20a7a4f77382d5c +Subproject commit 229f836da6616f2982ce1a7a5b18cf48f6c27e0c From 961e009e639d0667bed5e7eaa150816bf7bcca74 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 5 Apr 2024 04:13:08 +0100 Subject: [PATCH 08/12] HPCC-31568 Switch xz mirror for liblzma Tukaani-project/xz is currently suspended Signed-off-by: Gordon Smith --- vcpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcpkg b/vcpkg index e0042a2ed27..416adc933cf 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit e0042a2ed2734e621bd59c2723298f3d630825f7 +Subproject commit 416adc933cf704c71ee3c1a5b4add9a393ed531c From 1f9930ae497018a8a3bab2d01261f25f8a2563c6 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 5 Apr 2024 09:07:17 +0100 Subject: [PATCH 09/12] HPCC-31557 Documentation build should reuse vcpkg cache Signed-off-by: Gordon Smith --- .github/workflows/build-assets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index ae75fa6149b..767f8d65cba 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -226,7 +226,7 @@ jobs: sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ - cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES=OFF -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \ cmake --build /hpcc-dev/build --parallel $(nproc) --target all" docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" From 6c1435f82cf5ccdb8f4189aec5fdea15f1a2332d Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Thu, 28 Mar 2024 16:25:02 -0400 Subject: [PATCH 10/12] HPCC-31545 Thread pool wait time reporting Signed-off-by: Mark Kelly --- system/jlib/jthread.cpp | 40 ++++++++++++++++++++++---------------- thorlcr/slave/slavmain.cpp | 2 +- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/system/jlib/jthread.cpp b/system/jlib/jthread.cpp index ccd55f0580a..fd19894939e 100644 --- a/system/jlib/jthread.cpp +++ b/system/jlib/jthread.cpp @@ -1006,36 +1006,42 @@ class CThreadPool: public CThreadPoolBase, implements IThreadPool, public CInter PooledThreadHandle _start(void *param,const char *name, bool noBlock, unsigned timeout=0) { CCycleTimer startTimer; - bool timedout = defaultmax && !availsem.wait(noBlock ? 0 : (timeout>0?timeout:delay)); + bool waited = false; + bool timedout = false; + if (defaultmax) + { + waited = !availsem.wait(0); + if (noBlock) + timedout = waited; + else if (waited) + timedout = !availsem.wait(timeout>0?timeout:delay); + } PooledThreadHandle ret; { CriticalBlock block(crit); if (timedout) { - if (!availsem.wait(0)) { // make sure take allocated sem if has become available + if (!availsem.wait(0)) // make sure take allocated sem if has become available + { if (noBlock || timeout > 0) throw MakeStringException(0, "No threads available in pool %s", poolname.get()); IWARNLOG("Pool limit exceeded for %s", poolname.get()); } - else - timedout = false; } if (traceStartDelayPeriod) { ++startsInPeriod; - if (timedout) - { + if (waited) startDelayInPeriod += startTimer.elapsedCycles(); - if (overAllTimer.elapsedCycles() >= queryOneSecCycles()*traceStartDelayPeriod) // check avg. delay per minute - { - double totalDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod)))/1000000; - double avgDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod/startsInPeriod)))/1000000; - unsigned totalElapsedSecs = overAllTimer.elapsedMs()/1000; - PROGLOG("%s: %u threads started in last %u seconds, total delay = %0.2f milliseconds, average delay = %0.2f milliseconds, currently running = %u", poolname.get(), startsInPeriod, totalElapsedSecs, totalDelayMs, avgDelayMs, runningCount()); - startsInPeriod = 0; - startDelayInPeriod = 0; - overAllTimer.reset(); - } + if (overAllTimer.elapsedCycles() >= queryOneSecCycles()*traceStartDelayPeriod) // check avg. delay per minute + { + double totalDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod)))/1000000; + double avgDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod/startsInPeriod)))/1000000; + unsigned totalElapsedSecs = overAllTimer.elapsedMs()/1000; + PROGLOG("%s: %u threads started in last %u seconds, total delay = %0.2f milliseconds, average delay = %0.2f milliseconds, currently running = %u", poolname.get(), startsInPeriod, totalElapsedSecs, totalDelayMs, avgDelayMs, runningCount()); + startsInPeriod = 0; + startDelayInPeriod = 0; + overAllTimer.reset(); } } CPooledThreadWrapper &t = allocThread(); @@ -2683,4 +2689,4 @@ void PerfTracer::dostop() #else UNIMPLEMENTED; #endif -} \ No newline at end of file +} diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 83cf9b82cc7..adb46a42916 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -1185,7 +1185,7 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, { Owned factory = new CProcessorFactory(*this); processorPool.setown(createThreadPool("KJService processor pool", factory, this, keyLookupMaxProcessThreads, 10000)); - processorPool->setStartDelayTracing(60000); + processorPool->setStartDelayTracing(60); } public: IMPLEMENT_IINTERFACE_USING(CSimpleInterfaceOf); From 6f7f709eadbaa23bc6b81e9360aeea9b0db01730 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 4 Apr 2024 16:09:00 +0100 Subject: [PATCH 11/12] HPCC-31559 Add option to avoid renames (spray and thor for now) Introduce an option expert/avoidRename that components can use to avoid writing to a temp file, and renaming, and instead write directly to the target filename. This is an experiment in relation to HPCC-31427. Signed-off-by: Jake Smith --- helm/hpcc/values.schema.json | 54 ++++++++++++++++++++------- system/jlib/jfile.cpp | 19 ++++++++++ thorlcr/activities/thactivityutil.cpp | 2 +- thorlcr/thorutil/thormisc.hpp | 1 + 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index fdac6c23749..90ed24b37c1 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -232,8 +232,7 @@ "$ref" : "#/definitions/egress" }, "expert": { - "description": "Custom internal options usually reserved for internal testing", - "type": "object" + "$ref": "#/definitions/expert" }, "hpa": { "$ref": "#/definitions/hpa" @@ -342,7 +341,7 @@ } }, "expert": { - "description": "Settings for developers, debugging and testing" + "$ref": "#/definitions/expert" }, "misc": { "description": "Miscellaneous settings", @@ -1341,6 +1340,9 @@ "terminationGracePeriodSeconds": { "$ref": "#/definitions/terminationGracePeriodSeconds", "default": 3600 + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -1429,6 +1431,9 @@ }, "hpa": { "$ref": "#/definitions/hpa" + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -1466,6 +1471,9 @@ }, "terminationGracePeriodSeconds": { "$ref": "#/definitions/terminationGracePeriodSeconds" + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -1543,8 +1551,7 @@ "$ref": "#/definitions/hpa" }, "expert": { - "description": "Custom internal options usually reserved for internal testing", - "type": "object" + "$ref": "#/definitions/expert" } } }, @@ -1634,6 +1641,9 @@ "type": "boolean", "default": false, "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -2339,8 +2349,7 @@ "description": "Preemptively update secrets that are active and about to expire in the background" }, "expert": { - "description": "Custom internal options usually reserved for internal testing", - "type": "object" + "$ref": "#/definitions/expert" }, "egress": { "$ref" : "#/definitions/egress" @@ -2388,6 +2397,9 @@ }, "hpa": { "$ref": "#/definitions/hpa" + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -2581,8 +2593,7 @@ "$ref" : "#/definitions/componentCost" }, "expert": { - "description": "Custom internal options usually reserved for internal testing", - "type": "object" + "$ref": "#/definitions/expert" }, "egress": { "$ref" : "#/definitions/egress" @@ -2675,6 +2686,9 @@ }, "egress": { "$ref" : "#/definitions/egress" + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -2714,7 +2728,8 @@ "env": {}, "annotations": {}, "labels": {}, - "egress": {} + "egress": {}, + "expert": {} }, "additionalProperties": false } @@ -2762,7 +2777,8 @@ "labels": {}, "limit": {}, "cutoff": {}, - "egress": {} + "egress": {}, + "expert": {} }, "additionalProperties": false } @@ -2790,7 +2806,8 @@ "labels": {}, "limit": {}, "cutoff": {}, - "egress": {} + "egress": {}, + "expert": {} }, "additionalProperties": false } @@ -2816,7 +2833,8 @@ "labels": {}, "limit": {}, "cutoff": {}, - "egress": {} + "egress": {}, + "expert": {} }, "additionalProperties": false } @@ -2850,7 +2868,8 @@ "resources": {}, "annotations": {}, "labels": {}, - "egress": {} + "egress": {}, + "expert": {} }, "additionalProperties": false }, @@ -2935,6 +2954,9 @@ }, "hpa": { "$ref": "#/definitions/hpa" + }, + "expert": { + "$ref": "#/definitions/expert" } } }, @@ -3364,6 +3386,10 @@ "description": "Period permitted for component to terminate gracefully before being killed by Kubernetes", "default": 600, "minimum": 0 + }, + "expert": { + "description": "Settings for developers, debugging and testing", + "type": "object" } } } diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index cc7de5e2160..9791c60ff30 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7423,12 +7423,31 @@ void FileIOStats::trace() static constexpr FileSystemProperties linuxFileSystemProperties {true, true, true, true, 0x10000}; // 64K static constexpr FileSystemProperties defaultUrlFileSystemProperties{false, false, false, false, 0x400000}; // 4Mb +static constexpr FileSystemProperties linuxFileSystemNoRenameProperties{false, true, true, true, 0x10000}; // 64K + +static std::atomic avoidRename{-1}; +static CriticalSection avoidRenameCS; +static bool isAvoidRenameEnabled() +{ + if (-1 == avoidRename) + { + CriticalBlock b(avoidRenameCS); + if (-1 == avoidRename) + { + avoidRename = getComponentConfigSP()->getPropBool("expert/@avoidRename"); + DBGLOG("FileSystemProperties.canRename = %s", boolToStr(!avoidRename)); // NB: canRename if !avoidRename + } + } + return avoidRename; +} //This implementation should eventually make use of the file hook. const FileSystemProperties & queryFileSystemProperties(const char * filename) { if (isUrl(filename)) return defaultUrlFileSystemProperties; + else if (isAvoidRenameEnabled()) + return linuxFileSystemNoRenameProperties; else return linuxFileSystemProperties; } diff --git a/thorlcr/activities/thactivityutil.cpp b/thorlcr/activities/thactivityutil.cpp index c77f5ede002..cd7c323eb66 100644 --- a/thorlcr/activities/thactivityutil.cpp +++ b/thorlcr/activities/thactivityutil.cpp @@ -733,7 +733,7 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, partDesc.getFilename(0, rfn); StringBuffer primaryName; rfn.getPath(primaryName); - if (isUrl(primaryName)) + if (isUrl(primaryName) || activity->getOptBool(THOROPT_AVOID_RENAME)) // THOROPT_AVOID_RENAME see HPCC-31559 { twFlags &= ~TW_RenameToPrimary; twFlags |= TW_Direct; diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index ea131972dd0..6764a98599b 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -115,6 +115,7 @@ #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel" // The trace SOAP level (default=1) #define THOROPT_SORT_ALGORITHM "sortAlgorithm" // The algorithm used to sort records (quicksort/mergesort) #define THOROPT_COMPRESS_ALLFILES "compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) +#define THOROPT_AVOID_RENAME "avoidRename" // Avoid rename, write directly to target physical filenames (no temp file) #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000 // max of row matches before selfjoin emits warning From 0f2fb3bb51db9fe2219522f8ad55744c884d7367 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 4 Apr 2024 16:19:34 +0100 Subject: [PATCH 12/12] HPCC-29482 WsDfs should pass user to DFS::getFileTree When performing a ~remote file lookup, the WsDfs service looks up the logical file in Dali (via IDistributedFileDirectory::getFileTree). It should pass the active esp context user, but passes null. This can cause access denials. Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 6 +++--- dali/base/dadfs.hpp | 2 +- esp/services/ws_dfsservice/ws_dfsservice.cpp | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 81202ff10e1..1ad137e69ef 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -1256,7 +1256,7 @@ protected: friend class CDistributedFile; return ret; } virtual bool removePhysicalPartFiles(const char *logicalName, IFileDescriptor *fileDesc, IMultiException *mexcept, unsigned numParallelDeletes=0) override; - virtual void setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout); + virtual void setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout); }; @@ -11635,11 +11635,11 @@ void CDistributedFileDirectory::setFileAccessed(CDfsLogicalFileName &dlfn,IUserD checkDfsReplyException(mb); } -void CDistributedFileDirectory::setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout) +void CDistributedFileDirectory::setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout) { CDfsLogicalFileName dlfn; dlfn.set(logicalName); - setFileAccessed(dlfn, nullptr, dt, foreigndali, foreigndalitimeout); + setFileAccessed(dlfn, udesc, dt, foreigndali, foreigndalitimeout); } void CDistributedFileDirectory::setFileProtect(CDfsLogicalFileName &dlfn,IUserDescriptor *user, const char *owner, bool set, const INode *foreigndali,unsigned foreigndalitimeout) diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp index b18ae460bb0..003fca751c9 100644 --- a/dali/base/dadfs.hpp +++ b/dali/base/dadfs.hpp @@ -731,7 +731,7 @@ interface IDistributedFileDirectory: extends IInterface // useful to clearup after temporary unpublished file. virtual bool removePhysicalPartFiles(const char *logicalName, IFileDescriptor *fileDesc, IMultiException *mexcept, unsigned numParallelDeletes=0) = 0; - virtual void setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali=nullptr, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0; + virtual void setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali=nullptr, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0; }; diff --git a/esp/services/ws_dfsservice/ws_dfsservice.cpp b/esp/services/ws_dfsservice/ws_dfsservice.cpp index 866064e5e0d..d79e3afdd7a 100644 --- a/esp/services/ws_dfsservice/ws_dfsservice.cpp +++ b/esp/services/ws_dfsservice/ws_dfsservice.cpp @@ -50,7 +50,7 @@ enum LfnMetaOpts : byte LfnMOptTls = 0x02, }; BITMASK_ENUM(LfnMetaOpts); -static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, LfnMetaOpts opts, IPropertyTree *metaRoot, IPropertyTree *meta) +static void populateLFNMeta(IUserDescriptor *userDesc, const char *logicalName, unsigned __int64 leaseId, LfnMetaOpts opts, IPropertyTree *metaRoot, IPropertyTree *meta) { CDfsLogicalFileName lfn; lfn.set(logicalName); @@ -59,7 +59,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, L assertex(!lfn.isMulti()); // not supported, don't think needs to be/will be. - Owned tree = queryDistributedFileDirectory().getFileTree(logicalName, nullptr); + Owned tree = queryDistributedFileDirectory().getFileTree(logicalName, userDesc); if (!tree) return; if (hasMask(opts, LfnMOptRemap)) @@ -111,7 +111,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, L { IPropertyTree &sub = *(orderedSubFiles[f]); sub.getProp("@name", subname.clear()); - populateLFNMeta(subname, leaseId, opts, metaRoot, fileMeta); + populateLFNMeta(userDesc, subname, leaseId, opts, metaRoot, fileMeta); } } } @@ -182,7 +182,7 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r opts |= LfnMOptTls; Owned responseTree = createPTree(); - populateLFNMeta(logicalName, leaseId, opts, responseTree, responseTree); + populateLFNMeta(userDesc, logicalName, leaseId, opts, responseTree, responseTree); // serialize response MemoryBuffer respMb, compressedRespMb; @@ -198,7 +198,7 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r // Really this should be done at end (or at end as well), but this is same as existing DFS lookup. CDateTime dt; dt.setNow(); - queryDistributedFileDirectory().setFileAccessed(logicalName, dt); + queryDistributedFileDirectory().setFileAccessed(userDesc, logicalName, dt); LOG(MCauditInfo,",FileAccess,EspProcess,READ,%s,%u,%s", logicalName, timeoutSecs, userID.str()); }