From f4ea57423ca185b90be6cdbd40f240c32ea39dac Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 7 Mar 2024 17:27:44 +0000 Subject: [PATCH 01/23] Split off 9.2.68 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index b36187391e9..639d553b742 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.67-closedown0 +version: 9.2.69-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.67-closedown0 +appVersion: 9.2.69-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 6f1375ad63b..b57b5a84b92 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 257c73db7be..29e76b501cd 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 3b4fc294026..e2722829484 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 878cc0b50f7..6e608619a5a 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 36a490e5c8a..b26989c9cdc 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index c6715e0ebc8..d20bb478117 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 4602c3bfc1c..69ae68f7331 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 67af35b335c..3fc8e209313 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 07e0e22895b..459825ad1e8 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 062930aa659..71a30393f2f 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 4adf65c8685..6660e282188 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 78df1a6c1bc..0d95e79a84f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.67-closedown0 + helmVersion: 9.2.69-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index e633902a6f2..5323e18ba8f 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 67 ) +set ( HPCC_POINT 69 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-02-29T16:47:01Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-03-07T17:27:44Z" ) ### From eae0c2f2954e3b3aaf4256967025126211844f08 Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Wed, 6 Mar 2024 19:04:27 -0500 Subject: [PATCH 02/23] HPCC-31210 Fix publisher workunit task failure tracking gets out of s Publisher workunits seem to be over counting failures and setting failure state on the parent. This PR adds a subtask notation marker to make sure publisher workunits child task failures or completions only get counted once. Signed-off-by: Anthony Fishbeck --- dali/dfu/dfuwu.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp index 9a37925a2f0..e065ef5d43f 100644 --- a/dali/dfu/dfuwu.cpp +++ b/dali/dfu/dfuwu.cpp @@ -377,9 +377,14 @@ static StringBuffer &getSubTaskParentXPath(StringBuffer &wuRoot, const char *wui return wuRoot.append('/').append(wuid); } +static bool isSubTaskWuid(const char *subtaskWuid) +{ + return (!isEmptyString(subtaskWuid) && 'P'==*subtaskWuid && strchr(subtaskWuid, 'T')!=nullptr); +} + static bool notePublisherSubTaskState(const char *subtaskWuid, DFUstate state) { - if (!subtaskWuid || 'P'!=*subtaskWuid || !strchr(subtaskWuid, 'T')) + if (!isSubTaskWuid(subtaskWuid)) return false; switch (state) { @@ -699,6 +704,7 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress queryRoot()->removeProp("@kbpersecave"); queryRoot()->removeProp("@kbpersec"); queryRoot()->removeProp("@slavesdone"); + queryRoot()->removeProp("@noted"); parent->commit(); } void setDone(const char * timeTaken, unsigned kbPerSec, bool set100pc) @@ -718,6 +724,8 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress { CriticalBlock block(parent->crit); CDateTime dt; + bool noted = queryRoot()->getPropBool("@noted", false); + bool subtask = isSubTaskWuid(parent->queryId()); switch (state) { case DFUstate_started: dt.setNow(); @@ -737,13 +745,17 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress state = DFUstate_aborted; dt.setNow(); setTimeStopped(dt); + if (subtask && !noted) + queryRoot()->setPropBool("@noted", true); break; } StringBuffer s; encodeDFUstate(state,s); queryRoot()->setProp("@state",s.str()); + parent->commit(); - notePublisherSubTaskState(parent->queryId(), state); + if (subtask && !noted) + notePublisherSubTaskState(parent->queryId(), state); } void setTimeStarted(const CDateTime &val) { From 18462f800ebb65228e9ab0879332b898432f2c0f Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 8 Mar 2024 11:27:57 +0000 Subject: [PATCH 03/23] HPCC-31419 Fix dbglogYAML impl. prototype dbglogYAML's implementation prototype did not match header, flags should be declared as type byte, not integer. This has not caused any compilation errors so far, since this function has not yet been used. Signed-off-by: Jake Smith --- system/jlib/jptree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 994d916f422..28f769af44d 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -6607,7 +6607,7 @@ void printJSON(const IPropertyTree *tree, unsigned indent, byte flags) printf("%s", json.str()); } -void dbglogJSON(const IPropertyTree *tree, unsigned indent, unsigned flags) +void dbglogJSON(const IPropertyTree *tree, unsigned indent, byte flags) { StringBuffer json; toJSON(tree, json, indent, flags); From 1b386468ac2555c61470b201020ccdd6894f657b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 8 Mar 2024 17:36:10 +0000 Subject: [PATCH 04/23] Split off 9.6.0 Signed-off-by: Gavin Halliday --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 8 ++++---- 14 files changed, 27 insertions(+), 27 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 03823f22692..a8ed0383657 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.5.0-trunk0 +version: 9.6.1-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.5.0-trunk0 +appVersion: 9.6.1-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 06675ff8708..6f67c52e55c 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1505,7 +1505,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 1ac691ddc3d..2165f9020b9 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index c50a16224f7..c109a540047 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -84,7 +84,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index d7730ee37cd..357188399da 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 1a1f4099b8d..c2ae5734032 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -60,7 +60,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index d6dc1014868..ef0d318b5b5 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -143,7 +143,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 1267b40adbf..f25952447ea 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index e4330adad4e..28265b9ac55 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -125,7 +125,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index f8860b43f20..2918d159887 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 99226830c4c..b5a5d44aeb5 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -183,7 +183,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -245,7 +245,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -353,7 +353,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index d677bef133d..f1ad2fb0014 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 843f15731ce..2fb7bba4f9d 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -86,7 +86,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -151,7 +151,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -351,7 +351,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -416,7 +416,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.5.0-trunk0 + helmVersion: 9.6.1-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 34bcd2ea98d..b35bce67fec 100644 --- a/version.cmake +++ b/version.cmake @@ -4,9 +4,9 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) -set ( HPCC_MINOR 5 ) -set ( HPCC_POINT 0 ) -set ( HPCC_MATURITY "trunk" ) +set ( HPCC_MINOR 6 ) +set ( HPCC_POINT 1 ) +set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "1970-01-01T01:00:00Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-03-08T17:36:09Z" ) ### From 55655ddcbbeb9648a80c0a554d9836010dcf1913 Mon Sep 17 00:00:00 2001 From: Kunal Aswani Date: Wed, 28 Feb 2024 13:27:57 -0500 Subject: [PATCH 05/23] HPCC-31258 Initial File structure and page layout creation This is to create the initial design of Sasha in Cloud. Feature availability to come as it is created. Signed-off-by: Kunal Aswani --- esp/src/src-react/components/Menu.tsx | 1 + esp/src/src-react/components/Sasha.tsx | 96 ++++++++++++++++++++++++++ esp/src/src-react/routes.tsx | 12 +++- esp/src/src/nls/hpcc.ts | 10 +++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 esp/src/src-react/components/Sasha.tsx diff --git a/esp/src/src-react/components/Menu.tsx b/esp/src/src-react/components/Menu.tsx index 382df996894..1e34430c628 100644 --- a/esp/src/src-react/components/Menu.tsx +++ b/esp/src/src-react/components/Menu.tsx @@ -162,6 +162,7 @@ const subMenuItems: SubMenuItems = { { headerText: nlsHPCC.Security + " (L)", itemKey: "/topology/security" }, { headerText: nlsHPCC.DESDL + " (L)", itemKey: "/topology/desdl" }, { headerText: nlsHPCC.DaliAdmin, itemKey: "/topology/daliadmin" }, + { headerText: nlsHPCC.Sasha, itemKey: "/topology/sasha" }, ], "operations": [ { headerText: nlsHPCC.Topology + " (L)", itemKey: "/operations" }, diff --git a/esp/src/src-react/components/Sasha.tsx b/esp/src/src-react/components/Sasha.tsx new file mode 100644 index 00000000000..65ef5af0886 --- /dev/null +++ b/esp/src/src-react/components/Sasha.tsx @@ -0,0 +1,96 @@ +import * as React from "react"; +import { Dropdown, TextField, PrimaryButton } from "@fluentui/react"; +import nlsHPCC from "src/nlsHPCC"; + +interface SashaProps {} + +export const Sasha: React.FunctionComponent = ({ }) => { + const [selectedOption, setSelectedOption] = React.useState(""); + const [wuid, setWuid] = React.useState(""); + const [result, setResult] = React.useState(""); + + const handleOptionChange = (event: React.FormEvent, option: any) => { + setSelectedOption(option.key); + }; + + const handleSubmit = (event: React.FormEvent) => { + event.preventDefault(); + // Perform action based on selected option + switch (selectedOption) { + case "getVersion": + // Implement getVersion function call + break; + case "getLastServerMessage": + // Implement getLastServerMessage function call + break; + case "restoreECLWorkUnit": + // Implement restoreECLWorkUnit function call + break; + case "restoreDFUWorkUnit": + // Implement restoreDFUWorkUnit function call + break; + case "archiveECLWorkUnit": + // Implement archiveECLWorkUnit function call + break; + case "archiveDFUWorkUnit": + // Implement archiveDFUWorkUnit function call + break; + case "backupECLWorkUnit": + // Implement backupECLWorkUnit function call + break; + case "backupDFUWorkUnit": + // Implement backupDFUWorkUnit function call + break; + default: + console.log("Invalid option selected"); + } + // Reset form + setSelectedOption(""); + setWuid(""); + setResult(""); + }; + + // Conditional rendering for default value + const defaultValue = result ? null :
{nlsHPCC.noDataMessage}
; + + return ( +
+
+ + {["restoreECLWorkUnit", "restoreDFUWorkUnit", "archiveECLWorkUnit", "archiveDFUWorkUnit", "backupECLWorkUnit", "backupDFUWorkUnit"].includes(selectedOption) && ( +
+ , newValue?: string) => setWuid(newValue || "")} + styles={{ fieldGroup: { width: 400 } }} + /> +
+ )} + {nlsHPCC.Submit} + {/* Render defaultValue when result is empty */} + {defaultValue} + {/* Render result when available */} + {result &&
{nlsHPCC.Results}: {result}
} + +
+ ); +}; + +export default Sasha; diff --git a/esp/src/src-react/routes.tsx b/esp/src/src-react/routes.tsx index d5e36af6746..c1cf36bef86 100644 --- a/esp/src/src-react/routes.tsx +++ b/esp/src/src-react/routes.tsx @@ -390,7 +390,17 @@ export const routes: RoutesEx = [ }, ] }, - ] + { + path: "/sasha", + children: [ + { + path: "", action: (ctx, params) => import("./components/Sasha").then(_ => { + return <_.Sasha />; + }) + }, + ] + }, + ] }, { mainNav: ["operations"], diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index c000f4b115e..29a0a68078f 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -58,6 +58,8 @@ export = { AppendCluster: "Append Cluster", Apply: "Apply", AreYouSureYouWantToResetTheme: "Are you sure you want to reset to the default theme?", + ArchiveECLWorkunit: "Archive ECL Workunit", + ArchiveDFUWorkunit: "Archive DFU Workunit", ArchivedOnly: "Archived Only", ArchivedWarning: "Warning: please specify a small date range. If not, it may take some time to retrieve the workunits and the browser may be timed out.", Attach: "Attach", @@ -72,6 +74,8 @@ export = { AutoRefreshIncrement: "Auto Refresh Increment", AutoRefreshEvery: "Auto refresh every x minutes", Back: "Back", + BackupECLWorkunit: "Backup ECL Workunit", + BackupDFUWorkunit: "Backup DFU Workunit", BannerColor: "Banner Color", BannerColorTooltip: "Change the background color of the top navigation", BannerMessage: "Banner Message", @@ -372,10 +376,12 @@ export = { GetDFSCSV: "DFS CSV", GetDFSMap: "DFS Map", GetDFSParents: "DFS Parents", + GetLastServerMessage: "Get Last Server Message", GetLogicalFile: "Logical File", GetLogicalFilePart: "Logical File Part", GetProtectedList: "Protected List", GetValue: "Value", + GetVersion: "Get Version", GetPart: "Get Part", GetSoftwareInformation: "Get Software Information", Graph: "Graph", @@ -801,6 +807,8 @@ export = { Restarted: "Restarted", Restarts: "Restarts", Restore: "Restore", + RestoreECLWorkunit: "Restore ECL Workunit", + RestoreDFUWorkunit: "Restore DFU Workunit", Resubmit: "Resubmit", ResubmitTooltip: "Resubmit existing workunit", Resubmitted: "Resubmitted", @@ -820,6 +828,7 @@ export = { Sample: "Sample", SampleRequest: "Sample Request", SampleResponse: "Sample Response", + Sasha: "Sasha", Save: "Save", Scope: "Scope", SearchResults: "Search Results", @@ -829,6 +838,7 @@ export = { SecurityMessageHTML: "Only view HTML from trusted users. This workunit was created by \"{__placeholder__}\". \nRender HTML?", SeeConfigurationManager: "See Configuration Manager", SelectA: "Select a", + SelectAnOption: "Select an option", Selected: "Selected", SelectValue: "Select a value", SelectEllipsis: "Select...", From fb21f9420b191805fbbe28993632ee094ffdbba0 Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Mon, 11 Mar 2024 16:17:22 -0400 Subject: [PATCH 06/23] HPCC-30824 Coverity scan reported new defects related to ws_cloud_esp.ipp Changed code generator to initialize identified member variable when declared. Signed-Off-By: Kenneth Rowland kenneth.rowland@lexisnexisrisk.com --- tools/hidl/hidlcomp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/hidl/hidlcomp.cpp b/tools/hidl/hidlcomp.cpp index f9dba61e2d7..0111ee84948 100644 --- a/tools/hidl/hidlcomp.cpp +++ b/tools/hidl/hidlcomp.cpp @@ -5980,7 +5980,7 @@ void EspServInfo::write_esp_service_ipp() outf("\timplements IEsp%s\n", name_); outs("{\n"); outs("private:\n"); - outs("\tIEspContainer* m_container;\n"); + outs("\tIEspContainer* m_container = nullptr;\n"); outs("public:\n"); outs("\tIMPLEMENT_IINTERFACE;\n\n"); outf("\tC%s(){}\n\tvirtual ~C%s(){}\n", name_, name_); From bb97a32f586fd61bf430eab18962be765065d6bf Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 12 Mar 2024 08:02:05 +0000 Subject: [PATCH 07/23] HPCC-31444 Ensure MP accept() doesn't terminate accept thread Avoid an accept() exception causing the MP connect thread from terminating. Instead, warn, and pause for a short time, then continue. One of the reasons accept() may throw an exception is if there is handle exhaustion due to a rapid number of connections coming in very quickly. Signed-off-by: Jake Smith --- system/mp/mpcomm.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index 4c5d696f09d..8352399b3a2 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -2525,6 +2525,7 @@ int CMPConnectThread::run() #ifdef _TRACE LOG(MCdebugInfo, unknownJob, "MP: Connect Thread Starting - accept loop"); #endif + Owned exception; while (running) { Owned sock; @@ -2535,8 +2536,7 @@ int CMPConnectThread::run() } catch (IException *e) { - LOG(MCdebugInfo, unknownJob, e,"MP accept failed"); - throw; // error handling TBD + exception.setown(e); } if (sock) { @@ -2552,7 +2552,19 @@ int CMPConnectThread::run() else { if (running) - LOG(MCdebugInfo, unknownJob, "MP Connect Thread accept returned NULL"); + { + if (exception) + { + constexpr unsigned sleepSecs = 5; + // Log and pause for a few seconds, because accept loop may have failed due to handle exhaustion. + VStringBuffer msg("MP accept failed. Accept loop will be paused for %u seconds", sleepSecs); + EXCLOG(exception, msg.str()); + exception.clear(); + MilliSleep(sleepSecs * 1000); + } + else // not sure this can ever happen (no exception, still running, and sock==nullptr) + LOG(MCdebugInfo, unknownJob, "MP Connect Thread accept returned NULL"); + } } } #ifdef _TRACE From 79d019907c4c539f1cdba84a85fcdaec668fdbac Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 12 Mar 2024 13:39:40 +0000 Subject: [PATCH 08/23] HPCC-31437 Drop Win32 clienttools from release process Signed-off-by: Gordon Smith --- .github/workflows/build-assets.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 992737387f8..d4281be7525 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -436,11 +436,6 @@ jobs: strategy: matrix: include: - - os: "windows-2022" - mono: "" - sudo: "" - cmake_config_options: '-T host=x64 -A Win32 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' - cmake_build_options: "--config Release --parallel $NUMBER_OF_PROCESSORS" - os: "windows-2022" mono: "" sudo: "" From e1736eb09d19c8ac42996c8d98bed316646dc7c4 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 12 Mar 2024 14:50:05 +0000 Subject: [PATCH 09/23] HPCC-31438 Blas plugin package missing blas plugin Signed-off-by: Gordon Smith --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 621f77deaaa..663d17c0648 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ if ( PLUGIN ) HPCC_ADD_SUBDIRECTORY (system/security/securesocket) HPCC_ADD_SUBDIRECTORY (system/security/zcrypt) HPCC_ADD_SUBDIRECTORY (dali/base) + HPCC_ADD_SUBDIRECTORY (plugins/eclblas "ECLBLAS") HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED") HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED") HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED") From 69fc3b667fbfbcc28f2806818d88388aabb98272 Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Tue, 23 May 2023 15:56:30 -0400 Subject: [PATCH 10/23] HPCC-29904 Remove generated code that builds the XSD and WSDL Removed code from HIDL that generated the code that generated XSD and WSDL forms Signed-Off-By: Kenneth Rowland kenneth.rowland@lexisnexisrisk.com --- esp/bindings/SOAP/Platform/soapparam.hpp | 26 - esp/bindings/http/platform/httpbinding.cpp | 236 +--- esp/bindings/http/platform/httpbinding.hpp | 7 +- tools/hidl/hidlcomp.cpp | 1145 +------------------- 4 files changed, 27 insertions(+), 1387 deletions(-) diff --git a/esp/bindings/SOAP/Platform/soapparam.hpp b/esp/bindings/SOAP/Platform/soapparam.hpp index be999d8e10a..a0f1a0f6036 100644 --- a/esp/bindings/SOAP/Platform/soapparam.hpp +++ b/esp/bindings/SOAP/Platform/soapparam.hpp @@ -1079,32 +1079,6 @@ class SoapEnumParamNew UNIMPLEMENTED; } - void getXsdDefinition_(IEspContext &context, IHttpMessage *request, StringBuffer &schema, BoolHash &added, const char* descriptions[]) - { - if (added.getValue(typeName_)) - return; - - added.setValue(typeName_, 1); - - schema.appendf("", typeName_.get()); - - if ( descriptions && ((context.queryOptions()&ESPCTX_NO_ANNOTATION) == 0) ) - { - schema.append(""); - for (unsigned i=0; i", enumstrings.item(i), desc); - } - schema.append(""); - } - - schema.appendf("", baseType_.get()); - for (unsigned i=0; i", enumstrings.item(i)); - schema.append(""); - } }; template diff --git a/esp/bindings/http/platform/httpbinding.cpp b/esp/bindings/http/platform/httpbinding.cpp index cabdfd0630d..2d1c646f39f 100644 --- a/esp/bindings/http/platform/httpbinding.cpp +++ b/esp/bindings/http/platform/httpbinding.cpp @@ -50,6 +50,7 @@ #include "esdl_def_helper.hpp" + #define FILE_UPLOAD "FileUploadAccess" #define DEFAULT_HTTP_PORT 80 @@ -1453,14 +1454,7 @@ void EspHttpBinding::getSoapMessage(StringBuffer& soapmsg, IEspContext& ctx, CHt StringBuffer req, tag, schema, filtered; msg->marshall(req, NULL); - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(ctx, request, serviceQName, methodQName, getVersion(ctx), false, false, schema); - } - else - { - getSchema(schema,ctx,request,serviceQName,methodQName,false); - } + getServiceSchema(ctx, request, serviceQName, methodQName, getVersion(ctx), false, false, schema); getXMLMessageTag(ctx, true, methodQName, tag); filterXmlBySchema(req,schema,tag.str(),filtered); @@ -1756,88 +1750,10 @@ int EspHttpBinding::onGetVersion(IEspContext &context, CHttpRequest* request, CH bool EspHttpBinding::getSchema(StringBuffer& schema, IEspContext &ctx, CHttpRequest* req, const char *service, const char *method,bool standalone) { - StringBuffer serviceQName; - StringBuffer methodQName; - - if (!qualifyServiceName(ctx, service, method, serviceQName, &methodQName)) - return false; - - const char *sqName = serviceQName.str(); - const char *mqName = methodQName.str(); - - Owned namespaces = createPTree(); - appendSchemaNamespaces(namespaces, ctx, req, service, method); - Owned nsiter = namespaces->getElements("namespace"); + throw MakeStringException(-1, "getSchema deprecated in EspHttpBinding, use getServiceSchema instead"); +} - StringBuffer nstr; - generateNamespace(ctx, req, sqName, mqName, nstr); - schema.appendf("query(); - schema.appendf(" xmlns:%s=\"%s\"", ns.queryProp("@nsvar"), ns.queryProp("@ns")); - } - schema.append(">\n"); - ForEach(*nsiter) - { - IPropertyTree &ns = nsiter->query(); - if (ns.hasProp("@import")) - schema.appendf("", ns.queryProp("@ns"), ns.queryProp("@location")); - } - - - schema.append( - "" - "" - "" - "" - "" - "" - "" - "\n" - "" - "" - "" - "" - "" - "\n" - "\n" - ); - - if (ctx.queryOptions()&ESPCTX_WSDL_EXT) - { - schema.append( - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "\n" - ); - } - bool mda=(req->queryParameters()->getPropInt("mda")!=0); - getXsdDefinition(ctx, req, schema, sqName, mqName, mda); - schema.append("\n"); - schema.append(""); - return true; -} void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* request, const char *serviceQName, const char *methodQName, double version, bool isWsdl, bool addAnnotations, StringBuffer &schema) @@ -1848,6 +1764,12 @@ void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* reque Owned esdlDef = createEsdlDefinition(nullptr, nullptr); Owned defHelper = createEsdlDefinitionHelper(); + StringBuffer xmlFilename; + if (!getServiceXmlFilename(xmlFilename)) + { + throw MakeStringException(-1, "Unable to get service XML filename"); + } + StringBuffer nstr; generateEsdlNamespace(context, request, serviceQName, methodQName, nstr); @@ -1875,7 +1797,7 @@ void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* reque } std::string xmlFilePath(getCFD()); - xmlFilePath.append("esdl_files").append(PATHSEPSTR).append(serviceXmlFilename.str()); + xmlFilePath.append("esdl_files").append(PATHSEPSTR).append(xmlFilename.str()); esdlDef->addDefinitionsFromFile(xmlFilePath.c_str()); unsigned optFlags{DEPFLAG_COLLAPSE|DEPFLAG_ARRAYOF}; Owned structs = esdlDef->getDependencies(serviceQName, methodQName, ";", version, opts, optFlags ); @@ -1889,32 +1811,12 @@ void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* reque int EspHttpBinding::onGetXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method) { - int rc; - if (!serviceXmlFilename.isEmpty()) - { - rc = getServiceWsdlOrXsd(context, request, response, service, method, false); - } - else - { - rc = getWsdlOrXsd(context,request,response,service,method,false); - } - - return rc; + return getServiceWsdlOrXsd(context, request, response, service, method, false); } int EspHttpBinding::onGetWsdl(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method) { - int rc; - if (!serviceXmlFilename.isEmpty()) - { - rc = getServiceWsdlOrXsd(context, request, response, service, method, true); - } - else - { - rc = getWsdlOrXsd(context,request,response,service,method,true); - } - - return rc; + return getServiceWsdlOrXsd(context, request, response, service, method, true); } int EspHttpBinding::getServiceWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl) @@ -1952,81 +1854,6 @@ int EspHttpBinding::getServiceWsdlOrXsd(IEspContext &context, CHttpRequest* requ return 0; } -int EspHttpBinding::getWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl) -{ - bool mda=(request->queryParameters()->getPropInt("mda")!=0); - try - { - StringBuffer serviceQName; - StringBuffer methodQName; - - if (!qualifyServiceName(context, service, method, serviceQName, &methodQName)) - { - return onGetNotFound(context, request, response, service); - } - else - { - const char *sqName = serviceQName.str(); - const char *mqName = methodQName.str(); - StringBuffer ns; - generateNamespace(context, request, serviceQName.str(), methodQName.str(), ns); - - StringBuffer content(""); - if (context.queryRequestParameters()->hasProp("display")) - content.append(""); - else if (isWsdl && context.queryRequestParameters()->hasProp("wsdlviewer")) - content.append(""); - if (isWsdl) - { - content.appendf("", ns.str(), ns.str()); - content.append(""); - } - - getSchema(content,context,request,service,method,!isWsdl); - - if (isWsdl) - { - content.append(""); - - getWsdlMessages(context, request, content, sqName, mqName, mda); - getWsdlPorts(context, request, content, sqName, mqName, mda); - getWsdlBindings(context, request, content, sqName, mqName, mda); - - StringBuffer location(m_wsdlAddress.str()); - if (request->queryParameters()->hasProp("wsdl_destination_path")) - location.append(request->queryParameters()->queryProp("wsdl_destination_path")); - else - location.append('/').append(sqName).appendf("?ver_=%g", context.getClientVersion()); - - if (request->queryParameters()->hasProp("encode_results")) - { - const char *encval = request->queryParameters()->queryProp("encode_results"); - location.append("&").appendf("encode_=%s", (encval && *encval) ? encval : "1"); - } - - content.appendf("", sqName); - content.appendf("", sqName, sqName); - content.appendf("", location.str()); - content.append(""); - content.append(""); - content.append(""); - } - - response->setContent(content.length(), content.str()); - response->setContentType(HTTP_TYPE_APPLICATION_XML_UTF8); - response->setStatus(HTTP_STATUS_OK); - } - } - catch (IException *e) - { - return onGetException(context, request, response, *e); - } - - response->send(); - return 0; -} static void genSampleXml(StringStack& parent, IXmlType* type, StringBuffer& out, const char* tag, const char* ns=NULL) { @@ -2114,14 +1941,7 @@ void EspHttpBinding::generateSampleXml(bool isRequest, IEspContext &context, CHt return; } - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); - } - else - { - getSchema(schemaXml,context,request,serv,method,false); - } + getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); Owned schema; IXmlType* type = nullptr; @@ -2195,14 +2015,7 @@ void EspHttpBinding::generateSampleJson(bool isRequest, IEspContext &context, CH return; } - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); - } - else - { - getSchema(schemaXml, context, request, serv, method, false); - } + getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); Owned schema; IXmlType* type = nullptr; @@ -2677,14 +2490,7 @@ int EspHttpBinding::onGetXForm(IEspContext &context, CHttpRequest* request, CHtt double ver = getVersion(context); StringBuffer schema; - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schema); - } - else - { - getSchema(schema,context,request,serv,method,false); - } + getServiceSchema(context, request, serviceQName, methodQName, ver, false, true, schema); StringBuffer page; IXslProcessor* xslp = getXmlLibXslProcessor(); @@ -2988,14 +2794,7 @@ void EspHttpBinding::validateResponse(IEspContext& context, CHttpRequest* reques } // schema - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, xsd); - } - else - { - getSchema(xsd,context,request,service,method,false); - } + getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, xsd); // validation if (getEspLogLevel()>LogMax) @@ -3030,6 +2829,7 @@ void EspHttpBinding::validateResponse(IEspContext& context, CHttpRequest* reques unsigned len = temp.length(); // This has to be done before temp.detach is called! content.setBuffer(len, temp.detach(), true); } + void EspHttpBinding::sortResponse(IEspContext& context, CHttpRequest* request, MemoryBuffer& content, const char *serviceName, const char* methodName) { diff --git a/esp/bindings/http/platform/httpbinding.hpp b/esp/bindings/http/platform/httpbinding.hpp index 1dd35a2c6dc..fa94e6146df 100644 --- a/esp/bindings/http/platform/httpbinding.hpp +++ b/esp/bindings/http/platform/httpbinding.hpp @@ -25,6 +25,7 @@ #include "bindutil.hpp" #include "seclib.hpp" + class CMethodInfo : public CInterface { public: @@ -112,6 +113,7 @@ interface IEspWsdlSections // virtual MethodInfoArray & queryQualifiedNames(IEspContext& ctx)=0; virtual int getQualifiedNames(IEspContext& ctx, MethodInfoArray & methods)=0; virtual int getXsdDefinition(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; + virtual int getServiceXmlFilename(StringBuffer &filename)=0; virtual int getWsdlMessages(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; virtual int getWsdlPorts(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; virtual int getWsdlBindings(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; @@ -192,7 +194,6 @@ class esp_http_decl EspHttpBinding : Owned corsHelper; - void getXMLMessageTag(IEspContext& ctx, bool isRequest, const char *method, StringBuffer& tag); protected: @@ -201,7 +202,6 @@ class esp_http_decl EspHttpBinding : bool m_includeJsonTest; StringBuffer m_challenge_realm; StringAttr m_defaultSvcVersion; - StringBuffer serviceXmlFilename; public: EspHttpBinding(IPropertyTree* cfg, const char *bindname=NULL, const char *procname=NULL); @@ -348,6 +348,7 @@ class esp_http_decl EspHttpBinding : // MethodInfoArray &queryQualifiedNames(IEspContext& ctx) { m_methods.popAll(); getQualifiedNames(ctx,m_methods); return m_methods;}; int getXsdDefinition(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda){return 0;}; + int getServiceXmlFilename(StringBuffer &filename) {return 0;}; int getWsdlMessages(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda); int getWsdlPorts(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda); int getWsdlBindings(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda); @@ -356,7 +357,6 @@ class esp_http_decl EspHttpBinding : virtual int getMethodHelp(IEspContext &context, const char *serv, const char *method, StringBuffer &page); bool isMethodInService(IEspContext& context, const char *servname, const char *methname); - virtual int getMethodHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &page, bool bIncludeFormTag){return 0;} virtual bool hasSubService(IEspContext &context, const char *name); virtual IRpcRequestBinding *createReqBinding(IEspContext &context, IHttpMessage *ireq, const char *service, const char *method){return NULL;} @@ -445,7 +445,6 @@ class esp_http_decl EspHttpBinding : protected: virtual bool basicAuth(IEspContext* ctx); - int getWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl); int getServiceWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl); virtual bool getSchema(StringBuffer& schema, IEspContext &ctx, CHttpRequest* req, const char *service, const char *method,bool standalone); virtual void appendSchemaNamespaces(IPropertyTree *namespaces, IEspContext &ctx, CHttpRequest* req, const char *service, const char *method){} diff --git a/tools/hidl/hidlcomp.cpp b/tools/hidl/hidlcomp.cpp index f9dba61e2d7..047b0dedf63 100644 --- a/tools/hidl/hidlcomp.cpp +++ b/tools/hidl/hidlcomp.cpp @@ -633,40 +633,6 @@ esp_xlate_info *esp_xlat(const char *from, bool defaultToString) return (defaultToString) ? &esp_xlate_table[0] : NULL; } -//TODO: this is not bullet proof. better idea?? -// Return: NULL if no need to be defined, otherwise the type to be defined -// Note: the caller needs to free memory -static char* getToBeDefinedType(const char* type) -{ - const char* colon = strchr(type, ':'); - const char* bareType = colon ? colon+1 : type; - - if (strnicmp(type, "xsd",colon-type)==0) - return NULL; - - /* - for (unsigned i=0; esp_xlate_table[i].meta_type!=NULL; i++) - { - if (stricmp(bareType,esp_xlate_table[i].xsd_type)==0) - return NULL; - } - */ - - if (strnicmp(type, "tns", colon-type)!=0) - { - char msg[128]; - sprintf(msg, "*** unhandled type: %s", type); - outs(msg); - yyerror(msg); - return NULL; - } - - if (strncmp(bareType, "ArrayOf", sizeof("ArrayOf")-1)==0) - return strdup(bareType+sizeof("ArrayOf")-1); - else - return strdup(bareType); -} - static const char *MetaTypeToXsdType(const char *val) { esp_xlate_info *xlation=esp_xlat(val); @@ -2484,11 +2450,6 @@ void EspMessageInfo::write_esp_ipp() outf("\tCX%s(const char* defvalue_) : SoapEnumParamNew()\n", name_, name_); outf("\t{ doInit(); setDefaultValue(defvalue_); }\n"); - - // getXsdDefinition - outs("\tstatic void getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &schema, BoolHash &added)\n"); - outs("\t{ getSharedInstance().getXsdDefinitionInternal(context,request,schema,added); }\n"); - // getMapInfo() outs("\tstatic void getMapInfo(IMapInfo& info, BoolHash& added) { getSharedInstance().getMapInfo_(info,added); }\n\n"); outf("\tstatic const char* stringOf(C%s val) { return getSharedInstance().toString(val); }\n\n",name_); @@ -2503,47 +2464,9 @@ void EspMessageInfo::write_esp_ipp() // TODO: getMapInfo_() internal implementation outs("\tvoid getMapInfo_(IMapInfo& info, BoolHash& added) { }\n"); - // handle descriptions - { - outs("\tvoid getXsdDefinitionInternal(IEspContext &context, CHttpRequest* request, StringBuffer &schema, BoolHash &added)\n"); - bool hasDesc = false; - for (pi = getParams(); pi!=NULL; pi=pi->next) - { - const char* desc = pi->getMetaString("desc",NULL); - if (desc) { hasDesc = true; break; } - } - if (hasDesc) - { - outs("\t{\n\t\tconst char* descriptions [] = {"); - for (pi = getParams(); pi!=NULL; pi=pi->next) - { - const char* desc = pi->getMetaString("desc",NULL); - outf("%s,", desc ? desc : "NULL"); - } - outs("};\n"); - outs("\t\tgetXsdDefinition_(context,request,schema,added,descriptions);\n\t}\n"); - } - else - outf("\t{ getXsdDefinition_(context,request,schema,added,NULL); }\n"); - } - outf("\tvoid doInit();\n"); outs("};\n\n"); - - // array -#if 0 - //outf("MAKEValueArray(C%s, %sArray);\n\n", name_, name_); - outf("inline C%s Array__Member2Param(C%s &src) { return src; }\n", name_, name_); - outf("inline void Array__Assign(C%s & dest, C%s const & src){ dest = src; }\n", name_, name_); - outf("inline bool Array__Equal(C%s const & m, C%s const p) { return m==p; }\n", name_, name_); - outf("inline void Array__Destroy(C%s & /*next* /) { }\n", name_); - //outf("MAKECopyArrayOf(simple, simple, array) - outf("MAKEArrayOf(C%s, C%s, %sArray)\n\n", name_,name_,name_); - //outf("class %sArray : public ArrayOf {};\n\n",name_,name_,name_); - outs("\n"); -#endif - return; } @@ -2639,22 +2562,10 @@ void EspMessageInfo::write_esp_ipp() outf("\t\treturn \"%s\";\n", name_); outs("\t}\n\n"); - //method ==> getXsdDefinition - outs("\tstatic StringBuffer &getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &schema, BoolHash&added, const char *xns=\"xsd\", const char *wsns=\"tns\", unsigned flags=1)\n"); - outs("\t{\n"); - outs("\t\treturn getXsdDefinition(context, request, queryXsdElementName(), schema, added, xns, wsns, flags);\n"); - outs("\t}\n"); - - //method ==> getXsdDefinition - outs("\tstatic StringBuffer &getXsdDefinition(IEspContext &context, CHttpRequest* request, const char *msgTypeName, StringBuffer &schema, BoolHash&added, const char *xns=\"xsd\", const char *wsns=\"tns\", unsigned flags=1);\n"); - //method ==> getMapInfo outs("\tstatic void getMapInfo(IMapInfo& info);\n"); outs("\tstatic void getMapInfo(IMapInfo& info, BoolHash& added);\n"); - //method ==> getHtmlForm - outs("\tstatic StringBuffer &getHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &form, bool includeFormTag=true, const char *prefix=NULL);\n"); - //method ==> hasCustomHttpContent outs("\tstatic bool hasCustomHttpContent()\n"); outs("\t{\n"); @@ -2664,9 +2575,6 @@ void EspMessageInfo::write_esp_ipp() outs("\t\treturn false;\n"); outs("\t}\n"); - //method ==> serializeHtml - outs("\tStringBuffer &serializeHtml(IEspContext &context, const char *serv, const char *method, StringBuffer &html);\n"); - //method ==> serialize (IRpcMessage&) outs("\n\tvoid serialize(IRpcMessage& rpc_resp);\n"); @@ -2721,8 +2629,6 @@ void EspMessageInfo::write_esp_ipp() outs("\n\tbool unserialize(IEspContext* ctx, IProperties& params, MapStrToBuf *attachments, const char *basepath=NULL);\n"); } - //outs("\n\tvoid unserialize(const char * msg);\n"); - if (espm_type_==espm_response) { outs("\n\tvirtual void setRedirectUrl(const char *url)\n"); @@ -2959,516 +2865,6 @@ void EspMessageInfo::write_esp() outs("\tunserialize(ctx,*params,attachments, NULL);\n}\n"); } - //======================================================================================= - //method ==> getXsdDefinition - // @context: the context could affect the schema, e.g., the version, the URL parameters - // @flags: 0x001 - define the struct/enum as complexType. - // 0x010 - do not include group-type enclosure (i.e., just define the fields, e.g., as part of other complex type definition) - // 0x100 - do not include fields at all (only to include definitions of fields, recursively) - bool isExtSimpleType = getMetaInt("element", 0)!=0; - outf("\nStringBuffer &C%s::getXsdDefinition(IEspContext &context, CHttpRequest* request, const char *msgTypeName, StringBuffer &schema, BoolHash &added, const char *xns, const char *wsns, unsigned flags)\n", name_); - outs("{\n"); - - indentReset(1); - indentOuts("if (!(flags & 0x100))\n\t{\n"); - - indentOuts(1,"IProperties *props = request->queryParameters();\n"); - indentOuts("if(msgTypeName) {\n"); - indentOuts(1,"if(added.getValue(msgTypeName))\n"); - indentOuts1(1,"return schema;\n"); - indentOuts("else\n"); - indentOuts1(1,"added.setValue(msgTypeName, 1);\n"); - indentOuts(-1,"}\n"); - - indentOuts("if (flags & 0x01) {\n"); - bool isEmptyComplexType = true; //a complex type with no children, no attribute and no parent - if (isExtSimpleType || getParentName() || hasNonAttributeChild() || (espm_type_==espm_response && getMetaInt("exceptions_inline", 0))) - isEmptyComplexType = false; - else - { - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (!pi->getMetaInt("attribute", 0) && !pi->getMetaInt("hidden", 0) && !pi->getMetaInt("hidden_soap", 0)) - isEmptyComplexType = false; - } - } - - if (isEmptyComplexType) - { - if (espm_type_==espm_struct) - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - else - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - } - else if (espm_type_==espm_struct) - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - else - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - - if (isExtSimpleType) - indentOuts("schema.append(\"\\n\");\n"); - indentOuts(-1, "}\n"); - - // native arrays - typedef std::map EspNativeArrays; - EspNativeArrays nativeArrays; - // esp struct arrays - typedef std::set EspStructArrays; - EspStructArrays structArrays; - - //no element children for extended simple type - if (!isEmptyComplexType && !isExtSimpleType) - { - const char *xsdGroupType = getXsdGroupType(); - - bool hasChild = hasNonAttributeChild(); - bool exceptions = espm_type_==espm_response && getMetaInt("exceptions_inline", 0); - bool needGroupType = exceptions || hasChild || parent; - if (needGroupType) - { - indentOuts("if (!(flags & 0x10)) {\n"); - indentOutf1(1,"schema.append(\"\");\n", xsdGroupType); - if (exceptions) - indentOuts("schema.append(\"\");\n"); - indentOuts("}\n"); - } - - if (parent) - indentOutf("C%s::getXsdDefinition(context, request, NULL, schema, added, xns, wsns, 0x10);\n", parent); - - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (!pi->getMetaInt("attribute", 0) && !pi->getMetaInt("hidden", 0) && !pi->getMetaInt("hidden_soap", 0)) - { - enum {definedtype, inline_primitive_array, inline_array, complextype} complexity=definedtype; - StrBuffer buffer; - - const char *xsd_type = pi->getMetaXsdType(); - if (xsd_type) - { - buffer.append(xsd_type); - } - else if (pi->flags & PF_TEMPLATE) - { - if (!strcmp(pi->templ, "ESParray")) - { - if (pi->isPrimitiveArray()) - { - if (pi->getMetaString("item_tag", NULL)) - complexity=inline_primitive_array; - else { - const char* type = pi->getArrayImplType(); - nativeArrays[pi->getArrayItemXsdType()] = VStrBuffer("Esp%s", type).str(); - buffer.appendf("tns:Esp%s", type); - } - } - else - { - buffer.append("tns:"); - - if (pi->getMetaString("item_tag", NULL)) - complexity=inline_array; - else { - structArrays.insert(pi->typname); - buffer.append("ArrayOf"); - } - buffer.append(pi->typname); - } - } - else - buffer.append("xsd:string"); - } - else if (pi->kind==TK_ESPSTRUCT || pi->kind==TK_ESPENUM) - { - buffer.appendf("tns:%s",pi->typname); - } - else - { - buffer.append(pi->getXsdType()); - } - - - const char *xsdtype = buffer.str(); - const char *xsdns = ""; - - if (xsdtype) - { - int count= buffer.length(); - if (count>0 && buffer.charAt(count-1)=='\"') - buffer.setCharAt(count-1,0); - if (*xsdtype=='\"') - xsdtype++; - if (strchr(xsdtype, ':')==NULL) - xsdns="xsd:"; - } - - bool hasMapInfo = pi->hasMapInfo(); - if (hasMapInfo) - { - indentOutf("if (!context.suppressed(\"%s\",\"%s\")) {\n", this->name_, pi->name); - indentInc(1); - } - - const char *access=pi->getMetaString("access", NULL); - if (access) - { - indentOuts("SecAccessFlags acc;\n"); - indentOutf("if (context.authorizeFeature(%s, acc) && acc>=SecAccess_Read) {\n", access); - indentInc(1); - } - - StrBuffer xmlTag; - const char* tagName = pi->getMetaStringValue(xmlTag,"xml_tag") ? xmlTag.str() : pi->name; - - switch (complexity) - { - case inline_array: - { - indentOutf("schema.append(\"\\n\");\n", tagName); - indentOuts("schema.append(\"\\n\");\n"); - indentOutf("schema.append(\"\");\n", pi->getMetaString("item_tag", "Item"), xsdtype); - indentOuts("schema.append(\"\");\n"); - indentOutf("schema.append(\"\");\n"); - break; - } - case inline_primitive_array: - { - indentOutf("schema.append(\"\");\n", tagName); - indentOuts("schema.append(\"\");\n"); - indentOutf("schema.append(\"getMetaString("item_tag", "Item"), pi->getArrayItemXsdType()); - - int cols = pi->getMetaInt("cols",0); - int rows = pi->getMetaInt("rows",0); - if (cols>0 || rows>0) - { - outs("\");\n"); - indentOuts("if (context.queryOptions()&ESPCTX_ALL_ANNOTATION)\n"); - indentOuts(1,"schema.append(\"> 0) - outf(" formCols=\\\"%d\\\"", cols); - if (rows>0) - outf(" formRows=\\\"%d\\\"", rows); - outs("/>\\n\");\n"); - indentOuts(-1,"else\n"); - indentOuts("\tschema.append(\"/>\\n\");\n"); - } - else - outs("/>\\n\");\n"); - - indentOuts("schema.append(\"\\n\");\n"); - indentOutf("schema.append(\"\\n\");\n"); - break; - } - case definedtype: - { - indentOutf("schema.append(\"getMetaInt("required", 0)) ? 1 : 0; - if (minOccurs==0) - outs(" minOccurs=\\\"0\\\""); - - // default value - if (pi->hasMetaTag("default")) - { - if (pi->kind==TK_CHAR || pi->kind==TK_UNSIGNEDCHAR || strcmp(xsdtype,"string")==0) - { - const char* val = pi->getMetaString("default",NULL); - if (val && *val) - { - // remove quotes - if (*val=='"' || *val=='\'') - { - char* s = strdup(val); - *(s+strlen(s)-1) = 0; - outf(" default=\\\"%s\\\"", s+1); - free(s); - } - else - outf(" default=\\\"%s\\\"", val); - } - } - else if (pi->kind==TK_DOUBLE || pi->kind==TK_FLOAT || strcmp(xsdtype,"double")==0) - { - double val = pi->getMetaDouble("default"); - if (val==0) - val = pi->getMetaInt("default"); // auto conversion - if (val!=0) - outf(" default=\\\"%g\\\"", val); - } - else if (pi->kind==TK_BOOL || strcmp(xsdtype,"bool")==0) - { - outf(" default=\\\"%s\\\"", pi->getMetaInt("default") ? "true" : "false"); - } - else if (pi->kind==TK_ESPENUM) - { - const char* val = pi->getMetaString("default",NULL); - if (val && *val) - { - // remove quotes - if (*val=='"' || *val=='\'') - { - char* s = strdup(val); - *(s+strlen(s)-1) = 0; - outf(" default=\\\"%s\\\"", s+1); - free(s); - } - else - outf(" default=\\\"%s\\\"", val); - } - else - { - int value = pi->getMetaInt("default", -1); - if (value != -1) - outf(" default=\\\"%d\\\"", value); - } - } - else// assume it is integer - { - int val = pi->getMetaInt("default"); - if (val) - outf(" default=\\\"%d\\\"", val); - } - } - - // name & type - outf(" name=\\\"%s\\\" type=\\\"%s%s\\\"", tagName, xsdns, xsdtype); - - // check for annotations - StrBuffer annot; - - const char* formType = pi->getMetaInt("password") ? "password" : NULL; - if (formType) - annot.appendf(" formType=\\\"%s\\\"",formType); - - bool collapsed = pi->getMetaInt("collapsed")?true:false;; - if (collapsed) - annot.append(" collapsed=\\\"true\\\""); - - int cols = pi->getMetaInt("cols",0); - if (cols>0) - annot.appendf(" formCols=\\\"%d\\\"", cols); - - int rows = pi->getMetaInt("rows",0); - if (rows>0) - annot.appendf(" formRows=\\\"%d\\\"", rows); - - StrBuffer tmp; - pi->getMetaStringValue(tmp,"form_ui"); - if (tmp.length()) { - StrBuffer encoded; - encodeXML(tmp.str(),encoded); - annot.appendf(" ui=\\\"%s\\\"", printfEncode(encoded.str(), tmp.clear()).str()); - } - - pi->getMetaStringValue(tmp.clear(), "html_head"); - if (tmp.length()) { - StrBuffer encoded; - encodeXML(tmp.str(),encoded); - annot.appendf(" html_head=\\\"%s\\\"", printfEncode(encoded.str(), tmp.clear()).str()); - } - - if (annot.length()) - { - outs("\");\n"); - indentOuts("if (context.queryOptions()&ESPCTX_ALL_ANNOTATION)\n"); - indentOuts(1,"schema.append(\"> \\n\");\n"); - indentOuts(-1,"else\n"); - indentOuts("\tschema.append(\"/>\\n\");\n"); - } - else - outs("/>\\n\");\n"); - - break; - } - default: - break; - } - - if (access) - indentOuts(-1,"}\n"); - - //if (optional) - if (hasMapInfo) - indentOuts(-1,"}\n"); - } - } - - if (needGroupType) - { - indentOuts("if (!(flags & 0x10))\n"); - indentOutf1(1,"schema.append(\"\\n\");\n", xsdGroupType); - } - - } //!isEmptyComplexType && !isExtSimpleType - - if (!isEmptyComplexType) - { - //attributes last - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (pi->getMetaInt("attribute")!=0 && !pi->getMetaInt("hidden")) - { - StrBuffer tmp; - const char* tagName = pi->getMetaStringValue(tmp,"xml_tag") ? tmp.str() : pi->name; - indentOutf("schema.append(\"\");\n", tagName, pi->getXsdType()); - } - } - - indentOuts("if (flags & 0x01) {\n"); - if (isExtSimpleType) - indentOuts1(1,"schema.append(\"\\n\");\n"); - if (espm_type_==espm_struct) - indentOuts1(1,"schema.append(\"\\n\");\n"); - else - indentOuts1(1,"schema.append(\"\\n\");\n"); - indentOuts("}\n"); - } - - indentOuts(-1,"}\n"); // if (flags & 0x100) - //------------------------------------------------------------------------- - // try to figure out structs without ESPuses - // true if it is with map info, once a STRUCT or ENUM is there without map info, it is false - const int MIT_HasMap = 0x01; - const int MIT_IsEnum = 0x02; - const int MIT_IsStruct = 0x04; - enum MapInfoType { MIT_EnumFalse=MIT_IsEnum, MIT_EnumTrue=MIT_HasMap|MIT_IsEnum, - MIT_StructFalse=MIT_IsStruct,MIT_StructTrue=MIT_IsStruct|MIT_HasMap }; - typedef std::map TypeMap; - TypeMap typesNeeded; - - bool parenthesisOpened = false; - if (parent) - { - outs("\tif (!(flags & 0x10) || (flags & 0x100))\n"); - outs("\t{\n"); - parenthesisOpened = true; - outf("\t\tC%s::getXsdDefinition(context, request, schema, added, xns, wsns, 0x110);\n", parent); - typesNeeded[parent] = false; - } - for (pi=getParams();pi!=NULL;pi=pi->next) - { - // we still need to make it usable for the service writer - //if (pi->getMetaInt("hidden", 0) || pi->getMetaInt("hidden_soap", 0)) - // continue; - std::string thisType; - - const char *xsd_type=pi->getMetaXsdType(); - if (xsd_type) - { - char* buf = strdup(xsd_type[0]=='"' ? xsd_type+1 : xsd_type); - size_t count= strlen(buf); - if (count>0 && buf[count-1]=='\"') - buf[count-1]=0; - - char* typeName = getToBeDefinedType(buf); - if (typeName) - { - thisType = typeName; - free(typeName); - } - free(buf); - } - - else if (pi->kind==TK_ESPSTRUCT || pi->kind==TK_ESPENUM) - { - if (pi->typname) - thisType = pi->typname; - else - outs("\t\t*** pi->typname is empty!!\n"); - } - - else if (pi->flags & PF_TEMPLATE) - { - if (strcmp(pi->templ, "ESParray")==0) - { - if (pi->typname && strcmp(pi->typname, "string")!=0 && strcmp(pi->typname, "EspTextFile")!=0) - thisType = pi->typname; - } - else - outf("\t\t// *** skip field: name=%s, type=%s\n", pi->name, pi->typname); - } - - if (thisType.length() && strcmp(thisType.c_str(),name_)!=0) - { - TypeMap::const_iterator it = typesNeeded.find(thisType); - if (it==typesNeeded.end() || (*it).second & MIT_HasMap) // thisType is not in typesNeeded or it is there with version info - { - if (!parenthesisOpened) - { - outs("\tif (!(flags & 0x10) || (flags & 0x100))\n\t{\n"); - parenthesisOpened = true; - } - - - if (pi->hasMapInfo()) - { - outf("\t\tif (!context.suppressed(\"%s\",\"%s\"))\n\t", this->name_, pi->name); - typesNeeded[thisType] = (pi->kind==TK_ESPENUM) ? MIT_EnumTrue : MIT_StructTrue; - } - else - typesNeeded[thisType] = (pi->kind==TK_ESPENUM) ? MIT_EnumFalse : MIT_StructFalse; - if (pi->kind == TK_ESPENUM) - outf("\t\tCX%s::getXsdDefinition(context, request, schema, added);\n", thisType.c_str()); - else - outf("\t\tC%s::getXsdDefinition(context, request, schema, added);\n", thisType.c_str()); - } - } - } - - if (parenthesisOpened) - outs("\t}\n"); - - // native arrays - for (EspNativeArrays::const_iterator it = nativeArrays.begin(); it!=nativeArrays.end(); it++) - { - outf("\tif (added.getValue(\"%s\")==NULL) {\n", (*it).second.c_str()); - outf("\t\taddEspNativeArray(schema,\"%s\",\"%s\");\n", (*it).first.c_str(), (*it).second.c_str()); - outf("\t\tadded.setValue(\"%s\",1);\n", (*it).second.c_str()); - outf("\t}\n"); - } - - // struct arrays - for (EspStructArrays::const_iterator it2 = structArrays.begin(); it2!=structArrays.end(); it2++) - { - const char* type = (*it2).c_str(); - outf("\tif (added.getValue(\"%s\") && added.getValue(\"ArrayOf%s\")==NULL) {\n", type, type); - outf("\t\tschema.append(\"\\n\");\n",type); - outf("\t\tschema.append(\"\\n\");\n"); - outf("\t\tschema.append(\"\\n\");\n", type, type); - outf("\t\tschema.append(\"\\n\");\n"); - outf("\t\tschema.append(\"\\n\");\n"); - outf("\t\tadded.setValue(\"ArrayOf%s\",1);\n", type); - outf("\t}\n"); - } - - /* - //typesNeeded.erase(name_); // Do not include self: avoid recursive call - - if (typesNeeded.size()>0) - { - outs("\tif (!(flags & 0x10))\n"); - outs("\t{\n"); - for (TypeMap::const_iterator it = typesNeeded.begin(); it!=typesNeeded.end(); ++it) - { - const char* type = (*it).first.c_str(); - const char* fld = (*it).second.c_str(); - if (fld[0] != 0) - { - outf("\t\tif (!context.suppressed(\"%s\",\"%s\"))\n\t", this->name_, fld); - outf("\t\t\tC%s::getXsdDefinition(context, request, schema, added);\n", type); - } - else - outf("\t\tC%s::getXsdDefinition(context, request, schema, added);\n", type); - } - outs("\t}\n"); - } - */ - outs("\treturn schema;\n"); - outs("}\n"); - //======================================================================================= //method ==> getMapInfo outf("\nvoid C%s::getMapInfo(IMapInfo& info) { BoolHash added; getMapInfo(info, added); }\n",name_); @@ -3502,224 +2898,9 @@ void EspMessageInfo::write_esp() } } - if (typesNeeded.size()>0) - { - for (TypeMap::const_iterator it = typesNeeded.begin(); it!=typesNeeded.end(); ++it) - { - outf("\tif (!added.getValue(\"%s\"))\n", (*it).first.c_str()); - outf("\t{\n"); - outf("\t\tadded.setValue(\"%s\",1);\n", (*it).first.c_str()); - outf("\t\tC%s%s::getMapInfo(info,added);\n",((*it).second & MIT_IsEnum)?"X":"",(*it).first.c_str()); - outf("\t}\n"); - } - } - outs("}\n"); - //======================================================================================= - //method ==> getHtmlForm - //TODO: move includeFormTag into onGetForm() to reduce significant generated code indentReset(); - indentOutf("\nStringBuffer &C%s::getHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &form, bool includeFormTag, const char *prefix)\n", name_); - - indentOuts("{\n"); - indentOuts(1,"IProperties *props = request->queryParameters();\n"); - bool hasAttachment=false; - for (pi=getParams();pi;pi=pi->next) - { - if (pi->typname && !stricmp("EspTextFile", pi->typname)) - hasAttachment=true; - } - - indentOuts("if (includeFormTag) {\n"); - indentOutf(1,"StringBuffer params, versionTag;\n"); - indentOuts("bool hasVersion = getUrlParams(props,params);\n"); - indentOuts("if (!hasVersion) versionTag.appendf(\"%cver_=%g\",params.length()?'&':'?',context.getClientVersion());\n"); - indentOutf("form.appendf(\"\\n
\\n\", serv, method, params.str(), versionTag.str());\n", - hasAttachment ? "multipart/form-data" : "application/x-www-form-urlencoded"); - indentOutf(-1,"}\n"); - if (parent) - indentOutf("C%s::getHtmlForm(context, request, serv, method, form, false, prefix);\n", parent); - - indentOuts("StringBuffer extfix;\n"); - indentOuts("form.append(\" \\n\");\n"); - - for (pi=getParams();pi!=NULL;pi=pi->next) - { - bool hasMapInfo = pi->hasMapInfo(); - if (hasMapInfo) - { - indentOutf("if (!context.suppressed(\"%s\",\"%s\")) {\n", this->name_, pi->name); - indentInc(1); - } - - if (!pi->getMetaInt("hidden")) - { - int rows = pi->getMetaInt("rows", 5); - - StrBuffer label; - if (!pi->getMetaStringValue(label,"label")) - if (!pi->getMetaStringValue(label,"xml_tag")) - label = pi->name; - - if (pi->getMetaInt("rows") || (pi->flags & PF_TEMPLATE)) - { - int cols = pi->getMetaInt("cols", 50); - - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->name); - indentOutf("form.appendf(\"\");\n"); - } - else - { - esp_xlate_info *espinfo = esp_xlat(pi); - if (pi->kind==TK_ESPSTRUCT) - { - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->getXmlTag()); - indentOutf("form.append(\"\").append(\"\");\n"); - } - else if (espinfo->access_kind==TK_BOOL) - { - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->getXmlTag()); - - // handle default value - indentOutf("\n\tform.appendf(\" \\n\", extfix.str());\n", - label.str(), pi->getMetaInt("default",0)?"checked=\\\"1\\\"":""); - } - else if (espinfo->access_kind==TK_INT || espinfo->access_kind==TK_UNSIGNED - || espinfo->access_kind==TK_SHORT || espinfo->access_kind==TK_UNSIGNEDSHORT) - { - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix) extfix.append(prefix).append(\".\");\n"); - indentOutf("\textfix.append(\"%s\");\n", pi->getXmlTag()); - - int cols = pi->getMetaInt("cols", 20); - indentOutf("form.appendf(\" \\n\");\n"); - } - //TODO: handle default value for float, double, long, longlong etc - else - { - const char *xsdtype = pi->getXsdType(); - bool isDate = (!stricmp(xsdtype, "date")); - const char *inputType = "text"; - if (pi->typname && !strcmp(pi->typname, "EspTextFile")) - inputType="file"; - else if (pi->getMetaInt("password")) - inputType="password"; - - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->getXmlTag()); - - // handle default value - StrBuffer tmp; - const char* def = pi->getMetaStringValue(tmp,"default") ? tmp.str() : ""; - - int cols = pi->getMetaInt("cols", isDate ? 20 : 50); - indentOutf("form.appendf(\" \\n\");\n"); - } - } - } - if (hasMapInfo) - indentOuts(-1,"}\n"); - } - - indentReset(1); - indentOuts("if (includeFormTag) {\n"); - indentOuts(1,"form.append(\"\");\n"); - indentOuts(-1,"}\n"); - - indentOuts("form.append(\"
%s: \");\n", label.str()); - if (!pi->isEspStringArray()) - { - // handle default value - StrBuffer tmp; - const char* def = pi->getMetaStringValue(tmp,"default") ? tmp.str() : ""; - - indentOutf("form.appendf(\"\", extfix.str());\n", - cols, rows, def); - - indentOutf("form.append(\"
\");\n"); - } - else - { - indentOutf("form.appendf(\"\", extfix.str());\n", cols, rows); - } - - indentOutf("form.append(\"
%s:
\");\n", label.str()); - indentOutf("C%s::getHtmlForm(context, request, serv, method, form, false, extfix.str());\n", pi->typname); - indentOuts("form.append(\"
%s?
%s: getMetaInt("default"); - if (defValue!=0) - indentOutf(" value=\\\"%d\\\"", defValue); - - outs("/>\", extfix.str());\n"); - indentOuts("form.append(\"
%s: \", extfix.str());\n", label.str(), inputType, cols, def); - if (isDate) - indentOutf("form.append(\"\\\"Pick
\");\n", pi->name); - indentOuts("form.append(\"
\");\n"); - indentOuts("form.append(\"  
\");\n"); - indentOuts("if (includeFormTag)\n"); - indentOuts(1,"form.append(\"
\");\n"); - indentOuts(-1,"return form;\n"); - indentOuts(-1,"}\n"); - - //method ==> serializeHtml - outf("\nStringBuffer &C%s::serializeHtml(IEspContext &context, const char *serv, const char *method, StringBuffer &html)\n", name_); - outs("{\n"); - if (getMetaInt("serialize_html")) - { - outs("\thtml.append(\"

\");\n"); - - if (contentVar) - { - if (!(contentVar->flags & PF_TEMPLATE)) - { - if (contentVar->typname!=NULL && strcmp(contentVar->typname, "EspResultSet")==0) - { - outf("\thtml.appendf(\"
%s:
\");\n", contentVar->name); - outf("\tEspHttpBinding::formatHtmlResultSet(context, serv, method, m_%s->str(), html);", contentVar->name); - } - else if (contentVar->typname==NULL || strcmp(contentVar->typname, "binary")!=0) - { - outf("\thtml.appendf(\"
%s:
\");\n", contentVar->name, contentVar->name, contentVar->name); - } - } - } - else - { - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (!(pi->flags & PF_TEMPLATE) && (pi->kind!=TK_ESPSTRUCT)) - { - if (pi->typname!=NULL && strcmp(pi->typname, "EspResultSet")==0) - { - outf("\thtml.appendf(\"
%s:
\");\n", pi->name); - outf("\tEspHttpBinding::formatHtmlResultSet(context, serv, method, m_%s->str(), html);", pi->name); - } - else if (pi->typname==NULL || strcmp(pi->typname, "binary")!=0) - { - outf("\thtml.appendf(\"
%s:
\");\n", pi->name, pi->name, pi->name); - } - } - } - } - - outs("\thtml.append(\"

\");\n"); - } - outs("\treturn html;\n"); - outs("}\n"); - //method ==> serialize (IRpcMessage&) outf("\nvoid C%s::serialize(IRpcMessage& rpc_resp)\n{\n", name_); @@ -3988,9 +3169,7 @@ void EspMessageInfo::write_esp() outf("\tdouble clientVer = ctx ? ctx->getClientVersion() : -1;\n"); } -#if 1 // not respecting nil_remove: backward compatible - for (pi=getParams();pi!=NULL;pi=pi->next) { if (pi->getMetaInt("attribute")) @@ -4182,232 +3361,6 @@ void EspMessageInfo::write_esp() free(uname); } -#else - // respect nil_remove: may cause backward compatibility problem - - bool nilRemove = getMetaInt("nil_remove", 0)!=0; - - indentReset(1); - for (pi=getParams();pi!=NULL;pi=pi->next) - { - char *uname=strdup(pi->name); - *uname = upperchar(*uname); - - if (pi->hasMapInfo()) - pi->write_mapinfo_check(1,"ctx"); - - if (pi->flags & PF_TEMPLATE) // array - { - indentOutf("{\n"); - indentInc(1); - if (pi->isPrimitiveArray()) - { - const char *item_tag = pi->getMetaString("item_tag", "Item"); - const char *type = pi->getArrayImplType(); - indentOutf("%s& v = src.get%s();\n",type,uname); - indentOutf("int size = v.length();\n"); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"<%s>\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOuts("for (int i=0;ikind) - { - case TK_BOOL: - case TK_SHORT: - case TK_INT: fmt = "d"; break; - case TK_UNSIGNED: fmt = "u"; break; - case TK_LONG: fmt = "ld"; break; - case TK_UNSIGNEDLONG: fmt = "lu"; break; - case TK_FLOAT: - case TK_DOUBLE: fmt = "g"; break; - case TK_null: - case TK_CHAR: fmt = "s"; break; - default: - { - char buf[128]; - sprintf(buf,"Unhandled array type: %s (%s)", getTypeKindName(pi->kind), name_); - yyerror(buf); - } - } - - indentOutf1(1,"buffer.appendf(\"<%s>%%%s\",v.item(i));\n",item_tag,fmt,item_tag); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - else if (pi->typname) - { - indentOutf("IArrayOf& v = src.get%s();\n",pi->typname,uname); - indentOuts("int size = v.length();\n"); - const char *item_tag = pi->getMetaString("item_tag", pi->typname); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"<%s>\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("for (int i=0;i\");\n",item_tag); - indentOutf("C%s::serializer(ctx,v.item(i),buffer,false);\n",pi->typname); - indentOutf("buffer.append(\"\");\n",item_tag); - indentOutf(-1,"}\n"); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - else - { - indentOutf("**** TODO: unhandled array: type=%s, name=%s, xsd-type=%s\n", pi->typname, uname, pi->getXsdType()); - } - indentOutf(-1,"}\n"); - } - else if (pi->kind == TK_ESPSTRUCT) - { - indentOutf("{\n"); - indentInc(1); - if (nilRemove) - { - indentOutf("StringBuffer tmp;\n"); - indentOutf("C%s::serializer(ctx,src.get%s(), tmp, false);\n", pi->typname, uname); - indentOutf("if (tmp.length()>0)\n"); - const char* tag = pi->getXmlTag(); - indentOutf1(1,"buffer.appendf(\"<%s>%%s\",tmp.str());\n", tag, tag); - } - else - { - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("C%s::serializer(ctx,src.get%s(), buffer, false);\n", pi->typname, uname); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - indentOutf(-1,"}\n"); - } - else - { - esp_xlate_info* info = esp_xlat(pi); - switch(info->access_kind) - { - case TK_CHAR: - indentOuts("{\n"); - indentOutf(1,"const char* s = src.get%s();\n", uname); - if (nilRemove) - { - indentOutf("if (s && *s)\n"); - indentInc(1); - } - - if (!getMetaInt("encode",1)) - { - indentOutf("buffer.appendf(\"<%s>%%s\",s);\n",pi->name,pi->name); - } - else - { - if (nilRemove) - indentOutf1(-1,"{\n"); - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("encodeUtf8XML(s,buffer);\n"); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - if (nilRemove) - indentOutf1(-1,"}\n"); - } - if (nilRemove) - indentInc(-1); - indentOuts(-1,"}\n"); - break; - - case TK_INT: - case TK_LONG: - case TK_SHORT: - indentOuts("{\n"); - indentOutf(1,"%s n = src.get%s();\n", esp_xlat(pi)->access_type, uname); - if (nilRemove) - { - indentOuts("if (n)\n"); - indentInc(1); - } - indentOutf("buffer.appendf(\"<%s>%%d\", n);\n", pi->name,pi->name); - if (nilRemove) - indentInc(-1); - indentOuts(-1,"}\n"); - break; - - case TK_BOOL: - indentOuts("{\n"); - indentOutf(1,"%s b = src.get%s();\n", esp_xlat(pi)->access_type, uname); - if (nilRemove) - { - indentOuts("if (b)\n"); - indentInc(1); - } - const char* tag = pi->getXmlTag(); - indentOutf("buffer.append(\"<%s>1\");\n", tag,tag); - if (nilRemove) - indentInc(-1); - indentOuts(-1,"}\n"); - break; - - default: - if (pi->kind == TK_STRUCT && info->eam_type == EAM_jmbin) // binary - { - //TODO: should we encode binary data? - indentOuts("{\n"); - if (nilRemove) - { - indentOutf(1,"StringBuffer tmp;\n"); - indentOutf("JBASE64_Encode(src.get%s().toByteArray(), src.get%s().length(), tmp, true);\n", uname, uname); - indentOutf("if (tmp.length()>0)\n"); - const char* tag = pi->getXmlTag(); - indentOutf1(1,"buffer.appendf(\"<%s>%%s\",tmp.str());\n", tag,tag); - } - else - { - indentOutf(1,"buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("JBASE64_Encode(src.get%s().toByteArray(), src.get%s().length(), buffer, ture);\n", uname, uname); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - indentOuts(-1,"}\n"); - } - else - { - indentOuts("{\n"); - if (nilRemove) - { - indentOutf(1,"//*** default kind: %s; type=%s, name=%s\n", getTypeKindName(pi->kind), pi->typname, uname); - indentOutf("StringBuffer tmp(src.get%s());\n",uname); - indentOutf("if (tmp.length()>0)\n"); - indentOutf1(1,"buffer.appendf(\"<%s>%%s\",tmp.str());\n", pi->name,pi->name); - } - else - { - indentOutf(1,"//*** default kind: %s; type=%s, name=%s\n", getTypeKindName(pi->kind), pi->typname, uname); - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("buffer.append(src.get%s());\n", uname); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - indentOuts(-1,"}\n"); - } - break; - } - } - - free(uname); - } -#endif outf("\tif (keepRootTag)\n\t\tbuffer.append(\"\");\n", name_); outs("}\n"); @@ -4993,11 +3946,8 @@ void EspServInfo::write_esp_binding_ipp() //method ==> processRequest outs("\tvirtual int processRequest(IRpcMessage* rpc_call, IRpcMessage* rpc_response);\n"); - //method ==> getXsdDefinition - outs("\tint getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &content, const char *service, const char *method, bool mda);\n"); - - //method ==> getMethodHtmlForm - outs("\tvirtual int getMethodHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &page, bool bIncludeFormTag);\n"); + // method ===> getServiceXmlFilename + outs("\tint getServiceXmlFilename(StringBuffer &filename);\n"); //method ==> getQualifiedNames outs("\tint getQualifiedNames(IEspContext& ctx, MethodInfoArray & methods);\n"); @@ -5021,7 +3971,6 @@ void EspServInfo::write_esp_binding_ipp() outs("\tvirtual int onGetForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method);\n"); //Method ==> onGetXForm - //if (getMetaInt("use_new_form",0)) outs("\tvirtual int onGetXForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method);\n"); //Method ==> supportGeneratedForms @@ -5117,7 +4066,6 @@ void EspServInfo::write_esp_binding_ipp() void EspServInfo::write_esp_binding(const char *packagename) { EspMethodInfo *mthi=NULL; - int useMethodName = getMetaInt("use_method_name", 0); StrBuffer wsdlVer; bool hasVersion = getMetaVerInfo(tags,"version",wsdlVer); if (!hasVersion) @@ -5136,7 +4084,6 @@ void EspServInfo::write_esp_binding(const char *packagename) outf("{\n"); outf("\tinit_strings();\n"); outf("\tsetWsdlVersion(%s);\n", wsdlVer.str()); - outf("\tserviceXmlFilename.append(\"%s.xml\");\n", packagename); outf("}\n"); outf("\nC%sSoapBinding::C%sSoapBinding(IPropertyTree* cfg, const char *bindname, const char *procname, http_soap_log_level level):CHttpSoapBinding(cfg, bindname, procname, level)\n", name_, name_); @@ -5144,7 +4091,6 @@ void EspServInfo::write_esp_binding(const char *packagename) outf("\tinit_strings();\n"); outf("\tinit_metrics();\n"); outf("\tsetWsdlVersion(%s);\n", wsdlVer.str()); - outf("\tserviceXmlFilename.append(\"%s.xml\");\n", packagename); outf("}\n"); outf("\nvoid C%sSoapBinding::init_strings()\n", name_); @@ -5360,89 +4306,12 @@ void EspServInfo::write_esp_binding(const char *packagename) outs("\treturn -1;\n"); outs("}\n"); - //method ==> getXsdDefinition packagename is the base ecm filename - outf("\nint C%sSoapBinding::getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &content, const char *service, const char *method, bool mda)\n", name_); - outs("{\n"); - - outs("\tBoolHash added;\n"); - - - // version - if (hasVersion) - { - outs("\tif (context.getClientVersion()<=0)\n"); - outf("\t\tcontext.setClientVersion(%s);\n\n", wsdlVer.str()); - } - outs("\tDBGLOG(\"Client version: %g\", context.getClientVersion());\n"); - - // getXSDDefinition(StringBuffer &content, const char *packageName, const char *serviceName, double version, const char *method); -// outf( "getXSDDefinition(content, \"%s\", \"%s\", context.getClientVersion(), method);\n\n", packagename, name_); - - indentReset(1); - outf(1, "bool fullservice = (!Utils::strcasecmp(service, \"%s\"));\n", name_); - indentOuts("bool allMethods = (method==NULL || *method==0);\n"); - bool needPropsDeclare = true; - bool needAccessFlagDeclare = true; - for (mthi=methods;mthi!=NULL;mthi=mthi->next) - { - const char *requestName = (useMethodName) ? mthi->getName() : mthi->getReq(); - const char *optional=mthi->getMetaString("optional", NULL); - const char *access=mthi->getMetaString("access", NULL); - - if (optional) - { - if (needPropsDeclare) - { - indentOuts("IProperties *props = request->queryParameters();\n"); - needPropsDeclare = false; - } - indentOutf("if (props && props->hasProp(%s)) {\n", optional); - indentInc(1); - } - if (access) - { - if (needAccessFlagDeclare) - { - indentOuts("SecAccessFlags acc;\n"); - needAccessFlagDeclare = false; - } - indentOutf("if (context.authorizeFeature(%s, acc) && acc>=SecAccess_Read) {\n", access); - indentInc(1); - } - - indentOutf("if ((allMethods&&(fullservice||isMethodInSubService(context, service, \"%s\"))) || Utils::strcasecmp(method, \"%s\")==0)\n", mthi->getName(), mthi->getName()); - indentOuts("{\n"); - indentInc(1); - bool hasMap = mthi->write_mapinfo_check("context"); - indentOutf("C%s::getMapInfo(context.queryMapInfo());\n", mthi->getReq()); - indentOutf("C%s::getMapInfo(context.queryMapInfo());\n", mthi->getResp()); - indentOutf("C%s::getXsdDefinition(context, request, \"%s\", content, added);\n", mthi->getReq(), requestName); - indentOutf("C%s::getXsdDefinition(context, request, content, added);\n", mthi->getResp()); - if (hasMap) - indentOuts(-1,"}\n"); - indentOuts(-1,"}\n"); - if (access) - indentOuts(-1,"}\n"); - if (optional) - indentOuts(-1,"}\n"); - } - indentOuts("return 0;\n"); - indentOuts(-1,"}\n"); - //method ==> getMethodHtmlForm - outf("\nint C%sSoapBinding::getMethodHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &page, bool bIncludeFormTag)\n", name_); + //method ==> getServiceXmlFilename for xsd and wsdl transformations + outf("\nint C%sSoapBinding::getServiceXmlFilename(StringBuffer &filename)\n", name_); outs("{\n"); - - outf("\tDBGLOG(\"Client version: %%g\", context.getClientVersion());\n"); - - for (mthi=methods;mthi!=NULL;mthi=mthi->next) - { - outf("\tif (Utils::strcasecmp(method, \"%s\")==0)\n", mthi->getName()); - outs("\t{\n"); - outf("\t\tC%s::getHtmlForm(context, request, serv, method, page);\n", mthi->getReq()); - outs("\t}\n"); - } - outs("\treturn 0;\n"); + outf("\tfilename.append(\"%s.xml\");\n", packagename); + outs("\treturn 1;\n"); outs("}\n"); //method ==> getQualifiedNames @@ -6035,14 +4904,12 @@ void EspServInfo::write_esp_client_ipp() outs("\tStringBuffer soap_url;\n"); //dom - outs("\tStringBuffer soap_userid;\n"); outs("\tStringBuffer soap_password;\n"); outs("\tStringBuffer soap_realm;\n"); outs("\tStringBuffer soap_action;\n"); outs("\tlong soap_reqid = 0;\n"); - outs("\npublic:\n"); outs("\tIMPLEMENT_IINTERFACE;\n\n"); From d75097aa5b7dc2cdb64224c94f2f0fd5e11e09dc Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 12 Mar 2024 15:21:25 +0000 Subject: [PATCH 11/23] HPCC-31429 Exclude auxQueues from Thor cluster list Esp services use 'getContainerWUClusterInfo' to gather cluster info, as a starting point they use the target queues. In k8s we may now have auxQueues based queues too. These to not correspond to a cluster. (they are instead listened to by existing targets). Exclude them from the gather clustered list. Signed-off-by: Jake Smith --- esp/smc/SMCLib/TpContainer.cpp | 4 ++++ helm/hpcc/templates/_helpers.tpl | 1 + 2 files changed, 5 insertions(+) diff --git a/esp/smc/SMCLib/TpContainer.cpp b/esp/smc/SMCLib/TpContainer.cpp index 9a3d81fe562..02189e2e7a5 100644 --- a/esp/smc/SMCLib/TpContainer.cpp +++ b/esp/smc/SMCLib/TpContainer.cpp @@ -641,6 +641,10 @@ extern TPWRAPPER_API unsigned getContainerWUClusterInfo(CConstWUClusterInfoArray ForEach(*queues) { IPropertyTree& queue = queues->query(); + + // auxillary queues are additional queues and do not have a 1:1 mapping to clusters + if (queue.getPropBool("@isAuxQueue")) + continue; Owned cluster = new CContainerWUClusterInfo(queue.queryProp("@name"), queue.queryProp("@type"), queue.queryProp("@ldapUser"), (unsigned) queue.getPropInt("@width", 1), queue.getPropBool("@queriesOnly")); diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 493af724e17..4aea93c6980 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1164,6 +1164,7 @@ Generate instance queue names {{- else -}} {{- $newQueueItem := deepCopy $queueItem -}} {{- $_ := set $newQueueItem "name" $auxQueueName -}} + {{- $_ := set $newQueueItem "isAuxQueue" true -}} {{- $_ := set $auxThorQueues $auxQueueName $newQueueItem -}} {{- end -}} {{- end -}} From 0ffb3fe13508df4f491cc92b2f74284a0121fa00 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 12 Mar 2024 14:36:57 -0400 Subject: [PATCH 12/23] HPCC-31435 ECL Watch v9 WU details ECL tab files scrollbar Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- .../src-react/components/ECLArchiveTree.tsx | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/esp/src/src-react/components/ECLArchiveTree.tsx b/esp/src/src-react/components/ECLArchiveTree.tsx index c39ef9c8f1a..9319050584b 100644 --- a/esp/src/src-react/components/ECLArchiveTree.tsx +++ b/esp/src/src-react/components/ECLArchiveTree.tsx @@ -69,28 +69,29 @@ export const ECLArchiveTree: React.FunctionComponent = ({ }, [archive, setSelectedItem]); const { ...treeProps } = flatTree.getTreeProps(); - return - { - Array.from(flatTree.items(), flatTreeItem => { - console.log(flatTreeItem.getTreeItemProps()); - const { fileTimePct, content, ...treeItemProps } = flatTreeItem.getTreeItemProps(); - return - attrId.startsWith(content)) ? : : - selectedAttrIDs.some(attrId => attrId.startsWith(content)) ? : ) : - selectedAttrIDs.some(attrId => attrId === flatTreeItem.value) ? - : - - } - aside={} - > - {content} - - ; - }) - } - ; + return
+ + { + Array.from(flatTree.items(), flatTreeItem => { + const { fileTimePct, content, ...treeItemProps } = flatTreeItem.getTreeItemProps(); + return + attrId.startsWith(content)) ? : : + selectedAttrIDs.some(attrId => attrId.startsWith(content)) ? : ) : + selectedAttrIDs.some(attrId => attrId === flatTreeItem.value) ? + : + + } + aside={} + > + {content} + + ; + }) + } + +
; }; From 891b9300f4c1846347e04a4b92d2684484dfb5ac Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 19 Feb 2024 10:00:44 +0000 Subject: [PATCH 13/23] HPCC-31300 Clean up roxie metric code Remove unused timer thread for publishing metrics to SNMP Remove unused function Remove interval metric stuff altogether Signed-off-by: Richard Chapman --- roxie/ccd/ccdsnmp.cpp | 235 +++++++++--------------------------------- roxie/ccd/ccdsnmp.hpp | 3 +- 2 files changed, 50 insertions(+), 188 deletions(-) diff --git a/roxie/ccd/ccdsnmp.cpp b/roxie/ccd/ccdsnmp.cpp index ba6fc6c9dd3..169b6a17200 100644 --- a/roxie/ccd/ccdsnmp.cpp +++ b/roxie/ccd/ccdsnmp.cpp @@ -29,15 +29,13 @@ #include "roxiemem.hpp" #include "udplib.hpp" -#define DEFAULT_PULSE_INTERVAL 30 - RoxieQueryStats unknownQueryStats; RoxieQueryStats loQueryStats; RoxieQueryStats hiQueryStats; RoxieQueryStats slaQueryStats; RoxieQueryStats combinedQueryStats; -#define addMetric(a, b) doAddMetric(a, #a, b) +#define addMetric(a) doAddMetric(a, #a) interface INamedMetric : extends IInterface { @@ -46,11 +44,6 @@ interface INamedMetric : extends IInterface virtual void resetValue() = 0; }; -interface ITimerCallback : extends IInterface -{ - virtual void onTimer() = 0; -}; - class RelaxedAtomicMetric : implements INamedMetric, public CInterface { RelaxedAtomic &counter; @@ -123,101 +116,6 @@ class UserMetric : implements INamedMetric, public CInterface }; -class TickProvider : public Thread -{ - IArrayOf listeners; - CriticalSection crit; - Semaphore stopped; - - void doTicks() - { - CriticalBlock c(crit); - ForEachItemIn(idx, listeners) - { - listeners.item(idx).onTimer(); - } - } - -public: - TickProvider() : Thread("TickProvider") - { - } - int run() - { - for (;;) - { - if (stopped.wait(10000)) - break; - doTicks(); - } - return 0; - } - - void addListener(ITimerCallback *l) - { - CriticalBlock c(crit); - listeners.append(*LINK(l)); - } - - void stop() - { - stopped.signal(); - join(); - } -}; - -class IntervalMetric : implements INamedMetric, implements ITimerCallback, public CInterface -{ - Linked base; - CriticalSection crit; - unsigned lastSnapshotTime; - long lastSnapshotValue; - unsigned minInterval; - long value; - - void takeSnapshot() - { - CriticalBlock c(crit); - unsigned now = msTick(); - unsigned period = now - lastSnapshotTime; - if (period >= minInterval) - { - long newValue = base->getValue(); - value = ((newValue - lastSnapshotValue) * 1000) / period; - lastSnapshotTime = now; - lastSnapshotValue = newValue; - } - } - -public: - IMPLEMENT_IINTERFACE; - IntervalMetric(INamedMetric *_base, unsigned _minInterval=1000) : base(_base), minInterval(_minInterval) - { - lastSnapshotTime = msTick(); - lastSnapshotValue = 0; - value = 0; - } - virtual void onTimer() - { - takeSnapshot(); - } - virtual long getValue() - { - takeSnapshot(); - return value; - } - virtual bool isCumulative() { return false; } - - virtual void resetValue() - { - CriticalBlock c(crit); - lastSnapshotTime = msTick(); - lastSnapshotValue = 0; - value = 0; - } - -}; - class UnsignedRatioMetric : implements INamedMetric, public CInterface { RelaxedAtomic &counter; @@ -252,31 +150,27 @@ class CRoxieMetricsManager : implements IRoxieMetricsManager, public CInterface ~CRoxieMetricsManager(); virtual long getValue(const char * name); - void dumpMetrics(); StringBuffer &getMetrics(StringBuffer &xml); void resetMetrics(); - void doAddMetric(RelaxedAtomic &counter, const char *name, unsigned interval, bool isMinVal = false); - void doAddMetric(INamedMetric *n, const char *name, unsigned interval); - void doAddMetric(AccessorFunction function, const char *name, unsigned interval); + void doAddMetric(RelaxedAtomic &counter, const char *name, bool isMinVal = false); + void doAddMetric(INamedMetric *n, const char *name); + void doAddMetric(AccessorFunction function, const char *name); void addRatioMetric(RelaxedAtomic &counter, const char *name, RelaxedAtomic &elapsed); void addUserMetric(const char *name, const char *regex); private: MapStringToMyClassViaBase metricMap; - bool started; - - TickProvider ticker; }; -void RoxieQueryStats::addMetrics(CRoxieMetricsManager *snmpManager, const char *prefix, unsigned interval) +void RoxieQueryStats::addMetrics(CRoxieMetricsManager *snmpManager, const char *prefix) { StringBuffer name; - snmpManager->doAddMetric(count, name.clear().append(prefix).append("QueryCount"), interval, false); - snmpManager->doAddMetric(failedCount, name.clear().append(prefix).append("QueryFailed"), interval, false); - snmpManager->doAddMetric(active, name.clear().append(prefix).append("QueryActive"), 0, false); - snmpManager->doAddMetric(maxTime, name.clear().append(prefix).append("QueryMaxTime"), 0, false); - snmpManager->doAddMetric(minTime, name.clear().append(prefix).append("QueryMinTime"), 0, true); + snmpManager->doAddMetric(count, name.clear().append(prefix).append("QueryCount"), false); + snmpManager->doAddMetric(failedCount, name.clear().append(prefix).append("QueryFailed"), false); + snmpManager->doAddMetric(active, name.clear().append(prefix).append("QueryActive"), false); + snmpManager->doAddMetric(maxTime, name.clear().append(prefix).append("QueryMaxTime"), false); + snmpManager->doAddMetric(minTime, name.clear().append(prefix).append("QueryMinTime"), true); snmpManager->addRatioMetric(count, name.clear().append(prefix).append("QueryAverageTime"), totalTime); } @@ -287,77 +181,65 @@ using roxiemem::getDataBuffersActive; CRoxieMetricsManager::CRoxieMetricsManager() { - started = false; - unknownQueryStats.addMetrics(this, "unknown", 1000); - loQueryStats.addMetrics(this, "lo", 1000); - hiQueryStats.addMetrics(this, "hi", 1000); - slaQueryStats.addMetrics(this, "sla", 1000); - combinedQueryStats.addMetrics(this, "all", 1000); - addMetric(restarts, 0); - - addMetric(nodesLoaded, 1000); - - addMetric(blobCacheHits, 1000); - addMetric(blobCacheAdds, 1000); - addMetric(blobCacheDups, 1000); - addMetric(leafCacheHits, 1000); - addMetric(leafCacheAdds, 1000); - addMetric(leafCacheDups, 1000); - addMetric(nodeCacheHits, 1000); - addMetric(nodeCacheAdds, 1000); - addMetric(nodeCacheDups, 1000); - - addMetric(unwantedDiscarded, 1000); - - addMetric(getHeapAllocated, 0); - addMetric(getHeapPercentAllocated, 0); - addMetric(getDataBufferPages, 0); - addMetric(getDataBuffersActive, 0); + unknownQueryStats.addMetrics(this, "unknown"); + loQueryStats.addMetrics(this, "lo"); + hiQueryStats.addMetrics(this, "hi"); + slaQueryStats.addMetrics(this, "sla"); + combinedQueryStats.addMetrics(this, "all"); + addMetric(restarts); + + addMetric(nodesLoaded); + + addMetric(blobCacheHits); + addMetric(blobCacheAdds); + addMetric(blobCacheDups); + addMetric(leafCacheHits); + addMetric(leafCacheAdds); + addMetric(leafCacheDups); + addMetric(nodeCacheHits); + addMetric(nodeCacheAdds); + addMetric(nodeCacheDups); + + addMetric(unwantedDiscarded); + + addMetric(getHeapAllocated); + addMetric(getHeapPercentAllocated); + addMetric(getDataBufferPages); + addMetric(getDataBuffersActive); - addMetric(packetsResent, 1000); - addMetric(flowPermitsSent, 1000); - addMetric(flowRequestsReceived, 1000); - addMetric(dataPacketsReceived, 1000); - addMetric(flowRequestsSent, 1000); - addMetric(flowPermitsReceived, 1000); - addMetric(dataPacketsSent, 1000); - ticker.start(); + addMetric(packetsResent); + addMetric(flowPermitsSent); + addMetric(flowRequestsReceived); + addMetric(dataPacketsReceived); + addMetric(flowRequestsSent); + addMetric(flowPermitsReceived); + addMetric(dataPacketsSent); } -void CRoxieMetricsManager::doAddMetric(RelaxedAtomic &counter, const char *name, unsigned interval, bool isMinVal) +void CRoxieMetricsManager::doAddMetric(RelaxedAtomic &counter, const char *name, bool isMinVal) { - doAddMetric(new RelaxedAtomicMetric(counter, interval != 0, isMinVal), name, interval); + doAddMetric(new RelaxedAtomicMetric(counter, false, isMinVal), name); } -void CRoxieMetricsManager::doAddMetric(INamedMetric *n, const char *name, unsigned interval) +void CRoxieMetricsManager::doAddMetric(INamedMetric *n, const char *name) { - if (interval) - { - StringBuffer fname(name); - fname.append("/s"); - IntervalMetric *im = new IntervalMetric(n, interval); - ticker.addListener(im); - metricMap.setValue(fname.str(), im); - im->Release(); - } metricMap.setValue(name, n); n->Release(); } -void CRoxieMetricsManager::doAddMetric(AccessorFunction function, const char *name, unsigned interval) +void CRoxieMetricsManager::doAddMetric(AccessorFunction function, const char *name) { - assertex(interval==0); - doAddMetric(new FunctionMetric(function), name, interval); + doAddMetric(new FunctionMetric(function), name); } void CRoxieMetricsManager::addRatioMetric(RelaxedAtomic &counter, const char *name, RelaxedAtomic &elapsed) { - doAddMetric(new UnsignedRatioMetric(counter, elapsed), name, 0); + doAddMetric(new UnsignedRatioMetric(counter, elapsed), name); } void CRoxieMetricsManager::addUserMetric(const char *name, const char *regex) { - doAddMetric(new UserMetric(name, regex), name, 0); + doAddMetric(new UserMetric(name, regex), name); } long CRoxieMetricsManager::getValue(const char * name) @@ -375,25 +257,6 @@ long CRoxieMetricsManager::getValue(const char * name) CRoxieMetricsManager::~CRoxieMetricsManager() { - ticker.stop(); - if (started) - dumpMetrics(); -} - -void CRoxieMetricsManager::dumpMetrics() -{ - HashIterator metrics(metricMap); - ForEach(metrics) - { - IMapping &cur = metrics.query(); - INamedMetric *m = (INamedMetric *) *metricMap.mapToValue(&cur); - if (m->isCumulative()) - { - const char *name = (const char *) cur.getKey(); - long val = m->getValue(); - DBGLOG("TOTALS: %s = %ld", name, val); - } - } } StringBuffer &CRoxieMetricsManager::getMetrics(StringBuffer &xml) diff --git a/roxie/ccd/ccdsnmp.hpp b/roxie/ccd/ccdsnmp.hpp index 81ffaaa9ede..49a46db233f 100644 --- a/roxie/ccd/ccdsnmp.hpp +++ b/roxie/ccd/ccdsnmp.hpp @@ -64,7 +64,7 @@ class RoxieQueryStats active--; } - void addMetrics(CRoxieMetricsManager *mgr, const char *prefix, unsigned interval); + void addMetrics(CRoxieMetricsManager *mgr, const char *prefix); }; interface IQueryStatsAggregator : public IInterface @@ -86,7 +86,6 @@ extern RoxieQueryStats combinedQueryStats; interface IRoxieMetricsManager : extends IInterface { - virtual void dumpMetrics() = 0; virtual void addUserMetric(const char *name, const char *regex) = 0; virtual StringBuffer &getMetrics(StringBuffer &) = 0; virtual void resetMetrics() = 0; From f23ed697ef24a34005e48238a8988decd8ccabd0 Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Wed, 13 Mar 2024 15:33:42 -0400 Subject: [PATCH 14/23] HPCC-31451 Unversioned WSDL change in ESP services Removed enum TpMachineType since code was expecting a string type. Fixed misspelled type from strint to string in ws_smc service definition. Signed-Off-By: Kenneth Rowland kenneth.rowland@lexisnexisrisk.com --- esp/scm/ws_smc.ecm | 2 +- esp/scm/ws_topology.ecm | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/esp/scm/ws_smc.ecm b/esp/scm/ws_smc.ecm index 5f2d7cc206f..c8121f80c60 100644 --- a/esp/scm/ws_smc.ecm +++ b/esp/scm/ws_smc.ecm @@ -32,7 +32,7 @@ ESPStruct ActiveWorkunit string Owner; string Jobname; string Server; - strint Instance; + string Instance; string Priority; [min_ver("1.01")] string Extra; [min_ver("1.02")] string GraphName; diff --git a/esp/scm/ws_topology.ecm b/esp/scm/ws_topology.ecm index 5a5d5447759..fafa113e054 100644 --- a/esp/scm/ws_topology.ecm +++ b/esp/scm/ws_topology.ecm @@ -415,14 +415,10 @@ ESPresponse [exceptions_inline,encode(0)] TpClusterInfoResponse }; -ESPenum TpMachineType : int -{ - TPMACHINECLUSTER(0, "TPMACHINECLUSTER") -}; ESPrequest [] TpMachineQueryRequest { //Type of Node the parent is.... - TpMachineType Type; + string Type; string Cluster; string OldIP; //Path to Parent within Dali(Encoded to prevent back-engineering) @@ -437,7 +433,7 @@ ESPresponse [exceptions_inline] TpMachineQueryResponse { bool EnablePreflightInfo; [min_ver("1.15")] bool HasThorSpareProcess; - TpMachineType Type; + string Type; string Cluster; string OldIP; string LogDirectory; From 95e619ebc417c3480f329ca03b1108cf8a03eff1 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:48:44 -0400 Subject: [PATCH 15/23] HPCC-31436 ECL Watch v9 WU details ECL tab sort files Sort files & folders alphabetically Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/ECLArchiveTree.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/ECLArchiveTree.tsx b/esp/src/src-react/components/ECLArchiveTree.tsx index c39ef9c8f1a..a460b26c729 100644 --- a/esp/src/src-react/components/ECLArchiveTree.tsx +++ b/esp/src/src-react/components/ECLArchiveTree.tsx @@ -57,7 +57,7 @@ export const ECLArchiveTree: React.FunctionComponent = ({ fileTimePct: isAttribute(modAttr) && Math.round((archive?.sourcePathTime(modAttr.sourcePath) / archive?.timeTotalExecute) * 100), }); }); - setFlatTreeItems(flatTreeItems); + setFlatTreeItems(flatTreeItems.sort((a, b) => a.value.toString().localeCompare(b.value.toString(), undefined, { sensitivity: "base" }))); }, [archive, archive?.modAttrs, archive?.timeTotalExecute]); const onClick = React.useCallback(evt => { From 7d6a17336e937f1925f5710b126b2075b273103c Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Mon, 11 Mar 2024 15:38:07 -0400 Subject: [PATCH 16/23] HPCC-31428 ECL Watch v9 change ZAP dialog API endpoint should use /WsWorkunits/WUCreateAndDownloadZAPInfo changing the structure of LogFilter request params, as Rodrigo noted fixing a couple of Dropdowns that were not properly updating values Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- .../src-react/components/forms/ZAPDialog.tsx | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/esp/src/src-react/components/forms/ZAPDialog.tsx b/esp/src/src-react/components/forms/ZAPDialog.tsx index 3bfa9367b20..750e997efba 100644 --- a/esp/src/src-react/components/forms/ZAPDialog.tsx +++ b/esp/src/src-react/components/forms/ZAPDialog.tsx @@ -192,14 +192,14 @@ export const ZAPDialog: React.FunctionComponent = ({ if (key === "AbsoluteTimeRange") { const startDate = logFilter.AbsoluteTimeRange.StartDate ? new Date(logFilter.AbsoluteTimeRange.StartDate).toISOString() : ""; const endDate = logFilter.AbsoluteTimeRange.EndDate ? new Date(logFilter.AbsoluteTimeRange.EndDate).toISOString() : ""; - formData.append("LogFilter.AbsoluteTimeRange.StartDate", startDate); - formData.append("LogFilter.AbsoluteTimeRange.EndDate", endDate); + formData.append("LogFilter_AbsoluteTimeRange_StartDate", startDate); + formData.append("LogFilter_AbsoluteTimeRange_EndDate", endDate); } else { - formData.append(`LogFilter.${key}`, logFilter[key]); + formData.append(`LogFilter_${key}`, logFilter[key]); } } - fetch("/WsWorkunits/WUCreateZAPInfo", { + fetch("/WsWorkunits/WUCreateAndDownloadZAPInfo", { method: "POST", body: formData }) @@ -630,7 +630,7 @@ export const ZAPDialog: React.FunctionComponent = ({ = ({ = ({ { key: "0", text: "ASC" }, { key: "1", text: "DESC" }, ]} - defaultSelectedKey="1" + defaultSelectedKey={"1"} + selectedKey={value} onRenderLabel={(props: CustomLabelProps) => } + onChange={(evt, option) => { + onChange(option.key); + }} /> } /> = ({ { key: LogType.Progress, text: "Progress" }, { key: LogType.Metric, text: "Metric" }, ]} - defaultSelectedKey="ALL" + selectedKey={value} onRenderLabel={(props: CustomLabelProps) => } + onChange={(evt, option) => { + onChange(option.key); + }} /> } /> From 4aa343742acee07783599d9f53cef6e3dffccdd6 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 14 Mar 2024 13:45:34 +0000 Subject: [PATCH 17/23] HPCC-31455 Drop cmake presets VSCode is "auto" loading the presets which is causing issues. This can be revisited once there is a more complete set of opinionated presets Signed-off-by: Gordon Smith --- CMakePresets.json => CMakePresets.json.sample | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CMakePresets.json => CMakePresets.json.sample (100%) diff --git a/CMakePresets.json b/CMakePresets.json.sample similarity index 100% rename from CMakePresets.json rename to CMakePresets.json.sample From 2b6fa2faa41f722328dba326004b649baf38d8df Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Fri, 8 Mar 2024 14:24:50 +0000 Subject: [PATCH 18/23] HPCC-31424 fix xpath in CRemoteXmlReadActivity being set incorrectly --- fs/dafsserver/dafsserver.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 5cb9a26b118..72dbe2962bf 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -2001,11 +2001,13 @@ class CRemoteXmlReadActivity : public CRemoteMarkupReadActivity public: CRemoteXmlReadActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc, TAKxmlread) { - xpath.set("/Dataset/"); if (customRowTag.isEmpty()) // no override fileDesc->queryProperties().getProp("@rowTag", xpath); else + { + xpath.set("/Dataset/"); xpath.append(customRowTag); + } } }; From 11dee7fb890689dc8c040f97dd64330d0be8bf7b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 27 Feb 2024 15:37:40 +0000 Subject: [PATCH 19/23] HPCC-31390 Allow soaptext1.ecl to be deployed as a query to aks Signed-off-by: Gavin Halliday --- testing/regress/ecl-test-azure.json | 2 +- testing/regress/ecl/soaptext1.ecl | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/testing/regress/ecl-test-azure.json b/testing/regress/ecl-test-azure.json index c06fdaaf7ab..d57c20465e3 100644 --- a/testing/regress/ecl-test-azure.json +++ b/testing/regress/ecl-test-azure.json @@ -58,7 +58,7 @@ "roxiegzip.ecl:TargetIP=roxie", "roxiewhitespace.ecl:TargetIP=roxie", "roxiepipe.ecl:TargetIP=roxie", - "soaptext1.ecl:TargetIP=roxie" + "soaptext1.ecl:remoteRoxieService=roxie,remoteEspService=eclqueries" ], "engineParams":[ "failOnLeaks", diff --git a/testing/regress/ecl/soaptext1.ecl b/testing/regress/ecl/soaptext1.ecl index 97bfd53c212..2022c95cc23 100644 --- a/testing/regress/ecl/soaptext1.ecl +++ b/testing/regress/ecl/soaptext1.ecl @@ -31,14 +31,18 @@ import ^ as root; //Simplified configuration -roxieIP := '.' : STORED('TargetIP'); -espIP := '.' : STORED('TargetIP'); +string roxieIP := '.' : STORED('remoteRoxieService'); +string espIP := '.' : STORED('remoteEspService'); +boolean defaultEncryptConnection := false : stored('encryptConnection'); +boolean defaultPersistConnection := false : stored('persistConnection'); +boolean defaultCallDirect := false : stored('callDirect'); +boolean defaultCallViaEsp := false : stored('callViaEsp'); //Simplified call options - doesn't provide all combinations that should be tested eventually -boolean encryptConnection := #IFDEFINED(root.encryptConnection, false); -boolean persistConnection := #IFDEFINED(root.persistConnection, false); -boolean callDirect := #IFDEFINED(root.callDirect, false); -boolean callViaEsp := #IFDEFINED(root.callViaEsp, false); +boolean encryptConnection := #IFDEFINED(root.encryptConnection, defaultEncryptConnection); +boolean persistConnection := #IFDEFINED(root.persistConnection, defaultPersistConnection); +boolean callDirect := #IFDEFINED(root.callDirect, defaultCallDirect); +boolean callViaEsp := #IFDEFINED(root.callViaEsp, defaultCallViaEsp); //--- end of version configuration --- From 8a11e42dea70d067543b1bbe11df0dd663e7a173 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 14 Mar 2024 17:18:51 +0000 Subject: [PATCH 20/23] HPCC-31456 Prevent libmysql from building "host" targets Was adding 80 additional build steps to the vcpkg build process Signed-off-by: Gordon Smith --- vcpkg.json.in | 4 +- vcpkg_overlays/libmysql/cross-build.patch | 162 ++++++++++++++++++ vcpkg_overlays/libmysql/dependencies.patch | 140 +++++++++++++++ vcpkg_overlays/libmysql/fix_dup_symbols.patch | 20 +++ vcpkg_overlays/libmysql/install-exports.patch | 121 +++++++++++++ vcpkg_overlays/libmysql/libmysql-config.cmake | 6 + vcpkg_overlays/libmysql/portfile.cmake | 160 +++++++++++++++++ vcpkg_overlays/libmysql/usage | 4 + vcpkg_overlays/libmysql/vcpkg.json | 47 +++++ 9 files changed, 663 insertions(+), 1 deletion(-) create mode 100644 vcpkg_overlays/libmysql/cross-build.patch create mode 100644 vcpkg_overlays/libmysql/dependencies.patch create mode 100644 vcpkg_overlays/libmysql/fix_dup_symbols.patch create mode 100644 vcpkg_overlays/libmysql/install-exports.patch create mode 100644 vcpkg_overlays/libmysql/libmysql-config.cmake create mode 100644 vcpkg_overlays/libmysql/portfile.cmake create mode 100644 vcpkg_overlays/libmysql/usage create mode 100644 vcpkg_overlays/libmysql/vcpkg.json diff --git a/vcpkg.json.in b/vcpkg.json.in index 9afef0a36e2..310ac2cb38c 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -106,7 +106,9 @@ }, { "name": "libmysql", - "platform": "@VCPKG_MYSQLEMBED@ & !(windows & x86)" + "platform": "@VCPKG_MYSQLEMBED@ & !(windows & x86)", + "default-features": false, + "features": [] }, { "name": "librdkafka", diff --git a/vcpkg_overlays/libmysql/cross-build.patch b/vcpkg_overlays/libmysql/cross-build.patch new file mode 100644 index 00000000000..a91118b4f85 --- /dev/null +++ b/vcpkg_overlays/libmysql/cross-build.patch @@ -0,0 +1,162 @@ +diff --git a/cmake/os/Darwin.cmake b/cmake/os/Darwin.cmake +index 725b9bd..289bb3d 100644 +--- a/cmake/os/Darwin.cmake ++++ b/cmake/os/Darwin.cmake +@@ -31,13 +31,9 @@ ENDIF() + # We require at least XCode 10.0 + IF(NOT FORCE_UNSUPPORTED_COMPILER) + IF(MY_COMPILER_IS_CLANG) +- CHECK_C_SOURCE_RUNS(" +- int main() +- { +- return (__clang_major__ < 10); +- }" HAVE_SUPPORTED_CLANG_VERSION) +- IF(NOT HAVE_SUPPORTED_CLANG_VERSION) +- MESSAGE(FATAL_ERROR "XCode 10.0 or newer is required!") ++ IF (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10) ++ MESSAGE(FATAL_ERROR ++ "XCode 10.0 or newer is required! Compiler version was ${CMAKE_CXX_COMPILER_VERSION}") + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "Unsupported compiler!") +diff --git a/libmysql/CMakeLists.txt b/libmysql/CMakeLists.txt +index 8e22453..2b63413 100644 +--- a/libmysql/CMakeLists.txt ++++ b/libmysql/CMakeLists.txt +@@ -259,6 +259,7 @@ ELSEIF(FREEBSD) + MESSAGE(STATUS "BSD built in DNS SRV APIs") + ELSE() + FIND_LIBRARY(RESOLV_LIBRARY NAMES resolv) ++ FIND_LIBRARY(RESOLV_LIBRARY NAMES resolv PATHS ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES} NO_DEFAULT_PATH) + IF (RESOLV_LIBRARY) + LIST(APPEND LIBS_TO_LINK ${RESOLV_LIBRARY}) + SET(HAVE_UNIX_DNS_SRV 1 PARENT_SCOPE) +@@ -413,6 +414,7 @@ MYSQL_ADD_EXECUTABLE(libmysql_api_test + ${CMAKE_CURRENT_BINARY_DIR}/api_test.c + LINK_LIBRARIES libmysql ${LIBRT} + SKIP_INSTALL ++ EXCLUDE_FROM_ALL + ) + # Clang/UBSAN needs this on some platforms. + SET_TARGET_PROPERTIES(libmysql_api_test PROPERTIES LINKER_LANGUAGE CXX) +@@ -439,6 +441,6 @@ ADD_CUSTOM_COMMAND( + COMMAND libmysql_api_test + > ${CMAKE_CURRENT_BINARY_DIR}/libmysql_api_test.out + ) +-MY_ADD_CUSTOM_TARGET(run_libmysql_api_test ALL ++MY_ADD_CUSTOM_TARGET(run_libmysql_api_test + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libmysql_api_test.out + ) +diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt +index 8e93387..d971acb 100644 +--- a/scripts/CMakeLists.txt ++++ b/scripts/CMakeLists.txt +@@ -69,7 +69,6 @@ ADD_CUSTOM_COMMAND( + + # Add target for the above to be built + MY_ADD_CUSTOM_TARGET(GenFixPrivs +- ALL + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/mysql_fix_privilege_tables_sql.h + ) + +diff --git a/scripts/sys_schema/CMakeLists.txt b/scripts/sys_schema/CMakeLists.txt +index aaba357..a0e5265 100644 +--- a/scripts/sys_schema/CMakeLists.txt ++++ b/scripts/sys_schema/CMakeLists.txt +@@ -221,7 +221,6 @@ MY_ADD_CUSTOM_TARGET(sql_commands + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/sql_commands.h) + + MY_ADD_CUSTOM_TARGET(GenSysSchemaC +- ALL + DEPENDS comp_sql sql_commands + COMMENT "Generating Sys Schema C code" + ) +diff --git a/share/CMakeLists.txt b/share/CMakeLists.txt +index 28cde40..be916cb 100644 +--- a/share/CMakeLists.txt ++++ b/share/CMakeLists.txt +@@ -47,8 +47,12 @@ SET(dirs + ukrainian + ) + +-FOREACH (dir ${dirs}) +- INSTALL(DIRECTORY ${CMAKE_BINARY_DIR}/${INSTALL_MYSQLSHAREDIR}/${dir} ++ set(src_dir ${CMAKE_BINARY_DIR}/share/libmysql) ++ if(CMAKE_CROSSCOMPILING) ++ set(src_dir ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql) ++ endif() ++ FOREACH (dir ${dirs}) ++ INSTALL(DIRECTORY ${src_dir}/${dir} + DESTINATION ${INSTALL_MYSQLSHAREDIR} + COMPONENT Server + ) +diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt +index 665b35d..88f0743 100644 +--- a/strings/CMakeLists.txt ++++ b/strings/CMakeLists.txt +@@ -69,6 +69,13 @@ SET(ZH_HANS_SRC_FILE ${CMAKE_SOURCE_DIR}/strings/lang_data/zh_hans.txt) + SET(ZH_HANS_DST_FILE ${CMAKE_BINARY_DIR}/strings/uca900_zh_tbls.cc) + SET(JA_HANS_SRC_FILE ${CMAKE_SOURCE_DIR}/strings/lang_data/ja_hans.txt) + SET(JA_HANS_DST_FILE ${CMAKE_BINARY_DIR}/strings/uca900_ja_tbls.cc) ++if(CMAKE_CROSSCOMPILING) ++ file(COPY ++ "${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/uca900_zh_tbls.cc" ++ "${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/uca900_ja_tbls.cc" ++ DESTINATION "${CMAKE_BINARY_DIR}/strings" ++ ) ++else() + ADD_CUSTOM_COMMAND(OUTPUT ${ZH_HANS_DST_FILE} + ${JA_HANS_DST_FILE} + COMMAND uca9dump zh +@@ -79,6 +86,8 @@ ADD_CUSTOM_COMMAND(OUTPUT ${ZH_HANS_DST_FILE} + --out_file=${JA_HANS_DST_FILE} + DEPENDS uca9dump ${ZH_HANS_SRC_FILE} ${JA_HANS_SRC_FILE} + ) ++install(FILES "${ZH_HANS_DST_FILE}" "${JA_HANS_DST_FILE}" DESTINATION "share/libmysql/${PROJECT_VERSION}") ++endif() + + SET_SOURCE_FILES_PROPERTIES( + ${JA_HANS_DST_FILE} ${ZH_HANS_DST_FILE} +diff --git a/utilities/CMakeLists.txt b/utilities/CMakeLists.txt +index da34524..dc397da 100644 +--- a/utilities/CMakeLists.txt ++++ b/utilities/CMakeLists.txt +@@ -39,6 +39,21 @@ MYSQL_ADD_EXECUTABLE(comp_client_err + SKIP_INSTALL + ) + ++if(CMAKE_CROSSCOMPILING) ++ file(COPY ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/include/mysql/mysqld_error.h ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/mysqlclient_ername.h ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/mysqld_ername.h ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/mysqld_errmsg.h ++ DESTINATION ${PROJECT_BINARY_DIR}/include ++ ) ++ file(COPY ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/errmsg.sys ++ DESTINATION ${PROJECT_BINARY_DIR}/share/libmysql/english ++ ) ++ add_custom_target(GenClientError) ++ add_custom_target(GenError) ++else() + ADD_CUSTOM_COMMAND(OUTPUT ${PROJECT_BINARY_DIR}/include/mysqlclient_ername.h + COMMAND comp_client_err + --in_file=${PROJECT_SOURCE_DIR}/include/errmsg.h +@@ -97,6 +112,16 @@ MYSQL_ADD_EXECUTABLE(range_check_err + SKIP_INSTALL + ) + ++ install( ++ FILES ++ ${PROJECT_BINARY_DIR}/include/mysqlclient_ername.h ++ ${PROJECT_BINARY_DIR}/include/mysqld_ername.h ++ ${PROJECT_BINARY_DIR}/include/mysqld_errmsg.h ++ ${PROJECT_BINARY_DIR}/share/libmysql/english/errmsg.sys ++ DESTINATION share/libmysql/${PROJECT_VERSION} ++ ) ++endif() ++ + # Set InnoDB mutex type + ADD_DEFINITIONS(-DMUTEX_EVENT) + diff --git a/vcpkg_overlays/libmysql/dependencies.patch b/vcpkg_overlays/libmysql/dependencies.patch new file mode 100644 index 00000000000..0129d71e974 --- /dev/null +++ b/vcpkg_overlays/libmysql/dependencies.patch @@ -0,0 +1,140 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e17ec1d0..465e42f0 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -818,8 +818,12 @@ INCLUDE(fileutils) + INCLUDE(zlib) + INCLUDE(zstd) + INCLUDE(lz4) +-INCLUDE(icu) +-INCLUDE(libevent) ++macro(MYSQL_CHECK_ICU) ++endmacro() ++macro(MYSQL_CHECK_LIBEVENT) ++endmacro() ++macro(WARN_MISSING_SYSTEM_LIBEVENT) ++endmacro() + INCLUDE(ssl) + INCLUDE(sasl) + INCLUDE(ldap) +@@ -837,7 +841,8 @@ INCLUDE(curl) + INCLUDE(rapidjson) + INCLUDE(fprofile) + INCLUDE(gloves) +-INCLUDE(fido2) ++macro(MYSQL_CHECK_FIDO) ++endmacro() + INCLUDE(win_jemalloc) + + IF(UNIX) +@@ -1942,7 +1947,7 @@ MYSQL_CHECK_RAPIDJSON() + MYSQL_CHECK_FIDO() + MYSQL_CHECK_FIDO_DLLS() + +-IF(APPLE) ++IF(0) + GET_FILENAME_COMPONENT(HOMEBREW_BASE ${HOMEBREW_HOME} DIRECTORY) + IF(EXISTS ${HOMEBREW_BASE}/include/boost) + FOREACH(SYSTEM_LIB ICU LIBEVENT LZ4 PROTOBUF ZSTD FIDO) +diff --git a/cmake/boost.cmake b/cmake/boost.cmake +index c979055c..53e55fcf 100644 +--- a/cmake/boost.cmake ++++ b/cmake/boost.cmake +@@ -301,7 +301,7 @@ IF(NOT BOOST_MAJOR_VERSION EQUAL 10) + COULD_NOT_FIND_BOOST() + ENDIF() + +-IF(NOT BOOST_MINOR_VERSION EQUAL 77) ++IF(NOT BOOST_MINOR_VERSION EQUAL 77 AND NOT IGNORE_BOOST_VERSION) + MESSAGE(WARNING "Boost minor version found is ${BOOST_MINOR_VERSION} " + "we need 77" + ) +diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake +index 7107f4bd..bb7f3733 100644 +--- a/cmake/libutils.cmake ++++ b/cmake/libutils.cmake +@@ -534,7 +534,7 @@ MACRO(MERGE_CONVENIENCE_LIBRARIES TARGET_ARG) + + # On Windows, ssleay32.lib/libeay32.lib or libssl.lib/libcrypto.lib + # must be merged into mysqlclient.lib +- IF(WIN32 AND ${TARGET} STREQUAL "mysqlclient") ++ IF(0) + SET(LINKER_EXTRA_FLAGS "") + FOREACH(LIB ${SSL_LIBRARIES}) + STRING_APPEND(LINKER_EXTRA_FLAGS " ${LIB}") +diff --git a/cmake/lz4.cmake b/cmake/lz4.cmake +index 10e7e8c8..48772329 100644 +--- a/cmake/lz4.cmake ++++ b/cmake/lz4.cmake +@@ -46,7 +46,7 @@ FUNCTION(FIND_SYSTEM_LZ4) + FIND_PATH(LZ4_INCLUDE_DIR + NAMES lz4frame.h) + FIND_LIBRARY(LZ4_SYSTEM_LIBRARY +- NAMES lz4) ++ NAMES lz4d lz4 NAMES_PER_DIR) + IF (LZ4_INCLUDE_DIR AND LZ4_SYSTEM_LIBRARY) + SET(SYSTEM_LZ4_FOUND 1 CACHE INTERNAL "") + ADD_LIBRARY(lz4_interface INTERFACE) +diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake +index 04e3af87..8ae52c82 100644 +--- a/cmake/ssl.cmake ++++ b/cmake/ssl.cmake +@@ -273,7 +273,18 @@ MACRO (MYSQL_CHECK_SSL) + FIND_ALTERNATIVE_SYSTEM_SSL() + ENDIF() + +- IF(WITH_SSL STREQUAL "system" OR WITH_SSL_PATH OR ALTERNATIVE_SYSTEM_SSL) ++ IF(WITH_SSL STREQUAL "system") ++ find_package(OpenSSL REQUIRED) ++ set(OPENSSL_LIBRARY OpenSSL::SSL CACHE STRING "") ++ set(CRYPTO_LIBRARY OpenSSL::Crypto CACHE STRING "") ++ find_program(OPENSSL_EXECUTABLE openssl ++ DOC "path to the openssl executable") ++ set(SSL_DEFINES "-DHAVE_OPENSSL") ++ add_library(ext::openssl ALIAS OpenSSL::SSL) ++ set(SSL_LIBRARIES ext::openssl) ++ set(OPENSSL_APPLINK_C "${OPENSSL_APPLINK_SOURCE}") ++ include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) ++ ELSEIF(WITH_SSL STREQUAL "system" OR WITH_SSL_PATH OR ALTERNATIVE_SYSTEM_SSL) + IF((APPLE OR WIN32) AND WITH_SSL STREQUAL "system") + # FindOpenSSL.cmake knows about + # http://www.slproweb.com/products/Win32OpenSSL.html +diff --git a/cmake/zlib.cmake b/cmake/zlib.cmake +index 3781fe09..ac312ad0 100644 +--- a/cmake/zlib.cmake ++++ b/cmake/zlib.cmake +@@ -124,7 +124,7 @@ MACRO (MYSQL_CHECK_ZLIB) + MESSAGE(FATAL_ERROR "WITH_ZLIB must be bundled or system") + ENDIF() + +- ADD_LIBRARY(ext::zlib ALIAS zlib_interface) ++ ADD_LIBRARY(ext::zlib ALIAS ZLIB::ZLIB) + + IF(ZLIB_VERSION VERSION_LESS MIN_ZLIB_VERSION_REQUIRED) + MESSAGE(FATAL_ERROR +diff --git a/cmake/zstd.cmake b/cmake/zstd.cmake +index 425426d1..0ae1a907 100644 +--- a/cmake/zstd.cmake ++++ b/cmake/zstd.cmake +@@ -90,15 +90,16 @@ MACRO (MYSQL_CHECK_ZSTD) + IF(WITH_ZSTD STREQUAL "bundled") + MYSQL_USE_BUNDLED_ZSTD() + ELSEIF(WITH_ZSTD STREQUAL "system") +- FIND_SYSTEM_ZSTD() +- IF (NOT SYSTEM_ZSTD_FOUND) +- MESSAGE(FATAL_ERROR "Cannot find system zstd libraries.") +- ENDIF() ++ find_package(ZSTD NAMES zstd REQUIRED) + ELSE() + MESSAGE(FATAL_ERROR "WITH_ZSTD must be bundled or system") + ENDIF() + +- ADD_LIBRARY(ext::zstd ALIAS zstd_interface) ++ if(TARGET zstd::libzstd_shared) ++ add_library(ext::zstd ALIAS zstd::libzstd_shared) ++ else() ++ add_library(ext::zstd ALIAS zstd::libzstd_static) ++ endif() + + IF(ZSTD_VERSION VERSION_LESS MIN_ZSTD_VERSION_REQUIRED) + MESSAGE(FATAL_ERROR diff --git a/vcpkg_overlays/libmysql/fix_dup_symbols.patch b/vcpkg_overlays/libmysql/fix_dup_symbols.patch new file mode 100644 index 00000000000..352fd8af89f --- /dev/null +++ b/vcpkg_overlays/libmysql/fix_dup_symbols.patch @@ -0,0 +1,20 @@ +diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt +index 058967b..bcd8841 100644 +--- a/client/CMakeLists.txt ++++ b/client/CMakeLists.txt +@@ -43,7 +43,6 @@ MYSQL_ADD_EXECUTABLE(mysql + pattern_matcher.cc + readline.cc + client_query_attributes.cc +- multi_factor_passwordopt-vars.cc + ${CMAKE_CURRENT_SOURCE_DIR}/common/user_registration.cc + LINK_LIBRARIES mysqlclient client_base ${EDITLINE_LIBRARY} + ) +@@ -226,7 +226,6 @@ SET(MYSQLBINLOG_SOURCES + ${CMAKE_SOURCE_DIR}/sql/binlog_reader.cc + ${CMAKE_SOURCE_DIR}/sql/stream_cipher.cc + ${CMAKE_SOURCE_DIR}/sql/rpl_log_encryption.cc +- ${CMAKE_SOURCE_DIR}/libbinlogevents/src/trx_boundary_parser.cpp + ) + + SET(MYSQLBINLOG_LIBRARIES diff --git a/vcpkg_overlays/libmysql/install-exports.patch b/vcpkg_overlays/libmysql/install-exports.patch new file mode 100644 index 00000000000..3cf34ed3be5 --- /dev/null +++ b/vcpkg_overlays/libmysql/install-exports.patch @@ -0,0 +1,121 @@ +diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake +index baf49cd5..c45fda71 100644 +--- a/cmake/install_macros.cmake ++++ b/cmake/install_macros.cmake +@@ -113,8 +113,30 @@ FUNCTION(MYSQL_INSTALL_TARGET target_arg) + IF(ARG_NAMELINK_SKIP) + SET(LIBRARY_INSTALL_ARGS NAMELINK_SKIP) + ENDIF() ++ if(target STREQUAL "libmysql" OR target STREQUAL "mysqlclient") ++ target_include_directories(${target} INTERFACE $) ++ if(target STREQUAL "mysqlclient") ++ set_target_properties(${target} PROPERTIES EXPORT_NAME libmysql) # uniform ++ endif() ++ FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/unofficial-libmysql-config.cmake" ++"include(CMakeFindDependencyMacro) ++find_dependency(OpenSSL) ++find_dependency(Threads) ++find_dependency(ZLIB) ++find_dependency(zstd CONFIG) ++include(\"\${CMAKE_CURRENT_LIST_DIR}/unofficial-libmysql-targets.cmake\") ++if(NOT TARGET ${target}) # legacy vcpkg ++ add_library(${target} INTERFACE IMPORTED) ++ set_target_properties(${target} PROPERTIES INTERFACE_LINK_LIBRARIES unofficial::libmysql::libmysql) ++endif() ++") ++ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/unofficial-libmysql-config.cmake" DESTINATION "share/unofficial-libmysql") ++ install(EXPORT "unofficial-libmysql-targets" NAMESPACE unofficial::libmysql:: DESTINATION "share/unofficial-libmysql") ++ set(LIBRARY_EXPORT_ARGS EXPORT unofficial-libmysql-targets) ++ endif() + INSTALL(TARGETS ${target} +- RUNTIME DESTINATION ${ARG_DESTINATION} ${COMP} ++ ${LIBRARY_EXPORT_ARGS} ++ RUNTIME DESTINATION bin ${COMP} + ARCHIVE DESTINATION ${ARG_DESTINATION} ${COMP} + LIBRARY DESTINATION ${ARG_DESTINATION} ${COMP} ${LIBRARY_INSTALL_ARGS}) + SET(INSTALL_LOCATION ${ARG_DESTINATION} ) +diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake +index a5333987..c954bfb1 100644 +--- a/cmake/libutils.cmake ++++ b/cmake/libutils.cmake +@@ -385,8 +385,12 @@ MACRO(MERGE_LIBRARIES_SHARED TARGET_ARG) + IF(ARG_NAMELINK_SKIP) + SET(INSTALL_ARGS NAMELINK_SKIP) + ENDIF() ++ if(INSTALL_STATIC_LIBRARIES) ++ set_target_properties(${TARGET} PROPERTIES EXCLUDE_FROM_ALL 1) ++ else() + MYSQL_INSTALL_TARGET(${TARGET} DESTINATION "${DESTINATION}" ${COMP} + ${INSTALL_ARGS}) ++ endif() + ENDIF() + + IF(WIN32) +@@ -563,6 +567,8 @@ MACRO(MERGE_CONVENIENCE_LIBRARIES TARGET_ARG) + ENDIF() + IF(INSTALL_STATIC_LIBRARIES) + MYSQL_INSTALL_TARGET(${TARGET} DESTINATION "${INSTALL_LIBDIR}" ${COMP}) ++ else() ++ set_target_properties(${TARGET} PROPERTIES EXCLUDE_FROM_ALL 1) + ENDIF() + ENDIF() + ENDMACRO(MERGE_CONVENIENCE_LIBRARIES) +diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt +index c4ef5694..8e933877 100644 +--- a/scripts/CMakeLists.txt ++++ b/scripts/CMakeLists.txt +@@ -315,10 +315,14 @@ MACRO(EXTRACT_LINK_LIBRARIES target var) + FOREACH(lib ${TARGET_LIB_DEPENDS}) + IF (lib MATCHES "^\\-l") + SET(${var} "${${var}} ${lib} ") ++ elseif(lib MATCHES "^ext::(openssl|zlib|zstd)" AND NOT WIN32) ++ list(APPEND ext_targets "${CMAKE_MATCH_1}") + ELSEIF (lib MATCHES "^ext::") + STRING(REGEX MATCH "ext::([a-z]+)" MATCH_LIB "${lib}") + SET(SYSTEM_LIB ${CMAKE_MATCH_1}) + SET(${var} "${${var}} -l${SYSTEM_LIB} ") ++ elseif(TARGET "${lib}" AND NOT WIN32) ++ # merged or shared, not external + ELSEIF (lib MATCHES "^\\-L") + # Search directory. Test on FreeBSD: -L/usr/local/lib -lunwind + SET(${var} "${${var}} ${lib} ") +@@ -354,6 +358,7 @@ IF(MSVC) + GET_TARGET_PROPERTY(LIBMYSQL_OS_SHLIB_VERSION mysqlclient VERSION) + GET_TARGET_PROPERTY(LIBMYSQL_OS_OUTPUT_NAME mysqlclient OUTPUT_NAME) + ELSE() ++ EXTRACT_LINK_LIBRARIES(libmysql CLIENT_LIBS) + GET_TARGET_PROPERTY(LIBMYSQL_OS_SHLIB_VERSION libmysql VERSION) + GET_TARGET_PROPERTY(LIBMYSQL_OS_OUTPUT_NAME libmysql OUTPUT_NAME) + ENDIF() +@@ -373,6 +378,10 @@ IF (WITH_SSL STREQUAL "system") + "${CONFIG_LIBS_PRIVATE}") + ENDIF() + ENDIF() ++list(APPEND CONFIG_REQUIRES_PRIVATE ${ext_targets}) ++list(REMOVE_DUPLICATES CONFIG_REQUIRES_PRIVATE) ++list(JOIN CONFIG_REQUIRES_PRIVATE " " CONFIG_REQUIRES_PRIVATE) ++string(REPLACE "zstd" "libzstd" CONFIG_REQUIRES_PRIVATE "${CONFIG_REQUIRES_PRIVATE}" ) + + MESSAGE(STATUS "CONFIG_CLIENT_LIBS ${CONFIG_CLIENT_LIBS}") + MESSAGE(STATUS "CONFIG_LIBS_PRIVATE ${CONFIG_LIBS_PRIVATE}") +diff --git a/scripts/mysql_config.sh b/scripts/mysql_config.sh +index 6e8520be..bf1a45af 100644 +--- a/scripts/mysql_config.sh ++++ b/scripts/mysql_config.sh +@@ -119,6 +119,16 @@ fi + # Create options + libs="-L$pkglibdir@RPATH_OPTION@" + libs="$libs -l@LIBMYSQL_OS_OUTPUT_NAME@ @CONFIG_CLIENT_LIBS@" ++if test -z "${PKG_CONFIG}" ; then ++ if pkg-config --version >/dev/null 2>&1 ; then ++ PKG_CONFIG=pkg-config ++ elif pkgconf --version >/dev/null 2>&1 ; then ++ PKG_CONFIG=pkgconf ++ else ++ PKG_CONFIG=false ++ fi ++fi ++libs="$libs $(${PKG_CONFIG} --libs @CONFIG_REQUIRES_PRIVATE@)" + + cflags="-I$pkgincludedir @CFLAGS@" + cxxflags="-I$pkgincludedir @CXXFLAGS@" diff --git a/vcpkg_overlays/libmysql/libmysql-config.cmake b/vcpkg_overlays/libmysql/libmysql-config.cmake new file mode 100644 index 00000000000..ea2bbbe0ab0 --- /dev/null +++ b/vcpkg_overlays/libmysql/libmysql-config.cmake @@ -0,0 +1,6 @@ +file(READ "${CMAKE_CURRENT_LIST_DIR}/usage" usage) +message(WARNING "find_package(libmysql) is deprecated.\n${usage}") +include(CMakeFindDependencyMacro) +find_dependency(unofficial-libmysql CONFIG REQUIRED) +set(libmysql_FOUND 1) +set(MYSQL_LIBRARIES unofficial::libmysql::libmysql) diff --git a/vcpkg_overlays/libmysql/portfile.cmake b/vcpkg_overlays/libmysql/portfile.cmake new file mode 100644 index 00000000000..34565231170 --- /dev/null +++ b/vcpkg_overlays/libmysql/portfile.cmake @@ -0,0 +1,160 @@ +if (EXISTS "${CURRENT_INSTALLED_DIR}/include/mysql/mysql.h") + message(FATAL_ERROR "FATAL ERROR: ${PORT} and libmariadb are incompatible.") +endif() + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO mysql/mysql-server + REF mysql-${VERSION} + SHA512 5df45c1ce1e2c620856b9274666cf56738d6a0308c33c9c96583b494c987fb0e862e676301109b9e4732070d54e6086596a62ad342f35adc59ca9f749e37b561 + HEAD_REF master + PATCHES + dependencies.patch + install-exports.patch + fix_dup_symbols.patch + cross-build.patch +) + +vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS + INVERTED_FEATURES + host-tools FORCE_CMAKE_CROSSCOMPILING +) + +file(GLOB third_party "${SOURCE_PATH}/extra/*" "${SOURCE_PATH}/include/boost_1_70_0") +list(REMOVE_ITEM third_party "${SOURCE_PATH}/extra/libedit") +if (third_party) + file(REMOVE_RECURSE ${third_party}) +endif() + +#Skip the version check for Visual Studio +set(FORCE_UNSUPPORTED_COMPILER "") +if(VCPKG_TARGET_IS_WINDOWS) + set(FORCE_UNSUPPORTED_COMPILER 1) +endif() + +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" BUILD_STATIC_LIBS) +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "static" STATIC_CRT_LINKAGE) + +set(cross_options "") +if(VCPKG_CROSSCOMPILING) + list(APPEND cross_options + -DCMAKE_CROSSCOMPILING=1 + -DVCPKG_HOST_TRIPLET=${HOST_TRIPLET} + # required, skip try_run + -DHAVE_RAPIDJSON_WITH_STD_REGEX=1 + ) + if(NOT VCPKG_TARGET_IS_WINDOWS) + list(APPEND cross_options + # optimistic, skip try_run + -DHAVE_CLOCK_GETTIME=1 + -DHAVE_CLOCK_REALTIME=1 + # pessimistic, skip try_run + -DHAVE_C_FLOATING_POINT_FUSED_MADD=1 + -DHAVE_CXX_FLOATING_POINT_FUSED_MADD=1 + -DHAVE_SETNS=0 + ) + endif() +endif() +if ("-DFORCE_CMAKE_CROSSCOMPILING=ON" IN_LIST FEATURE_OPTIONS) + list(APPEND cross_options + -DCMAKE_CROSSCOMPILING=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + ${cross_options} + -DINSTALL_INCLUDEDIR=include/mysql + -DINSTALL_DOCDIR=share/${PORT}/doc + -DINSTALL_MANDIR=share/${PORT}/doc + -DINSTALL_INFODIR=share/${PORT}/doc + -DINSTALL_DOCREADMEDIR=share/${PORT} + -DINSTALL_SHAREDIR=share + -DINSTALL_MYSQLSHAREDIR=share/${PORT} + -DWITHOUT_SERVER=ON + -DWITH_BUILD_ID=OFF + -DWITH_UNIT_TESTS=OFF + -DENABLED_PROFILING=OFF + -DWIX_DIR=OFF + -DIGNORE_BOOST_VERSION=ON + -DWITH_TEST_TRACE_PLUGIN=OFF + -DMYSQL_MAINTAINER_MODE=OFF + -DBUNDLE_RUNTIME_LIBRARIES=OFF + -DDOWNLOAD_BOOST=OFF + -DWITH_CURL=none + -DWITH_EDITLINE=bundled # not in vcpkg + -DWITH_LZ4=system + -DWITH_RAPIDJSON=system + -DWITH_SSL=system + -DWITH_SYSTEMD=OFF + -DWITH_ZLIB=system + -DWITH_ZSTD=system + -DFORCE_UNSUPPORTED_COMPILER=${FORCE_UNSUPPORTED_COMPILER} + -DINSTALL_STATIC_LIBRARIES=${BUILD_STATIC_LIBS} + -DLINK_STATIC_RUNTIME_LIBRARIES=${STATIC_CRT_LINKAGE} + MAYBE_UNUSED_VARIABLES + BUNDLE_RUNTIME_LIBRARIES # only on windows + LINK_STATIC_RUNTIME_LIBRARIES # only on windows + WIX_DIR # only on windows + WITH_BUILD_ID # only on windows +) + +vcpkg_cmake_install(ADD_BIN_TO_PATH) +vcpkg_cmake_config_fixup(PACKAGE_NAME unofficial-libmysql) +vcpkg_fixup_pkgconfig() + +set(MYSQL_TOOLS + my_print_defaults + mysql + mysql_config_editor + mysql_migrate_keyring + mysql_secure_installation + mysql_ssl_rsa_setup + mysqladmin + mysqlbinlog + mysqlcheck + mysqldump + mysqlimport + mysqlpump + mysqlshow + mysqlslap + mysqltest + perror + zlib_decompress +) +if (NOT VCPKG_CROSSCOMPILING) + list(APPEND MYSQL_TOOLS + comp_err + ) +endif() +if (VCPKG_TARGET_IS_WINDOWS) + list(APPEND MYSQL_TOOLS + echo + ) +else() + list(APPEND MYSQL_TOOLS + mysql_config + ) +endif() + +vcpkg_copy_tools(TOOL_NAMES ${MYSQL_TOOLS} AUTO_CLEAN) + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/lib/debug" +) + +if (NOT VCPKG_TARGET_IS_WINDOWS) + set(MYSQL_CONFIG_FILE "${CURRENT_PACKAGES_DIR}/tools/libmysql/mysql_config") + vcpkg_replace_string("${MYSQL_CONFIG_FILE}" "/bin/mysql_.*config" "/tools/libmysql/mysql_.*config") + vcpkg_replace_string("${MYSQL_CONFIG_FILE}" "'${CURRENT_PACKAGES_DIR}" "\"\$basedir\"\'") +endif() + +file(INSTALL "${CURRENT_PORT_DIR}/libmysql-config.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") + +set(libedit_copying "${SOURCE_PATH}/COPYING for libedit") +file(COPY_FILE "${SOURCE_PATH}/extra/libedit/libedit-20210910-3.1/COPYING" "${libedit_copying}") +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE" "${libedit_copying}") diff --git a/vcpkg_overlays/libmysql/usage b/vcpkg_overlays/libmysql/usage new file mode 100644 index 00000000000..7ea8c37e177 --- /dev/null +++ b/vcpkg_overlays/libmysql/usage @@ -0,0 +1,4 @@ +libmysql provides CMake targets: + + find_package(unofficial-libmysql REQUIRED) + target_link_libraries(main PRIVATE unofficial::libmysql::libmysql) diff --git a/vcpkg_overlays/libmysql/vcpkg.json b/vcpkg_overlays/libmysql/vcpkg.json new file mode 100644 index 00000000000..b143f59ef7f --- /dev/null +++ b/vcpkg_overlays/libmysql/vcpkg.json @@ -0,0 +1,47 @@ +{ + "name": "libmysql", + "version": "8.0.34", + "port-version": 1, + "description": "A MySQL client library for C development", + "homepage": "https://github.com/mysql/mysql-server", + "license": "GPL-2.0-or-later", + "supports": "!android & !mingw & !uwp & !xbox", + "dependencies": [ + "boost-algorithm", + "boost-functional", + "boost-geometry", + "boost-graph", + "boost-optional", + "lz4", + { + "name": "ncurses", + "platform": "!windows | mingw" + }, + "openssl", + "rapidjson", + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "zlib", + "zstd" + ], + "default-features": [ + "host-tools" + ], + "features": { + "host-tools": { + "description": "Build CLI tools", + "dependencies": [ + { + "name": "libmysql", + "host": true + } + ] + } + } +} \ No newline at end of file From 1c1d3db357512f0379addeac2bdb065fca7d918e Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 8 Mar 2024 15:36:36 +0000 Subject: [PATCH 21/23] HPCC-31425 Add support for tracking the active span in the thread context Signed-off-by: Gavin Halliday --- system/jlib/jthread.cpp | 19 ++++++++++++++++++- system/jlib/jthread.hpp | 9 +++++++++ system/jlib/jtrace.cpp | 42 +++++++++++++++++++++++++++++++++++++++++ system/jlib/jtrace.hpp | 25 ++++++++++++++++-------- 4 files changed, 86 insertions(+), 9 deletions(-) diff --git a/system/jlib/jthread.cpp b/system/jlib/jthread.cpp index 177e539e1e2..ef433a8f212 100644 --- a/system/jlib/jthread.cpp +++ b/system/jlib/jthread.cpp @@ -2684,12 +2684,14 @@ static TraceFlags defaultTraceFlags = TraceFlags::Standard; static thread_local LogMsgJobId defaultJobId = UnknownJob; static thread_local TraceFlags threadTraceFlags = TraceFlags::Standard; static thread_local const IContextLogger *default_thread_logctx = nullptr; +static thread_local ISpan * threadActiveSpan = nullptr; void saveThreadContext(SavedThreadContext & saveCtx) { saveCtx.jobId = defaultJobId; saveCtx.logctx = default_thread_logctx; saveCtx.traceFlags = threadTraceFlags; + saveCtx.activeSpan = threadActiveSpan; } void restoreThreadContext(const SavedThreadContext & saveCtx) @@ -2700,9 +2702,9 @@ void restoreThreadContext(const SavedThreadContext & saveCtx) defaultJobId = saveCtx.jobId; default_thread_logctx = saveCtx.logctx; threadTraceFlags = saveCtx.traceFlags; + threadActiveSpan = saveCtx.activeSpan; } - LogMsgJobId queryThreadedJobId() { return defaultJobId; @@ -2718,6 +2720,21 @@ const IContextLogger * queryThreadedContextLogger() return default_thread_logctx; } +ISpan * queryThreadedActiveSpan() +{ + ISpan * result = threadActiveSpan; + if (!result) + result = queryNullSpan(); + return result; +} + +ISpan * setThreadedActiveSpan(ISpan * span) +{ + ISpan * ret = threadActiveSpan; + threadActiveSpan = span; + return ret; +} + //--------------------------- bool doTrace(TraceFlags featureFlag, TraceFlags level) diff --git a/system/jlib/jthread.hpp b/system/jlib/jthread.hpp index 3390dacd27d..5d312aa9d2f 100644 --- a/system/jlib/jthread.hpp +++ b/system/jlib/jthread.hpp @@ -34,18 +34,27 @@ #endif // Functions used to reset thread-local context variables, when a threadpool starts +//NOTE: Currently activeSpan is not linked in the SavedThreadContext. +//That will not cause any problems as long as the lifetime of the thread executing with that saved span value is less than the lifetime of the span. +//It would be trivial to use a linked pointer inside the savedthread context, at the expense of a little inefficiency. +//Revisit when this feature starts being used in anger. typedef unsigned __int64 LogMsgJobId; interface IContextLogger; +interface ISpan; struct jlib_decl SavedThreadContext { const IContextLogger * logctx = nullptr; LogMsgJobId jobId = (LogMsgJobId)-1; TraceFlags traceFlags = queryDefaultTraceFlags(); + ISpan * activeSpan = nullptr; }; extern void restoreThreadContext(const SavedThreadContext & saveCtx); extern void saveThreadContext(SavedThreadContext & saveCtx); +extern jlib_decl ISpan * queryThreadedActiveSpan(); +extern jlib_decl ISpan * setThreadedActiveSpan(ISpan * span); + //-------------------------------------------------------------- interface jlib_decl IThread : public IInterface diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 1d619407d4f..04bdaf19e98 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -1458,6 +1458,43 @@ ISpan * CTraceManager::createServerSpan(const char * name, const IProperties * h //--------------------------------------------------------------------------------------------------------------------- +OwnedSpanScope::OwnedSpanScope(ISpan * _ptr) : span(_ptr) +{ + if (_ptr) + prevSpan = setThreadedActiveSpan(_ptr); +} + +void OwnedSpanScope::setown(ISpan * _span) +{ + assertex(_span); + //Just in case the span is already set, ensure it is ended and that the previous span is restored. + clear(); + span.setown(_span); + prevSpan = setThreadedActiveSpan(_span); +} + +void OwnedSpanScope::set(ISpan * _span) +{ + setown(LINK(_span)); +} + +void OwnedSpanScope::clear() +{ + if (span) + { + setThreadedActiveSpan(prevSpan); + span->endSpan(); + span.clear(); + } +} + +OwnedSpanScope::~OwnedSpanScope() +{ + clear(); +} + +//--------------------------------------------------------------------------------------------------------------------- + MODULE_INIT(INIT_PRIORITY_STANDARD) { return true; @@ -1475,6 +1512,11 @@ ISpan * getNullSpan() return nullSpan.getLink(); } +ISpan * queryNullSpan() +{ + return nullSpan; +} + void initTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig) { theTraceManager.query([=] () { return new CTraceManager(componentName, componentConfig, globalConfig); }); diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 5ada18fc74b..fbdb6cc40ce 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -134,16 +134,24 @@ interface ISpan : extends IInterface virtual const char* queryLocalId() const = 0; }; -class OwnedSpanScope : public Owned +class jlib_decl OwnedSpanScope { public: - inline OwnedSpanScope() { } - inline OwnedSpanScope(ISpan * _ptr) : Owned(_ptr) { } - ~OwnedSpanScope() - { - if (get()) - get()->endSpan(); - } + OwnedSpanScope() = default; + OwnedSpanScope(ISpan * _ptr); + ~OwnedSpanScope(); + + inline ISpan * operator -> () const { return span; } + inline operator ISpan *() const { return span; } + + void clear(); + ISpan * query() const { return span; } + void set(ISpan * _span); + void setown(ISpan * _span); + +private: + Owned span; + ISpan * prevSpan = nullptr; }; extern jlib_decl IProperties * getClientHeaders(const ISpan * span); @@ -180,6 +188,7 @@ interface ITraceManager : extends IInterface virtual bool isTracingEnabled() const = 0; }; +extern jlib_decl ISpan * queryNullSpan(); extern jlib_decl ISpan * getNullSpan(); extern jlib_decl void initTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig); extern jlib_decl ITraceManager & queryTraceManager(); From 632212b1980050f61e3a1325944efba9070e673b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 15 Feb 2024 16:22:06 +0000 Subject: [PATCH 22/23] HPCC-31294 Refactor the way job ids are passed to the logging Signed-off-by: Gavin Halliday --- common/dllserver/dllserver.cpp | 2 +- common/environment/environment.cpp | 12 +- common/remote/rmtspawn.cpp | 48 ++-- common/thorhelper/roxiehelper.ipp | 6 +- common/thorhelper/thorcommon.hpp | 9 +- common/thorhelper/thorsoapcall.cpp | 2 +- common/thorhelper/thortparse.cpp | 14 +- common/workunit/workunit.cpp | 4 +- dali/base/dacoven.cpp | 2 +- dali/base/dadfs.cpp | 4 +- dali/base/danqs.cpp | 2 +- dali/base/dasds.cpp | 48 ++-- dali/base/dasess.cpp | 2 +- dali/dfu/dfurun.cpp | 8 +- dali/dfu/dfuserver.cpp | 2 +- dali/ft/daft.cpp | 4 +- dali/ft/daftformat.cpp | 93 ++++---- dali/ft/daftformat.ipp | 2 +- dali/ft/daftprogress.cpp | 4 +- dali/ft/filecopy.cpp | 126 +++++------ dali/ft/filecopy.ipp | 2 - dali/ft/ftbase.cpp | 12 +- dali/ft/ftslavelib.cpp | 2 +- dali/ft/fttransform.cpp | 26 +-- dali/sasha/sacoalescer.cpp | 2 +- dali/sasha/saserver.cpp | 10 +- dali/sasha/saverify.cpp | 6 +- dali/server/daserver.cpp | 6 +- ecl/eclagent/eclagent.cpp | 54 ++--- ecl/eclcc/eclcc.cpp | 4 +- ecl/hql/hqlexpr.cpp | 2 +- ecl/hqlcpp/hqlhtcpp.cpp | 10 +- ecl/hthor/hthorkey.cpp | 4 +- ecl/wutest/wujobqtest.cpp | 4 +- esp/bindings/SOAP/Platform/soapservice.cpp | 6 +- esp/esdllib/esdl_def.cpp | 2 +- esp/platform/espcfg.cpp | 2 +- esp/platform/espcontext.cpp | 4 +- esp/platform/espp.hpp | 2 +- esp/services/ws_fileio/ws_fileioservice.cpp | 8 +- esp/services/ws_fs/ws_fsService.cpp | 2 +- .../ws_workunits/ws_workunitsHelpers.cpp | 2 +- esp/smc/SMCLib/LogicFileWrapper.hpp | 2 +- esp/smc/SMCLib/TpCommon.cpp | 2 +- fs/dafsserver/dafsserver.cpp | 56 ++--- plugins/fileservices/fileservices.cpp | 2 +- roxie/ccd/ccd.hpp | 8 +- roxie/ccd/ccdcontext.cpp | 4 +- roxie/ccd/ccdlistener.cpp | 2 + roxie/ccd/ccdqueue.cpp | 4 + roxie/ccd/ccdserver.cpp | 16 +- rtl/eclrtl/eclrtl.cpp | 2 +- rtl/eclrtl/rtlbcdtest.cpp | 4 +- system/jlib/jcontainerized.cpp | 6 +- system/jlib/jdebug.cpp | 8 +- system/jlib/jfile.cpp | 2 +- system/jlib/jlog.cpp | 214 ++++-------------- system/jlib/jlog.hpp | 127 +++-------- system/jlib/jlog.ipp | 11 +- system/jlib/jmisc.cpp | 8 +- system/mp/mpcomm.cpp | 102 ++++----- .../hashdistrib/thhashdistribslave.cpp | 4 +- .../activities/indexread/thindexreadslave.cpp | 4 +- .../keyedjoin/thkeyedjoinslave-legacy.cpp | 2 +- .../activities/keyedjoin/thkeyedjoinslave.cpp | 8 +- thorlcr/activities/msort/thsortu.cpp | 16 +- .../activities/soapcall/thsoapcallslave.cpp | 8 +- thorlcr/activities/thdiskbase.cpp | 2 +- thorlcr/graph/thgraph.cpp | 26 +-- thorlcr/graph/thgraphmaster.cpp | 30 +-- thorlcr/graph/thgraphslave.cpp | 4 +- thorlcr/master/mawatchdog.cpp | 16 +- thorlcr/master/thactivitymaster.cpp | 2 +- thorlcr/master/thdemonserver.cpp | 14 +- thorlcr/master/thgraphmanager.cpp | 42 ++-- thorlcr/master/thmastermain.cpp | 41 ++-- thorlcr/mfilemanager/thmfilemanager.cpp | 2 +- thorlcr/msort/tsortl.cpp | 24 +- thorlcr/msort/tsortmp.cpp | 6 +- thorlcr/msort/tsorts.cpp | 6 +- thorlcr/msort/tsorts1.cpp | 30 +-- thorlcr/slave/backup.cpp | 4 +- thorlcr/slave/slavmain.cpp | 21 +- thorlcr/slave/slwatchdog.cpp | 8 +- thorlcr/slave/thslavemain.cpp | 39 ++-- thorlcr/thorutil/thbuf.cpp | 4 +- thorlcr/thorutil/thormisc.cpp | 28 ++- thorlcr/thorutil/thormisc.hpp | 6 +- tools/swapnode/swapnodelib.cpp | 4 +- 89 files changed, 678 insertions(+), 869 deletions(-) diff --git a/common/dllserver/dllserver.cpp b/common/dllserver/dllserver.cpp index 028ee42c324..46cef2609f4 100644 --- a/common/dllserver/dllserver.cpp +++ b/common/dllserver/dllserver.cpp @@ -632,7 +632,7 @@ ILoadedDllEntry * DllServerBase::doLoadDll(const char * name, DllLocationType ty location->getDllFilename(rfile); StringBuffer x; rfile.getPath(x); - LOG(MCdebugInfo, unknownJob, "Loading dll (%s) from location %s", name, x.str()); + LOG(MCdebugInfo, "Loading dll (%s) from location %s", name, x.str()); return createDllEntry(x.str(), false, NULL, resourcesOnly); } diff --git a/common/environment/environment.cpp b/common/environment/environment.cpp index d3d33cf2a2e..53ef6a8e44d 100644 --- a/common/environment/environment.cpp +++ b/common/environment/environment.cpp @@ -1877,7 +1877,7 @@ bool CLocalEnvironment::getRunInfo(IStringVal & path, IStringVal & dir, const ch Owned machine = getMachineByAddress(machineaddr); if (!machine) { - LOG(MCdebugInfo, unknownJob, "Unable to find machine for %s", machineaddr); + LOG(MCdebugInfo, "Unable to find machine for %s", machineaddr); return false; } @@ -1885,7 +1885,7 @@ bool CLocalEnvironment::getRunInfo(IStringVal & path, IStringVal & dir, const ch Owned domain = machine->getDomain(); if (!domain) { - LOG(MCdebugInfo, unknownJob, "Unable to find domain for %s", machineaddr); + LOG(MCdebugInfo, "Unable to find domain for %s", machineaddr); return false; } @@ -1896,7 +1896,7 @@ bool CLocalEnvironment::getRunInfo(IStringVal & path, IStringVal & dir, const ch Owned instance = getInstance(tag, version, targetdomain); if (!instance) { - LOG(MCdebugInfo, unknownJob, "Unable to find process %s for domain %s", tag, targetdomain.get()); + LOG(MCdebugInfo, "Unable to find process %s for domain %s", tag, targetdomain.get()); return false; } return instance->getRunInfo(path,dir,defprogname); @@ -1944,7 +1944,7 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * ne unsigned dropzonePathLen = _MAX_PATH + 1; #ifdef _DEBUG - LOG(MCdebugInfo, unknownJob, "Netaddress: '%s', targetFilePath: '%s'", netaddress, targetFilePath); + LOG(MCdebugInfo, "Netaddress: '%s', targetFilePath: '%s'", netaddress, targetFilePath); #endif // Check the directory path first @@ -1973,7 +1973,7 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * ne #ifdef _DEBUG StringBuffer serverIpString; serverIP.getHostText(serverIpString); - LOG(MCdebugInfo, unknownJob, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str()); + LOG(MCdebugInfo, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str()); #endif if (strisame(netaddress, dropzoneServer) || targetIp.ipequals(serverIP)) { @@ -2551,7 +2551,7 @@ unsigned getAccessibleServiceURLList(const char *serviceType, std::vector espBindingIter = espProcessIter->query().getElements("EspBinding"); ForEach(*espBindingIter) { - xpath.setf("Software/EspService[@name=\"%s\"]/Properties/@type", espBindingIter->query().queryProp("@service")); + xpath.setf("Software/EspService[@name=\"%s\"]/Properties/@type", espBindingIter->query().queryProp("@service")); if (strisame(env->queryProp(xpath), serviceType)) { diff --git a/common/remote/rmtspawn.cpp b/common/remote/rmtspawn.cpp index 64f213b1fcb..fb029ab023c 100644 --- a/common/remote/rmtspawn.cpp +++ b/common/remote/rmtspawn.cpp @@ -137,7 +137,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint if (abort && abort->abortRequested()) { - LOG(MCdetailDebugInfo, unknownJob, "Action aborted before connecting to slave (%3d)", replyTag); + LOG(MCdetailDebugInfo, "Action aborted before connecting to slave (%3d)", replyTag); return NULL; } @@ -169,7 +169,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint unsigned attempts = 20; SocketEndpoint connectEP(childEP); connectEP.port = port; - LOG(MCdetailDebugInfo, unknownJob, "Start connect to correct slave (%3d)", replyTag); + LOG(MCdetailDebugInfo, "Start connect to correct slave (%3d)", replyTag); IException * error = NULL; ISocket * result = NULL; while (!result && attempts) @@ -178,7 +178,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint { StringBuffer tmp; connectEP.getEndpointHostText(tmp); - LOG(MCdetailDebugInfo, unknownJob, "Try to connect to slave %s",tmp.str()); + LOG(MCdetailDebugInfo, "Try to connect to slave %s",tmp.str()); Owned socket = ISocket::connect_wait(connectEP,MASTER_CONNECT_SLAVE_TIMEOUT); if (socket) { @@ -202,7 +202,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint if (connected) { assertex(slaveTag == replyTag); - LOG(MCdetailDebugInfo, unknownJob, "Connected to correct slave (%3d)", replyTag); + LOG(MCdetailDebugInfo, "Connected to correct slave (%3d)", replyTag); result = socket.getClear(); break; } @@ -218,9 +218,9 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint break; } if (slaveKind != kind) - LOG(MCdetailDebugInfo, unknownJob, "Connected to wrong kind of slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); + LOG(MCdetailDebugInfo, "Connected to wrong kind of slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); else - LOG(MCdetailDebugInfo, unknownJob, "Failed to connect to correct slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); + LOG(MCdetailDebugInfo, "Failed to connect to correct slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); //Wrong slave listening, need to leave time for the other, don't count as an attempt MilliSleep(fastRand() % 5000 + 5000); @@ -230,7 +230,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint StringBuffer s; s.appendf("Retry after exception talking to slave (%d): ",replyTag); e->errorMessage(s); - LOG(MCdetailDebugInfo, unknownJob, "%s", s.str()); + LOG(MCdetailDebugInfo, "%s", s.str()); e->Release(); //Probably another element just connected, and the listening socket has just been killed. //So try again. Wait just long enough to give another thread a chance. @@ -241,7 +241,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint catch (IException * e) { StringBuffer s; - LOG(MCdetailDebugInfo, unknownJob, e, s.appendf("Failed to connect to slave (%d) (try again): ", replyTag).str()); + LOG(MCdetailDebugInfo, e, s.appendf("Failed to connect to slave (%d) (try again): ", replyTag).str()); e->Release(); // No socket listening or contention - try again fairly soon MilliSleep(fastRand()%400+100); @@ -288,7 +288,7 @@ bool CRemoteParentInfo::processCommandLine(int argc, const char * * argv, String void CRemoteParentInfo::log() { StringBuffer temp; - LOG(MCdebugProgress, unknownJob, "Starting remote slave. Master=%s reply=%d port=%d", parent.getEndpointHostText(temp).str(), replyTag, port); + LOG(MCdebugProgress, "Starting remote slave. Master=%s reply=%d port=%d", parent.getEndpointHostText(temp).str(), replyTag, port); } bool CRemoteParentInfo::sendReply(unsigned version) @@ -300,7 +300,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) { try { - LOG(MCdetailDebugInfo, unknownJob, "Ready to listen. reply=%d port=%d", replyTag, port); + LOG(MCdetailDebugInfo, "Ready to listen. reply=%d port=%d", replyTag, port); Owned listen = ISocket::create(port, 1); if (listen) { @@ -313,11 +313,11 @@ bool CRemoteParentInfo::sendReply(unsigned version) { try { - LOG(MCdetailDebugInfo, unknownJob, "Ready to accept connection. reply=%d", replyTag); + LOG(MCdetailDebugInfo, "Ready to accept connection. reply=%d", replyTag); if (!listen->wait_read(SLAVE_LISTEN_FOR_MASTER_TIMEOUT)) { - LOG(MCdetailDebugInfo, unknownJob, "Gave up waiting for a connection. reply=%d", replyTag); + LOG(MCdetailDebugInfo, "Gave up waiting for a connection. reply=%d", replyTag); return false; } @@ -333,7 +333,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) buffer.read(connectTag); masterIP.getHostText(masterIPtext.clear()); - LOG(MCdetailDebugInfo, unknownJob, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str()); + LOG(MCdetailDebugInfo, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str()); same = (kind == connectKind) && masterIP.ipequals(parent) && (connectTag == replyTag); } @@ -344,7 +344,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) //can remove when all .exes have new code. if (connectKind != kind) { - LOG(MCdetailDebugInfo, unknownJob, "Connection for wrong slave kind (%u vs %u)- ignore", connectKind, kind); + LOG(MCdetailDebugInfo, "Connection for wrong slave kind (%u vs %u)- ignore", connectKind, kind); } } @@ -358,12 +358,12 @@ bool CRemoteParentInfo::sendReply(unsigned version) if (same) { socket.setown(connect.getClear()); - LOG(MCdetailDebugInfo, unknownJob, "Connection matched - continue...."); + LOG(MCdetailDebugInfo, "Connection matched - continue...."); return true; } if ((connectKind == kind) && (version != connectVersion)) { - LOG(MCdebugInfo, unknownJob, "Version mismatch - terminating slave process expected %d got %d", version, connectVersion); + LOG(MCdebugInfo, "Version mismatch - terminating slave process expected %d got %d", version, connectVersion); return false; } } @@ -419,7 +419,7 @@ void CRemoteSlave::run(int argc, const char * * argv) logFile.append(".log"); attachStandardFileLogMsgMonitor(logFile.str(), 0, MSGFIELD_STANDARD, MSGAUD_all, MSGCLS_all, TopDetail, LOGFORMAT_table, true, true); queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler()); // no point logging output to screen if run remote! - LOG(MCdebugProgress, unknownJob, "Starting %s %s %s %s %s %s %s",slaveName.get(),(argc>1)?argv[1]:"",(argc>2)?argv[2]:"",(argc>3)?argv[3]:"",(argc>4)?argv[4]:"",(argc>5)?argv[5]:"",(argc>6)?argv[6]:""); + LOG(MCdebugProgress, "Starting %s %s %s %s %s %s %s",slaveName.get(),(argc>1)?argv[1]:"",(argc>2)?argv[2]:"",(argc>3)?argv[3]:"",(argc>4)?argv[4]:"",(argc>5)?argv[5]:"",(argc>6)?argv[6]:""); #else setupContainerizedLogMsgHandler(); #endif @@ -485,7 +485,7 @@ void CRemoteSlave::run(int argc, const char * * argv) } catch (RELEASE_CATCH_ALL) { - LOG(MCwarning, unknownJob, "Server seems to have crashed - close done gracefully"); + LOG(MCwarning, "Server seems to have crashed - close done gracefully"); error.setown(MakeStringException(999, "Server seems to have crashed - close done gracefully")); } @@ -495,25 +495,25 @@ void CRemoteSlave::run(int argc, const char * * argv) msg.append(results); catchWriteBuffer(masterSocket, msg); - LOG(MCdebugProgress, unknownJob, "Results sent from slave %d", info.replyTag); + LOG(MCdebugProgress, "Results sent from slave %d", info.replyTag); //Acknowledgement before closing down... msg.clear(); if (catchReadBuffer(masterSocket, msg, RMTTIME_RESPONSE_MASTER)) { - LOG(MCdebugProgress, unknownJob, "Terminate acknowledgement received from master for slave %d", info.replyTag); + LOG(MCdebugProgress, "Terminate acknowledgement received from master for slave %d", info.replyTag); msg.read(ok); assertex(ok); } else - LOG(MCdebugProgress, unknownJob, "No terminate acknowledgement received from master for slave %d", info.replyTag); + LOG(MCdebugProgress, "No terminate acknowledgement received from master for slave %d", info.replyTag); if (error) break; timeOut = 24*60*60*1000; } while (stayAlive); - LOG(MCdebugProgress, unknownJob, "Terminate acknowledgement received from master for slave %d", info.replyTag); + LOG(MCdebugProgress, "Terminate acknowledgement received from master for slave %d", info.replyTag); } } catch (IException * e) @@ -522,7 +522,7 @@ void CRemoteSlave::run(int argc, const char * * argv) e->Release(); } } - LOG(MCdebugProgress, unknownJob, "Stopping %s", slaveName.get()); + LOG(MCdebugProgress, "Stopping %s", slaveName.get()); } @@ -570,7 +570,7 @@ void checkForRemoteAbort(ISocket * socket) bool sendRemoteAbort(ISocket * socket) { - LOG(MCdebugInfo, unknownJob, "Send abort to remote slave (%d)", isAborting()); + LOG(MCdebugInfo, "Send abort to remote slave (%d)", isAborting()); MemoryBuffer msg; msg.append(true); diff --git a/common/thorhelper/roxiehelper.ipp b/common/thorhelper/roxiehelper.ipp index 33eaa5f662c..398c9c3902c 100644 --- a/common/thorhelper/roxiehelper.ipp +++ b/common/thorhelper/roxiehelper.ipp @@ -61,12 +61,12 @@ interface IRoxieContextLogger : extends IContextLogger { // Override base interface with versions that add prefix // We could consider moving some or all of these down into IContextLogger - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { StringBuffer text, prefix; getLogPrefix(prefix); text.valist_appendf(format, args); - CTXLOGa(LOG_TRACING, cat, job, code, prefix.str(), text.str()); + CTXLOGa(LOG_TRACING, cat, code, prefix.str(), text.str()); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -77,7 +77,7 @@ interface IRoxieContextLogger : extends IContextLogger virtual StringBuffer &getLogPrefix(StringBuffer &ret) const = 0; virtual bool isIntercepted() const = 0; - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const = 0; + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const = 0; void CTXLOGae(IException *E, const char *file, unsigned line, const char *prefix, const char *format, ...) const __attribute__((format(printf, 6, 7))); virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const = 0; virtual void CTXLOGl(LogItem *) const = 0; diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp index 2c3a63452e1..3381bc08316 100644 --- a/common/thorhelper/thorcommon.hpp +++ b/common/thorhelper/thorcommon.hpp @@ -398,14 +398,14 @@ class CStatsContextLogger : public CSimpleInterfaceOf Owned activeSpan = getNullSpan(); mutable CRuntimeStatisticCollection stats; public: - CStatsContextLogger(const CRuntimeStatisticCollection &_mapping, const LogMsgJobInfo & _job=unknownJob) : job(_job), stats(_mapping) {} + CStatsContextLogger(const CRuntimeStatisticCollection &_mapping) : stats(_mapping) {} void reset() { stats.reset(); } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { - VALOG(cat, job, code, format, args); + VALOG(cat, code, format, args); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -419,7 +419,7 @@ class CStatsContextLogger : public CSimpleInterfaceOf E->errorMessage(ss.append(": ")); if (format) ss.append(": ").valist_appendf(format, args); - LOG(MCoperatorProgress, queryJob(), "%s", ss.str()); + LOG(MCoperatorProgress, "%s", ss.str()); } virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const override { @@ -485,7 +485,6 @@ class CStatsContextLogger : public CSimpleInterfaceOf { previous.updateDelta(to, stats); } - virtual const LogMsgJobInfo & queryJob() const override { return job; } }; extern THORHELPER_API bool isActivitySink(ThorActivityKind kind); diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index c8707a01ad6..50d71b2bbcc 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -1950,7 +1950,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (httpheaders && *httpheaders) { if (soapTraceLevel > 6 || master->logXML) - master->logctx.mCTXLOG("%s: Adding HTTP Headers(%s)", master->wscCallTypeText(), httpheaders); + master->logctx.mCTXLOG("%s: Adding HTTP Headers(%s)", master->wscCallTypeText(), httpheaders); request.append(httpheaders); } diff --git a/common/thorhelper/thortparse.cpp b/common/thorhelper/thortparse.cpp index 8271bd476c4..b13f368274f 100644 --- a/common/thorhelper/thortparse.cpp +++ b/common/thorhelper/thortparse.cpp @@ -574,7 +574,7 @@ bool LRActiveState::mergePackedNode(unsigned stateId, StackElement * next, bool { cur->shifted.setown(curSymbol->createMerged(nextSymbol)); #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Nodes Merged: %p = %p, %p", cur->shifted.get(), curSymbol, nextSymbol); + LOG(MCdebugProgress, "Nodes Merged: %p = %p, %p", cur->shifted.get(), curSymbol, nextSymbol); #endif return true; } @@ -704,11 +704,11 @@ StackElement * LRParser::createState(StackElement * prev, state_id nextState, Gr newState = new StackElement(shifted, nextState, prev, this); } #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "%p: Push state %d symbol %d[%p] previous: %d[%p]", newState, nextState, shifted ? shifted->id : -1, shifted, prev ? prev->state : -1, prev); + LOG(MCdebugProgress, "%p: Push state %d symbol %d[%p] previous: %d[%p]", newState, nextState, shifted ? shifted->id : -1, shifted, prev ? prev->state : -1, prev); StringBuffer s; newState->getDebugText(s); s.newline(); - LOG(MCdebugProgress, unknownJob, s.str()); + LOG(MCdebugProgress, s.str()); #endif return newState; } @@ -769,7 +769,7 @@ void LRParser::doReductions(GrammarSymbol * next, bool singleToken) void LRParser::process(GrammarSymbol * next, bool singleToken) { #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Process token '%.*s' %d at position %d", next->queryEndPtr()-next->queryStartPtr(), next->queryStartPtr(), next->id, next->queryStartPtr()-rowState.inputText); + LOG(MCdebugProgress, "Process token '%.*s' %d at position %d", next->queryEndPtr()-next->queryStartPtr(), next->queryStartPtr(), next->id, next->queryStartPtr()-rowState.inputText); #endif doReductions(next, singleToken); selectEndPosition((size32_t)(next->queryEndPtr()-rowState.inputText)); @@ -791,7 +791,7 @@ void LRParser::expandReduction(StackElement & element, LRProduction * production StringBuffer s; for (unsigned i = 0; i < production->getNumSymbols(); i++) s.appendf("%p ", reducedArgs[i]); - LOG(MCdebugProgress, unknownJob, "Reduce by production %d new element %p[%s]", reduced->id, reduced.get(), s.str()); + LOG(MCdebugProgress, "Reduce by production %d new element %p[%s]", reduced->id, reduced.get(), s.str()); #endif //MORE: Some kind of recursion checking needed? if (activeOutput->okToAddReduction(nextState)) @@ -848,7 +848,7 @@ void LRParser::doShifts(LRActiveState * active, GrammarSymbol * next) if (nextState != NO_STATE) { #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Shift to state %d", nextState); + LOG(MCdebugProgress, "Shift to state %d", nextState); #endif activeOutput->addElementOwn(createState(&cur, nextState, next), chooseBest); } @@ -859,7 +859,7 @@ void LRParser::doShifts(LRActiveState * active, GrammarSymbol * next) GrammarSymbol * sym = accept->shifted; accepted.append(*LINK(sym)); #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Accepted %p[%p]", accept, sym); + LOG(MCdebugProgress, "Accepted %p[%p]", accept, sym); #endif } } diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index e5c391e9731..ec3b3590e70 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -9569,7 +9569,7 @@ void CLocalWorkUnit::deleteTempFiles(const char *graph, bool deleteOwned, bool d if(needDelete) { const char *name = file.queryProp("@name"); - LOG(MCdebugProgress, unknownJob, "Removing workunit file %s from DFS", name); + LOG(MCdebugProgress, "Removing workunit file %s from DFS", name); queryDistributedFileDirectory().removeEntry(name, queryUserDescriptor()); toRemove.append(file); } @@ -9705,7 +9705,7 @@ void CLocalWorkUnit::releaseFile(const char *fileName) if (!name.isEmpty()&&(1 == usageCount)) { if (queryDistributedFileDirectory().removeEntry(fileName, queryUserDescriptor())) - LOG(MCdebugProgress, unknownJob, "Removed (released) file %s from DFS", name.get()); + LOG(MCdebugProgress, "Removed (released) file %s from DFS", name.get()); } } if (!more) diff --git a/dali/base/dacoven.cpp b/dali/base/dacoven.cpp index 30cd69fa001..081f5f2d73c 100644 --- a/dali/base/dacoven.cpp +++ b/dali/base/dacoven.cpp @@ -254,7 +254,7 @@ class CCovenBase: implements ICoven, public CInterface if (server) { _ServerVersion.set(ServerVersion); - LOG(MCdebugProgress, unknownJob, "Server Version = %s, required minimum client version %s", ServerVersion, MinClientVersion); + LOG(MCdebugProgress, "Server Version = %s, required minimum client version %s", ServerVersion, MinClientVersion); } else { diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 887cafe07f3..1ff1c0fee5d 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -13767,12 +13767,12 @@ bool CDistributedFileDirectory::removePhysicalPartFiles(const char *logicalName, { unsigned start = msTick(); if (!partfile->remove()&&(copy==0)&&!islazy) // only warn about missing primary files - LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); + LOG(MCwarning, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); else { unsigned t = msTick()-start; if (t>5*1000) - LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); + LOG(MCwarning, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } } catch (IException *e) diff --git a/dali/base/danqs.cpp b/dali/base/danqs.cpp index bf8bd7885fe..d22cf9a4fac 100644 --- a/dali/base/danqs.cpp +++ b/dali/base/danqs.cpp @@ -646,7 +646,7 @@ class CDaliNamedQueueServer: public IDaliServer, public Thread, implements IConn catch (IException *e) { // should just be aborted exceptions, but remove subscription whatever the error. - LOG(MCdebugInfo, unknownJob, e, "Named Queue Server - MQR_ADD_QUEUE notify"); + LOG(MCdebugInfo, e, "Named Queue Server - MQR_ADD_QUEUE notify"); e->Release(); stubs.remove(i,true); toremove.append(stub); diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index dcbc2bd7aa4..5e5907cfd5a 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -1219,7 +1219,7 @@ class CDeltaWriter : implements IThreaded { exception.setown(e); StringBuffer err("Saving external (backup): "); - LOG(MCoperatorError, unknownJob, e, err.append(rL).str()); + LOG(MCoperatorError, e, err.append(rL).str()); } if (!exception.get()) break; @@ -1252,7 +1252,7 @@ class CDeltaWriter : implements IThreaded { exception.setown(e); StringBuffer err("Removing external (backup): "); - LOG(MCoperatorWarning, unknownJob, e, err.append(rL).str()); + LOG(MCoperatorWarning, e, err.append(rL).str()); } if (!exception.get()) break; @@ -1580,7 +1580,7 @@ class CLegacyBinaryFileExternal : public CExternalFile, implements IExternalHand if (*_name) s.append("in property ").append(_name); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); if (withValue) { StringBuffer str("EXTERNAL BINARY FILE: \""); @@ -1662,7 +1662,7 @@ class CXMLFileExternal : public CExternalFile, implements IExternalHandler if (name && *name) s.append("in property ").append(name); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); StringBuffer str("EXTERNAL XML FILE: \""); str.append(filename.str()).append("\" MISSING"); tree.setown(createPTree(owner.queryName())); @@ -2190,7 +2190,7 @@ void CBinaryFileExternal::readValue(const char *name, MemoryBuffer &mb) { StringBuffer s("Missing external file "); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); StringBuffer str("EXTERNAL BINARY FILE: \""); str.append(filename.str()).append("\" MISSING"); CPTValue v(str.length()+1, str.str(), false); @@ -2236,7 +2236,7 @@ void CBinaryFileExternal::read(const char *name, IPropertyTree &owner, MemoryBuf if (*_name) s.append("in property ").append(_name); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); StringBuffer str("EXTERNAL BINARY FILE: \""); str.append(filename.str()).append("\" MISSING"); CPTValue v(str.length()+1, str.str(), false); @@ -2468,7 +2468,7 @@ CServerConnection::~CServerConnection() void CServerConnection::aborted(SessionId id) { - LOG(MCdebugInfo, unknownJob, "CServerConnection: connection aborted (%" I64F "x) sessId=%" I64F "x",connectionId, id); + LOG(MCdebugInfo, "CServerConnection: connection aborted (%" I64F "x) sessId=%" I64F "x",connectionId, id); #if 0 // JCSMORE - think this is ok, but concerned about deadlock, change later. Owned lockBlock = new CLCLockBlock(((CCovenSDSManager &)manager).dataRWLock, false, readWriteTimeout, __FILE__, __LINE__); SDSManager->disconnect(connectionId, false); @@ -2685,7 +2685,7 @@ class CServerRemoteTree : public CRemoteTreeBase try { SDSManager->deleteExternal(index); } catch (IException *e) { - LOG(MCoperatorWarning, unknownJob, e, StringBuffer("Deleting external reference for ").append(queryName()).str()); + LOG(MCoperatorWarning, e, StringBuffer("Deleting external reference for ").append(queryName()).str()); e->Release(); } } @@ -4811,7 +4811,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) mb.append(e->errorMessage(s).str()); StringBuffer clientUrl("EXCEPTION in reply to client "); mb.getSender().getEndpointHostText(clientUrl); - LOG(MCoperatorError, unknownJob, e); + LOG(MCoperatorError, e); } try { CheckTime block10("DAMP_REQUEST reply"); @@ -4837,7 +4837,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) try { coven.reply(mb); - LOG(MCdebugInfo, unknownJob, "Failed to reply, but succeeded sending initial reply error to client"); + LOG(MCdebugInfo, "Failed to reply, but succeeded sending initial reply error to client"); } catch (IException *e) { @@ -5047,7 +5047,7 @@ IPropertyTree *loadStore(const char *storeFilename, unsigned edition, IPTreeMake catch (DALI_CATCHALL) { IException *e = MakeStringException(0, "Unknown exception - loading store file : %s", storeFilename); - LOG(MCoperatorDisaster, unknownJob, e, ""); + LOG(MCoperatorDisaster, e, ""); if (!logErrorsOnly) throw; e->Release(); @@ -5715,7 +5715,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface } virtual void saveStore(IPropertyTree *root, unsigned *_newEdition, bool currentEdition=false) { - LOG(MCdebugProgress, unknownJob, "Saving store"); + LOG(MCdebugProgress, "Saving store"); refreshStoreInfo(); @@ -5798,7 +5798,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface try { renameDelta(edition, newEdition, remoteBackupLocation); } catch (IException *e) { - LOG(MCoperatorError, unknownJob, e, "Failure handling backup"); + LOG(MCoperatorError, e, "Failure handling backup"); e->Release(); } } @@ -5828,7 +5828,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface catch (IException *e) { StringBuffer s; - LOG(MCoperatorError, unknownJob, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation)); + LOG(MCoperatorError, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation)); e->Release(); } @@ -5836,7 +5836,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface *_newEdition = newEdition; done = true; - LOG(MCdebugProgress, unknownJob, "Store saved"); + LOG(MCdebugProgress, "Store saved"); } catch (IException *e) { @@ -6247,7 +6247,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) if (!root) { StringBuffer s(storeName); - LOG(MCdebugInfo, unknownJob, "Store %d does not exist, creating new store", iStoreHelper->queryCurrentEdition()); + LOG(MCdebugInfo, "Store %d does not exist, creating new store", iStoreHelper->queryCurrentEdition()); root = new CServerRemoteTree("SDS"); } bool errors; @@ -6260,12 +6260,12 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) if (deltaE.get()) throw LINK(deltaE); } - LOG(MCdebugInfo, unknownJob, "store and deltas loaded"); + LOG(MCdebugInfo, "store and deltas loaded"); const char *environment = config.queryProp("@environment"); if (environment && *environment) { - LOG(MCdebugInfo, unknownJob, "loading external Environment from: %s", environment); + LOG(MCdebugInfo, "loading external Environment from: %s", environment); Owned envFile = createIFile(environment); if (!envFile->exists()) throw MakeStringException(0, "'%s' does not exist", environment); @@ -6559,14 +6559,14 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) unsigned items = treeMaker.convertQueue.ordinality(); if (items) { - LOG(MCdebugInfo, unknownJob, "Converting %d items larger than threshold size %d, to external definitions", items, externalSizeThreshold); + LOG(MCdebugInfo, "Converting %d items larger than threshold size %d, to external definitions", items, externalSizeThreshold); ForEachItemIn(i, treeMaker.convertQueue) SDSManager->writeExternal(treeMaker.convertQueue.item(i), true); saveNeeded = true; } if (saveNeeded) { - LOG(MCdebugInfo, unknownJob, "Saving converted store"); + LOG(MCdebugInfo, "Saving converted store"); SDSManager->saveStore(); } } @@ -6591,7 +6591,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) if (remoteBackupLocation.length()) { try { validateBackup(); } - catch (IException *e) { LOG(MCoperatorError, unknownJob, e, "Validating backup"); e->Release(); } + catch (IException *e) { LOG(MCoperatorError, e, "Validating backup"); e->Release(); } StringBuffer deltaFilename(dataPath); iStoreHelper->getCurrentDeltaFilename(deltaFilename); @@ -8785,7 +8785,7 @@ bool CCovenSDSManager::fireException(IException *e) { if (handled) { - LOG(MCoperatorDisaster, unknownJob, e, "FATAL, too many exceptions"); + LOG(MCoperatorDisaster, e, "FATAL, too many exceptions"); return false; // did not successfully handle. } IERRLOG(e, "Exception while restarting or shutting down"); @@ -8822,7 +8822,7 @@ bool CCovenSDSManager::fireException(IException *e) } manager.unhandledThread.clear(); } - catch (IException *_e) { LOG(MCoperatorError, unknownJob, _e, "Exception while restarting or shutting down"); _e->Release(); } + catch (IException *_e) { LOG(MCoperatorError, _e, "Exception while restarting or shutting down"); _e->Release(); } catch (DALI_CATCHALL) { IERRLOG("Unknown exception while restarting or shutting down"); } if (!restart) { @@ -8914,7 +8914,7 @@ class CDaliSDSServer: implements IDaliServer, public CInterface try { manager->loadStore(NULL, &cancelLoad); } catch (IException *) { - LOG(MCdebugInfo, unknownJob, "Failed to load main store"); + LOG(MCdebugInfo, "Failed to load main store"); throw; } storeLoaded = true; diff --git a/dali/base/dasess.cpp b/dali/base/dasess.cpp index e356731ea86..8b40da715f8 100644 --- a/dali/base/dasess.cpp +++ b/dali/base/dasess.cpp @@ -1191,7 +1191,7 @@ class CLdapWorkItem : public Thread ret = ldapconn->getPermissions(key,obj,udesc,flags); } catch(IException *e) { - LOG(MCoperatorError, unknownJob, e, "CLdapWorkItem"); + LOG(MCoperatorError, e, "CLdapWorkItem"); e->Release(); } ready.signal(); diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 5e4ae47c28f..160c51cf07e 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -453,7 +453,7 @@ class CDFUengine: public CInterface, implements IDFUengine const char * pfilePath = filePath.str(); #ifdef _DEBUG - LOG(MCdebugInfo, unknownJob, "File path is '%s'", filePath.str()); + LOG(MCdebugInfo, "File path is '%s'", filePath.str()); #endif const char pathSep = filename.getPathSeparator(); @@ -488,7 +488,7 @@ class CDFUengine: public CInterface, implements IDFUengine if (env->isDropZoneRestrictionEnabled()) throwError2(DFTERR_NoMatchingDropzonePath, netaddress.str(), pfilePath); else - LOG(MCdebugInfo, unknownJob, "No matching drop zone path on '%s' to file path: '%s'", netaddress.str(), pfilePath); + LOG(MCdebugInfo, "No matching drop zone path on '%s' to file path: '%s'", netaddress.str(), pfilePath); } #ifdef _DEBUG else @@ -496,7 +496,7 @@ class CDFUengine: public CInterface, implements IDFUengine SCMStringBuffer dropZoneName; dropZone->getName(dropZoneName); - LOG(MCdebugInfo, unknownJob, "Drop zone path '%s' is %svisible in ECLWatch." + LOG(MCdebugInfo, "Drop zone path '%s' is %svisible in ECLWatch." , dropZoneName.str() , (dropZone->isECLWatchVisible() ? "" : "not ") ); @@ -1212,6 +1212,8 @@ class CDFUengine: public CInterface, implements IDFUengine OWARNLOG("DFURUN: Workunit %s not found",dfuwuid); return DFUstate_unknown; } + + JobNameScope jobName(dfuwuid); if (dfuServerName.length()) wu->setDFUServerName(dfuServerName.str()); StringBuffer logname; diff --git a/dali/dfu/dfuserver.cpp b/dali/dfu/dfuserver.cpp index dee00b31dd7..8777cf8e4af 100644 --- a/dali/dfu/dfuserver.cpp +++ b/dali/dfu/dfuserver.cpp @@ -292,7 +292,7 @@ int main(int argc, const char *argv[]) engine->joinListeners(); if (replserver.get()) replserver->stopServer(); - LOG(MCprogress, unknownJob, "Exiting"); + LOG(MCprogress, "Exiting"); } } diff --git a/dali/ft/daft.cpp b/dali/ft/daft.cpp index e2e1cd5d1b8..051375099ac 100644 --- a/dali/ft/daft.cpp +++ b/dali/ft/daft.cpp @@ -183,7 +183,7 @@ offset_t CDistributedFileSystem::getSize(IDistributedFile * file, bool forceget, lock.queryAttributes().setPropInt64("@size", totalSize); } } - //LOG(MCdebugInfo(1000), unknownJob, "DFS: getSize(%s)=%" I64F "d", file->queryLogicalName(), totalSize); + //LOG(MCdebugInfo(1000), "DFS: getSize(%s)=%" I64F "d", file->queryLogicalName(), totalSize); return totalSize; } @@ -241,7 +241,7 @@ offset_t CDistributedFileSystem::getSize(IDistributedFilePart * part, bool force } } - //LOG(MCdebugInfo(2000), unknownJob, "DFS: getSize(%s)=%" I64F "d", part->queryPartName(), size); + //LOG(MCdebugInfo(2000), "DFS: getSize(%s)=%" I64F "d", part->queryPartName(), size); return size; } diff --git a/dali/ft/daftformat.cpp b/dali/ft/daftformat.cpp index ecd55b91eaf..6cc8230ae56 100644 --- a/dali/ft/daftformat.cpp +++ b/dali/ft/daftformat.cpp @@ -105,7 +105,7 @@ void CPartitioner::commonCalcPartitions() if (endOffset == totalSize) lastSplit = numParts-1; if (lastSplit >= numParts) lastSplit = numParts-1; // very rare with variable length records, last file is very small or copying a couple of records 50 ways. - //LOG(MCdebugInfo, unknownJob, "commonCalcPartitions: partSize:%lld, endOffset: %lld, firstSplit: %d, lastSplit: %d ",partSize ,endOffset, firstSplit, lastSplit); + //LOG(MCdebugInfo, "commonCalcPartitions: partSize:%lld, endOffset: %lld, firstSplit: %d, lastSplit: %d ",partSize ,endOffset, firstSplit, lastSplit); JSON_DBGLOG("commonCalcPartitions: partSize:%lld, endOffset: %lld, firstSplit: %d, lastSplit: %d ",partSize ,endOffset, firstSplit, lastSplit); if (!partSeparator.isEmpty() && appendingContent) //appending to existing content, add a separator if necessary @@ -215,7 +215,7 @@ void CPartitioner::getRecordStructure(StringBuffer & _recordStructure) CSimpleFixedPartitioner::CSimpleFixedPartitioner(unsigned _recordSize, bool _noTranslation) { - LOG(MCdebugProgressDetail, unknownJob, "CSimpleFixedPartitioner::CSimpleFixedPartitioner( _recordSize:%d, _noTranslation:%d)", _recordSize, _noTranslation); + LOG(MCdebugProgressDetail, "CSimpleFixedPartitioner::CSimpleFixedPartitioner( _recordSize:%d, _noTranslation:%d)", _recordSize, _noTranslation); recordSize = _recordSize; noTranslation = _noTranslation; } @@ -241,7 +241,7 @@ void CSimpleFixedPartitioner::setPartitionRange(offset_t _totalSize, offset_t _t CSimpleBlockedPartitioner::CSimpleBlockedPartitioner(bool _noTranslation) : CSimpleFixedPartitioner(EFX_BLOCK_SIZE, _noTranslation) { - LOG(MCdebugProgressDetail, unknownJob, "CSimpleBlockedPartitioner::CSimpleBlockedPartitioner( _noTranslation:%d)", _noTranslation); + LOG(MCdebugProgressDetail, "CSimpleBlockedPartitioner::CSimpleBlockedPartitioner( _noTranslation:%d)", _noTranslation); } //---------------------------------------------------------------------------- @@ -298,7 +298,7 @@ void CInputBasePartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor { // Display progress oldInputOffset = nextInputOffset; - LOG(MCdebugProgressDetail, unknownJob, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, (double)100.0*(double)nextInputOffset/(double)splitOffset); + LOG(MCdebugProgressDetail, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, (double)100.0*(double)nextInputOffset/(double)splitOffset); } inputOffset = nextInputOffset; @@ -315,7 +315,7 @@ void CInputBasePartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor if (size > bufferSize) { - LOG(MCdebugProgressDetail, unknownJob, "Split record size %d (0x%08x) is larger than the buffer size: %d", size, size, bufferSize); + LOG(MCdebugProgressDetail, "Split record size %d (0x%08x) is larger than the buffer size: %d", size, size, bufferSize); throwError2(DFTERR_WrongSplitRecordSize, size, size); } @@ -347,7 +347,7 @@ void CInputBasePartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor cursor.inputOffset = inputOffset; cursor.nextInputOffset = nextInputOffset; - LOG(MCdebugProgressDetail, unknownJob, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, 100.0); + LOG(MCdebugProgressDetail, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, 100.0); } @@ -466,7 +466,7 @@ unsigned CInputBasePartitioner::transformBlock(offset_t endOffset, TransformCurs CFixedPartitioner::CFixedPartitioner(size32_t _recordSize) : CInputBasePartitioner(0, _recordSize) { recordSize = _recordSize; - LOG(MCdebugProgressDetail, unknownJob, "CFixedPartitioner::CFixedPartitioner( recordSize:%d)", recordSize); + LOG(MCdebugProgressDetail, "CFixedPartitioner::CFixedPartitioner( recordSize:%d)", recordSize); } size32_t CFixedPartitioner::getSplitRecordSize(const byte * record, unsigned maxToRead, bool processFullBuffer) @@ -484,7 +484,7 @@ size32_t CFixedPartitioner::getTransformRecordSize(const byte * record, unsigned CBlockedPartitioner::CBlockedPartitioner() : CFixedPartitioner(EFX_BLOCK_SIZE) { - LOG(MCdebugProgressDetail, unknownJob, "CBlockedPartitioner::CBlockedPartitioner()"); + LOG(MCdebugProgressDetail, "CBlockedPartitioner::CBlockedPartitioner()"); } @@ -498,7 +498,7 @@ void CBlockedPartitioner::setTarget(IOutputProcessor * _target) CVariablePartitioner::CVariablePartitioner(bool _bigendian) : CInputBasePartitioner(sizeof(varLenType), EXPECTED_VARIABLE_LENGTH) { - LOG(MCdebugProgressDetail, unknownJob, "CVariablePartitioner::CVariablePartitioner(_bigendian:%d)", _bigendian); + LOG(MCdebugProgressDetail, "CVariablePartitioner::CVariablePartitioner(_bigendian:%d)", _bigendian); assertex(sizeof(varLenType) == 4); bigendian = _bigendian; } @@ -539,7 +539,7 @@ void CVariablePartitioner::setTarget(IOutputProcessor * _target) CRECFMvbPartitioner::CRECFMvbPartitioner(bool blocked) : CInputBasePartitioner(blocked?BDW_SIZE:RDW_SIZE, EXPECTED_VARIABLE_LENGTH) { - LOG(MCdebugProgressDetail, unknownJob, "CRECFMvbPartitioner::CRECFMvbPartitioner(blocked:%d)", blocked); + LOG(MCdebugProgressDetail, "CRECFMvbPartitioner::CRECFMvbPartitioner(blocked:%d)", blocked); isBlocked = blocked; } @@ -553,7 +553,7 @@ size32_t CRECFMvbPartitioner::getRecordSize(const byte * record, unsigned maxToR if (rest) { - LOG(MCdebugProgressDetail, unknownJob, "Wrong RECFMv RDW info: size:%d (0x%04x) rest:%d (0x%04x) at pos :%d", recordsize ,recordsize, rest, rest, unsigned (record - bufferBase()) ); + LOG(MCdebugProgressDetail, "Wrong RECFMv RDW info: size:%d (0x%04x) rest:%d (0x%04x) at pos :%d", recordsize ,recordsize, rest, rest, unsigned (record - bufferBase()) ); throwError1(DFTERR_WrongRECFMvRecordDescriptorWord, rest); } @@ -613,7 +613,7 @@ size32_t CRECFMvbPartitioner::getTransformRecordSize(const byte * record, unsign unsigned CRECFMvbPartitioner::transformBlock(offset_t endOffset, TransformCursor & cursor) { - LOG(MCdebugProgressDetail, unknownJob, "CRECFMvbPartitioner::transformBlock(offset_t endOffset: %" I64F "d (0x%016" I64F "x), TransformCursor & cursor)", endOffset ,endOffset); + LOG(MCdebugProgressDetail, "CRECFMvbPartitioner::transformBlock(offset_t endOffset: %" I64F "d (0x%016" I64F "x), TransformCursor & cursor)", endOffset ,endOffset); const byte *buffer = bufferBase(); offset_t startOffset = cursor.inputOffset; offset_t inputOffset = startOffset; @@ -680,11 +680,11 @@ unsigned CRECFMvbPartitioner::transformBlock(offset_t endOffset, TransformCursor CCsvPartitioner::CCsvPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize) { - LOG(MCdebugProgressDetail, unknownJob, "CCsvPartitioner::CCsvPartitioner(_format :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); - LOG(MCdebugProgressDetail, unknownJob, " separator :'%s'", _format.separate.get()); - LOG(MCdebugProgressDetail, unknownJob, " quote :'%s'", _format.quote.get()); - LOG(MCdebugProgressDetail, unknownJob, " terminator:'%s'", _format.terminate.get()); - LOG(MCdebugProgressDetail, unknownJob, " escape :'%s'", _format.escape.get()); + LOG(MCdebugProgressDetail, "CCsvPartitioner::CCsvPartitioner(_format :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); + LOG(MCdebugProgressDetail, " separator :'%s'", _format.separate.get()); + LOG(MCdebugProgressDetail, " quote :'%s'", _format.quote.get()); + LOG(MCdebugProgressDetail, " terminator:'%s'", _format.terminate.get()); + LOG(MCdebugProgressDetail, " escape :'%s'", _format.escape.get()); maxElementLength = 1; format.set(_format); @@ -700,7 +700,7 @@ CCsvPartitioner::CCsvPartitioner(const FileFormat & _format) : CInputBasePartiti if (escape && *escape) { if (quote && (*escape == *quote)) - LOG(MCdebugProgressDetail, unknownJob, "The quote ('%s') and the escape ('%s') are same, ignore escape.", quote, escape); + LOG(MCdebugProgressDetail, "The quote ('%s') and the escape ('%s') are same, ignore escape.", quote, escape); else addActionList(matcher, escape, ESCAPE, &maxElementLength); } @@ -744,14 +744,14 @@ void CCsvPartitioner::getRecordStructure(StringBuffer & _recordStructure) if (size > bufferSize) { - LOG(MCdebugProgressDetail, unknownJob, "First record size %d (0x%08x) is larger than the buffer size: %d. Please check the separator and terminator parameters.", size, size, bufferSize); + LOG(MCdebugProgressDetail, "First record size %d (0x%08x) is larger than the buffer size: %d. Please check the separator and terminator parameters.", size, size, bufferSize); throwError2(DFTERR_WrongSplitRecordSize, size, size); } else if (isFirstRow) { // Possibly reached the end of the file but the processing is still in first row // so no terminator detected - LOG(MCdebugProgressDetail, unknownJob, "%d (0x%08x) bytes processed, but there is no terminator found. Please check the separator and terminator parameters.", size, size); + LOG(MCdebugProgressDetail, "%d (0x%08x) bytes processed, but there is no terminator found. Please check the separator and terminator parameters.", size, size); throwError1(DFTERR_EndOfCsvRecordNotFound, size); } } @@ -970,7 +970,7 @@ size32_t CCsvPartitioner::getSplitRecordSize(const byte * start, unsigned maxToR throwError(DFTERR_EndOfRecordNotFound); - LOG(MCdebugProgress, unknownJob, "CSV splitRecordSize(%d) at end of file", (unsigned) (end - start)); + LOG(MCdebugProgress, "CSV splitRecordSize(%d) at end of file", (unsigned) (end - start)); if (++numOfBufferOverrun > maxNumberOfBufferOverrun) throwError1(DFTERR_EndOfCsvRecordNotFound, numOfProcessedBytes); @@ -995,7 +995,7 @@ void CCsvPartitioner::setTarget(IOutputProcessor * _target) CCsvQuickPartitioner::CCsvQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CCsvPartitioner(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CCsvQuickPartitioner::CCsvQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); + LOG(MCdebugProgressDetail, "CCsvQuickPartitioner::CCsvQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); noTranslation = _noTranslation; } @@ -1050,13 +1050,13 @@ void CCsvQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor if (ensureSize == format.maxRecordSize) throw; e->Release(); - LOG(MCdebugProgress, unknownJob, "Failed to find split after reading %d", ensureSize); + LOG(MCdebugProgress, "Failed to find split after reading %d", ensureSize); ensureSize += blockSize; if (ensureSize > format.maxRecordSize) ensureSize = format.maxRecordSize; } } - LOG(MCdebugProgress, unknownJob, "Found split after reading %d", ensureSize); + LOG(MCdebugProgress, "Found split after reading %d", ensureSize); } } } @@ -1088,10 +1088,10 @@ void CCsvQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor CUtfPartitioner::CUtfPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize) { - LOG(MCdebugProgressDetail, unknownJob, "CUtfPartitioner::CUtfPartitioner(_format.type :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); - LOG(MCdebugProgressDetail, unknownJob, " separator :'%s'", _format.separate.get()); - LOG(MCdebugProgressDetail, unknownJob, " quote :'%s'", _format.quote.get()); - LOG(MCdebugProgressDetail, unknownJob, " terminator:'%s'", _format.terminate.get()); + LOG(MCdebugProgressDetail, "CUtfPartitioner::CUtfPartitioner(_format.type :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); + LOG(MCdebugProgressDetail, " separator :'%s'", _format.separate.get()); + LOG(MCdebugProgressDetail, " quote :'%s'", _format.quote.get()); + LOG(MCdebugProgressDetail, " terminator:'%s'", _format.terminate.get()); maxElementLength = 1; format.set(_format); @@ -1311,7 +1311,7 @@ size32_t CUtfPartitioner::getSplitRecordSize(const byte * start, unsigned maxToR numOfProcessedBytes += (unsigned)(end - start); - LOG(MCdebugProgress, unknownJob, "UTF splitRecordSize(%d) at end of file", (unsigned) (end - start)); + LOG(MCdebugProgress, "UTF splitRecordSize(%d) at end of file", (unsigned) (end - start)); if (++numOfBufferOverrun > maxNumberOfBufferOverrun) throwError1(DFTERR_EndOfUtfRecordNotFound, numOfProcessedBytes); @@ -1332,7 +1332,7 @@ void CUtfPartitioner::setTarget(IOutputProcessor * _target) CUtfQuickPartitioner::CUtfQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CUtfPartitioner(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CUtfQuickPartitioner::CUtfQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); + LOG(MCdebugProgressDetail, "CUtfQuickPartitioner::CUtfQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); noTranslation = _noTranslation; } @@ -1389,13 +1389,13 @@ void CUtfQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor if (ensureSize == format.maxRecordSize) throw; e->Release(); - LOG(MCdebugProgress, unknownJob, "Failed to find split after reading %d", ensureSize); + LOG(MCdebugProgress, "Failed to find split after reading %d", ensureSize); ensureSize += blockSize; if (ensureSize > format.maxRecordSize) ensureSize = format.maxRecordSize; } } - LOG(MCdebugProgress, unknownJob, "Found split after reading %d", ensureSize); + LOG(MCdebugProgress, "Found split after reading %d", ensureSize); } } } @@ -1463,7 +1463,7 @@ size32_t BufferedDirectReader::ensure(size32_t required) XmlSplitter::XmlSplitter(const FileFormat & format) { - LOG(MCdebugProgressDetail, unknownJob, "XmlSplitter::XmlSplitter(_format.type :'%s', format.rowTag:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); + LOG(MCdebugProgressDetail, "XmlSplitter::XmlSplitter(_format.type :'%s', format.rowTag:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); maxElementLength = 1; utfFormat = getUtfFormatType(format.type); StringBuffer openTag, closeTag, endTag, endCloseTag; @@ -1686,7 +1686,7 @@ offset_t XmlSplitter::getFooterLength(BufferedDirectReader & reader, offset_t si CJsonInputPartitioner::CJsonInputPartitioner(const FileFormat & _format) { - LOG(MCdebugProgressDetail, unknownJob, "CJsonInputPartitioner::CJsonInputPartitioner(format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), _format.getUnitSize()); + LOG(MCdebugProgressDetail, "CJsonInputPartitioner::CJsonInputPartitioner(format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), _format.getUnitSize()); format.set(_format); CriticalBlock block(openfilecachesect); @@ -1750,7 +1750,7 @@ CJsonPartitioner::CJsonPartitioner(const FileFormat & _format) : CJsonInputParti CXmlPartitioner::CXmlPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize), splitter(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlPartitioner::CXmlPartitioner(_format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), format.getUnitSize()); + LOG(MCdebugProgressDetail, "CXmlPartitioner::CXmlPartitioner(_format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), format.getUnitSize()); format.set(_format); unitSize = format.getUnitSize(); utfFormat = getUtfFormatType(format.type); @@ -1782,7 +1782,7 @@ void CXmlPartitioner::setTarget(IOutputProcessor * _target) CXmlQuickPartitioner::CXmlQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CXmlPartitioner(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::CXmlQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); + LOG(MCdebugProgressDetail, "CXmlQuickPartitioner::CXmlQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); noTranslation = _noTranslation; } @@ -1792,7 +1792,7 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor numInBuffer = bufferOffset = 0; if (splitOffset != 0) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::findSplitPoint(splitOffset:%" I64F "d)", splitOffset); + LOG(MCdebugProgressDetail, "CXmlQuickPartitioner::findSplitPoint(splitOffset:%" I64F "d)", splitOffset); unsigned delta = (unsigned)(splitOffset & (unitSize-1)); if (delta) splitOffset += (unitSize - delta); @@ -1818,16 +1818,16 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor } if (sizeAvailable >= format.maxRecordSize) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::findSplitPoint: record size (>%d bytes) is larger than expected maxRecordSize (%d bytes) [and blockSize (%d bytes)]", sizeRecord, format.maxRecordSize, blockSize); + LOG(MCdebugProgressDetail, "CXmlQuickPartitioner::findSplitPoint: record size (>%d bytes) is larger than expected maxRecordSize (%d bytes) [and blockSize (%d bytes)]", sizeRecord, format.maxRecordSize, blockSize); throwError3(DFTERR_EndOfXmlRecordNotFound, splitOffset+bufferOffset, sizeRecord, format.maxRecordSize); } - LOG(MCdebugProgress, unknownJob, "Failed to find split after reading %d", ensureSize); + LOG(MCdebugProgress, "Failed to find split after reading %d", ensureSize); ensureSize += blockSize; if (ensureSize > format.maxRecordSize) ensureSize = format.maxRecordSize; ensureBuffered(ensureSize); } - LOG(MCdebugProgress, unknownJob, "Found split after reading %d", ensureSize); + LOG(MCdebugProgress, "Found split after reading %d", ensureSize); } else if (splitOffset - thisOffset < thisSize) throwError2(DFTERR_UnexpectedReadFailure, fullPath.get(), splitOffset-thisOffset+thisHeaderSize); @@ -1842,7 +1842,7 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor CRemotePartitioner::CRemotePartitioner(FileSprayer &_sprayer, const SocketEndpoint & _ep, const FileFormat & _srcFormat, const FileFormat & _tgtFormat, const char * _slave, const char *_wuid) : wuid(_wuid), sprayer(_sprayer) { - LOG(MCdebugProgressDetail, unknownJob, "CRemotePartitioner::CRemotePartitioner(_srcFormat.type :'%s', _tgtFormat.type:'%s', _slave:'%s', _wuid:'%s')", _srcFormat.getFileFormatTypeString(), _tgtFormat.getFileFormatTypeString(), _slave, _wuid); + LOG(MCdebugProgressDetail, "CRemotePartitioner::CRemotePartitioner(_srcFormat.type :'%s', _tgtFormat.type:'%s', _slave:'%s', _wuid:'%s')", _srcFormat.getFileFormatTypeString(), _tgtFormat.getFileFormatTypeString(), _slave, _wuid); ep.set(_ep); srcFormat.set(_srcFormat); tgtFormat.set(_tgtFormat); @@ -1887,7 +1887,6 @@ void CRemotePartitioner::callRemote() HANDLE localFtSlaveHandle = 0; // used only if ftslave is launched on this host try { - LogMsgJobInfo job(unknownJob); StringBuffer url, tmp; ep.getEndpointHostText(url); @@ -1931,7 +1930,7 @@ void CRemotePartitioner::callRemote() sendDaFsFtSlaveCmd(socket, msg); } - LOG(MCdebugProgressDetail, job, "Remote partition part %s[%d]", url.str(), whichInput); + LOG(MCdebugProgressDetail, "Remote partition part %s[%d]", url.str(), whichInput); msg.clear(); if (!catchReadBuffer(socket, msg, FTTIME_PARTITION)) @@ -1952,7 +1951,7 @@ void CRemotePartitioner::callRemote() catchWriteBuffer(socket, msg); } - LOG(MCdebugProgressDetail, job, "Remote partition calculated %s[%d] ok(%d)", url.str(), whichInput, ok); + LOG(MCdebugProgressDetail, "Remote partition calculated %s[%d] ok(%d)", url.str(), whichInput, ok); } catch (IException * e) { @@ -2300,7 +2299,7 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil { IFormatProcessor * partitioner; bool sameFormats = srcFormat.equals(tgtFormat); - LOG(MCdebugProgressDetail, unknownJob, "createFormatProcessor(srcFormat:'%s', tgtFormat:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); + LOG(MCdebugProgressDetail, "createFormatProcessor(srcFormat:'%s', tgtFormat:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); switch (srcFormat.type) { case FFTfixed: @@ -2360,7 +2359,7 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil IOutputProcessor * createOutputProcessor(const FileFormat & format) { - LOG(MCdebugProgressDetail, unknownJob, "createOutputProcessor(format.type:'%s')", format.getFileFormatTypeString()); + LOG(MCdebugProgressDetail, "createOutputProcessor(format.type:'%s')", format.getFileFormatTypeString()); switch (format.type) { case FFTfixed: @@ -2382,7 +2381,7 @@ IOutputProcessor * createOutputProcessor(const FileFormat & format) IFormatPartitioner * createFormatPartitioner(FileSprayer &sprayer, const SocketEndpoint & ep, const FileFormat & srcFormat, const FileFormat & tgtFormat, bool calcOutput, const char * slave, const char *wuid) { bool sameFormats = sameEncoding(srcFormat, tgtFormat); - LOG(MCdebugProgressDetail, unknownJob, "createFormatProcessor(srcFormat.type:'%s', tgtFormat.type:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); + LOG(MCdebugProgressDetail, "createFormatProcessor(srcFormat.type:'%s', tgtFormat.type:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); if (sameFormats) { switch (srcFormat.type) diff --git a/dali/ft/daftformat.ipp b/dali/ft/daftformat.ipp index 34c2e85328e..3db0465ee24 100644 --- a/dali/ft/daftformat.ipp +++ b/dali/ft/daftformat.ipp @@ -379,7 +379,7 @@ public: JsonSplitter(const FileFormat & format, IFileIOStream &stream) : headerLength(0), pathPos(0), tangent(0), rowDepth(0), rowStart((offset_t)-1), rowEnd(0), footerLength((offset_t)-1), newRowSet(true), hasRootArray(false) { - LOG(MCdebugProgressDetail, unknownJob, "JsonSplitter::JsonSplitter(format.type :'%s', rowPath:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); + LOG(MCdebugProgressDetail, "JsonSplitter::JsonSplitter(format.type :'%s', rowPath:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); size = stream.size(); const char *rowPath = format.rowTag; diff --git a/dali/ft/daftprogress.cpp b/dali/ft/daftprogress.cpp index 5070d8614a0..a83a81ec1a1 100644 --- a/dali/ft/daftprogress.cpp +++ b/dali/ft/daftprogress.cpp @@ -136,11 +136,11 @@ void DemoProgress::displayProgress(unsigned percentDone, unsigned secsLeft, cons unsigned kbPerSecondAve, unsigned kbPerSecondRate, unsigned numNodes) { - LOG(MCdebugProgress, unknownJob, "Progress: %d%% done, %s left. (%" I64F "d/%" I64F "d%s @Ave(%dKB/s) Rate(%dKB/s) [%d/%d]", + LOG(MCdebugProgress, "Progress: %d%% done, %s left. (%" I64F "d/%" I64F "d%s @Ave(%dKB/s) Rate(%dKB/s) [%d/%d]", percentDone, timeLeft, scaledDone, scaledTotal, scale, kbPerSecondAve, kbPerSecondRate, numNodes, totalNodes); } void DemoProgress::displaySummary(const char * timeTaken, unsigned kbPerSecond) { - LOG(MCdebugProgress, unknownJob, "Summary: Total time taken %s, Average transfer %dKB/sec", timeTaken, kbPerSecond); + LOG(MCdebugProgress, "Summary: Total time taken %s, Average transfer %dKB/sec", timeTaken, kbPerSecond); } diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 267457c6e2f..885dc2b2168 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -207,7 +207,6 @@ FileTransferThread::FileTransferThread(FileSprayer & _sprayer, byte _action, con // progressInfo = _progressInfo; sem = NULL; ok = false; - job = unknownJob; allDone = false; started = false; } @@ -424,7 +423,7 @@ bool FileTransferThread::launchFtSlaveCmd() newProgress.deserializeExtra(msg, 2); sprayer.updateProgress(newProgress); - LOG(MCdebugProgress(10000), job, "Update %s: %d %" I64F "d->%" I64F "d", url.str(), newProgress.whichPartition, newProgress.inputLength, newProgress.outputLength); + LOG(MCdebugProgress(10000), "Update %s: %d %" I64F "d->%" I64F "d", url.str(), newProgress.whichPartition, newProgress.inputLength, newProgress.outputLength); if (isAborting()) { if (!sendRemoteAbort(socket)) @@ -434,7 +433,7 @@ bool FileTransferThread::launchFtSlaveCmd() msg.read(ok); setErrorOwn(deserializeException(msg)); - LOG(MCdebugProgressDetail, job, "Finished generating part %s [%p] ok(%d) error(%d)", url.str(), this, (int)ok, (int)(error!=NULL)); + LOG(MCdebugProgressDetail, "Finished generating part %s [%p] ok(%d) error(%d)", url.str(), this, (int)ok, (int)(error!=NULL)); // if communicating with ftslave, the process has a final ack wait if (sprayer.useFtSlave) @@ -463,7 +462,7 @@ bool FileTransferThread::launchFtSlaveCmd() bool FileTransferThread::performTransfer() { - LOG(MCdebugProgress, job, "Transferring part %s [%p]", url.str(), this); + LOG(MCdebugProgress, "Transferring part %s [%p]", url.str(), this); started = true; allDone = true; if (sprayer.isSafeMode || action == FTactionpush) @@ -493,19 +492,19 @@ bool FileTransferThread::performTransfer() if (allDone) { - LOG(MCdebugInfo, job, "Creation of part %s already completed", url.str()); + LOG(MCdebugInfo, "Creation of part %s already completed", url.str()); return true; } if (partition.empty()) { - LOG(MCdebugInfo, job, "No elements to transfer for this slave"); + LOG(MCdebugInfo, "No elements to transfer for this slave"); return true; } - LOG(MCdebugProgressDetail, job, "Start generate part %s [%p]", url.str(), this); + LOG(MCdebugProgressDetail, "Start generate part %s [%p]", url.str(), this); bool ok = launchFtSlaveCmd(); - LOG(MCdebugProgressDetail, job, "Stopped generate part %s [%p]", url.str(), this); + LOG(MCdebugProgressDetail, "Stopped generate part %s [%p]", url.str(), this); allDone = true; return ok; @@ -530,7 +529,7 @@ bool FileTransferThread::transferAndSignal() } catch (IException * e) { - FLLOG(MCexception(e), job, e, "Transferring files"); + FLLOG(MCexception(e), e, "Transferring files"); setErrorOwn(e); } } @@ -674,7 +673,6 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR allowRecovery = !options->getPropBool(ANnoRecover) && !querySplitPrefix(); isRecovering = allowRecovery && progressTree->hasProp(ANhasProgress); isSafeMode = options->getPropBool(ANsafe); - job = unknownJob; progressReport = NULL; abortChecker = NULL; sizeToBeRead = 0; @@ -692,7 +690,7 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR copyCompressed = false; transferBufferSize = options->getPropInt(ANtransferBufferSize); if (transferBufferSize) - LOG(MCdebugProgressDetail, job, "Using transfer buffer size %d", transferBufferSize); + LOG(MCdebugProgressDetail, "Using transfer buffer size %d", transferBufferSize); else // zero is default transferBufferSize = DEFAULT_STD_BUFFER_SIZE; progressDone = false; @@ -710,7 +708,7 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR fileUmask = (int)strtol(umaskStr, &eptr, 8); if (errno || *eptr != '\0') { - LOG(MCdebugInfo, job, "Invalid umask value <%s> ignored", umaskStr); + LOG(MCdebugInfo, "Invalid umask value <%s> ignored", umaskStr); fileUmask = -1; } else @@ -763,7 +761,7 @@ void FileSprayer::addEmptyFilesToPartition(unsigned from, unsigned to) { for (unsigned i = from; i < to ; i++) { - LOG(MCdebugProgressDetail, job, "Insert a dummy entry for target %d", i); + LOG(MCdebugProgressDetail, "Insert a dummy entry for target %d", i); PartitionPoint & next = createLiteral(0, NULL, 0); next.whichOutput = i; partition.append(next); @@ -791,7 +789,7 @@ void FileSprayer::afterTransfer() { if (calcInputCRC()) { - LOG(MCdebugProgressDetail, job, "Checking input CRCs"); + LOG(MCdebugProgressDetail, "Checking input CRCs"); CRC32Merger partCRC; unsigned startCurSource = 0; @@ -802,7 +800,7 @@ void FileSprayer::afterTransfer() if (!curProgress.hasInputCRC) { - LOG(MCdebugProgressDetail, job, "Could not calculate input CRCs - cannot check"); + LOG(MCdebugProgressDetail, "Could not calculate input CRCs - cannot check"); break; } partCRC.addChildCRC(curProgress.inputLength, curProgress.inputCRC, false); @@ -969,9 +967,9 @@ void FileSprayer::beforeTransfer() } } catch (IException *e) { - FLLOG(MCexception(e), job, e, "Creating Directory"); + FLLOG(MCexception(e), e, "Creating Directory"); e->Release(); - LOG(MCdebugInfo, job, "Ignoring create directory error"); + LOG(MCdebugInfo, "Ignoring create directory error"); } // If pushing files, and not recovering, then need to delete the target files, because the slaves might be writing in any order @@ -1041,7 +1039,7 @@ void FileSprayer::beforeTransfer() if (machine->getOS() == MachineOsW2K) { throttleNicSpeed = machine->getNicSpeedMbitSec(); - LOG(MCdebugInfo, job, "Throttle target speed to %dMbit/sec", throttleNicSpeed); + LOG(MCdebugInfo, "Throttle target speed to %dMbit/sec", throttleNicSpeed); } } } @@ -1082,7 +1080,7 @@ bool FileSprayer::calcInputCRC() void FileSprayer::calculateOne2OnePartition() { - LOG(MCdebugProgressDetail, job, "Setting up one2One partition"); + LOG(MCdebugProgressDetail, "Setting up one2One partition"); if (sources.ordinality() != targets.ordinality()) throwError(DFTERR_ReplicateNumPartsDiffer); if (!srcFormat.equals(tgtFormat)) @@ -1153,7 +1151,7 @@ void FileSprayer::calculateSplitPrefixPartition(const char * splitPrefix) if (!srcFormat.equals(tgtFormat)) throwError(DFTERR_SplitPrefixSameFormat); - LOG(MCdebugProgressDetail, job, "Setting up split prefix partition"); + LOG(MCdebugProgressDetail, "Setting up split prefix partition"); Owned target = &targets.popGet(); // remove target, add lots of new ones RemoteFilename blobTarget; @@ -1190,7 +1188,7 @@ void FileSprayer::calculateSplitPrefixPartition(const char * splitPrefix) void FileSprayer::calculateMany2OnePartition() { - LOG(MCdebugProgressDetail, job, "Setting up many2one partition"); + LOG(MCdebugProgressDetail, "Setting up many2one partition"); const char *partSeparator = srcFormat.getPartSeparatorString(); offset_t partSeparatorLength = ( partSeparator == nullptr ? 0 : strlen(partSeparator)); offset_t lastContentLength = 0; @@ -1222,7 +1220,7 @@ void FileSprayer::calculateMany2OnePartition() void FileSprayer::calculateNoSplitPartition() { - LOG(MCdebugProgressDetail, job, "Setting up no split partition"); + LOG(MCdebugProgressDetail, "Setting up no split partition"); if (!usePullOperation() && !srcFormat.equals(tgtFormat)) throwError(DFTERR_NoSplitPushChangeFormat); @@ -1285,7 +1283,7 @@ void FileSprayer::calculateNoSplitPartition() void FileSprayer::calculateSprayPartition() { - LOG(MCdebugProgressDetail, job, "Calculating N:M partition"); + LOG(MCdebugProgressDetail, "Calculating N:M partition"); bool calcOutput = needToCalcOutput(); FormatPartitionerArray partitioners; @@ -1347,7 +1345,7 @@ IFormatPartitioner * FileSprayer::createPartitioner(aindex_t index, bool calcOut StringBuffer remoteFilename; FilePartInfo & cur = sources.item(index); cur.filename.getRemotePath(remoteFilename.clear()); - LOG(MCdebugInfoDetail, job, "Partition %d(%s)", index, remoteFilename.str()); + LOG(MCdebugInfoDetail, "Partition %d(%s)", index, remoteFilename.str()); srcFormat.quotedTerminator = options->getPropBool("@quotedTerminator", true); const SocketEndpoint & ep = cur.filename.queryEndpoint(); @@ -1379,7 +1377,7 @@ void FileSprayer::examineCsvStructure() storeCsvRecordStructure(*partitioner); } else - LOG(MCdebugInfoDetail, job, "No source CSV file to examine."); + LOG(MCdebugInfoDetail, "No source CSV file to examine."); } void FileSprayer::calculateOutputOffsets() @@ -1670,7 +1668,7 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize) FilePartInfo & cur = sources.item(idx); StringBuffer s; cur.filename.getPath(s); - LOG(MCdebugInfo, job, "Examine header of file %s", s.str()); + LOG(MCdebugInfo, "Examine header of file %s", s.str()); Owned file = createIFile(cur.filename); Owned io = file->open(IFOread); @@ -1928,14 +1926,14 @@ void FileSprayer::displayPartition() #ifdef _DEBUG if ((partition.item(idx).whichInput >= 0) && (partition.item(idx).whichInput < sources.ordinality()) ) - LOG(MCdebugInfoDetail, unknownJob, + LOG(MCdebugInfoDetail, " Header size: %" I64F "u, XML header size: %" I64F "u, XML footer size: %" I64F "u", sources.item(partition.item(idx).whichInput).headerSize, sources.item(partition.item(idx).whichInput).xmlHeaderLength, sources.item(partition.item(idx).whichInput).xmlFooterLength ); else - LOG(MCdebugInfoDetail, unknownJob," No source file for this partition"); + LOG(MCdebugInfoDetail, " No source file for this partition"); #endif } } @@ -1963,7 +1961,7 @@ void FileSprayer::gatherFileSizes(bool errorIfMissing) { FilePartInfoArray fileSizeQueue; - LOG(MCdebugProgress, job, "Start gathering file sizes..."); + LOG(MCdebugProgress, "Start gathering file sizes..."); ForEachItemIn(idx, sources) { @@ -1973,7 +1971,7 @@ void FileSprayer::gatherFileSizes(bool errorIfMissing) } gatherFileSizes(fileSizeQueue, errorIfMissing); - LOG(MCdebugProgress, job, "Finished gathering file sizes..."); + LOG(MCdebugProgress, "Finished gathering file sizes..."); } void FileSprayer::afterGatherFileSizes() @@ -1985,7 +1983,7 @@ void FileSprayer::afterGatherFileSizes() { FilePartInfo & cur = sources.item(idx2); - LOG(MCdebugProgress, job, "%9u:%s (size: %llu bytes)", + LOG(MCdebugProgress, "%9u:%s (size: %llu bytes)", idx2, cur.filename.getTail(tailStr.clear()).str(), cur.size ); cur.offset = totalSize; @@ -1999,8 +1997,8 @@ void FileSprayer::afterGatherFileSizes() throwError2(DFTERR_InputIsInvalidMultiple, cur.filename.getRemotePath(s).str(), srcFormat.getUnitSize()); } } - LOG(MCdebugProgress, job, "----------------------------------------------"); - LOG(MCdebugProgress, job, "All together: %llu bytes in %u file(s)", totalSize, sources.ordinality()); + LOG(MCdebugProgress, "----------------------------------------------"); + LOG(MCdebugProgress, "All together: %llu bytes in %u file(s)", totalSize, sources.ordinality()); } } @@ -2016,7 +2014,7 @@ void FileSprayer::gatherFileSizes(FilePartInfoArray & fileSizeQueue, bool errorI unsigned numThreads = (unsigned)sqrt((float)fileSizeQueue.ordinality()); if (numThreads>20) numThreads = 20; - LOG(MCdebugProgress, job, "Gathering %d file sizes on %d threads", fileSizeQueue.ordinality(), numThreads); + LOG(MCdebugProgress, "Gathering %d file sizes on %d threads", fileSizeQueue.ordinality(), numThreads); unsigned idx; for (idx = 0; idx < numThreads; idx++) threads.append(*new FileSizeThread(fileSizeQueue, fileSizeCS, compressedInput&&!copyCompressed, errorIfMissing)); @@ -2098,7 +2096,7 @@ void FileSprayer::gatherMissingSourceTarget(IFileDescriptor * source) } else if (secondarySize != -1) { - LOG(MCwarning, unknownJob, "Replicate - primary and secondary copies have different sizes (%" I64F "d v %" I64F "d) for part %u", primarySize, secondarySize, idx); + LOG(MCwarning, "Replicate - primary and secondary copies have different sizes (%" I64F "d v %" I64F "d) for part %u", primarySize, secondarySize, idx); continue; // ignore copy } } @@ -2106,7 +2104,7 @@ void FileSprayer::gatherMissingSourceTarget(IFileDescriptor * source) { if (primarySize == -1) { - LOG(MCwarning, unknownJob, "Replicate - neither primary or secondary copies exist for part %u", idx); + LOG(MCwarning, "Replicate - neither primary or secondary copies exist for part %u", idx); primarySize = 0; // to stop later failure to gather the file size } continue; // ignore copy @@ -2482,7 +2480,7 @@ void FileSprayer::insertHeaders() } addPrefix(filePrefix.length(), filePrefix.toByteArray(), idx, partitionWork); } - LOG(MCdebugProgress, job, "Publish headers"); + LOG(MCdebugProgress, "Publish headers"); partition.swapWith(partitionWork); } } @@ -2519,7 +2517,7 @@ void FileSprayer::calcNumConcurrentTransfers() numConcurrentTransfers = numTransferParts/3; if (numConcurrentTransfers > numTransferParts) numConcurrentTransfers = numTransferParts; - LOG(MCdebugInfo, job, "numTransferParts: %u, numConcurrentTransfers: %u", numTransferParts, numConcurrentTransfers); + LOG(MCdebugInfo, "numTransferParts: %u, numConcurrentTransfers: %u", numTransferParts, numConcurrentTransfers); } void FileSprayer::performTransfer() @@ -2541,7 +2539,7 @@ void FileSprayer::performTransfer() progressReport->setRange(getSizeReadAlready(), sizeToBeRead, transferSlaves.ordinality()); - LOG(MCdebugInfo, job, "Begin to transfer parts (%d threads)\n", numConcurrentTransfers); + LOG(MCdebugInfo, "Begin to transfer parts (%d threads)\n", numConcurrentTransfers); //Throttle maximum number of concurrent transfers by starting n threads, and //then waiting for one to complete before going on to the next @@ -2588,7 +2586,7 @@ void FileSprayer::performTransfer() void FileSprayer::pullParts() { bool needCalcCRC = calcCRC(); - LOG(MCdebugInfoDetail, job, "Calculate CRC = %d", needCalcCRC); + LOG(MCdebugInfoDetail, "Calculate CRC = %d", needCalcCRC); ForEachItemIn(idx, targets) { FileTransferThread & next = * new FileTransferThread(*this, FTactionpull, targets.item(idx).filename.queryEndpoint(), needCalcCRC, wuid); @@ -2610,7 +2608,7 @@ void FileSprayer::pullParts() void FileSprayer::pushWholeParts() { bool needCalcCRC = calcCRC(); - LOG(MCdebugInfoDetail, job, "Calculate CRC = %d", needCalcCRC); + LOG(MCdebugInfoDetail, "Calculate CRC = %d", needCalcCRC); //Create a slave for each of the target files, but execute it on the node corresponding to the first source file //For container mode this will need to execute on this node, or on a load balanced service ForEachItemIn(idx, targets) @@ -2662,7 +2660,7 @@ void FileSprayer::pushParts() void FileSprayer::transferUsingAPI(IAPICopyClient * copyClient) { - LOG(MCdebugInfo, job, "Transfer files using api: %s", copyClient->name()); + LOG(MCdebugInfo, "Transfer files using api: %s", copyClient->name()); OwnedPointerArrayOf apiClients; unsigned failCount = 0; @@ -2773,7 +2771,7 @@ void FileSprayer::transferUsingAPI(IAPICopyClient * copyClient) void FileSprayer::removeSource() { - LOG(MCwarning, job, "Source file removal not yet implemented"); + LOG(MCwarning, "Source file removal not yet implemented"); } bool FileSprayer::restorePartition() @@ -2940,7 +2938,7 @@ void FileSprayer::setSource(IFileDescriptor * source, unsigned copy, unsigned mi } if (sources.ordinality() == 0) - LOG(MCuserWarning, unknownJob, "The wildcarded source did not match any filenames"); + LOG(MCuserWarning, "The wildcarded source did not match any filenames"); // throwError(DFTERR_NoFilesMatchWildcard); //Now get the size of the files directly (to check they exist). If they don't exist then switch to the backup instead. @@ -3005,7 +3003,7 @@ void FileSprayer::setTarget(IDistributedFile * target) compressOutput = target->isCompressed(); - LOG(MCdebugInfo, unknownJob, "FileSprayer::setTarget: compressedInput:%s, compressOutput:%s", + LOG(MCdebugInfo, "FileSprayer::setTarget: compressedInput:%s, compressOutput:%s", boolToStr(compressedInput), boolToStr(compressOutput)); @@ -3119,7 +3117,7 @@ void FileSprayer::waitForTransferSem(Semaphore & sem) unsigned timeSinceProgress = msTick() - lastProgressTick; if (timeSinceProgress > EXPECTED_RESPONSE_TIME) { - LOG(MCwarning, unknownJob, "No response from any slaves in last %d seconds.", timeSinceProgress/1000); + LOG(MCwarning, "No response from any slaves in last %d seconds.", timeSinceProgress/1000); CriticalBlock block(soFarCrit); StringBuffer list; @@ -3158,7 +3156,7 @@ bool FileSprayer::isAborting() { if (abortChecker->abortRequested()) { - LOG(MCdebugInfo, unknownJob, "Abort requested via callback"); + LOG(MCdebugInfo, "Abort requested via callback"); aborting = true; } lastAbortCheckTick = nowTick; @@ -3258,15 +3256,15 @@ void FileSprayer::spray() progressTree->setPropBool("@noFileMatch", true); } - LOG(MCdebugInfo, job, "compressedInput:%d, compressOutput:%d", compressedInput, compressOutput); - LOG(MCdebugInfo, job, "noCommon:%s", boolToStr(options->getPropBool(ANnocommon))); - LOG(MCdebugInfo, job, "maxConnections option:%d", options->getPropInt(ANmaxConnections)); + LOG(MCdebugInfo, "compressedInput:%d, compressOutput:%d", compressedInput, compressOutput); + LOG(MCdebugInfo, "noCommon:%s", boolToStr(options->getPropBool(ANnocommon))); + LOG(MCdebugInfo, "maxConnections option:%d", options->getPropInt(ANmaxConnections)); LocalAbortHandler localHandler(daftAbortHandler); if (allowRecovery && progressTree->getPropBool(ANcomplete)) { - LOG(MCdebugInfo, job, "Command completed successfully in previous invocation"); + LOG(MCdebugInfo, "Command completed successfully in previous invocation"); return; } @@ -3315,11 +3313,11 @@ void FileSprayer::spray() if (restorePartition()) { - LOG(MCdebugProgress, job, "Partition restored from recovery information"); + LOG(MCdebugProgress, "Partition restored from recovery information"); } else { - LOG(MCdebugProgress, job, "Calculate partition information"); + LOG(MCdebugProgress, "Calculate partition information"); if (replicate || copySource) calculateOne2OnePartition(); else if (!allowSplit()) @@ -3337,7 +3335,7 @@ void FileSprayer::spray() assignPartitionFilenames(); // assign source filenames - used in insertHeaders.. if (!replicate && !copySource) { - LOG(MCdebugProgress, job, "Insert headers"); + LOG(MCdebugProgress, "Insert headers"); insertHeaders(); } addEmptyFilesToPartition(); @@ -3395,7 +3393,7 @@ void FileSprayer::spray() copyEventText.append(distributedTarget->queryLogicalName()); //MORE: use new interface to send 'file copied' event - //LOG(MCevent, unknownJob, EVENT_FILECOPIED, copyEventText.str()); + //LOG(MCevent, EVENT_FILECOPIED, copyEventText.str()); cleanupRecovery(); } @@ -3961,7 +3959,7 @@ bool FileSprayer::calcUsePull() const if (allowRecovery && progressTree->hasProp(ANpull)) { bool usePull = progressTree->getPropBool(ANpull); - LOG(MCdebugInfo, job, "Pull = %d from recovery", (int)usePull); + LOG(MCdebugInfo, "Pull = %d from recovery", (int)usePull); return usePull; } @@ -3970,12 +3968,12 @@ bool FileSprayer::calcUsePull() const if (options->getPropBool(ANpull, false)) { - LOG(MCdebugInfo, job, "Use pull since explicitly specified"); + LOG(MCdebugInfo, "Use pull since explicitly specified"); return true; } if (options->getPropBool(ANpush, false)) { - LOG(MCdebugInfo, job, "Use push since explicitly specified"); + LOG(MCdebugInfo, "Use push since explicitly specified"); return false; } @@ -3985,7 +3983,7 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; sources.item(idx2).filename.queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use pull operation because %s cannot push", s.str()); + LOG(MCdebugInfo, "Use pull operation because %s cannot push", s.str()); return true; } } @@ -3993,7 +3991,7 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; sources.item(0).filename.queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use pull operation because %s doesn't appear to have an ftslave", s.str()); + LOG(MCdebugInfo, "Use pull operation because %s doesn't appear to have an ftslave", s.str()); return true; } @@ -4003,7 +4001,7 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; targets.item(idx).queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use push operation because %s cannot pull", s.str()); + LOG(MCdebugInfo, "Use push operation because %s cannot pull", s.str()); return false; } } @@ -4012,18 +4010,18 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; targets.item(0).queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use push operation because %s doesn't appear to have an ftslave", s.str()); + LOG(MCdebugInfo, "Use push operation because %s doesn't appear to have an ftslave", s.str()); return false; } //Use push if going to a single node. if ((targets.ordinality() == 1) && (sources.ordinality() > 1)) { - LOG(MCdebugInfo, job, "Use push operation because going to a single node from many"); + LOG(MCdebugInfo, "Use push operation because going to a single node from many"); return false; } - LOG(MCdebugInfo, job, "Use pull operation as default"); + LOG(MCdebugInfo, "Use pull operation as default"); return true; } diff --git a/dali/ft/filecopy.ipp b/dali/ft/filecopy.ipp index 36d6f24cda5..e6005431304 100644 --- a/dali/ft/filecopy.ipp +++ b/dali/ft/filecopy.ipp @@ -88,7 +88,6 @@ protected: Semaphore * sem; byte action; bool calcCRC; - LogMsgJobInfo job; bool allDone; bool started; StringAttr wuid; @@ -318,7 +317,6 @@ protected: OutputProgressArray progress; IDaftProgress * progressReport; IAbortRequestCallback * abortChecker; - LogMsgJobInfo job; offset_t totalSize; unsigned __int64 sizeToBeRead; bool replicate; diff --git a/dali/ft/ftbase.cpp b/dali/ft/ftbase.cpp index 8417af2f355..47cd917ddb9 100644 --- a/dali/ft/ftbase.cpp +++ b/dali/ft/ftbase.cpp @@ -64,7 +64,7 @@ void renameDfuTempToFinal(const RemoteFilename & realname) EXCLOG(e, "Failed to rename target file"); StringBuffer oldName; realname.getPath(oldName); - LOG(MCdebugInfoDetail, unknownJob, "Error: Rename %s->%s failed - tring to delete target and rename again", oldName.str(), newTailname.str()); + LOG(MCdebugInfoDetail, "Error: Rename %s->%s failed - tring to delete target and rename again", oldName.str(), newTailname.str()); e->Release(); OwnedIFile old = createIFile(realname); old->remove(); @@ -124,7 +124,7 @@ void PartitionPoint::deserialize(MemoryBuffer & in) void PartitionPoint::display() { StringBuffer fulli, fullo; - LOG(MCdebugInfoDetail, unknownJob, + LOG(MCdebugInfoDetail, "Partition %s{%d}[%" I64F "d size %" I64F "d]->%s{%d}[%" I64F "d size %" I64F "d]", inputName.getPath(fulli).str(), whichInput, inputOffset, inputLength, outputName.getPath(fullo).str(), whichOutput, outputOffset, outputLength); @@ -581,7 +581,7 @@ MemoryBuffer & OutputProgress::deserializeExtra(MemoryBuffer & in, unsigned vers static const char * const statusText[] = {"Init","Active","Copied","Renamed"}; void OutputProgress::trace() { - LOG(MCdebugInfoDetail, unknownJob, "Chunk %d status: %s input length: %" I64F "d[CRC:%x] -> output length:%" I64F "d[CRC:%x]", whichPartition, statusText[status], inputLength, inputCRC, outputLength, outputCRC); + LOG(MCdebugInfoDetail, "Chunk %d status: %s input length: %" I64F "d[CRC:%x] -> output length:%" I64F "d[CRC:%x]", whichPartition, statusText[status], inputLength, inputCRC, outputLength, outputCRC); } MemoryBuffer & OutputProgress::serializeCore(MemoryBuffer & out) @@ -666,7 +666,7 @@ void OutputProgress::save(IPropertyTree * tree) void displayProgress(OutputProgressArray & progress) { - LOG(MCdebugInfoDetail, unknownJob, "Progress:"); + LOG(MCdebugInfoDetail, "Progress:"); ForEachItemIn(idx, progress) progress.item(idx).trace(); } @@ -675,7 +675,7 @@ void displayProgress(OutputProgressArray & progress) void displayPartition(PartitionPointArray & partition) { - LOG(MCdebugInfoDetail, unknownJob, "Partition:"); + LOG(MCdebugInfoDetail, "Partition:"); ForEachItemIn(idx, partition) partition.item(idx).display(); } @@ -747,7 +747,7 @@ size32_t CrcIOStream::write(size32_t len, const void * data) static int breakCount; bool daftAbortHandler() { - LOG(MCprogress, unknownJob, "Aborting..."); + LOG(MCprogress, "Aborting..."); // hit ^C 3 times to really stop it... if (breakCount++ >= 2) { diff --git a/dali/ft/ftslavelib.cpp b/dali/ft/ftslavelib.cpp index 6e17857a4f3..7ebf3cbe59b 100644 --- a/dali/ft/ftslavelib.cpp +++ b/dali/ft/ftslavelib.cpp @@ -89,7 +89,7 @@ bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryB StringBuffer text; fullPath.getRemotePath(text); - LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str()); + LOG(MCdebugProgress, "Process partition %d(%s)", whichInput, text.str()); Owned processor = createFormatProcessor(srcFormat, tgtFormat, true); Owned target = createOutputProcessor(tgtFormat); processor->setTarget(target); diff --git a/dali/ft/fttransform.cpp b/dali/ft/fttransform.cpp index ee3ab53a3ee..4e8ca2fdec8 100644 --- a/dali/ft/fttransform.cpp +++ b/dali/ft/fttransform.cpp @@ -538,7 +538,7 @@ void TransferServer::appendTransformed(unsigned chunkIndex, ITransformer * input offset_t outputOffset = out->tell(); offset_t inputOffset = input->tell(); if (totalLengthToRead) - LOG(MCdebugProgress, unknownJob, "Progress: %d%% done. [%" I64F "u]", (unsigned)(totalLengthRead*100/totalLengthToRead), (unsigned __int64)totalLengthRead); + LOG(MCdebugProgress, "Progress: %d%% done. [%" I64F "u]", (unsigned)(totalLengthRead*100/totalLengthToRead), (unsigned __int64)totalLengthRead); curProgress.status = (gotLength == 0) ? OutputProgress::StatusCopied : OutputProgress::StatusActive; curProgress.inputLength = input->tell()-startInputOffset; @@ -570,7 +570,7 @@ void TransferServer::appendTransformed(unsigned chunkIndex, ITransformer * input #ifdef _WIN32 if (gpfFrequency && ((fastRand() % gpfFrequency) == 0)) { - LOG(MCdebugInfo, unknownJob, "About to crash...."); + LOG(MCdebugInfo, "About to crash...."); *(char *)0 = 0; } #endif @@ -611,15 +611,15 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action) if (action == FTactionpull) { partition.item(0).outputName.getPath(localFilename); - LOG(MCdebugProgress, unknownJob, "Process Pull Command: %s", localFilename.str()); + LOG(MCdebugProgress, "Process Pull Command: %s", localFilename.str()); } else { partition.item(0).inputName.getPath(localFilename); - LOG(MCdebugProgress, unknownJob, "Process Push Command: %s", localFilename.str()); + LOG(MCdebugProgress, "Process Push Command: %s", localFilename.str()); } - LOG(MCdebugProgress, unknownJob, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency); - LOG(MCdebugProgress, unknownJob, "copySourceTimeStamp(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", copySourceTimeStamp, mirror, isSafeMode, calcInputCRC, calcOutputCRC); + LOG(MCdebugProgress, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency); + LOG(MCdebugProgress, "copySourceTimeStamp(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", copySourceTimeStamp, mirror, isSafeMode, calcInputCRC, calcOutputCRC); displayPartition(partition); @@ -658,7 +658,7 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action) progress.item(i2).deserializeExtra(msg, 2); } - LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize); + LOG(MCdebugProgress, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize); PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0); PROGLOG("encrypt(%d), decrypt(%d)", encryptKey.isEmpty()?0:1, decryptKey.isEmpty()?0:1); if (fileUmask != -1) @@ -691,7 +691,7 @@ void TransferServer::transferChunk(unsigned chunkIndex) StringBuffer targetPath; curPartition.outputName.getPath(targetPath); - LOG(MCdebugProgress, unknownJob, "Begin to transfer chunk %d (offset: %" I64F "d, size: %" I64F "d) to target:'%s' (offset: %" I64F "d, size: %" I64F "d) ", + LOG(MCdebugProgress, "Begin to transfer chunk %d (offset: %" I64F "d, size: %" I64F "d) to target:'%s' (offset: %" I64F "d, size: %" I64F "d) ", chunkIndex, curPartition.inputOffset, curPartition.inputLength, targetPath.str(), curPartition.outputOffset, curPartition.outputLength); const unsigned __int64 startOutOffset = out->tell(); if (startOutOffset != curPartition.outputOffset+curProgress.outputLength) @@ -779,7 +779,7 @@ bool TransferServer::pull() unsigned __int64 progressOffset = curOutputOffset + curProgress.outputLength; if (progressOffset > size) { - LOG(MCwarning, unknownJob, "Recovery information seems to be invalid (%" I64F "d %" I64F "d) start copying from the beginning", + LOG(MCwarning, "Recovery information seems to be invalid (%" I64F "d %" I64F "d) start copying from the beginning", size, progressOffset); //reset any remaining partitions... for (i = start; i < numPartitions; i++) @@ -796,7 +796,7 @@ bool TransferServer::pull() StringBuffer localFilename; localTempFilename.getPath(localFilename); - LOG(MCdebugProgress, unknownJob, "Continue pulling to file: %s from recovery position", localFilename.str()); + LOG(MCdebugProgress, "Continue pulling to file: %s from recovery position", localFilename.str()); start = i; goto processedProgress; // break out of both loops } @@ -835,7 +835,7 @@ bool TransferServer::pull() curOutput = curPartition.whichOutput; if (curProgress.status == OutputProgress::StatusRenamed) { - LOG(MCdebugProgress, unknownJob, "Renamed file found - must be CRC recovery"); + LOG(MCdebugProgress, "Renamed file found - must be CRC recovery"); idx = queryLastOutput(curOutput); continue; } @@ -868,7 +868,7 @@ bool TransferServer::pull() outio.setown(createCompressedFileWriter(outio, false, 0, true, compressor, COMPRESS_METHOD_LZ4)); } - LOG(MCdebugProgress, unknownJob, "Start pulling to file: %s", localFilename.str()); + LOG(MCdebugProgress, "Start pulling to file: %s", localFilename.str()); //Find the last partition entry that refers to the same file. if (!compressOutput && fsProperties.preExtendOutput) @@ -881,7 +881,7 @@ bool TransferServer::pull() stat_type prevWrites = outio->getStatistic(StNumDiskWrites); outio->write(lastOffset-sizeof(null),sizeof(null),&null); curProgress.numWrites += (outio->getStatistic(StNumDiskWrites)-prevWrites); - LOG(MCdebugProgress, unknownJob, "Extend length of target file to %" I64F "d", lastOffset); + LOG(MCdebugProgress, "Extend length of target file to %" I64F "d", lastOffset); } } diff --git a/dali/sasha/sacoalescer.cpp b/dali/sasha/sacoalescer.cpp index 7341411429a..95b885a025f 100644 --- a/dali/sasha/sacoalescer.cpp +++ b/dali/sasha/sacoalescer.cpp @@ -159,7 +159,7 @@ void coalesceDatastore(IPropertyTree *coalesceProps, bool force) } catch (IException *e) { - LOG(MCoperatorError, unknownJob, e, "COALESCER: Unexpected exception, coalesce component halted"); + LOG(MCoperatorError, e, "COALESCER: Unexpected exception, coalesce component halted"); throw; } } diff --git a/dali/sasha/saserver.cpp b/dali/sasha/saserver.cpp index 8dcc464b59d..16f808542cd 100644 --- a/dali/sasha/saserver.cpp +++ b/dali/sasha/saserver.cpp @@ -84,7 +84,7 @@ static void stopServer() ForEachItemInRev(i,servers) { ISashaServer &server=servers.item(i); - LOG(MCprogress, unknownJob, "Stopping %d",i); + LOG(MCprogress, "Stopping %d",i); server.stop(); } ForEachItemInRev(j,servers) @@ -96,10 +96,10 @@ static void stopServer() static bool actionOnAbort() { - LOG(MCprogress, unknownJob, "Stop signalled"); + LOG(MCprogress, "Stop signalled"); if (stopped) { - LOG(MCprogress, unknownJob, "Previously marked stopped. Killing process.."); + LOG(MCprogress, "Previously marked stopped. Killing process.."); queryLogMsgManager()->flushQueue(10*1000); #ifdef _WIN32 TerminateProcess(GetCurrentProcess(), 1); @@ -118,8 +118,8 @@ static bool actionOnAbort() void requestStop(IException *e) { if (e) - LOG(MCoperatorError, unknownJob, e, "SASERVER: Unexpected exception, saserver terminating"); - LOG(MCprogress, unknownJob, "Stop requested"); + LOG(MCoperatorError, e, "SASERVER: Unexpected exception, saserver terminating"); + LOG(MCprogress, "Stop requested"); stopSem.signal(); } diff --git a/dali/sasha/saverify.cpp b/dali/sasha/saverify.cpp index 774fc4c7855..53131d7b571 100644 --- a/dali/sasha/saverify.cpp +++ b/dali/sasha/saverify.cpp @@ -543,9 +543,9 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread ep.getEndpointHostText(epstr); CriticalBlock block(sect); if (failurelimit) { - LOG(MCoperatorError, unknownJob,"DAFSMON: dafilesrv on %s cannot be contacted",epstr.str()); + LOG(MCoperatorError, "DAFSMON: dafilesrv on %s cannot be contacted",epstr.str()); if (--failurelimit==0) - LOG(MCoperatorError, unknownJob,"DAFSMON: monitoring suspended for cluster %s (too many failures)",clustername); + LOG(MCoperatorError, "DAFSMON: monitoring suspended for cluster %s (too many failures)",clustername); } } } @@ -557,7 +557,7 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread ep.getEndpointHostText(epstr); StringBuffer s; s.appendf("DAFSMON: dafilesrv %s",epstr.str()); - LOG(MCoperatorError, unknownJob, e, s.str()); + LOG(MCoperatorError, e, s.str()); e->Release(); return; } diff --git a/dali/server/daserver.cpp b/dali/server/daserver.cpp index d02451e3ca6..3e4aac3c583 100644 --- a/dali/server/daserver.cpp +++ b/dali/server/daserver.cpp @@ -95,13 +95,13 @@ static void stopServer() ForEachItemInRev(h,servers) { IDaliServer &server=servers.item(h); - LOG(MCprogress, unknownJob, "Suspending %d",h); + LOG(MCprogress, "Suspending %d",h); server.suspend(); } ForEachItemInRev(i,servers) { IDaliServer &server=servers.item(i); - LOG(MCprogress, unknownJob, "Stopping %d",i); + LOG(MCprogress, "Stopping %d",i); server.stop(); } closeCoven(); @@ -649,7 +649,7 @@ int main(int argc, const char* argv[]) catch (IException *e) { StringBuffer s("Failure whilst preparing dali backup location: "); - LOG(MCoperatorError, unknownJob, e, s.append(mirrorPath).append(". Backup disabled").str()); + LOG(MCoperatorError, e, s.append(mirrorPath).append(". Backup disabled").str()); serverConfig->removeProp("SDS/@remoteBackupLocation"); e->Release(); } diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index 6d4f793b19d..4fd82e96076 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -128,7 +128,7 @@ static const char * getResultText(StringBuffer & s, const char * stepname, unsig void logGetResult(const char * name, const char * stepname, unsigned sequence) { - LOG(MCgetresult, unknownJob, "getResult%s(%s,%d)", name, nullText(stepname), sequence); + LOG(MCgetresult, "getResult%s(%s,%d)", name, nullText(stepname), sequence); } //======================================================================================= @@ -725,7 +725,7 @@ IConstWUResult *EclAgent::getResultForGet(const char *name, unsigned sequence) IConstWUResult *EclAgent::getExternalResult(const char * wuid, const char *name, unsigned sequence) { - LOG(MCsetresult, unknownJob, "EclAgent::getExternalResult(wuid:'%s',name='%s',sequence:%d)", nullText(wuid), nullText(name), sequence); + LOG(MCsetresult, "EclAgent::getExternalResult(wuid:'%s',name='%s',sequence:%d)", nullText(wuid), nullText(name), sequence); Owned factory = getWorkUnitFactory(); Owned externalWU = factory->openWorkUnit(wuid); if (externalWU) @@ -774,7 +774,7 @@ void EclAgent::outputFormattedResult(const char * name, unsigned sequence, bool void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val, unsigned size) { - LOG(MCsetresult, unknownJob, "setResultInt(%s,%d,%" I64F "d)", nullText(name), sequence, val); + LOG(MCsetresult, "setResultInt(%s,%d,%" I64F "d)", nullText(name), sequence, val); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -798,7 +798,7 @@ void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val, u void EclAgent::setResultUInt(const char * name, unsigned sequence, unsigned __int64 val, unsigned size) { - LOG(MCsetresult, unknownJob, "setResultUInt(%s,%d,%" I64F "u)", nullText(name), sequence, val); + LOG(MCsetresult, "setResultUInt(%s,%d,%" I64F "u)", nullText(name), sequence, val); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -823,7 +823,7 @@ void EclAgent::setResultUInt(const char * name, unsigned sequence, unsigned __in void EclAgent::setResultReal(const char *name, unsigned sequence, double val) { // Still a bit of a mess - variables vs results - LOG(MCsetresult, unknownJob, "setResultReal(%s,%d,%6f)", nullText(name), sequence, val); + LOG(MCsetresult, "setResultReal(%s,%d,%6f)", nullText(name), sequence, val); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1123,7 +1123,7 @@ void EclAgent::setResultData(const char * stepname, unsigned sequence, int len, void EclAgent::doSetResultString(type_t type, const char *name, unsigned sequence, int len, const char *val) { - LOG(MCsetresult, unknownJob, "setResultString(%s,%d,(%d bytes))", nullText(name), sequence, len); + LOG(MCsetresult, "setResultString(%s,%d,(%d bytes))", nullText(name), sequence, len); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1148,7 +1148,7 @@ void EclAgent::doSetResultString(type_t type, const char *name, unsigned sequenc //used to output a row void EclAgent::setResultRaw(const char * name, unsigned sequence, int len, const void *val) { - LOG(MCsetresult, unknownJob, "setResultRaw(%s,%d,(%d bytes))", nullText(name), sequence, len); + LOG(MCsetresult, "setResultRaw(%s,%d,(%d bytes))", nullText(name), sequence, len); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1172,7 +1172,7 @@ void EclAgent::setResultRaw(const char * name, unsigned sequence, int len, const void EclAgent::setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *xform) { - LOG(MCsetresult, unknownJob, "setResultSet(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultSet(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1232,7 +1232,7 @@ void EclAgent::setResultSet(const char * name, unsigned sequence, bool isAll, si void EclAgent::setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val) { - LOG(MCsetresult, unknownJob, "setResultUnicode(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultUnicode(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); @@ -1260,7 +1260,7 @@ void EclAgent::setResultUnicode(const char * name, unsigned sequence, int len, U void EclAgent::setResultBool(const char *name, unsigned sequence, bool val) { - LOG(MCsetresult, unknownJob, "setResultBool(%s,%d,%s)", nullText(name), sequence, val ? "true" : "false"); + LOG(MCsetresult, "setResultBool(%s,%d,%s)", nullText(name), sequence, val ? "true" : "false"); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); @@ -1285,7 +1285,7 @@ void EclAgent::setResultBool(const char *name, unsigned sequence, bool val) void EclAgent::setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val) { - LOG(MCsetresult, unknownJob, "setResultDecimal(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultDecimal(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1314,7 +1314,7 @@ void EclAgent::setResultDecimal(const char *name, unsigned sequence, int len, in void EclAgent::setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend) { - LOG(MCsetresult, unknownJob, "setResultDataset(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultDataset(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned result = updateWorkUnitResult(w, name, sequence); if (!result) @@ -1866,11 +1866,11 @@ void EclAgent::doProcess() CCycleTimer elapsedTimer; try { - LOG(MCrunlock, unknownJob, "Waiting for workunit lock"); + LOG(MCrunlock, "Waiting for workunit lock"); unsigned eclccCodeVersion; { WorkunitUpdate w = updateWorkUnit(); - LOG(MCrunlock, unknownJob, "Obtained workunit lock"); + LOG(MCrunlock, "Obtained workunit lock"); if (w->hasDebugValue("traceLevel")) traceLevel = w->getDebugValueInt("traceLevel", 10); w->setTracingValue("EclAgentBuild", hpccBuildInfo.buildTag); @@ -2266,9 +2266,9 @@ void EclAgent::runProcess(IEclProcess *process) debugContext->checkBreakpoint(DebugStateFinished, NULL, NULL); debugContext->debugTerminate(); } - LOG(MCrunlock, unknownJob, "Releasing persist read locks"); + LOG(MCrunlock, "Releasing persist read locks"); persistReadLocks.kill(); - LOG(MCrunlock, unknownJob, "Released persist read locks"); + LOG(MCrunlock, "Released persist read locks"); } unsigned EclAgent::getWorkflowIdDeprecated() @@ -2301,7 +2301,7 @@ IRemoteConnection *EclAgentWorkflowMachine::startPersist(const char * logicalNam { persistLock = persistCache.getValue(logicalName); persistCache.setValue(logicalName, NULL); - LOG(MCrunlock, unknownJob, "Decached persist read lock for %s", logicalName); + LOG(MCrunlock, "Decached persist read lock for %s", logicalName); } else persistLock = agent.startPersist(logicalName); @@ -2383,7 +2383,7 @@ void EclAgentWorkflowMachine::prelockPersists() char const * name = names.item(idx); Owned persistLock = agent.startPersist(name); persistCache.setValue(name, persistLock); - LOG(MCrunlock, unknownJob, "Cached persist read lock for %s", name); + LOG(MCrunlock, "Cached persist read lock for %s", name); } persistsPrelocked = true; } @@ -2424,16 +2424,16 @@ void EclAgentWorkflowMachine::obtainRunlock() { StringBuffer xpath; xpath.append("/WorkUnitRunLocks/").append(agent.wuid.get()); - LOG(MCrunlock, unknownJob, "Waiting for run lock"); + LOG(MCrunlock, "Waiting for run lock"); runlock.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_CREATE | RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, INFINITE)); - LOG(MCrunlock, unknownJob, "Obtained run lock"); + LOG(MCrunlock, "Obtained run lock"); if(!runlock) agent.fail(0, "EclAgent could not get a lock to run the workunit"); } void EclAgentWorkflowMachine::releaseRunlock() { - LOG(MCrunlock, unknownJob, "Releasing run lock"); + LOG(MCrunlock, "Releasing run lock"); if(runlock && queryDaliServerVersion().compare("1.3") < 0) runlock->close(true); runlock.clear(); @@ -2448,7 +2448,7 @@ IRemoteConnection *EclAgentWorkflowMachine::obtainCriticalLock(const char *name) void EclAgentWorkflowMachine::releaseCriticalLock(IRemoteConnection *criticalLock) { - LOG(MCrunlock, unknownJob, "Releasing critical lock"); + LOG(MCrunlock, "Releasing critical lock"); if(criticalLock && queryDaliServerVersion().compare("1.3") < 0) criticalLock->close(true); } @@ -2546,7 +2546,7 @@ void EclAgentWorkflowMachine::doExecutePersistItem(IRuntimeWorkflowItem & item) { persistLock.set(persistCache.getValue(logicalName)); persistCache.setValue(logicalName, NULL); - LOG(MCrunlock, unknownJob, "Decached persist read lock for %s", logicalName); + LOG(MCrunlock, "Decached persist read lock for %s", logicalName); } else persistLock.setown(agent.startPersist(logicalName)); @@ -2804,7 +2804,7 @@ bool EclAgent::checkPersistUptoDate(IRuntimeWorkflowItem & item, const char * lo bool EclAgent::changePersistLockMode(IRemoteConnection *persistLock, unsigned mode, const char * name, bool repeat) { - LOG(MCrunlock, unknownJob, "Waiting to change persist lock to %s for %s", (mode == RTM_LOCK_WRITE) ? "write" : "read", name); + LOG(MCrunlock, "Waiting to change persist lock to %s for %s", (mode == RTM_LOCK_WRITE) ? "write" : "read", name); //When converting a read lock to a write lock so the persist can be rebuilt hold onto the lock as short as //possible. Otherwise lots of workunits each trying to convert read locks to write locks will mean //that the read lock is never released by all the workunits at the same time, so no workunit can progress. @@ -2850,7 +2850,7 @@ IRemoteConnection *EclAgent::getPersistReadLock(const char * logicalName) for (const char * cur = name;*cur;cur++) xpath.append(isalnum(*cur) ? *cur : '_'); - LOG(MCrunlock, unknownJob, "Waiting for persist read lock for %s", name); + LOG(MCrunlock, "Waiting for persist read lock for %s", name); Owned persistLock; for (;;) { @@ -2972,7 +2972,7 @@ bool EclAgent::alreadyLockedPersist(const char * persistName) void EclAgent::finishPersist(const char * persistName, IRemoteConnection *persistLock) { - LOG(MCrunlock, unknownJob, "Finished persists - add to read lock list"); + LOG(MCrunlock, "Finished persists - add to read lock list"); processedPersists.append(persistName); persistReadLocks.append(*persistLock); } @@ -3289,7 +3289,7 @@ void EclAgent::reportProgress(const char *progress, unsigned flags) if (progress) { // MORE - think about how to best do this - LOG(MCdebugProgress, unknownJob,"%s", progress); + LOG(MCdebugProgress, "%s", progress); // WorkunitUpdate wu = updateWorkUnit(); // wu->reportProgress(progress, flags); } diff --git a/ecl/eclcc/eclcc.cpp b/ecl/eclcc/eclcc.cpp index eda80d7d39e..ece9f7717d7 100644 --- a/ecl/eclcc/eclcc.cpp +++ b/ecl/eclcc/eclcc.cpp @@ -1022,7 +1022,7 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char void EclCC::getComplexity(IWorkUnit *wu, IHqlExpression * query, IErrorReceiver & errs) { double complexity = getECLcomplexity(query, &errs, wu, optTargetClusterType); - LOG(MCoperatorProgress, unknownJob, "Complexity = %g", complexity); + LOG(MCoperatorProgress, "Complexity = %g", complexity); } //========================================================================================= @@ -2317,7 +2317,7 @@ void EclCC::traceError(char const * format, ...) UERRLOG("", encoded.str()); } else - VALOG(MCuserError, unknownJob, format, args); + VALOG(MCuserError, format, args); va_end(args); } diff --git a/ecl/hql/hqlexpr.cpp b/ecl/hql/hqlexpr.cpp index 7b86bb23ab3..90e2ef1d91a 100644 --- a/ecl/hql/hqlexpr.cpp +++ b/ecl/hql/hqlexpr.cpp @@ -15469,7 +15469,7 @@ extern HQL_API IHqlExpression * getFileViewerRecord(IHqlExpression * record, boo } catch (IException * e) { - LOG(MCwarning, unknownJob, e); + LOG(MCwarning, e); e->Release(); } return NULL; diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp index 2b8a188b6bc..56139d12077 100644 --- a/ecl/hqlcpp/hqlhtcpp.cpp +++ b/ecl/hqlcpp/hqlhtcpp.cpp @@ -19562,11 +19562,11 @@ static void logECL(const LogMsgCategory & category, size32_t len, const char * e unsigned size = next-ecl; if (ecl[size-1] == '\r') size--; - LOG(category, unknownJob, "%.*s", size, ecl); + LOG(category, "%.*s", size, ecl); len -= (next+1-ecl); ecl = next+1; } - LOG(category, unknownJob, "%s", ecl); + LOG(category, "%s", ecl); } @@ -19580,7 +19580,7 @@ void HqlCppTranslator::traceExpression(const char * title, IHqlExpression * expr if (!queryTrace(title)) return; - LOG(MCdebugInfo, unknownJob, "Tracing expressions: %s", title); + LOG(MCdebugInfo, "Tracing expressions: %s", title); if (options.traceIR) { EclIR::dbglogIR(expr); @@ -19607,7 +19607,7 @@ void HqlCppTranslator::traceExpressions(const char * title, WorkflowArray & work if (!queryTrace(title)) return; - LOG(MCdebugInfo, unknownJob, "Tracing expressions: %s", title); + LOG(MCdebugInfo, "Tracing expressions: %s", title); ForEachItemIn(idx1, workflow) { WorkflowItem & cur = workflow.item(idx1); @@ -19615,7 +19615,7 @@ void HqlCppTranslator::traceExpressions(const char * title, WorkflowArray & work if (compound) { - LOG(MCdebugInfo, unknownJob, "%s: #%d: id[%d]", title, idx1, cur.queryWfid()); + LOG(MCdebugInfo, "%s: #%d: id[%d]", title, idx1, cur.queryWfid()); if (options.traceIR) { diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp index 241641bd0f6..7fb6c7952ef 100644 --- a/ecl/hthor/hthorkey.cpp +++ b/ecl/hthor/hthorkey.cpp @@ -681,7 +681,7 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist //MORE: We could introduce a more efficient way of checking this that does not create a translator Owned actualTranslator = createRecordTranslator(expectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true)); - DBGLOG("Record layout translator created for %s", f->queryLogicalName()); + DBGLOG("Record layout translator created for %s", f->queryLogicalName()); actualTranslator->describe(); if (actualTranslator->keyedTranslated()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName()); @@ -2458,7 +2458,7 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc if (actualDiskMeta) { translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true))); - DBGLOG("Record layout translator created for %s", f->queryLogicalName()); + DBGLOG("Record layout translator created for %s", f->queryLogicalName()); translator->describe(); if (translator->canTranslate()) { diff --git a/ecl/wutest/wujobqtest.cpp b/ecl/wutest/wujobqtest.cpp index 276f8d0f07f..40f2e679f5b 100644 --- a/ecl/wutest/wujobqtest.cpp +++ b/ecl/wutest/wujobqtest.cpp @@ -155,11 +155,11 @@ void testEnqueue(unsigned nthreads,const char *qname) { unsigned start = msTick(); if (!partfile->remove()&&(copy==0)) // only warn about missing primary files - LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); + LOG(MCwarning, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); else { unsigned t = msTick()-start; if (t>5*1000) - LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); + LOG(MCwarning, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } } diff --git a/esp/bindings/SOAP/Platform/soapservice.cpp b/esp/bindings/SOAP/Platform/soapservice.cpp index cee7fda5e52..4a58cd3c51e 100644 --- a/esp/bindings/SOAP/Platform/soapservice.cpp +++ b/esp/bindings/SOAP/Platform/soapservice.cpp @@ -195,7 +195,7 @@ int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp) rpc_call->preunmarshall(xpp.get()); rpc_call->unmarshall(xpp.get(), multipart.get()); const char* userId = ctx->queryUserId(); - DBGLOG("JSON method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", + DBGLOG("JSON method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown"); ctx->setHTTPMethod("JSON"); } @@ -243,12 +243,12 @@ int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp) } catch (...) { response.set_status(SOAP_CLIENT_ERROR); response.set_err("Unknown error when parsing soap body XML"); - IERRLOG("SOAP request from %s@%s. Unknown error when parsing: %s", (userId&&*userId)?userId:"unknown", + IERRLOG("SOAP request from %s@%s. Unknown error when parsing: %s", (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown", requeststr.str()); return 0; } - DBGLOG("SOAP method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", + DBGLOG("SOAP method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown"); ctx->setHTTPMethod("SOAP"); } diff --git a/esp/esdllib/esdl_def.cpp b/esp/esdllib/esdl_def.cpp index dce6db127f8..62ee2797892 100644 --- a/esp/esdllib/esdl_def.cpp +++ b/esp/esdllib/esdl_def.cpp @@ -1669,7 +1669,7 @@ unsigned EsdlDefinition::walkChildrenDepthFirst( AddedObjs& foundByName, EsdlDef children->next(); } } //else { DBGLOG("%s<%s> no Children", StringBuffer(level*2, " ").str(), esdlObj->queryName()); } - } //else { DBGLOG("%s<%s> is not Struct", StringBuffer(level*2, " ").str(), esdlObj->queryName()); } + } //else { DBGLOG("%s<%s> is not Struct", StringBuffer(level*2, " ").str(), esdlObj->queryName()); } return stateOut; } diff --git a/esp/platform/espcfg.cpp b/esp/platform/espcfg.cpp index 60c495aa97f..911e5e163b9 100644 --- a/esp/platform/espcfg.cpp +++ b/esp/platform/espcfg.cpp @@ -689,7 +689,7 @@ void CEspConfig::loadBinding(binding_cfg &xcfg) { IEspRpcBinding* bind = xproc(xcfg.name.str(), xcfg.type.str(), m_envpt.get(), m_process.str()); if (bind) - LOG(MCoperatorInfo, unknownJob,"Load binding %s (type: %s, process: %s) succeeded", xcfg.name.str(), xcfg.type.str(), m_process.str()); + LOG(MCoperatorInfo, "Load binding %s (type: %s, process: %s) succeeded", xcfg.name.str(), xcfg.type.str(), m_process.str()); else OERRLOG("Failed to load binding %s (type: %s, process: %s)", xcfg.name.str(), xcfg.type.str(), m_process.str()); xcfg.bind.setown(bind); diff --git a/esp/platform/espcontext.cpp b/esp/platform/espcontext.cpp index 3270273fe5b..ea2c90f2d23 100755 --- a/esp/platform/espcontext.cpp +++ b/esp/platform/espcontext.cpp @@ -1029,7 +1029,7 @@ void ESPLOG(LogLevel level, const char* fmt, ...) { va_list args; va_start(args,fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } } @@ -1040,7 +1040,7 @@ void ESPLOG(IEspContext* ctx, LogLevel level, const char* fmt, ...) { va_list args; va_start(args,fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } } diff --git a/esp/platform/espp.hpp b/esp/platform/espp.hpp index 22fc9b742ef..27885454463 100644 --- a/esp/platform/espp.hpp +++ b/esp/platform/espp.hpp @@ -186,7 +186,7 @@ class CEspServer : public CInterface, { va_list args; va_start(args, fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } } diff --git a/esp/services/ws_fileio/ws_fileioservice.cpp b/esp/services/ws_fileio/ws_fileioservice.cpp index 304adaa908d..00335be3c88 100644 --- a/esp/services/ws_fileio/ws_fileioservice.cpp +++ b/esp/services/ws_fileio/ws_fileioservice.cpp @@ -159,13 +159,13 @@ bool CWsFileIOEx::onReadFileData(IEspContext &context, IEspReadFileDataRequest & if (io->read(offset, (int)dataToRead, buf) != dataToRead) { resp.setResult("ReadFileData error."); - LOG(MCprogress, unknownJob, "ReadFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "ReadFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } else { resp.setData(membuf); resp.setResult("ReadFileData done."); - LOG(MCprogress, unknownJob, "ReadFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "ReadFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } return true; @@ -237,12 +237,12 @@ bool CWsFileIOEx::onWriteFileData(IEspContext &context, IEspWriteFileDataRequest if (fileio->write(offset, len, srcdata.readDirect(len)) != len) { resp.setResult("WriteFileData error."); - LOG(MCprogress, unknownJob, "WriteFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "WriteFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } else { resp.setResult("WriteFileData done."); - LOG(MCprogress, unknownJob, "WriteFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "WriteFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } return true; diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp index 9ac375c37ff..4a83c2cc65c 100644 --- a/esp/services/ws_fs/ws_fsService.cpp +++ b/esp/services/ws_fs/ws_fsService.cpp @@ -1904,7 +1904,7 @@ IPropertyTree *CFileSprayEx::getAndValidateDropZone(const char *path, const char if (env->isDropZoneRestrictionEnabled()) throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "DropZone Plane not found for host %s path %s.", host, path); - LOG(MCdebugInfo, unknownJob, "No matching drop zone path on '%s' to file path: '%s'", host, path); + LOG(MCdebugInfo, "No matching drop zone path on '%s' to file path: '%s'", host, path); #endif return nullptr; } diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 309183dfd82..4b6ff05ad2b 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -467,7 +467,7 @@ unsigned WsWuInfo::sendComponentLogContent(IEspContext* context, IRemoteLogAcces if (abortCallback.abortRequested()) { - LOG(MCdebugInfo, unknownJob, "WsWuInfo::sendComponentLogContent(): abort requested via callback"); + LOG(MCdebugInfo, "WsWuInfo::sendComponentLogContent(): abort requested via callback"); break; } } diff --git a/esp/smc/SMCLib/LogicFileWrapper.hpp b/esp/smc/SMCLib/LogicFileWrapper.hpp index 032ae8f969d..3fc70350911 100644 --- a/esp/smc/SMCLib/LogicFileWrapper.hpp +++ b/esp/smc/SMCLib/LogicFileWrapper.hpp @@ -95,7 +95,7 @@ struct DeleteTask: public CInterface, implements ITask { StringBuffer e; e.appendf("Failed to remove file part %s\n",file->queryFilename()); - LOG(MCerror, unknownJob, "%s", e.str()); + LOG(MCerror, "%s", e.str()); errs.append(e); } } diff --git a/esp/smc/SMCLib/TpCommon.cpp b/esp/smc/SMCLib/TpCommon.cpp index ea498ebfcb0..d31c6ecd32f 100644 --- a/esp/smc/SMCLib/TpCommon.cpp +++ b/esp/smc/SMCLib/TpCommon.cpp @@ -90,7 +90,7 @@ extern TPWRAPPER_API void throwOrLogDropZoneLookUpError(int code, char const* fo if (env->isDropZoneRestrictionEnabled()) throw makeStringExceptionVA(code, format, args); - VALOG(MCdebugInfo, unknownJob, format, args); + VALOG(MCdebugInfo, format, args); #endif va_end(args); } diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index fa22badf415..560685e03f2 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -3881,7 +3881,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface file->setShareMode((IFSHmode)sMode); } if (TF_TRACE_PRE_IO) - PROGLOG("before open file '%s', (%d,%d,%d,%d,0%o)",name->text.get(),(int)mode,(int)share,extraFlags,sMode,cFlags); + PROGLOG("before open file '%s', (%d,%d,%d,%d,0%o)",name->text.get(),(int)mode,(int)share,extraFlags,sMode,cFlags); Owned fileio; #ifndef _WIN32 @@ -3943,9 +3943,9 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface if (TF_TRACE) { #ifndef _WIN32 - PROGLOG("open file '%s', (%d,%d) handle = %d",name->text.get(),(int)mode,(int)share,handle); + PROGLOG("open file '%s', (%d,%d) handle = %d",name->text.get(),(int)mode,(int)share,handle); #else - PROGLOG("open file '%s', (%d,%d) handle = %d, retries = %u, time(ms) = %u",name->text.get(),(int)mode,(int)share,handle,retries,timer.elapsedMs()); + PROGLOG("open file '%s', (%d,%d) handle = %d, retries = %u, time(ms) = %u",name->text.get(),(int)mode,(int)share,handle,retries,timer.elapsedMs()); #endif } return true; @@ -3958,7 +3958,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface IFileIO *fileio; checkFileIOHandle(reply, handle, fileio, true); if (TF_TRACE) - PROGLOG("close file, handle = %d",handle); + PROGLOG("close file, handle = %d",handle); reply.append(RFEnoerror); return true; } @@ -3978,13 +3978,13 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface size32_t numRead; unsigned posOfLength = reply.length(); if (TF_TRACE_PRE_IO) - PROGLOG("before read file, handle = %d, toread = %d",handle,len); + PROGLOG("before read file, handle = %d, toread = %d",handle,len); reply.reserve(sizeof(numRead)); void *data = reply.reserve(len); numRead = fileio->read(pos,len,data); stats.addRead(len); if (TF_TRACE) - PROGLOG("read file, handle = %d, pos = %" I64F "d, toread = %d, read = %d",handle,pos,len,numRead); + PROGLOG("read file, handle = %d, pos = %" I64F "d, toread = %d, read = %d",handle,pos,len,numRead); reply.setLength(posOfLength + sizeof(numRead) + numRead); reply.writeEndianDirect(posOfLength,sizeof(numRead),&numRead); } @@ -3998,7 +3998,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface __int64 size = fileio->size(); reply.append((unsigned)RFEnoerror).append(size); if (TF_TRACE) - PROGLOG("size file, handle = %d, size = %" I64F "d",handle,size); + PROGLOG("size file, handle = %d, size = %" I64F "d",handle,size); } void cmdSetSize(MemoryBuffer & msg, MemoryBuffer & reply) @@ -4008,7 +4008,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface msg.read(handle).read(size); IFileIO *fileio; if (TF_TRACE) - PROGLOG("set size file, handle = %d, size = %" I64F "d",handle,size); + PROGLOG("set size file, handle = %d, size = %" I64F "d",handle,size); checkFileIOHandle(reply, handle, fileio); fileio->setSize(size); reply.append((unsigned)RFEnoerror); @@ -4024,11 +4024,11 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface checkFileIOHandle(reply, handle, fileio); const byte *data = (const byte *)msg.readDirect(len); if (TF_TRACE_PRE_IO) - PROGLOG("before write file, handle = %d, towrite = %d",handle,len); + PROGLOG("before write file, handle = %d, towrite = %d",handle,len); size32_t numWritten = fileio->write(pos,len,data); stats.addWrite(numWritten); if (TF_TRACE) - PROGLOG("write file, handle = %d, towrite = %d, written = %d",handle,len,numWritten); + PROGLOG("write file, handle = %d, towrite = %d, written = %d",handle,len,numWritten); reply.append((unsigned)RFEnoerror).append(numWritten); } @@ -4037,7 +4037,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("exists, '%s'",name.get()); + PROGLOG("exists, '%s'",name.get()); Owned file=createIFile(name); bool e = file->exists(); reply.append((unsigned)RFEnoerror).append(e); @@ -4048,7 +4048,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("remove, '%s'",name.get()); + PROGLOG("remove, '%s'",name.get()); Owned file=createIFile(name); bool e = file->remove(); reply.append((unsigned)RFEnoerror).append(e); @@ -4077,7 +4077,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr toname; msg.read(toname); if (TF_TRACE) - PROGLOG("rename, '%s' to '%s'",fromname.get(),toname.get()); + PROGLOG("rename, '%s' to '%s'",fromname.get(),toname.get()); Owned file=createIFile(fromname); file->rename(toname); reply.append((unsigned)RFEnoerror); @@ -4090,7 +4090,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr toname; msg.read(toname); if (TF_TRACE) - PROGLOG("move, '%s' to '%s'",fromname.get(),toname.get()); + PROGLOG("move, '%s' to '%s'",fromname.get(),toname.get()); Owned file=createIFile(fromname); file->move(toname); reply.append((unsigned)RFEnoerror); @@ -4103,7 +4103,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr toname; msg.read(toname); if (TF_TRACE) - PROGLOG("copy, '%s' to '%s'",fromname.get(),toname.get()); + PROGLOG("copy, '%s' to '%s'",fromname.get(),toname.get()); copyFile(toname, fromname); reply.append((unsigned)RFEnoerror); } @@ -4122,7 +4122,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface __int64 written = fileio->appendFile(file,pos,len); stats.addWrite(written); if (TF_TRACE) - PROGLOG("append file, handle = %d, file=%s, pos = %" I64F "d len = %" I64F "d written = %" I64F "d",handle,srcname.get(),pos,len,written); + PROGLOG("append file, handle = %d, file=%s, pos = %" I64F "d len = %" I64F "d written = %" I64F "d",handle,srcname.get(),pos,len,written); reply.append((unsigned)RFEnoerror).append(written); } @@ -4131,7 +4131,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("isFile, '%s'",name.get()); + PROGLOG("isFile, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = (unsigned)file->isFile(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4142,7 +4142,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("isDir, '%s'",name.get()); + PROGLOG("isDir, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = (unsigned)file->isDirectory(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4153,7 +4153,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("isReadOnly, '%s'",name.get()); + PROGLOG("isReadOnly, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = (unsigned)file->isReadOnly(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4166,7 +4166,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface msg.read(name).read(set); if (TF_TRACE) - PROGLOG("setReadOnly, '%s' %d",name.get(),(int)set); + PROGLOG("setReadOnly, '%s' %d",name.get(),(int)set); Owned file=createIFile(name); file->setReadOnly(set); reply.append((unsigned)RFEnoerror); @@ -4178,7 +4178,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface unsigned fPerms; msg.read(name).read(fPerms); if (TF_TRACE) - PROGLOG("setFilePerms, '%s' 0%o",name.get(),fPerms); + PROGLOG("setFilePerms, '%s' 0%o",name.get(),fPerms); Owned file=createIFile(name); file->setFilePermissions(fPerms); reply.append((unsigned)RFEnoerror); @@ -4189,7 +4189,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("getTime, '%s'",name.get()); + PROGLOG("getTime, '%s'",name.get()); Owned file=createIFile(name); CDateTime createTime; CDateTime modifiedTime; @@ -4225,7 +4225,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface accessedTime.deserialize(msg); if (TF_TRACE) - PROGLOG("setTime, '%s'",name.get()); + PROGLOG("setTime, '%s'",name.get()); Owned file=createIFile(name); bool ret = file->setTime(creategot?&createTime:NULL,modifiedgot?&modifiedTime:NULL,accessedgot?&accessedTime:NULL); @@ -4237,7 +4237,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("CreateDir, '%s'",name.get()); + PROGLOG("CreateDir, '%s'",name.get()); Owned dir=createIFile(name); bool ret = dir->createDirectory(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4258,7 +4258,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface client.opendir.clear(); } if (TF_TRACE) - PROGLOG("GetDir, '%s', '%s', stream='%u'",name.get(),mask.get(),stream); + PROGLOG("GetDir, '%s', '%s', stream='%u'",name.get(),mask.get(),stream); if (!stream && !containsFileWildcard(mask)) { // if no streaming, and mask contains no wildcard, it is much more efficient to get the info without a directory iterator! @@ -4333,7 +4333,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface prev.setown(createRemoteDirectorIterator(ep, name, msg)); } if (TF_TRACE) - PROGLOG("MonitorDir, '%s' '%s'",name.get(),mask.get()); + PROGLOG("MonitorDir, '%s' '%s'",name.get(),mask.get()); Owned dir=createIFile(name); Owned iter=dir->monitorDirectory(prev,mask.length()?mask.get():NULL,sub,includedir,checkinterval,timeout); reply.append((unsigned)RFEnoerror); @@ -4526,7 +4526,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("getCRC, '%s'",name.get()); + PROGLOG("getCRC, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = file->getCRC(); reply.append((unsigned)RFEnoerror).append(ret); @@ -5105,7 +5105,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface msg.append(results); catchWriteBuffer(client.socket, msg); - LOG(MCdebugProgress, unknownJob, "Results sent from slave: %s", client.peerName.str()); + LOG(MCdebugProgress, "Results sent from slave: %s", client.peerName.str()); } void formatException(MemoryBuffer &reply, IException *e, RemoteFileCommandType cmd, bool testSocketFlag, unsigned _dfsErrorCode, CRemoteClientHandler *client) diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index c2fe3f389f8..45f3abd4458 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -2795,7 +2795,7 @@ static IPropertyTree *checkPlaneOrHost(const char *planeName,const char *host,co if (env->isDropZoneRestrictionEnabled()) throw makeStringExceptionV(-1,"DropZone Plane not found for host %s path %s.",host,path); - LOG(MCdebugInfo, unknownJob, "No matching drop zone path on '%s' to file path: '%s'",host,path); + LOG(MCdebugInfo, "No matching drop zone path on '%s' to file path: '%s'",host,path); #endif return nullptr; diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 9ea5ccb8378..86a64edd0f3 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -623,15 +623,15 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface writer.outputEndArray("Log"); } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { StringBuffer text, prefix; getLogPrefix(prefix); text.valist_appendf(format, args); - CTXLOGa(LOG_TRACING, cat, job, code, prefix.str(), text.str()); + CTXLOGa(LOG_TRACING, cat, code, prefix.str(), text.str()); } - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { LogContextScope ls(nullptr); if (category == LOG_TRACING) @@ -659,7 +659,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface { text.append(": ").valist_appendf(format, args); } - LOG(MCoperatorProgress, unknownJob, "[%s] %s", prefix, text.str()); + LOG(MCoperatorProgress, "[%s] %s", prefix, text.str()); if (intercept) { CriticalBlock b(crit); diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 68169884ae0..110b8c0f12a 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -1332,9 +1332,9 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext { globalStats.recordStatistics(progress, false); } - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { - logctx.CTXLOGa(category, cat, job, code, prefix, text); + logctx.CTXLOGa(category, cat, code, prefix, text); } virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index a6524342480..69b231fca0a 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -1197,6 +1197,8 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker daliHelper.setown(connectToDali()); wu.setown(daliHelper->attachWorkunit(wuid.get())); } + + JobNameScope jobName(wuid); Owned logctx = new StringContextLogger(wuid.get()); Owned traceHeaders = extractTraceDebugOptions(wu); diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index a396eca03cc..37c3496e307 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -1739,7 +1739,11 @@ class CRoxieWorker : public CInterface, implements IPooledThread Owned next = queue->dequeue(); if (next) { + JobNameScope jobName; logctx.set(next); + const char * wuid = logctx.queryWuid(); + if (wuid) + jobName.set(wuid); logctx.setStatistic(StTimeAgentQueue, nsTick()-next->queryEnqueuedTimeStamp()); #ifdef NEW_IBYTI logctx.setStatistic(StTimeIBYTIDelay, next->queryIBYTIDelayTime()); diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index 555d8ac29ae..d151b9f1969 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -240,13 +240,13 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { return ctx->collectingDetailedStatistics(); } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { - ctx->CTXLOGva(cat, job, code, format, args); + ctx->CTXLOGva(cat, code, format, args); } - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { - ctx->CTXLOGa(category, cat, job, code, prefix, text); + ctx->CTXLOGa(category, cat, code, prefix, text); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -1331,14 +1331,14 @@ class CRoxieServerActivity : implements CInterfaceOf, impl } // MORE - most of this is copied from ccd.hpp - can't we refactor? - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { if (ctx) - ctx->CTXLOGa(category, cat, job, code, prefix, text); + ctx->CTXLOGa(category, cat, code, prefix, text); else { LogContextScope ls(nullptr); - LOG(cat, job, code, "[%s] %s", prefix, text); + LOG(cat, code, "[%s] %s", prefix, text); } } @@ -1361,7 +1361,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { ss.append(": ").valist_appendf(format, args); } - LOG(MCoperatorProgress, unknownJob, "%s", ss.str()); + LOG(MCoperatorProgress, "%s", ss.str()); } } diff --git a/rtl/eclrtl/eclrtl.cpp b/rtl/eclrtl/eclrtl.cpp index b5d73720409..1b8a9f1d313 100644 --- a/rtl/eclrtl/eclrtl.cpp +++ b/rtl/eclrtl/eclrtl.cpp @@ -4211,7 +4211,7 @@ ECLRTL_API unsigned rtlSleep(unsigned delay) ECLRTL_API unsigned rtlDisplay(unsigned len, const char * src) { - LOG(MCprogress, unknownJob, "%.*s", len, src); + LOG(MCprogress, "%.*s", len, src); return 0; } diff --git a/rtl/eclrtl/rtlbcdtest.cpp b/rtl/eclrtl/rtlbcdtest.cpp index ff9f8684522..ffdf112a3bb 100644 --- a/rtl/eclrtl/rtlbcdtest.cpp +++ b/rtl/eclrtl/rtlbcdtest.cpp @@ -36,7 +36,7 @@ static bool check(bool condition, const char *fmt, ...) { va_list args; va_start(args, fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } return condition; @@ -50,7 +50,7 @@ static void cppunit_assert(bool condition, const char *fmt, ...) { va_list args; va_start(args, fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); CPPUNIT_ASSERT(!"Please refer to the errors above"); } diff --git a/system/jlib/jcontainerized.cpp b/system/jlib/jcontainerized.cpp index e80154ed7e6..5cbde1439c5 100644 --- a/system/jlib/jcontainerized.cpp +++ b/system/jlib/jcontainerized.cpp @@ -319,13 +319,13 @@ void runKubectlCommand(const char *title, const char *cmd, const char *input, St output = &_output; unsigned ret = runExternalCommand(title, *output, error, cmd, input, ".", nullptr); if (output->length()) - MLOG(MCdebugInfo, unknownJob, "%s: ret=%u, stdout=%s", cmd, ret, output->trimRight().str()); + MLOG(MCdebugInfo, "%s: ret=%u, stdout=%s", cmd, ret, output->trimRight().str()); if (error.length()) - MLOG(MCdebugError, unknownJob, "%s: ret=%u, stderr=%s", cmd, ret, error.trimRight().str()); + MLOG(MCdebugError, "%s: ret=%u, stderr=%s", cmd, ret, error.trimRight().str()); if (ret) { if (input) - MLOG(MCdebugError, unknownJob, "Using input %s", input); + MLOG(MCdebugError, "Using input %s", input); throw makeStringExceptionV(0, "Failed to run %s: error %u: %s", cmd, ret, error.str()); } } diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 2ca2586b909..78b9813c4a0 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -2642,7 +2642,7 @@ class CUdpStatsReporter if (drops > ret->drops) { if (!firstTime) - LOG(MCoperatorError, unknownJob, "DROPPED UDP PACKETS: port %d rx_queue=%u (peak %u) drops=%u (total %i)", port, queue, ret->rx_queue, drops-ret->drops, drops); + LOG(MCoperatorError, "DROPPED UDP PACKETS: port %d rx_queue=%u (peak %u) drops=%u (total %i)", port, queue, ret->rx_queue, drops-ret->drops, drops); ret->drops = drops; } } @@ -2709,10 +2709,10 @@ class CSnmpStatsReporter { firstCall = false; if (errors) - LOG(MCoperatorWarning, unknownJob, "UDP Initial InError total: %u", errors); + LOG(MCoperatorWarning, "UDP Initial InError total: %u", errors); } else if (errors > prevErrors) - LOG(MCoperatorError, unknownJob, "UDP InErrors: %u (total %u)", errors-prevErrors, errors); + LOG(MCoperatorError, "UDP InErrors: %u (total %u)", errors-prevErrors, errors); prevErrors = errors; } } @@ -3001,7 +3001,7 @@ static class CMemoryUsageReporter: public Thread udpStats.reportUdpInfo(0); } if(traceMode&&str.length()) { - MLOG(MCdebugInfo, unknownJob, "SYS: %s", str.str()); + MLOG(MCdebugInfo, "SYS: %s", str.str()); #ifndef _WIN32 if (traceMode&PerfMonExtended) { diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 417fcff3404..360db1cf1b1 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -2287,7 +2287,7 @@ void CCheckingFileIO::report(const char * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugError, unknownJob, format, args); + VALOG(MCdebugError, format, args); va_end(args); if (!traced) { diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 66d94c8135b..177e206825a 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -293,19 +293,9 @@ void LogMsgJobInfo::deserialize(MemoryBuffer & in) //-------------------------------------------------------------------------------------------------------------------- -const LogMsgJobInfo unknownJob(UnknownJob, UnknownUser); - -static thread_local LogMsgJobInfo tempJobInfo; -const LogMsgJobInfo & checkDefaultJobInfo(const LogMsgJobInfo & _jobInfo) +static const LogMsgJobInfo queryDefaultJobInfo() { - if (&_jobInfo != &unknownJob) - return _jobInfo; - - //Using a thread local as a temporary is a short term change - //the next step is to remove the parameter and always return a stack object - LogMsgJobInfo & result = tempJobInfo; - result.setJobID(queryThreadedJobId()); - return result; + return LogMsgJobInfo(queryThreadedJobId(), UnknownUser); } @@ -315,21 +305,21 @@ LogMsg::LogMsg(LogMsgJobId id, const char *job) : category(MSGAUD_programmer, jo text.append(job); } -LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session) - : category(_cat), sysInfo(_id, port, session), jobInfo(checkDefaultJobInfo(_jobInfo)), msgCode(_code), remoteFlag(false) +LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session) + : category(_cat), sysInfo(_id, port, session), jobInfo(queryDefaultJobInfo()), msgCode(_code), remoteFlag(false) { text.append(_text); } -LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session) - : category(_cat), sysInfo(_id, port, session), jobInfo(checkDefaultJobInfo(_jobInfo)), msgCode(_code), remoteFlag(false) +LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session) + : category(_cat), sysInfo(_id, port, session), jobInfo(queryDefaultJobInfo()), msgCode(_code), remoteFlag(false) { text.append(sz, _text); } -LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * format, va_list args, +LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * format, va_list args, unsigned port, LogMsgSessionId session) - : category(_cat), sysInfo(_id, port, session), jobInfo(checkDefaultJobInfo(_jobInfo)), msgCode(_code), remoteFlag(false) + : category(_cat), sysInfo(_id, port, session), jobInfo(queryDefaultJobInfo()), msgCode(_code), remoteFlag(false) { text.valist_appendf(format, args); } @@ -1259,7 +1249,7 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const char * format, .. buff.append(file).append("(").append(line).append(") : ").append(format); va_list args; va_start(args, format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); va_end(args); } @@ -1267,7 +1257,7 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, const char * format, { StringBuffer buff; buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); } void LogMsgPrepender::report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) @@ -1276,7 +1266,7 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, LogMsgCode code, const buff.append(file).append("(").append(line).append(") : ").append(format); va_list args; va_start(args, format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); va_end(args); } @@ -1284,7 +1274,7 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, LogMsgCode code, con { StringBuffer buff; buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); } void LogMsgPrepender::report(const LogMsgCategory & cat, const IException * exception, const char * prefix) @@ -1293,55 +1283,12 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const IException * exce buff.append(file).append("(").append(line).append(") : "); if(prefix) buff.append(prefix).append(" : "); exception->errorMessage(buff); - queryLogMsgManager()->report(cat, unknownJob, exception->errorCode(), "%s", buff.str()); -} - -void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - va_list args; - va_start(args, format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); - va_end(args); -} - -void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); -} - -void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - va_list args; - va_start(args, format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); - va_end(args); -} - -void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); -} - -void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * exception, const char * prefix) -{ - StringBuffer txt; - if (prefix) - txt.append(prefix).append(" : "); - exception->errorMessage(txt); - queryLogMsgManager()->report(cat, job, exception->errorCode(), "%s(%d) : %s", file, line, txt.str()); + queryLogMsgManager()->report(cat, exception->errorCode(), "%s", buff.str()); } IException * LogMsgPrepender::report(IException * e, const char * prefix, LogMsgClass cls) { - report(MCexception(e, cls), unknownJob, e, prefix); + report(MCexception(e, cls), e, prefix); return e; } @@ -1542,14 +1489,14 @@ void CLogMsgManager::report(const LogMsgCategory & cat, const char * format, ... if(rejectsCategory(cat)) return; va_list args; va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), unknownJob, NoLogMsgCode, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, format, args, port, session)); va_end(args); } void CLogMsgManager::report_va(const LogMsgCategory & cat, const char * format, va_list args) { if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), unknownJob, NoLogMsgCode, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, format, args, port, session)); } void CLogMsgManager::report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) @@ -1557,17 +1504,17 @@ void CLogMsgManager::report(const LogMsgCategory & cat, LogMsgCode code, const c if(rejectsCategory(cat)) return; va_list args; va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), unknownJob, code, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), code, format, args, port, session)); va_end(args); } void CLogMsgManager::report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) { if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), unknownJob, code, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), code, format, args, port, session)); } -void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg) +void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const char * msg) { if(rejectsCategory(cat)) return; const char *cursor = msg; @@ -1578,17 +1525,17 @@ void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const LogMsgJobI { case '\0': if (cursor != lineStart || cursor==msg) - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); return; case '\r': // NB: \r or \r\n translated into newline - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); if ('\n' == *(cursor+1)) cursor++; lineStart = cursor+1; break; case '\n': - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); lineStart = cursor+1; break; } @@ -1596,12 +1543,12 @@ void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const LogMsgJobI } } -void CLogMsgManager::mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) +void CLogMsgManager::mreport_va(const LogMsgCategory & cat, const char * format, va_list args) { if(rejectsCategory(cat)) return; StringBuffer log; log.limited_valist_appendf(1024*1024, format, args); - mreport_direct(cat, job, log); + mreport_direct(cat, log); } void CLogMsgManager::report(const LogMsgCategory & cat, const IException * exception, const char * prefix) @@ -1610,46 +1557,7 @@ void CLogMsgManager::report(const LogMsgCategory & cat, const IException * excep StringBuffer buff; if(prefix) buff.append(prefix).append(" : "); exception->errorMessage(buff); - pushMsg(new LogMsg(cat, getNextID(), unknownJob, exception->errorCode(), buff.str(), port, session)); -} - -void CLogMsgManager::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - if(rejectsCategory(cat)) return; - va_list args; - va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, format, args, port, session)); - va_end(args); -} - -void CLogMsgManager::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, format, args, port, session)); -} - -void CLogMsgManager::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) -{ - if(rejectsCategory(cat)) return; - va_list args; - va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), job, code, format, args, port, session)); - va_end(args); -} - -void CLogMsgManager::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) -{ - if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), job, code, format, args, port, session)); -} - -void CLogMsgManager::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * exception, const char * prefix) -{ - if(rejectsCategory(cat)) return; - StringBuffer buff; - if(prefix) buff.append(prefix).append(" : "); - exception->errorMessage(buff); - pushMsg(new LogMsg(cat, getNextID(), job, exception->errorCode(), buff.str(), port, session)); + pushMsg(new LogMsg(cat, getNextID(), exception->errorCode(), buff.str(), port, session)); } void CLogMsgManager::pushMsg(LogMsg * _msg) @@ -2295,13 +2203,8 @@ class CNullManager : implements ILogMsgManager virtual void report(const LogMsgCategory & cat, LogMsgCode code , const char * format, ...) override {} virtual void report_va(const LogMsgCategory & cat, LogMsgCode code , const char * format, va_list args) override {} virtual void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL) override {} - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) override {} - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) override {} - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) override {} - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, va_list args) override {} - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL) override {} - virtual void mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg) override {} - virtual void mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) override {} + virtual void mreport_direct(const LogMsgCategory & cat, const char * msg) override {} + virtual void mreport_va(const LogMsgCategory & cat, const char * format, va_list args) override {} virtual void report(const LogMsg & msg) const override {} virtual LogMsgId getNextID() override { return 0; } virtual bool rejectsCategory(const LogMsgCategory & cat) const override { return true; } @@ -2751,7 +2654,7 @@ void IContextLogger::CTXLOG(const char *format, ...) const { va_list args; va_start(args, format); - CTXLOGva(MCdebugInfo, queryJob(), NoLogMsgCode, format, args); + CTXLOGva(MCdebugInfo, NoLogMsgCode, format, args); va_end(args); } @@ -2812,9 +2715,9 @@ class DummyLogCtx : implements IContextLogger virtual void Link() const {} virtual bool Release() const { return false; } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { - VALOG(cat, job, code, format, args); + VALOG(cat, code, format, args); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -2828,7 +2731,7 @@ class DummyLogCtx : implements IContextLogger E->errorMessage(ss.append(": ")); if (format) ss.append(": ").valist_appendf(format, args); - LOG(MCoperatorProgress, queryJob(), "%s", ss.str()); + LOG(MCoperatorProgress, "%s", ss.str()); } virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const { @@ -3336,24 +3239,27 @@ void setDefaultJobName(const char * name) } -JobNameTranslator::JobNameTranslator(const char * name) +JobNameScope::JobNameScope(const char * name) { set(name); } -void JobNameTranslator::clear() +void JobNameScope::clear() { - if (id) + if (id != UnknownJob) { theManager->removeJobId(id); - id = 0; + id = prevId; + setDefaultJobId(prevId); } } -void JobNameTranslator::set(const char * name) +void JobNameScope::set(const char * name) { clear(); id = theManager->addJobId(name); + prevId = queryThreadedJobId(); + setDefaultJobId(id); } //--------------------------------------------------------------------------------------------------------------------- @@ -3389,65 +3295,39 @@ void ctxlogReport(const LogMsgCategory & cat, const char * format, ...) { va_list args; va_start(args, format); - ctxlogReportVA(cat, unknownJob, NoLogMsgCode, format, args); + ctxlogReportVA(cat, NoLogMsgCode, format, args); va_end(args); } void ctxlogReportVA(const LogMsgCategory & cat, const char * format, va_list args) { - ctxlogReportVA(cat, unknownJob, NoLogMsgCode, format, args); + ctxlogReportVA(cat, NoLogMsgCode, format, args); } void ctxlogReport(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) { va_list args; va_start(args, format); - ctxlogReportVA(cat, unknownJob, code, format, args); + ctxlogReportVA(cat, code, format, args); va_end(args); } void ctxlogReportVA(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) -{ - ctxlogReportVA(cat, unknownJob, code, format, args); -} -void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * prefix) -{ - ctxlogReport(cat, unknownJob, e, prefix); -} -void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - va_list args; - va_start(args, format); - ctxlogReportVA(cat, job, NoLogMsgCode, format, args); - va_end(args); -} -void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - ctxlogReportVA(cat, job, NoLogMsgCode, format, args); -} -void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) -{ - va_list args; - va_start(args, format); - ctxlogReportVA(cat, job, code, format, args); - va_end(args); -} -void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) { if (queryThreadedContextLogger()) { LogContextScope ls(nullptr); - ls.prev->CTXLOGva(cat, job, code, format, args); + ls.prev->CTXLOGva(cat, code, format, args); } else - queryLogMsgManager()->report_va(cat, job, code, format, args); + queryLogMsgManager()->report_va(cat, code, format, args); } -void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix) +void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * prefix) { StringBuffer buff; e->errorMessage(buff); - ctxlogReport(cat, job, e->errorCode(), "%s%s%s", prefix ? prefix : "", prefix ? prefix : " : ", buff.str()); + ctxlogReport(cat, e->errorCode(), "%s%s%s", prefix ? prefix : "", prefix ? prefix : " : ", buff.str()); } IException * ctxlogReport(IException * e, const char * prefix, LogMsgClass cls) { - ctxlogReport(MCexception(e, cls), unknownJob, e, prefix); + ctxlogReport(MCexception(e, cls), e, prefix); return e; } diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index 4a609630b7e..c79d2d56eba 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -593,10 +593,10 @@ class jlib_decl LogMsg : public CInterface public: LogMsg() : category(), sysInfo(), jobInfo(), remoteFlag(false) {} LogMsg(LogMsgJobId id, const char *job); // Used for tracking job ids - LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session); - LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session); - LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * format, va_list args, - unsigned port, LogMsgSessionId session) __attribute__((format(printf,6, 0))); + LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session); + LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session); + LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * format, va_list args, + unsigned port, LogMsgSessionId session) __attribute__((format(printf,5, 0))); StringBuffer & toStringPlain(StringBuffer & out, unsigned fields) const; StringBuffer & toStringXML(StringBuffer & out, unsigned fields) const; StringBuffer & toStringJSON(StringBuffer & out, unsigned fields) const; @@ -722,13 +722,8 @@ interface jlib_decl ILogMsgManager : public ILogMsgListener virtual void report(const LogMsgCategory & cat, LogMsgCode code , const char * format, ...) __attribute__((format(printf, 4, 5))) = 0; virtual void report_va(const LogMsgCategory & cat, LogMsgCode code , const char * format, va_list args) = 0; virtual void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL) = 0; - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 4, 5))) = 0; - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) = 0; - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) __attribute__((format(printf, 5, 6))) = 0; - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, va_list args) = 0; - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL) = 0; - virtual void mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg) = 0; - virtual void mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) = 0; + virtual void mreport_direct(const LogMsgCategory & cat, const char * msg) = 0; + virtual void mreport_va(const LogMsgCategory & cat, const char * format, va_list args) = 0; virtual void report(const LogMsg & msg) const = 0; virtual LogMsgId getNextID() = 0; virtual bool rejectsCategory(const LogMsgCategory & cat) const = 0; @@ -751,12 +746,7 @@ class jlib_decl LogMsgPrepender void report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 4, 5))); void report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args); void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 4, 5))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 5, 6))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL); - IException * report(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), unknownJob, handy for EXCLOG + IException * report(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), handy for EXCLOG private: char const * file; unsigned line; @@ -870,8 +860,6 @@ inline LogMsgCategory MCexception(IException * e, LogMsgClass cls = MSGCLS_error #define MCwarning MCuserWarning #define MCprogress MCuserProgress -extern jlib_decl const LogMsgJobInfo unknownJob; - // Function to return manager, standard handler and the reporters, and the handler's message fields extern jlib_decl ILogMsgManager * queryLogMsgManager(); @@ -883,19 +871,20 @@ extern jlib_decl void setupContainerizedLogMsgHandler(); extern jlib_decl void setDefaultJobName(const char *id); //A class for mapping job names to ids -class jlib_decl JobNameTranslator +class jlib_decl JobNameScope { public: - JobNameTranslator() = default; - JobNameTranslator(const char * name); - ~JobNameTranslator() { clear(); } + JobNameScope() = default; + JobNameScope(const char * name); + ~JobNameScope() { clear(); } void clear(); void set(const char * name); LogMsgJobId queryId() const { return id; } protected: - LogMsgJobId id = 0; + LogMsgJobId id = UnknownJob; + LogMsgJobId prevId = UnknownJob; }; // Macros to make logging as simple as possible @@ -926,30 +915,10 @@ inline void LOG(const LogMsgCategory & cat, LogMsgCode code , const char * forma LOGMSGREPORTER->report_va(cat, code , format, args); va_end(args); } -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 3, 4))); -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - va_list args; - va_start(args, format); - LOGMSGREPORTER->report_va(cat, job, format, args); - va_end(args); -} -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) __attribute__((format(printf, 4, 5))); -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) -{ - va_list args; - va_start(args, format); - LOGMSGREPORTER->report_va(cat, job, code , format, args); - va_end(args); -} inline void LOG(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL) { LOGMSGREPORTER->report(cat, e, prefix); } -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL) -{ - LOGMSGREPORTER->report(cat, job, e, prefix); -} inline void VALOG(const LogMsgCategory & cat, const char * format, va_list args) { LOGMSGREPORTER->report_va(cat, format, args); @@ -958,14 +927,6 @@ inline void VALOG(const LogMsgCategory & cat, LogMsgCode code , const char * for { LOGMSGREPORTER->report_va(cat, code , format, args); } -inline void VALOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - LOGMSGREPORTER->report_va(cat, job, format, args); -} -inline void VALOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, va_list args) -{ - LOGMSGREPORTER->report_va(cat, job, code , format, args); -} #else #define LOG LOGMSGREPORTER->report #define VALOG LOGMSGREPORTER->report_va @@ -976,12 +937,7 @@ extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, const char * fo extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 3, 4))); extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf, 3, 0))); extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL); -extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 3, 4))); -extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) __attribute__((format(printf, 3, 0))); -extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 4, 5))); -extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf, 4, 0))); -extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL); -extern jlib_decl IException * ctxlogReport(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), unknownJob, handy for EXCLOG +extern jlib_decl IException * ctxlogReport(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), handy for EXCLOG #define LOG ::ctxlogReport #define VALOG ::ctxlogReportVA @@ -992,7 +948,7 @@ inline void DBGLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugInfo, unknownJob, format, args); + VALOG(MCdebugInfo, format, args); va_end(args); } @@ -1001,7 +957,7 @@ inline void DISLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorDisaster, unknownJob, format, args); + VALOG(MCoperatorDisaster, format, args); va_end(args); } @@ -1010,7 +966,7 @@ inline void UERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserError, unknownJob, format, args); + VALOG(MCuserError, format, args); va_end(args); } @@ -1022,7 +978,7 @@ inline void OERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorError, unknownJob, format, args); + VALOG(MCoperatorError, format, args); va_end(args); } @@ -1031,7 +987,7 @@ inline void IERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugError, unknownJob, format, args); + VALOG(MCdebugError, format, args); va_end(args); } @@ -1040,7 +996,7 @@ inline void AERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCauditError, unknownJob, format, args); + VALOG(MCauditError, format, args); va_end(args); } @@ -1050,7 +1006,7 @@ inline void UWARNLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserWarning, unknownJob, format, args); + VALOG(MCuserWarning, format, args); va_end(args); } @@ -1062,7 +1018,7 @@ inline void OWARNLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorWarning, unknownJob, format, args); + VALOG(MCoperatorWarning, format, args); va_end(args); } @@ -1071,7 +1027,7 @@ inline void IWARNLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugWarning, unknownJob, format, args); + VALOG(MCdebugWarning, format, args); va_end(args); } @@ -1080,28 +1036,20 @@ inline void PROGLOG(const char * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserProgress, unknownJob, format, args); + VALOG(MCuserProgress, format, args); va_end(args); } inline void MLOG(const LogMsgCategory & cat, const char * msg) { - LOGMSGREPORTER->mreport_direct(cat, unknownJob, msg); -} - -inline void MLOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - va_list args; - va_start(args, format); - LOGMSGREPORTER->mreport_va(cat, job, format, args); - va_end(args); + LOGMSGREPORTER->mreport_direct(cat, msg); } inline void MLOG(const LogMsgCategory & cat, const char * format, ...) { va_list args; va_start(args, format); - LOGMSGREPORTER->mreport_va(cat, unknownJob, format, args); + LOGMSGREPORTER->mreport_va(cat, format, args); va_end(args); } @@ -1110,7 +1058,7 @@ inline void MLOG(const char * format, ...) { va_list args; va_start(args, format); - LOGMSGREPORTER->mreport_va(MCuserProgress, unknownJob, format, args); + LOGMSGREPORTER->mreport_va(MCuserProgress, format, args); va_end(args); } @@ -1119,7 +1067,7 @@ inline void DBGLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugInfo, unknownJob, code, format, args); + VALOG(MCdebugInfo, code, format, args); va_end(args); } @@ -1128,7 +1076,7 @@ inline void DISLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorDisaster, unknownJob, code, format, args); + VALOG(MCoperatorDisaster, code, format, args); va_end(args); } @@ -1137,7 +1085,7 @@ inline void UWARNLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserWarning, unknownJob, code, format, args); + VALOG(MCuserWarning, code, format, args); va_end(args); } @@ -1146,7 +1094,7 @@ inline void OWARNLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorWarning, unknownJob, code, format, args); + VALOG(MCoperatorWarning, code, format, args); va_end(args); } @@ -1155,7 +1103,7 @@ inline void IWARNLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugWarning, unknownJob, code, format, args); + VALOG(MCdebugWarning, code, format, args); va_end(args); } @@ -1179,7 +1127,7 @@ inline void OERRLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorError, unknownJob, code, format, args); + VALOG(MCoperatorError, code, format, args); va_end(args); } @@ -1188,7 +1136,7 @@ inline void IERRLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugError, unknownJob, code, format, args); + VALOG(MCdebugError, code, format, args); va_end(args); } @@ -1197,7 +1145,7 @@ inline void UERRLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserError, unknownJob, code, format, args); + VALOG(MCuserError, code, format, args); va_end(args); } @@ -1206,7 +1154,7 @@ inline void PROGLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserProgress, unknownJob, code, format, args); + VALOG(MCuserProgress, code, format, args); va_end(args); } @@ -1274,7 +1222,7 @@ interface jlib_decl IContextLogger : extends IInterface { virtual void CTXLOG(const char *format, ...) const __attribute__((format(printf, 2, 3))); virtual void mCTXLOG(const char *format, ...) const __attribute__((format(printf, 2, 3))); - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const __attribute__((format(printf,5,0))) = 0; + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const __attribute__((format(printf,4,0))) = 0; void logOperatorException(IException *E, const char *file, unsigned line, const char *format, ...) const __attribute__((format(printf, 5, 6))); virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) = 0; virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const = 0; @@ -1293,7 +1241,6 @@ interface jlib_decl IContextLogger : extends IInterface virtual void setSpanAttribute(const char *name, const char *value) const = 0; virtual void setSpanAttribute(const char *name, __uint64 value) const = 0; virtual void recordStatistics(IStatisticGatherer &progress) const = 0; - virtual const LogMsgJobInfo & queryJob() const { return unknownJob; } }; using ContextSpanScope = ContextSpanScopeImp; diff --git a/system/jlib/jlog.ipp b/system/jlib/jlog.ipp index 361d823cc2c..31a6f05dfeb 100644 --- a/system/jlib/jlog.ipp +++ b/system/jlib/jlog.ipp @@ -740,16 +740,11 @@ public: bool flushQueue(unsigned timeout) { if(processor) return processor->flush(timeout); else return true; } void report(const LogMsgCategory & cat, const char * format, ...) __attribute__((format(printf,3,4))); void report_va(const LogMsgCategory & cat, const char * format, va_list args) __attribute__((format(printf,3,0))); - void mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg); - void mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) __attribute__((format(printf,4,0))); + void mreport_direct(const LogMsgCategory & cat, const char * msg); + void mreport_va(const LogMsgCategory & cat, const char * format, va_list args) __attribute__((format(printf,3,0))); void report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf,4,5))); void report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf,4,0))); void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf,4,5))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) __attribute__((format(printf,4,0))); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) __attribute__((format(printf,5,6))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf,5,0))); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL); void report(const LogMsg & msg) const { if(prefilter.includeCategory(msg.queryCategory())) doReport(msg); } bool addMonitor(ILogMsgHandler * handler, ILogMsgFilter * filter); bool addMonitorOwn(ILogMsgHandler * handler, ILogMsgFilter * filter); @@ -814,7 +809,7 @@ private: class DropLogMsg : public LogMsg { public: - DropLogMsg(CLogMsgManager * owner, LogMsgId id, unsigned _count) : LogMsg(dropWarningCategory, id, unknownJob, NoLogMsgCode, "MISSING LOG MESSAGES: ", owner->port, owner->session), count(_count) + DropLogMsg(CLogMsgManager * owner, LogMsgId id, unsigned _count) : LogMsg(dropWarningCategory, id, NoLogMsgCode, "MISSING LOG MESSAGES: ", owner->port, owner->session), count(_count) { text.append("message queue length exceeded, dropped ").append(count).append(" messages"); } diff --git a/system/jlib/jmisc.cpp b/system/jlib/jmisc.cpp index 7f0d55d5e74..9d1fbf5d504 100644 --- a/system/jlib/jmisc.cpp +++ b/system/jlib/jmisc.cpp @@ -251,7 +251,7 @@ jlib_decl void PrintMemoryStatusLog() #ifdef _WIN32 MEMORYSTATUS mS; GlobalMemoryStatus(&mS); - LOG(MCdebugInfo, unknownJob, "Available Physical Memory = %dK", (unsigned)(mS.dwAvailPhys/1024)); + LOG(MCdebugInfo, "Available Physical Memory = %dK", (unsigned)(mS.dwAvailPhys/1024)); #ifdef FRAGMENTATION_CHECK // see if fragmented size32_t sz = MAX_TRY_SIZE; @@ -267,7 +267,7 @@ jlib_decl void PrintMemoryStatusLog() } sz *= 2; if ((szRelease(); } } @@ -969,7 +969,7 @@ protected: friend class CMPPacketReader; { StringBuffer str; #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connecting to %s role: %" I64F "u", remoteep.getEndpointHostText(str).str(), parent->getRole()); + LOG(MCdebugInfo, "MP: connecting to %s role: %" I64F "u", remoteep.getEndpointHostText(str).str(), parent->getRole()); #endif if (((int)tm.timeout)<0) remaining = CONNECT_TIMEOUT; @@ -1005,7 +1005,7 @@ protected: friend class CMPPacketReader; newsock->set_keep_alive(true); #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: connect after socket connect, retrycount = %d", retrycount); + LOG(MCdebugInfo, "MP: connect after socket connect, retrycount = %d", retrycount); #endif SocketEndpoint hostep; @@ -1024,13 +1024,13 @@ protected: friend class CMPPacketReader; connectHdr.id[0].getEndpointHostText(tmp1); tmp1.append(' '); connectHdr.id[1].getEndpointHostText(tmp1); - LOG(MCdebugInfo, unknownJob, "MP: connect after socket write %s",tmp1.str()); + LOG(MCdebugInfo, "MP: connect after socket write %s",tmp1.str()); #endif size32_t rd = 0; #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connect after socket write, waiting for read"); + LOG(MCdebugInfo, "MP: connect after socket write, waiting for read"); #endif // Wait for connection reply but also check for A<->B deadlock (where both processes are here @@ -1110,7 +1110,7 @@ protected: friend class CMPPacketReader; // if other side closes, connect again if (e->errorCode() == JSOCKERR_graceful_close) { - LOG(MCdebugInfo, unknownJob, "MP: Retrying (other side closed connection, probably due to clash)"); + LOG(MCdebugInfo, "MP: Retrying (other side closed connection, probably due to clash)"); e->Release(); break; } @@ -1118,7 +1118,7 @@ protected: friend class CMPPacketReader; e->Release(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); + LOG(MCdebugInfo, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); #endif } else @@ -1145,7 +1145,7 @@ protected: friend class CMPPacketReader; size32_t replyVal; memcpy(&replyVal, (size32_t *)replyBuf, sizeof(size32_t)); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connect after socket read replyVal=%u, sizeof(connectHdr)=%lu", replyVal, sizeof(connectHdr)); + LOG(MCdebugInfo, "MP: connect after socket read replyVal=%u, sizeof(connectHdr)=%lu", replyVal, sizeof(connectHdr)); #endif if (replyVal > sizeof(ConnectHdr)) { @@ -1191,7 +1191,7 @@ protected: friend class CMPPacketReader; if (attachSocket(newsock,remoteep,hostep,true,NULL,addrval)) { #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connected to %s",str.str()); + LOG(MCdebugInfo, "MP: connected to %s",str.str()); #endif lastxfer = msTick(); closed = false; @@ -1222,7 +1222,7 @@ protected: friend class CMPPacketReader; #ifdef _TRACE StringBuffer str; str.clear(); - LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); + LOG(MCdebugInfo, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); #endif } @@ -1276,7 +1276,7 @@ protected: friend class CMPPacketReader; CriticalBlock block(connectsect); if (closed) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "WritePacket closed on entry"); + LOG(MCdebugInfo, "WritePacket closed on entry"); PrintStackReport(); #endif if (!checkReconnect(tm)) @@ -1285,7 +1285,7 @@ protected: friend class CMPPacketReader; if (!channelsock) { if (!connect(tm)) { #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "WritePacket connect failed"); + LOG(MCdebugInfo, "WritePacket connect failed"); #endif return false; } @@ -1311,7 +1311,7 @@ protected: friend class CMPPacketReader; #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "WritePacket(target=%s,(%d,%d,%d))",remoteep.getEndpointHostText(ep1).str(),hdrsize,hdr2size,bodysize); + LOG(MCdebugInfo, "WritePacket(target=%s,(%d,%d,%d))",remoteep.getEndpointHostText(ep1).str(),hdrsize,hdr2size,bodysize); unsigned t2 = msTick(); #endif unsigned n = 0; @@ -1330,17 +1330,17 @@ protected: friend class CMPPacketReader; sizes[n++] = bodysize; } if (!dest) { - LOG(MCdebugInfo, unknownJob, "MP Warning: WritePacket unexpected NULL socket"); + LOG(MCdebugInfo, "MP Warning: WritePacket unexpected NULL socket"); return false; } dest->write_multiple(n,bufs,sizes); lastxfer = msTick(); #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "WritePacket(timewaiting=%d,timesending=%d)",t2-t1,lastxfer-t2); + LOG(MCdebugInfo, "WritePacket(timewaiting=%d,timesending=%d)",t2-t1,lastxfer-t2); #endif } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP writepacket"); + FLLOG(MCoperatorWarning, e,"MP writepacket"); closeSocket(false, true); throw; } @@ -1383,11 +1383,11 @@ protected: friend class CMPPacketReader; return false; } if (tm.timedout()) { - LOG(MCdebugInfo, unknownJob, "MP: verify, ping failed to %s",ep.str()); + LOG(MCdebugInfo, "MP: verify, ping failed to %s",ep.str()); closeSocket(); return false; } - LOG(MCdebugInfo, unknownJob, "MP: verify, ping failed to %s, retrying",ep.str()); + LOG(MCdebugInfo, "MP: verify, ping failed to %s, retrying",ep.str()); unsigned remaining; if (!pingtm.timedout(&remaining)&&remaining) Sleep(remaining); @@ -1501,7 +1501,7 @@ class UserPacketHandler // default #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, "MP: send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif return channel->writepacket(&hdr,sizeof(hdr),mb.toByteArray(),mb.length(),tm); } @@ -1565,7 +1565,7 @@ class MultiPacketHandler // TAG_SYS_MULTI otherMhdr->getDetails(errorMsg).newline(); } msg.getDetails(errorMsg); - LOG(MCerror, unknownJob, "MultiPacketHandler: protocol error (%d) %s", code, errorMsg.str()); + LOG(MCerror, "MultiPacketHandler: protocol error (%d) %s", code, errorMsg.str()); } lastErrMs = ms; } @@ -1633,7 +1633,7 @@ class MultiPacketHandler // TAG_SYS_MULTI #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: multi-send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, "MP: multi-send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif PacketHeader outhdr; outhdr = hdr; @@ -1651,13 +1651,13 @@ class MultiPacketHandler // TAG_SYS_MULTI if (i+1==mhdr.numparts) mhdr.size = mhdr.total-mhdr.ofs; #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: multi-send block=%d, num blocks=%d, ofs=%d, size=%d",i,mhdr.numparts,mhdr.ofs,mhdr.size); + LOG(MCdebugInfo, "MP: multi-send block=%d, num blocks=%d, ofs=%d, size=%d",i,mhdr.numparts,mhdr.ofs,mhdr.size); #endif outhdr.initseq(); outhdr.size = sizeof(outhdr)+sizeof(mhdr)+mhdr.size; if (!channel->writepacket(&outhdr,sizeof(outhdr),&mhdr,sizeof(mhdr),p,mhdr.size,tm)) { #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: multi-send failed"); + LOG(MCdebugInfo, "MP: multi-send failed"); #endif return false; } @@ -1746,7 +1746,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface if (pc) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPPacketReader::notifySelected() about to close socket, mode = 0x%x", selected); + LOG(MCdebugInfo, "CMPPacketReader::notifySelected() about to close socket, mode = 0x%x", selected); #endif pc->closeSocket(false, true); } @@ -1766,7 +1766,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface // assumes packet header will arrive in one go if (sizeavailread(&hdr,sizeof(hdr),sizeof(hdr),szread,60); // I don't *really* want to block here but not much else can do @@ -1787,7 +1787,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: ReadPacket(sender=%s,target=%s,tag=%d,replytag=%d,size=%d)",hdr.sender.getEndpointHostText(ep1).str(),hdr.target.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, "MP: ReadPacket(sender=%s,target=%s,tag=%d,replytag=%d,size=%d)",hdr.sender.getEndpointHostText(ep1).str(),hdr.target.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif remaining = hdr.size-sizeof(hdr); activemsg = new CMessageBuffer(remaining); // will get from low level IO at some stage @@ -1807,7 +1807,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface if (remaining==0) { // we have the packet so process #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: ReadPacket(timetaken = %d,select iterations=%d)",msTick()-parent->startxfer,parent->numiter); + LOG(MCdebugInfo, "MP: ReadPacket(timetaken = %d,select iterations=%d)",msTick()-parent->startxfer,parent->numiter); #endif do { switch (activemsg->getTag()) { @@ -1843,7 +1843,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface } catch (IException *e) { if (e->errorCode()!=JSOCKERR_graceful_close) - FLLOG(MCoperatorWarning, unknownJob, e,"MP(Packet Reader)"); + FLLOG(MCoperatorWarning, e,"MP(Packet Reader)"); e->Release(); } // error here, so close socket (ignore error as may be closed already) @@ -1939,30 +1939,30 @@ bool CMPChannel::attachSocket(ISocket *newsock,const SocketEndpoint &_remoteep,c StringBuffer ep2; _localep.getEndpointHostText(ep1); _remoteep.getEndpointHostText(ep2); - LOG(MCdebugInfo, unknownJob, "MP: Possible clash between %s->%s %d(%d)",ep1.str(),ep2.str(),(int)ismaster,(int)master); + LOG(MCdebugInfo, "MP: Possible clash between %s->%s %d(%d)",ep1.str(),ep2.str(),(int)ismaster,(int)master); try { if (ismaster!=master) { if (ismaster) { - LOG(MCdebugInfo, unknownJob, "MP: resolving socket attach clash (master)"); + LOG(MCdebugInfo, "MP: resolving socket attach clash (master)"); return false; } else { Sleep(50); // give the other side some time to close CTimeMon tm(10000); if (verifyConnection(tm,false)) { - LOG(MCdebugInfo, unknownJob, "MP: resolving socket attach clash (verified)"); + LOG(MCdebugInfo, "MP: resolving socket attach clash (verified)"); return false; } } } } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP attachsocket(1)"); + FLLOG(MCoperatorWarning, e,"MP attachsocket(1)"); e->Release(); } try { - LOG(MCdebugInfo, unknownJob, "Message Passing - removing stale socket to %s",ep2.str()); + LOG(MCdebugInfo, "Message Passing - removing stale socket to %s",ep2.str()); CriticalUnblock unblock(connectsect); closeSocket(true, true); #ifdef REFUSE_STALE_CONNECTION @@ -1972,7 +1972,7 @@ bool CMPChannel::attachSocket(ISocket *newsock,const SocketEndpoint &_remoteep,c Sleep(100); // pause to allow close socket triggers to run } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP attachsocket(2)"); + FLLOG(MCoperatorWarning, e,"MP attachsocket(2)"); e->Release(); } } @@ -2010,7 +2010,7 @@ bool CMPChannel::send(MemoryBuffer &mb, mptag_t tag, mptag_t replytag, CTimeMon if (closed||(reply&&!isConnected())) // flag error if has been disconnected { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPChannel::send closed on entry %d",(int)closed); + LOG(MCdebugInfo, "CMPChannel::send closed on entry %d",(int)closed); PrintStackReport(); #endif if (!checkReconnect(tm)) @@ -2087,7 +2087,7 @@ bool CMPChannel::sendPing(CTimeMon &tm) ret = parent->pingpackethandler->send(this,hdr,tm)&&!tm.timedout(&remaining); } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP ping(1)"); + FLLOG(MCoperatorWarning, e,"MP ping(1)"); e->Release(); } sendmutex.unlock(); @@ -2117,7 +2117,7 @@ bool CMPChannel::sendPingReply(unsigned timeout,bool identifyself) ret = parent->pingreplypackethandler->send(this,hdr,mb,mon); } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP ping reply(1)"); + FLLOG(MCoperatorWarning, e,"MP ping reply(1)"); e->Release(); ret = false; } @@ -2225,7 +2225,7 @@ CMPConnectThread::CMPConnectThread(CMPServer *_parent, unsigned port, bool _list listensock = NULL; // delay create till running parent->setPort(port); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP Connect Thread Init Port = %d", port); + LOG(MCdebugInfo, "MP Connect Thread Init Port = %d", port); #endif running = false; @@ -2415,7 +2415,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, // not sure how to get here as this is not one of the possible outcomes of above: rd == 0 or rd == sizeof(id) or an exception StringBuffer errMsg("MP Connect Thread: invalid number of connection bytes serialized from "); peerEp.getEndpointHostText(errMsg); - FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str()); + FLLOG(MCoperatorWarning, "%s", errMsg.str()); sock->close(); return false; // did not timeout } @@ -2472,7 +2472,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, // JCSMORE, I think _remoteep really must/should match a IP of this local host errMsg.append("MP Connect Thread: invalid remote and/or host ep serialized from "); peerEp.getEndpointHostText(errMsg); - FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str()); + FLLOG(MCoperatorWarning, "%s", errMsg.str()); } else if (parent->mpTraceLevel >= MPVerboseMsgThreshold) { @@ -2504,7 +2504,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, #ifdef _TRACE StringBuffer str1; StringBuffer str2; - LOG(MCdebugInfo, unknownJob, "MP Connect Thread: connected to %s",_remoteep.getEndpointHostText(str1).str()); + LOG(MCdebugInfo, "MP Connect Thread: connected to %s",_remoteep.getEndpointHostText(str1).str()); #endif } #ifdef _FULLTRACE @@ -2513,7 +2513,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP Connect Thread: Failed to make connection(1)"); + FLLOG(MCoperatorWarning, e,"MP Connect Thread: Failed to make connection(1)"); sock->close(); e->Release(); } @@ -2523,7 +2523,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, int CMPConnectThread::run() { #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Connect Thread Starting - accept loop"); + LOG(MCdebugInfo, "MP: Connect Thread Starting - accept loop"); #endif Owned exception; while (running) @@ -2563,12 +2563,12 @@ int CMPConnectThread::run() MilliSleep(sleepSecs * 1000); } else // not sure this can ever happen (no exception, still running, and sock==nullptr) - LOG(MCdebugInfo, unknownJob, "MP Connect Thread accept returned NULL"); + LOG(MCdebugInfo, "MP Connect Thread accept returned NULL"); } } } #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP Connect Thread Stopping"); + LOG(MCdebugInfo, "MP Connect Thread Stopping"); #endif return 0; } @@ -2680,7 +2680,7 @@ CMPServer::~CMPServer() StringBuffer buf; getReceiveQueueDetails(buf); if (buf.length()) - LOG(MCdebugInfo, unknownJob, "MP: Orphan check\n%s",buf.str()); + LOG(MCdebugInfo, "MP: Orphan check\n%s",buf.str()); #endif _releaseAll(); selecthandler->stop(true); @@ -2758,7 +2758,7 @@ bool CMPServer::recv(CMessageBuffer &mbuf, const SocketEndpoint *ep, mptag_t tag } if (nfy.aborted) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPserver::recv closed on notify"); + LOG(MCdebugInfo, "CMPserver::recv closed on notify"); PrintStackReport(); #endif IMP_Exception *e=new CMPException(MPERR_link_closed,*nfy.ep); @@ -2844,7 +2844,7 @@ unsigned CMPServer::probe(const SocketEndpoint *ep, mptag_t tag,CTimeMon &tm,Soc } if (nfy.aborted) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPserver::probe closed on notify"); + LOG(MCdebugInfo, "CMPserver::probe closed on notify"); PrintStackReport(); #endif IMP_Exception *e=new CMPException(MPERR_link_closed,*ep); @@ -2932,7 +2932,7 @@ void CMPServer::notifyClosed(SocketEndpoint &ep, bool trace) if (trace) { StringBuffer url; - LOG(MCdebugInfo, unknownJob, "MP: CMPServer::notifyClosed %s",ep.getEndpointHostText(url).str()); + LOG(MCdebugInfo, "MP: CMPServer::notifyClosed %s",ep.getEndpointHostText(url).str()); PrintStackReport(); } #endif @@ -3633,7 +3633,7 @@ void stopMPServer() { stopLogMsgReceivers(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Stopping MP Server"); + LOG(MCdebugInfo, "MP: Stopping MP Server"); #endif _globalMPServer = globalMPServer; globalMPServer = NULL; @@ -3644,7 +3644,7 @@ void stopMPServer() _globalMPServer->stop(); _globalMPServer->Release(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Stopped MP Server"); + LOG(MCdebugInfo, "MP: Stopped MP Server"); #endif CriticalBlock block(CGlobalMPServer::sect); initMyNode(0); diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp index 1ce8551fd34..0886da12708 100644 --- a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp @@ -2487,7 +2487,7 @@ class IndexDistributeSlaveActivity : public HashDistributeSlaveBase } *lookup; public: - IndexDistributeSlaveActivity(CGraphElementBase *container) : PARENT(container, indexDistribActivityStatistics), lookup(NULL), contextLogger(jhtreeCacheStatistics, thorJob), statsUpdater(jhtreeCacheStatistics, *this, contextLogger) + IndexDistributeSlaveActivity(CGraphElementBase *container) : PARENT(container, indexDistribActivityStatistics), lookup(NULL), contextLogger(jhtreeCacheStatistics), statsUpdater(jhtreeCacheStatistics, *this, contextLogger) { } ~IndexDistributeSlaveActivity() @@ -3422,7 +3422,7 @@ bool CBucket::flush(bool critical) { if (clearHashTable(critical)) { - LOG(MCthorDetailedDebugInfo, thorJob, "Flushed%s bucket %d - %d elements", critical?"(critical)":"", queryBucketNumber(), count); + LOG(MCthorDetailedDebugInfo, "Flushed%s bucket %d - %d elements", critical?"(critical)":"", queryBucketNumber(), count); return true; } } diff --git a/thorlcr/activities/indexread/thindexreadslave.cpp b/thorlcr/activities/indexread/thindexreadslave.cpp index dd6cc9257d5..f276bc2051b 100644 --- a/thorlcr/activities/indexread/thindexreadslave.cpp +++ b/thorlcr/activities/indexread/thindexreadslave.cpp @@ -670,11 +670,11 @@ class CIndexReadSlaveBase : public CSlaveActivity { // Only superfiles required multiple context loggers to track sublevel stats for(unsigned i = 0; i < parts; ++i) - contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics, thorJob)); + contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics)); } else { - contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics, thorJob)); + contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics)); } } } diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp index 8c6f06ced04..e239d91319a 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp @@ -1253,7 +1253,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem public: IMPLEMENT_IINTERFACE_USING(CSimpleInterface); - CKeyLocalLookup(CKeyedJoinSlave &_owner, const RtlRecord &_keyRecInfo) : owner(_owner), keyRecInfo(_keyRecInfo), indexReadFieldsRow(_owner.indexInputAllocator), contextLogger(jhtreeCacheStatistics, thorJob) + CKeyLocalLookup(CKeyedJoinSlave &_owner, const RtlRecord &_keyRecInfo) : owner(_owner), keyRecInfo(_keyRecInfo), indexReadFieldsRow(_owner.indexInputAllocator), contextLogger(jhtreeCacheStatistics) { tlkManager.setown(owner.keyHasTlk ? createLocalKeyManager(keyRecInfo, nullptr, &contextLogger, owner.helper->hasNewSegmentMonitors(), false) : nullptr); reset(); diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp index 36b53ed6f1e..d00cd89595b 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp @@ -1030,7 +1030,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem { StringBuffer log; getInfo(log).append(": ").append(msg); - LOG(MCthorDetailedDebugInfo, thorJob, "%s", log.str()); + LOG(MCthorDetailedDebugInfo, "%s", log.str()); } virtual StringBuffer &getInfo(StringBuffer &info) const { @@ -1377,7 +1377,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem else { for (unsigned i=0; i keyManager{nullptr}; public: - CKeyLookupMergeHandler(CKeyedJoinSlave &_activity) : CKeyLookupLocalBase(_activity), contextLogger(jhtreeCacheStatistics, thorJob) + CKeyLookupMergeHandler(CKeyedJoinSlave &_activity) : CKeyLookupLocalBase(_activity), contextLogger(jhtreeCacheStatistics) { limiter = &activity.lookupThreadLimiter; translators.push_back(nullptr); @@ -1646,7 +1646,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem else { for (unsigned i=0; ijhelper->nextRow(); assertex(!row); @@ -1647,7 +1647,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase } for (unsigned i=0;inumworkers;i++) parent->addWork(NULL,NULL); - LOG(MCthorDetailedDebugInfo, thorJob, "CMultiCoreJoinHelper::cReader exit"); + LOG(MCthorDetailedDebugInfo, "CMultiCoreJoinHelper::cReader exit"); return 0; } } reader; @@ -1671,7 +1671,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase } int run() { - LOG(MCthorDetailedDebugInfo, thorJob, "CMultiCoreJoinHelper::cWorker started"); + LOG(MCthorDetailedDebugInfo, "CMultiCoreJoinHelper::cWorker started"); Owned rowIf = parent->activity.getRowInterfaces(); Owned allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RHFpacked|roxiemem::RHFunique)); @@ -1704,7 +1704,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase rowWriter->putRow(NULL); // end-of-stream rowWriter->flush(); stopped = true; // NB: will not get past flush(), until all read - LOG(MCthorDetailedDebugInfo, thorJob, "CMultiCoreJoinHelper::cWorker exit"); + LOG(MCthorDetailedDebugInfo, "CMultiCoreJoinHelper::cWorker exit"); return 0; } bool isStopped() const { return stopped; } @@ -1848,7 +1848,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase } int run() { - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cReader started"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cReader started"); try { const void * row = parent->jhelper->nextRow(); assertex(!row); @@ -1857,7 +1857,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase parent->setException(e,"CMulticoreUnorderedJoinHelper::cReader"); } parent->stopWorkers(); - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cReader exit"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cReader exit"); return 0; } } reader; @@ -1876,7 +1876,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase Owned allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RHFpacked|roxiemem::RHFunique)); Owned rowWriter = parent->multiWriter->getWriter(); - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cWorker started"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cWorker started"); for (;;) { cWorkItem *work = parent->workqueue.dequeue(); @@ -1897,7 +1897,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase break; } } - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cWorker exit"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cWorker exit"); return 0; } } **workers; diff --git a/thorlcr/activities/soapcall/thsoapcallslave.cpp b/thorlcr/activities/soapcall/thsoapcallslave.cpp index cbb89484c42..b0b81f64e16 100644 --- a/thorlcr/activities/soapcall/thsoapcallslave.cpp +++ b/thorlcr/activities/soapcall/thsoapcallslave.cpp @@ -45,7 +45,7 @@ class CWscRowCallSlaveActivity : public CSlaveActivity, implements IWSCRowProvid public: IMPLEMENT_IINTERFACE_USING(CSlaveActivity); - CWscRowCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + CWscRowCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { buildAuthToken(queryJob().queryUserDescriptor(), authToken); setRequireInitData(false); @@ -141,7 +141,7 @@ class SoapDatasetCallSlaveActivity : public CSlaveActivity, implements IWSCRowPr public: IMPLEMENT_IINTERFACE_USING(CSlaveActivity); - SoapDatasetCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + SoapDatasetCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { setRequireInitData(false); appendOutputLinked(this); @@ -235,7 +235,7 @@ class SoapRowActionSlaveActivity : public ProcessSlaveActivity, implements IWSCR public: IMPLEMENT_IINTERFACE_USING(PARENT); - SoapRowActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + SoapRowActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { setRequireInitData(false); } @@ -294,7 +294,7 @@ class SoapDatasetActionSlaveActivity : public ProcessSlaveActivity, implements I public: IMPLEMENT_IINTERFACE_USING(PARENT); - SoapDatasetActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + SoapDatasetActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { setRequireInitData(false); } diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp index 0feb326a37c..a2159f3683f 100644 --- a/thorlcr/activities/thdiskbase.cpp +++ b/thorlcr/activities/thdiskbase.cpp @@ -328,7 +328,7 @@ void CWriteMasterBase::publish() { StringBuffer clusterName; fileDesc->getClusterGroupName(clusterIdx, clusterName, &queryNamedGroupStore()); - LOG(MCthorDetailedDebugInfo, thorJob, "Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str()); + LOG(MCthorDetailedDebugInfo, "Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str()); unsigned p=0; while (pnumParts()) { diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index 056d9f82d58..a95ddc26b91 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -1200,12 +1200,12 @@ void traceMemUsage() { StringBuffer memStatsStr; roxiemem::memstats(memStatsStr); - LOG(MCthorDetailedDebugInfo, thorJob, "Roxiemem stats: %s", memStatsStr.str()); + LOG(MCthorDetailedDebugInfo, "Roxiemem stats: %s", memStatsStr.str()); memsize_t heapUsage = getMapInfo("heap"); if (heapUsage) // if 0, assumed to be unavailable { memsize_t rmtotal = roxiemem::getTotalMemoryLimit(); - LOG(MCthorDetailedDebugInfo, thorJob, "Heap usage (excluding Roxiemem) : %" I64F "d bytes", (unsigned __int64)(heapUsage-rmtotal)); + LOG(MCthorDetailedDebugInfo, "Heap usage (excluding Roxiemem) : %" I64F "d bytes", (unsigned __int64)(heapUsage-rmtotal)); } } @@ -1412,7 +1412,7 @@ void CGraphBase::executeSubGraph(size32_t parentExtractSz, const byte *parentExt { StringBuffer s; toXML(&queryXGMML(), s, 2); - MLOG(MCthorDetailedDebugInfo, thorJob, "Running graph [%s] : %s", isGlobal()?"global":"local", s.str()); + MLOG(MCthorDetailedDebugInfo, "Running graph [%s] : %s", isGlobal()?"global":"local", s.str()); } } if (localResults) @@ -2296,7 +2296,7 @@ IThorGraphResults *CGraphBase::createThorGraphResults(unsigned num) CFileUsageEntry * CGraphTempHandler::registerFile(const char *name, graph_id graphId, unsigned usageCount, bool temp, WUFileKind fileKind, StringArray *clusters) { assertex(temp); - LOG(MCdebugProgress, thorJob, "registerTmpFile name=%s, usageCount=%d", name, usageCount); + LOG(MCdebugProgress, "registerTmpFile name=%s, usageCount=%d", name, usageCount); CriticalBlock b(crit); if (tmpFiles.find(name)) throw MakeThorException(TE_FileAlreadyUsedAsTempFile, "File already used as temp file (%s)", name); @@ -2307,7 +2307,7 @@ CFileUsageEntry * CGraphTempHandler::registerFile(const char *name, graph_id gra void CGraphTempHandler::deregisterFile(const char *name, bool kept) { - LOG(MCdebugProgress, thorJob, "deregisterTmpFile name=%s", name); + LOG(MCdebugProgress, "deregisterTmpFile name=%s", name); CriticalBlock b(crit); CFileUsageEntry *fileUsage = tmpFiles.find(name); if (!fileUsage) @@ -2324,9 +2324,9 @@ void CGraphTempHandler::deregisterFile(const char *name, bool kept) try { if (!removeTemp(name)) - LOG(MCwarning, thorJob, "Failed to delete tmp file : %s (not found)", name); + LOG(MCwarning, "Failed to delete tmp file : %s (not found)", name); } - catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, thorJob, e, s.append(name).str()); } + catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, e, s.append(name).str()); } } else fileUsage->decUsage(); @@ -2343,9 +2343,9 @@ void CGraphTempHandler::clearTemps() try { if (!removeTemp(tmpname)) - LOG(MCwarning, thorJob, "Failed to delete tmp file : %s (not found)", tmpname); + LOG(MCwarning, "Failed to delete tmp file : %s (not found)", tmpname); } - catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, thorJob, e, s.append(tmpname).str()); } + catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, e, s.append(tmpname).str()); } } iter.clear(); tmpFiles.kill(); @@ -2778,7 +2778,7 @@ CActivityBase &CJobBase::queryChannelActivity(unsigned c, graph_id gid, activity void CJobBase::startJob() { - LOG(MCdebugProgress, thorJob, "New Graph started : %s", graphName.get()); + LOG(MCdebugProgress, "New Graph started : %s", graphName.get()); perfmonhook.setown(createThorMemStatsPerfMonHook(*this, getOptInt(THOROPT_MAX_KERNLOG, 3))); setPerformanceMonitorHook(perfmonhook); PrintMemoryStatusLog(); @@ -2825,7 +2825,7 @@ void CJobBase::endJob() jobEnded = true; setPerformanceMonitorHook(nullptr); - LOG(MCdebugProgress, thorJob, "Job ended : %s", graphName.get()); + LOG(MCdebugProgress, "Job ended : %s", graphName.get()); clearKeyStoreCache(true); PrintMemoryStatusLog(); @@ -3018,7 +3018,7 @@ mptag_t CJobChannel::deserializeMPTag(MemoryBuffer &mb) deserializeMPtag(mb, tag); if (TAG_NULL != tag) { - LOG(MCthorDetailedDebugInfo, thorJob, "deserializeMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "deserializeMPTag: tag = %d", (int)tag); jobComm->flush(tag); } return tag; @@ -3057,7 +3057,7 @@ void CJobChannel::clean() if (!REJECTLOG(MCthorDetailedDebugInfo)) { queryRowManager()->reportMemoryUsage(false); - LOG(MCthorDetailedDebugInfo, thorJob, "CJobBase resetting memory manager"); + LOG(MCthorDetailedDebugInfo, "CJobBase resetting memory manager"); } if (graphExecutor) diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 665c8778a39..8867425a8c2 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -78,7 +78,7 @@ class CFatalHandler : public CTimeoutTrigger, implements IFatalHandler virtual bool action() override { StringBuffer s("FAILED TO RECOVER FROM EXCEPTION, STOPPING THOR"); - FLLOG(MCoperatorWarning, thorJob, exception, s.str()); + FLLOG(MCoperatorWarning, exception, s.str()); Owned jobManager = getJobManager(); if (jobManager) { @@ -298,7 +298,7 @@ void CSlaveMessageHandler::threadmain() } case smt_getPhysicalName: { - LOG(MCdebugProgress, thorJob, "getPhysicalName called from node %d", sender-1); + LOG(MCdebugProgress, "getPhysicalName called from node %d", sender-1); StringAttr logicalName; unsigned partNo; bool create; @@ -317,7 +317,7 @@ void CSlaveMessageHandler::threadmain() } case smt_getFileOffset: { - LOG(MCdebugProgress, thorJob, "getFileOffset called from node %d", sender-1); + LOG(MCdebugProgress, "getFileOffset called from node %d", sender-1); StringAttr logicalName; unsigned partNo; msg.read(logicalName); @@ -330,7 +330,7 @@ void CSlaveMessageHandler::threadmain() } case smt_actMsg: { - LOG(MCdebugProgress, thorJob, "smt_actMsg called from node %d", sender-1); + LOG(MCdebugProgress, "smt_actMsg called from node %d", sender-1); graph_id gid; msg.read(gid); activity_id id; @@ -348,7 +348,7 @@ void CSlaveMessageHandler::threadmain() { unsigned slave; msg.read(slave); - LOG(MCdebugProgress, thorJob, "smt_getresult called from slave %d", slave); + LOG(MCdebugProgress, "smt_getresult called from slave %d", slave); graph_id gid; msg.read(gid); activity_id ownerId; @@ -1001,7 +1001,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase return getWorkUnitResult(workunit, name, sequence); } #define PROTECTED_GETRESULT(STEPNAME, SEQUENCE, KIND, KINDTEXT, ACTION) \ - LOG(MCdebugProgress, thorJob, "getResult%s(%s,%d)", KIND, STEPNAME?STEPNAME:"", SEQUENCE); \ + LOG(MCdebugProgress, "getResult%s(%s,%d)", KIND, STEPNAME?STEPNAME:"", SEQUENCE); \ Owned r = getResultForGet(STEPNAME, SEQUENCE); \ try \ { \ @@ -1266,7 +1266,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase { try { - LOG(MCdebugProgress, thorJob, "getExternalResultRaw %s", stepname); + LOG(MCdebugProgress, "getExternalResultRaw %s", stepname); Owned r = getExternalResult(wuid, stepname, sequence); return r->getResultHash(); @@ -1308,7 +1308,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase tgt = NULL; try { - LOG(MCdebugProgress, thorJob, "getExternalResultRaw %s", stepname); + LOG(MCdebugProgress, "getExternalResultRaw %s", stepname); Variable2IDataVal result(&tlen, &tgt); Owned r = getExternalResult(wuid, stepname, sequence); @@ -1586,7 +1586,7 @@ mptag_t CJobMaster::allocateMPTag() { mptag_t tag = allocateClusterMPTag(); queryJobChannel(0).queryJobComm().flush(tag); - LOG(MCthorDetailedDebugInfo, thorJob, "allocateMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "allocateMPTag: tag = %d", (int)tag); return tag; } @@ -1595,7 +1595,7 @@ void CJobMaster::freeMPTag(mptag_t tag) if (TAG_NULL != tag) { freeClusterMPTag(tag); - LOG(MCthorDetailedDebugInfo, thorJob, "freeMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "freeMPTag: tag = %d", (int)tag); queryJobChannel(0).queryJobComm().flush(tag); } } @@ -1869,7 +1869,7 @@ bool CJobMaster::go() Owned factory = getWorkUnitFactory(); if (factory->isAborting(wu.queryWuid())) { - LOG(MCwarning, thorJob, "ABORT detected from user"); + LOG(MCwarning, "ABORT detected from user"); unsigned code = TE_WorkUnitAborting; // default if (job.getOptBool("dumpInfoOnUserAbort", false)) @@ -2089,13 +2089,13 @@ bool CJobMaster::fireException(IException *e) { case tea_warning: { - LOG(MCwarning, thorJob, e); + LOG(MCwarning, e); reportExceptionToWorkunitCheckIgnore(*workunit, e); break; } default: { - LOG(MCerror, thorJob, e); + LOG(MCerror, e); queryJobManager().replyException(*this, e); fatalHandler->inform(LINK(e)); try { abort(e); } @@ -2341,13 +2341,13 @@ bool CMasterGraph::fireException(IException *e) { case tea_warning: { - LOG(MCwarning, thorJob, e); + LOG(MCwarning, e); reportExceptionToWorkunitCheckIgnore(job.queryWorkUnit(), e); break; } default: { - LOG(MCerror, thorJob, e); + LOG(MCerror, e); if (NULL != fatalHandler) fatalHandler->inform(LINK(e)); if (owner) diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 4c339769d5f..6c1d9868eba 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1804,7 +1804,7 @@ void CJobSlave::reportGraphEnd(graph_id gid) getSocketStatistics(stats); StringBuffer s; getSocketStatisticsString(stats,s); - MLOG(MCthorDetailedDebugInfo, thorJob, "Graph[%" GIDPF "u] - Socket statistics : %s\n", gid, s.str()); + MLOG(MCthorDetailedDebugInfo, "Graph[%" GIDPF "u] - Socket statistics : %s\n", gid, s.str()); } resetSocketStatistics(); } @@ -1850,7 +1850,7 @@ mptag_t CJobSlave::deserializeMPTag(MemoryBuffer &mb) deserializeMPtag(mb, tag); if (TAG_NULL != tag) { - LOG(MCthorDetailedDebugInfo, thorJob, "CJobSlave::deserializeMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "CJobSlave::deserializeMPTag: tag = %d", (int)tag); for (unsigned c=0; cmarkdead = true; - LOG(MCdebugProgress, thorJob, "Watchdog : Marking Machine as Down! [%s]", epstr.str()); + LOG(MCdebugProgress, "Watchdog : Marking Machine as Down! [%s]", epstr.str()); //removeSlave(mstate->ep); // more TBD } } @@ -202,7 +202,7 @@ void CMasterWatchdog::stopReading() void CMasterWatchdog::threadmain() { - LOG(MCdebugProgress, thorJob, "Started watchdog"); + LOG(MCdebugProgress, "Started watchdog"); unsigned lastbeat=msTick(); unsigned lastcheck=lastbeat; @@ -234,7 +234,7 @@ void CMasterWatchdog::threadmain() { StringBuffer epstr; hb.sender.getEndpointHostText(epstr); - LOG(MCdebugProgress, thorJob, "Watchdog : Unknown Machine! [%s]", epstr.str()); //TBD + LOG(MCdebugProgress, "Watchdog : Unknown Machine! [%s]", epstr.str()); //TBD } } unsigned now=msTick(); @@ -253,7 +253,7 @@ void CMasterWatchdog::threadmain() { if (MPERR_link_closed != e->errorCode()) { - FLLOG(MCexception(e), thorJob, e,"Watchdog Server Exception"); + FLLOG(MCexception(e), e,"Watchdog Server Exception"); e->Release(); } else @@ -266,7 +266,7 @@ void CMasterWatchdog::threadmain() } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"Watchdog Server Exception"); + FLLOG(MCexception(e), e,"Watchdog Server Exception"); e->Release(); // NB: it is important to continue with master watchdog, to continue to consume packets from workers } diff --git a/thorlcr/master/thactivitymaster.cpp b/thorlcr/master/thactivitymaster.cpp index b7ca5b248f8..82ca2d647a9 100644 --- a/thorlcr/master/thactivitymaster.cpp +++ b/thorlcr/master/thactivitymaster.cpp @@ -701,7 +701,7 @@ void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned ex Owned e = MakeActivityException(activity, TE_FormatCrcMismatch, "%s: Layout does not match published layout. %s", kindStr.str(), fileStr.str()); if (index && !f->queryAttributes().hasProp("_record_layout")) // Cannot verify if _true_ crc mismatch if soft layout missing anymore - LOG(MCwarning, thorJob, e); + LOG(MCwarning, e); else { if (activity->queryContainer().queryJob().getWorkUnitValueBool("skipFileFormatCrcCheck", false)) diff --git a/thorlcr/master/thdemonserver.cpp b/thorlcr/master/thdemonserver.cpp index 5b4094c3315..2087ff76098 100644 --- a/thorlcr/master/thdemonserver.cpp +++ b/thorlcr/master/thdemonserver.cpp @@ -76,7 +76,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer catch (IException *e) { StringBuffer s; - LOG(MCwarning, thorJob, "Failed to update progress information: %s", e->errorMessage(s).str()); + LOG(MCwarning, "Failed to update progress information: %s", e->errorMessage(s).str()); e->Release(); } } @@ -101,7 +101,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer const cost_type totalCost = workunitCost + sgCost + graph.getDiskAccessCost(); if (costLimit>0 && totalCost > costLimit) { - LOG(MCwarning, thorJob, "ABORT job cost exceeds limit"); + LOG(MCwarning, "ABORT job cost exceeds limit"); graph.fireException(MakeThorException(TE_CostExceeded, "Job cost exceeds limit")); } } @@ -166,7 +166,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer catch (IException *E) { StringBuffer s; - LOG(MCwarning, thorJob, "Failed to update progress information: %s", E->errorMessage(s).str()); + LOG(MCwarning, "Failed to update progress information: %s", E->errorMessage(s).str()); E->Release(); } } @@ -194,7 +194,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer catch (IException *e) { StringBuffer s; - LOG(MCwarning, thorJob, "Failed to update progress information: %s", e->errorMessage(s).str()); + LOG(MCwarning, "Failed to update progress information: %s", e->errorMessage(s).str()); e->Release(); } } @@ -215,7 +215,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer if (0 == activeGraphs.ordinality()) { StringBuffer urlStr; - LOG(MCdebugProgress, thorJob, "heartbeat packet received with no active graphs"); + LOG(MCdebugProgress, "heartbeat packet received with no active graphs"); return; } size32_t compressedProgressSz = progressMb.remaining(); @@ -233,12 +233,12 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer ForEachItemIn(g, activeGraphs) if (activeGraphs.item(g).queryGraphId() == graphId) graph = (CMasterGraph *)&activeGraphs.item(g); if (!graph) { - LOG(MCdebugProgress, thorJob, "heartbeat received from unknown graph %" GIDPF "d", graphId); + LOG(MCdebugProgress, "heartbeat received from unknown graph %" GIDPF "d", graphId); break; } if (!graph->deserializeStats(slave, uncompressedMb)) { - LOG(MCdebugProgress, thorJob, "heartbeat error in graph %" GIDPF "d", graphId); + LOG(MCdebugProgress, "heartbeat error in graph %" GIDPF "d", graphId); break; } } diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 1b6ff4ca6a6..2aa393c5d5c 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -265,7 +265,7 @@ class CJobManager : public CSimpleInterface, implements IJobManager, implements } else if (strncmp(command,"quit", 4) == 0) { - LOG(MCwarning, thorJob, "ABORT detected from user during debug session"); + LOG(MCwarning, "ABORT detected from user during debug session"); Owned e = MakeThorException(TE_WorkUnitAborting, "User signalled abort during debug session"); job->fireException(e); response.appendf(""); @@ -384,7 +384,7 @@ void CJobManager::stop() { if (!stopped) { - LOG(MCdebugProgress, thorJob, "Stopping jobManager"); + LOG(MCdebugProgress, "Stopping jobManager"); stopped = true; if (jobq) { @@ -578,7 +578,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char void CJobManager::run() { - LOG(MCdebugProgress, thorJob, "Listening for graph"); + LOG(MCdebugProgress, "Listening for graph"); setWuid(NULL); #ifndef _CONTAINERIZED @@ -671,7 +671,7 @@ void CJobManager::run() if (exclusiveLockName.length()) { if (globals->getPropBool("@multiThorPriorityLock")) - FLLOG(MCoperatorWarning, thorJob, "multiThorPriorityLock cannot be used in conjunction with multiThorExclusionLockName"); + FLLOG(MCoperatorWarning, "multiThorPriorityLock cannot be used in conjunction with multiThorExclusionLockName"); else { PROGLOG("Multi-Thor exclusive lock defined: %s", exclusiveLockName.str()); @@ -836,7 +836,7 @@ void CJobManager::run() } catch (IException *e) { - FLLOG(MCoperatorWarning, thorJob, e, "CJobManager::run"); + FLLOG(MCoperatorWarning, e, "CJobManager::run"); continue; } Owned factory; @@ -887,11 +887,9 @@ bool CJobManager::doit(IConstWorkUnit *workunit, const char *graphName, const So StringAttr wuid(workunit->queryWuid()); StringAttr user(workunit->queryUser()); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + JobNameScope activeJobName(wuid); - LOG(MCdebugInfo, thorJob, "Processing wuid=%s, graph=%s from agent: %s", wuid.str(), graphName, agentep.getEndpointHostText(s).str()); + LOG(MCdebugInfo, "Processing wuid=%s, graph=%s from agent: %s", wuid.str(), graphName, agentep.getEndpointHostText(s).str()); LOG(MCauditInfo,",Progress,Thor,Start,%s,%s,%s,%s,%s,%s", queryServerStatus().queryProperties()->queryProp("@thorname"), wuid.str(), @@ -914,10 +912,6 @@ bool CJobManager::doit(IConstWorkUnit *workunit, const char *graphName, const So queryServerStatus().queryProperties()->queryProp("@nodeGroup"), queryServerStatus().queryProperties()->queryProp("@queue")); - thorJob.setJobID(UnknownJob); - setDefaultJobId(UnknownJob); - queryLogMsgManager()->removeJobId(thorJobId); - if (e.get()) throw e.getClear(); return allDone; } @@ -942,12 +936,12 @@ void CJobManager::setWuid(const char *wuid, const char *cluster) } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e, "WARNING: Failed to set wuid in SDS:"); + FLLOG(MCexception(e), e, "WARNING: Failed to set wuid in SDS:"); e->Release(); } catch (CATCHALL) { - FLLOG(MCerror, thorJob, "WARNING: Failed to set wuid in SDS: Unknown error"); + FLLOG(MCerror, "WARNING: Failed to set wuid in SDS: Unknown error"); } } @@ -1089,7 +1083,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e, "Failed to write query dll - ignoring!"); + FLLOG(MCexception(e), e, "Failed to write query dll - ignoring!"); e->Release(); } sendSo = getExpertOptBool("dllsToSlaves", true); @@ -1213,7 +1207,7 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob) } EXCLOG(e,"abortThor"); } - LOG(MCdebugProgress, thorJob, "abortThor called"); + LOG(MCdebugProgress, "abortThor called"); if (jM) jM->stop(); if (thorQueue) @@ -1225,7 +1219,7 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob) if (2 > aborting && abortCurrentJob) { aborting = 2; - LOG(MCdebugProgress, thorJob, "aborting any current active job"); + LOG(MCdebugProgress, "aborting any current active job"); if (jM) { if (!e) @@ -1237,7 +1231,7 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob) } if (errCode == TEC_Clean) { - LOG(MCdebugProgress, thorJob, "Removing sentinel upon normal shutdown"); + LOG(MCdebugProgress, "Removing sentinel upon normal shutdown"); Owned sentinelFile = createSentinelTarget(); removeSentinelFile(sentinelFile); } @@ -1425,6 +1419,8 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam while (true) { + JobNameScope activeJobName(currentWuid.str()); + PROGLOG("Executing: wuid=%s, graph=%s", currentWuid.str(), currentGraphName.str()); { @@ -1469,10 +1465,8 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam { if (!streq(currentWuid, wuid)) { - queryLogMsgManager()->removeJobId(thorJob.queryJobID()); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + activeJobName.set(wuid); + // perhaps slightly overkill, but avoid checking/locking wuid to add pod info. // if this instance has already done so. auto it = publishedPodWuids.find(wuid.str()); @@ -1521,7 +1515,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"ThorMaster"); + FLLOG(MCexception(e), e,"ThorMaster"); e->Release(); } if (multiThorMemoryThreshold) diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 748c6011a08..8cebd0ca7b7 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -556,7 +556,7 @@ bool checkClusterRelicateDAFS(IGroup &grp) ep.port = 0; StringBuffer ips; ep.getHostText(ips); - FLLOG(MCoperatorError, thorJob, "VALIDATE FAILED(%d) %s : %s",failedcodes.item(i),ips.str(),failedmessages.item(i)); + FLLOG(MCoperatorError, "VALIDATE FAILED(%d) %s : %s",failedcodes.item(i),ips.str(),failedmessages.item(i)); } PROGLOG("Cluster replicate nodes check completed in %dms",msTick()-start); return (failures.ordinality()==0); @@ -579,7 +579,7 @@ bool ControlHandler(ahType type) { if (firstCtrlC) { - LOG(MCdebugProgress, thorJob, "CTRL-C detected"); + LOG(MCdebugProgress, "CTRL-C detected"); firstCtrlC = false; { Owned registry = CRegistryServer::getRegistryServer(); @@ -590,7 +590,7 @@ bool ControlHandler(ahType type) } else { - LOG(MCdebugProgress, thorJob, "2nd CTRL-C detected - terminating process"); + LOG(MCdebugProgress, "2nd CTRL-C detected - terminating process"); if (auditStartLogged) { @@ -607,7 +607,7 @@ bool ControlHandler(ahType type) // ahTerminate else { - LOG(MCdebugProgress, thorJob, "SIGTERM detected, shutting down"); + LOG(MCdebugProgress, "SIGTERM detected, shutting down"); Owned registry = CRegistryServer::getRegistryServer(); if (registry) registry->stop(); @@ -653,7 +653,7 @@ int main( int argc, const char *argv[] ) StringBuffer daliServer; if (!globals->getProp("@daliServers", daliServer)) { - LOG(MCerror, thorJob, "No Dali server list specified in THOR.XML (daliServers=iport,iport...)\n"); + LOG(MCerror, "No Dali server list specified in THOR.XML (daliServers=iport,iport...)\n"); return 0; // no recycle } @@ -712,14 +712,14 @@ int main( int argc, const char *argv[] ) queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler()); #endif - LOG(MCdebugProgress, thorJob, "Opened log file %s", logUrl.str()); + LOG(MCdebugProgress, "Opened log file %s", logUrl.str()); } #else setupContainerizedLogMsgHandler(); logHandler = queryStderrLogMsgHandler(); logUrl.set("stderr"); #endif - LOG(MCdebugProgress, thorJob, "Build %s", hpccBuildInfo.buildTag); + LOG(MCdebugProgress, "Build %s", hpccBuildInfo.buildTag); Owned serverGroup = createIGroupRetry(daliServer.str(), DALI_SERVER_PORT); @@ -728,7 +728,7 @@ int main( int argc, const char *argv[] ) { try { - LOG(MCdebugProgress, thorJob, "calling initClientProcess %d", thorEp.port); + LOG(MCdebugProgress, "calling initClientProcess %d", thorEp.port); initClientProcess(serverGroup, DCR_ThorMaster, thorEp.port, nullptr, nullptr, MP_WAIT_FOREVER, true); if (0 == thorEp.port) thorEp.port = queryMyNode()->endpoint().port; @@ -742,11 +742,11 @@ int main( int argc, const char *argv[] ) { if ((e->errorCode()!=JSOCKERR_port_in_use)) throw; - FLLOG(MCexception(e), thorJob, e,"InitClientProcess"); + FLLOG(MCexception(e), e,"InitClientProcess"); if (retry++>10) throw; e->Release(); - LOG(MCdebugProgress, thorJob, "Retrying"); + LOG(MCdebugProgress, "Retrying"); Sleep(retry*2000); } } @@ -950,7 +950,7 @@ int main( int argc, const char *argv[] ) } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"ThorMaster"); + FLLOG(MCexception(e), e,"ThorMaster"); e->Release(); return -1; } @@ -995,12 +995,11 @@ int main( int argc, const char *argv[] ) if (isContainerized()) { saveWuidToFile(workunit); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(workunit); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + JobNameScope activeJobName(workunit); + StringBuffer thorEpStr; - LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); - LOG(MCdebugProgress, thorJob, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); + LOG(MCdebugProgress, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); + LOG(MCdebugProgress, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); unsigned numWorkersPerPod = 1; if (!globals->hasProp("@numWorkers")) @@ -1045,8 +1044,8 @@ int main( int argc, const char *argv[] ) else { StringBuffer thorEpStr; - LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); - LOG(MCdebugProgress, thorJob, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); + LOG(MCdebugProgress, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); + LOG(MCdebugProgress, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); unsigned localThorPortInc = globals->getPropInt("@localThorPortInc", DEFAULT_SLAVEPORTINC); unsigned slaveBasePort = globals->getPropInt("@slaveport", DEFAULT_THORSLAVEPORT); Owned rawGroup = getClusterNodeGroup(thorname, "ThorCluster"); @@ -1069,7 +1068,7 @@ int main( int argc, const char *argv[] ) // bare-metal - check health of dafilesrv's on the Thor cluster. if (globals->getPropBool("@replicateOutputs")&&globals->getPropBool("@validateDAFS",true)&&!checkClusterRelicateDAFS(queryNodeGroup())) { - FLLOG(MCoperatorError, thorJob, "ERROR: Validate failure(s) detected, exiting Thor"); + FLLOG(MCoperatorError, "ERROR: Validate failure(s) detected, exiting Thor"); return globals->getPropBool("@validateDAFSretCode"); // default is no recycle! } } @@ -1141,11 +1140,11 @@ int main( int argc, const char *argv[] ) // NB: workunit/graphName only set in one-shot mode (if isCloud()) thorMain(logHandler, workunit, graphName); LOG(MCauditInfo, ",Progress,Thor,Terminate,%s,%s,%s",thorname,nodeGroup.str(),queueName.str()); - LOG(MCdebugProgress, thorJob, "ThorMaster terminated OK"); + LOG(MCdebugProgress, "ThorMaster terminated OK"); } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"ThorMaster"); + FLLOG(MCexception(e), e,"ThorMaster"); exception.setown(e); } if (isContainerized()) diff --git a/thorlcr/mfilemanager/thmfilemanager.cpp b/thorlcr/mfilemanager/thmfilemanager.cpp index b26528dce45..f16549d4769 100644 --- a/thorlcr/mfilemanager/thmfilemanager.cpp +++ b/thorlcr/mfilemanager/thmfilemanager.cpp @@ -388,7 +388,7 @@ class CFileManager : public CSimpleInterface, implements IThorFileManager bool extend = 0 != (helperFlags&TDWextend); bool jobTemp = 0 != (helperFlags&TDXjobtemp); - LOG(MCdebugInfo, thorJob, "createLogicalFile ( %s )", logicalName); + LOG(MCdebugInfo, "createLogicalFile ( %s )", logicalName); Owned efile; CDfsLogicalFileName dlfn; diff --git a/thorlcr/msort/tsortl.cpp b/thorlcr/msort/tsortl.cpp index 00d33a9aeb2..75a70955103 100644 --- a/thorlcr/msort/tsortl.cpp +++ b/thorlcr/msort/tsortl.cpp @@ -119,11 +119,11 @@ class CSocketRowStream: public CSimpleInterface, implements IRowStream if (dsz.eos()) { inbuf.clear(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.nextRow recv (%d,%x)",id,(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.nextRow recv (%d,%x)",id,(unsigned)(memsize_t)socket.get()); #endif size32_t sz = socket->receive_block_size(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.nextRow(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.nextRow(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); #endif if (sz==0) { // eof so terminate (no need to confirm) @@ -135,7 +135,7 @@ class CSocketRowStream: public CSimpleInterface, implements IRowStream socket->receive_block(buf,sz); assertex(!dsz.eos()); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.nextRow got (%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.nextRow got (%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); #endif } RtlDynamicRowBuilder rowBuilder(allocator); @@ -151,12 +151,12 @@ class CSocketRowStream: public CSimpleInterface, implements IRowStream stopped = true; try { #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.stop(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.stop(%x)",(unsigned)(memsize_t)socket.get()); #endif bool eof = true; socket->write(&eof,sizeof(eof)); // confirm stop #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.stopped(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.stopped(%x)",(unsigned)(memsize_t)socket.get()); #endif } catch (IException *e) { @@ -203,7 +203,7 @@ class CSocketRowWriter: public CSimpleInterface, implements ISocketRowWriter preallocated = bufsize+initSize; #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter(%d,%x) preallocated = %d",id,(unsigned)(memsize_t)socket.get(),preallocated); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter(%d,%x) preallocated = %d",id,(unsigned)(memsize_t)socket.get(),preallocated); #endif } @@ -224,11 +224,11 @@ class CSocketRowWriter: public CSimpleInterface, implements ISocketRowWriter flush(); try { #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter.stop(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter.stop(%x)",(unsigned)(memsize_t)socket.get()); #endif socket->send_block(NULL,0); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter.stopped(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter.stopped(%x)",(unsigned)(memsize_t)socket.get()); #endif } catch (IJSOCK_Exception *e) { // already gone! @@ -257,11 +257,11 @@ class CSocketRowWriter: public CSimpleInterface, implements ISocketRowWriter { size32_t l = outbuf.length(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter.flush(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),l); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter.flush(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),l); #endif if (l) { if (!socket->send_block(outbuf.bufferBase(),l)) { - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter remote stop"); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter remote stop"); stopped = true; } pos += l; @@ -289,7 +289,7 @@ IRowStream *ConnectMergeRead(unsigned id, IThorRowInterfaces *rowif,SocketEndpoi #ifdef _FULL_TRACE StringBuffer s; nodeaddr.getEndpointHostText(s); - LOG(MCthorDetailedDebugInfo, thorJob, "ConnectMergeRead(%d,%s,%x,%" RCPF "d,%" RCPF "u)",id,s.str(),(unsigned)(memsize_t)socket.get(),startrec,numrecs); + LOG(MCthorDetailedDebugInfo, "ConnectMergeRead(%d,%s,%x,%" RCPF "d,%" RCPF "u)",id,s.str(),(unsigned)(memsize_t)socket.get(),startrec,numrecs); #endif socket->write(&hdr,sizeof(hdr)); return new CSocketRowStream(id,rowif->queryRowAllocator(),rowif->queryRowDeserializer(),socket); @@ -337,7 +337,7 @@ ISocketRowWriter *ConnectMergeWrite(IThorRowInterfaces *rowif,ISocket *socket,si #ifdef _FULL_TRACE char name[100]; int port = socket->peer_name(name,sizeof(name)); - LOG(MCthorDetailedDebugInfo, thorJob, "ConnectMergeWrite(%d,%s:%d,%x,%" RCPF "d,%" RCPF "u)",hdr.id,name,port,(unsigned)(memsize_t)socket,startrec,numrecs); + LOG(MCthorDetailedDebugInfo, "ConnectMergeWrite(%d,%s:%d,%x,%" RCPF "d,%" RCPF "u)",hdr.id,name,port,(unsigned)(memsize_t)socket,startrec,numrecs); #endif return new CSocketRowWriter(hdr.id,rowif,socket,bufsize); } diff --git a/thorlcr/msort/tsortmp.cpp b/thorlcr/msort/tsortmp.cpp index 957ec55b238..5e6c52d53c3 100644 --- a/thorlcr/msort/tsortmp.cpp +++ b/thorlcr/msort/tsortmp.cpp @@ -290,7 +290,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag rank_t sender; comm->recv(mb,0,tag,&sender); // NB only recv from master if (mb.length()==0) { - LOG(MCthorDetailedDebugInfo, thorJob, "Stopping SortSlaveMP::marshall"); + LOG(MCthorDetailedDebugInfo, "Stopping SortSlaveMP::marshall"); return false; } byte fn; @@ -301,7 +301,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag mbout.append(okout); #ifdef FULLTRACE StringBuffer tmp1; - LOG(MCthorDetailedDebugInfo, thorJob, ">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getEndpointHostText(tmp1).str(),tag,mb.getReplyTag()); + LOG(MCthorDetailedDebugInfo, ">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getEndpointHostText(tmp1).str(),tag,mb.getReplyTag()); #endif bool replydone = false; Owned err; @@ -531,7 +531,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag if (!replydone) { #ifdef FULLTRACE StringBuffer tmp1; - LOG(MCthorDetailedDebugInfo, thorJob, "reply(mbout); } diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 266d3a4d740..aef0ab2625b 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -106,7 +106,7 @@ class CWriteIntercept : public CSimpleInterface { StringBuffer err; err.append("Cannot create ").append(idxFile->queryFilename()); - LOG(MCerror, thorJob, "%s", err.str()); + LOG(MCerror, "%s", err.str()); throw MakeActivityException(&activity, -1, "%s", err.str()); } idxFileStream.setown(createBufferedIOStream(idxFileIO,0x100000)); @@ -1155,10 +1155,10 @@ class CThorSorter : public CSimpleInterface, implements IThorSorter, implements startmergesem.signal(); ActPrintLog(activity, "StartMiniSort output started"); traceWait("finishedmergesem(2)",finishedmergesem); - LOG(MCthorDetailedDebugInfo, thorJob, "StartMiniSort output done"); + LOG(MCthorDetailedDebugInfo, "StartMiniSort output done"); merger.clear(); intercept.clear(); - LOG(MCthorDetailedDebugInfo, thorJob, "StartMiniSort exit"); + LOG(MCthorDetailedDebugInfo, "StartMiniSort exit"); } virtual void Close() { diff --git a/thorlcr/msort/tsorts1.cpp b/thorlcr/msort/tsorts1.cpp index d5353f973f5..c0ab5c1df1c 100644 --- a/thorlcr/msort/tsorts1.cpp +++ b/thorlcr/msort/tsorts1.cpp @@ -60,7 +60,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream endpoint = targetep; char url[100]; targetep.getEndpointHostText(url,sizeof(url)); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) %s, pos=%" RCPF "d len=%" RCPF "u",streamno,url,startrec,numrecs); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: Stream(%u) %s, pos=%" RCPF "d len=%" RCPF "u",streamno,url,startrec,numrecs); SocketEndpoint mergeep = targetep; mergeep.port+=SOCKETSERVERINC; @@ -86,7 +86,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream stream = ConnectMergeRead(streamno,rowif,mergeep,startrec,numrecs,socket); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) connected to %s",streamno,url); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: Stream(%u) connected to %s",streamno,url); } virtual ~CMergeReadStream() { @@ -108,7 +108,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream #ifdef _FULL_TRACE char url[100]; endpoint.getEndpointHostText(url,sizeof(url)); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: EOS for %s",url); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: EOS for %s",url); #endif eos(); } @@ -121,7 +121,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream #ifdef _FULL_TRACE char url[100]; endpoint.getEndpointHostText(url,sizeof(url)); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: stop for %s",url); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: stop for %s",url); #endif stream->stop(); eos(); @@ -163,12 +163,12 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify ~CSortMerge() { #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "~CSortMerge in"); + LOG(MCthorDetailedDebugInfo, "~CSortMerge in"); #endif if (started) closedown(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "~CSortMerge out"); + LOG(MCthorDetailedDebugInfo, "~CSortMerge out"); #endif } void init() @@ -178,7 +178,7 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify char name[64]; int port = socket->peer_name(name,sizeof(name)); url.append(name).append(':').append(port); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge WRITE: start %s, pos=%" RCPF "d, len=%" RCPF "d",url.str(),poscount,numrecs); + LOG(MCthorDetailedDebugInfo, "SORT Merge WRITE: start %s, pos=%" RCPF "d, len=%" RCPF "d",url.str(),poscount,numrecs); rowcount_t pos=poscount; try { @@ -228,7 +228,7 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify char peer[16]; if (socket) { socket->peer_name(peer,sizeof(peer)-1); - LOG(MCthorDetailedDebugInfo, thorJob, "waitdone %s",peer); + LOG(MCthorDetailedDebugInfo, "waitdone %s",peer); } else peer[0] = 0; @@ -237,7 +237,7 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify if (exception) throw exception.getClear(); if (peer[0]) - LOG(MCthorDetailedDebugInfo, thorJob, "waitdone exit"); + LOG(MCthorDetailedDebugInfo, "waitdone exit"); } bool notifySelected(ISocket *sock,unsigned selected) { @@ -246,11 +246,11 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify if (closing) { closing = false; #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "notifySelected calling closedown"); + LOG(MCthorDetailedDebugInfo, "notifySelected calling closedown"); #endif closedown(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "notifySelected called closedown"); + LOG(MCthorDetailedDebugInfo, "notifySelected called closedown"); #endif done = true; donesem.signal(); @@ -515,7 +515,7 @@ protected: friend class CSortMerge; respos += vMAPL(j,i)-vMAPL(j,i-1); // note we are adding up all of the lower as we want start rowcount_t totalrows = resnum; - LOG(MCthorDetailedDebugInfo, thorJob, "Output start = %" RCPF "d, num = %" RCPF "u",respos,resnum); + LOG(MCthorDetailedDebugInfo, "Output start = %" RCPF "d, num = %" RCPF "u",respos,resnum); IArrayOf readers; IException *exc = NULL; @@ -530,7 +530,7 @@ protected: friend class CSortMerge; { if (i==partno) { - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) local, pos=%" RCPF "u len=%" RCPF "u",i,sstart,snum); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: Stream(%u) local, pos=%" RCPF "u len=%" RCPF "u",i,sstart,snum); readers.append(*slave.createMergeInputStream(sstart,snum)); } else @@ -576,7 +576,7 @@ void CSortMerge::closedown() { CriticalBlock block(crit); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge: closing %s",url.str()); + LOG(MCthorDetailedDebugInfo, "SORT Merge: closing %s",url.str()); #endif if (!socket) return; @@ -611,7 +611,7 @@ void CSortMerge::closedown() throw; } started = false; - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge: finished %s, %d rows merged",url.str(),ndone); + LOG(MCthorDetailedDebugInfo, "SORT Merge: finished %s, %d rows merged",url.str(),ndone); } IMergeTransferServer *createMergeTransferServer(ISortSlaveBase *parent) diff --git a/thorlcr/slave/backup.cpp b/thorlcr/slave/backup.cpp index 43c39a32c30..d466cc8c726 100644 --- a/thorlcr/slave/backup.cpp +++ b/thorlcr/slave/backup.cpp @@ -105,14 +105,14 @@ class CThorBackupHandler : public CSimpleInterface, implements IBackup, implemen { StringBuffer errMsg; if (!currentAbort) - LOG(MCwarning, thorJob, "%s", errMsg.append("Backup inconsistency detected, backup aborted: ").append(item->dst).str()); + LOG(MCwarning, "%s", errMsg.append("Backup inconsistency detected, backup aborted: ").append(item->dst).str()); backupIFile->remove(); } } catch (IException *e) { StringBuffer errMsg("copying: "); - LOG(MCwarning, thorJob, e, errMsg.append(item->src)); + LOG(MCwarning, e, errMsg.append(item->src)); try { backupIFile->remove(); } catch (IException *e) { EXCLOG(e); e->Release(); } if (!ignoreError) throw; diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 049f61b34a7..719899dc6a9 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -71,7 +71,7 @@ void enableThorSlaveAsDaliClient() { try { - LOG(MCdebugProgress, thorJob, "calling initClientProcess"); + LOG(MCdebugProgress, "calling initClientProcess"); initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp)); break; } @@ -79,11 +79,11 @@ void enableThorSlaveAsDaliClient() { if ((e->errorCode()!=JSOCKERR_port_in_use)) throw; - FLLOG(MCexception(e), thorJob, e,"InitClientProcess"); + FLLOG(MCexception(e), e,"InitClientProcess"); if (retry++>10) throw; e->Release(); - LOG(MCdebugProgress, thorJob, "Retrying"); + LOG(MCdebugProgress, "Retrying"); Sleep(retry*2000); } } @@ -450,7 +450,7 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, public: CKMContainer(CKJService &_service, CKeyLookupContext *_ctx) - : service(_service), ctx(_ctx), contextLogger(jhtreeCacheStatistics, thorJob) + : service(_service), ctx(_ctx), contextLogger(jhtreeCacheStatistics) { keyManager.setown(ctx->createKeyManager(&contextLogger)); StringBuffer tracing; @@ -1562,11 +1562,11 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, } virtual void reset() override { - LOG(MCthorDetailedDebugInfo, thorJob, "KJService reset()"); + LOG(MCthorDetailedDebugInfo, "KJService reset()"); processorPool->stopAll(true); processorPool->joinAll(false); clearAll(); - LOG(MCthorDetailedDebugInfo, thorJob, "KJService reset() done"); + LOG(MCthorDetailedDebugInfo, "KJService reset() done"); } virtual void start() override { @@ -1577,7 +1577,7 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, { if (aborted) return; - LOG(MCthorDetailedDebugInfo, thorJob, "KJService stop()"); + LOG(MCthorDetailedDebugInfo, "KJService stop()"); queryNodeComm().cancel(RANK_ALL, keyLookupMpTag); processorPool->stopAll(true); processorPool->joinAll(true); @@ -1774,6 +1774,7 @@ class CJobListener : public CSimpleInterface bool doReply; OwnedPtr perf; + JobNameScope activeJobName; while (!stopped && queryNodeComm().recv(msg, 0, masterSlaveMpTag)) { doReply = true; @@ -1892,10 +1893,8 @@ class CJobListener : public CSimpleInterface ILogMsgFilter *existingLogHandler = queryLogMsgManager()->queryMonitorFilter(logHandler); dbgassertex(existingLogHandler); verifyex(queryLogMsgManager()->changeMonitorFilterOwn(logHandler, getCategoryLogMsgFilter(existingLogHandler->queryAudienceMask(), existingLogHandler->queryClassMask(), maxLogDetail))); - queryLogMsgManager()->removeJobId(thorJob.queryJobID()); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + + activeJobName.set(wuid); PROGLOG("Started wuid=%s, user=%s, graph=%s [log detail level=%u]\n", wuid.get(), user.str(), graphName.get(), maxLogDetail); PROGLOG("Using query: %s", soPath.str()); diff --git a/thorlcr/slave/slwatchdog.cpp b/thorlcr/slave/slwatchdog.cpp index c3dd6078efc..30536b3ea68 100644 --- a/thorlcr/slave/slwatchdog.cpp +++ b/thorlcr/slave/slwatchdog.cpp @@ -106,7 +106,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement CriticalBlock b(crit); activeGraphs.append(*LINK(&graph)); StringBuffer str("Watchdog: Start Job "); - LOG(MCthorDetailedDebugInfo, thorJob, "%s", str.append(graph.queryGraphId()).str()); + LOG(MCthorDetailedDebugInfo, "%s", str.append(graph.queryGraphId()).str()); } virtual void stopGraph(CGraphBase &graph, MemoryBuffer *mb) override { @@ -119,7 +119,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement else { StringBuffer str("Watchdog: Stop Job "); - LOG(MCthorDetailedDebugInfo, thorJob, "%s", str.append(graph.queryGraphId()).str()); + LOG(MCthorDetailedDebugInfo, "%s", str.append(graph.queryGraphId()).str()); if (mb) { DelayedSizeMarker sizeMark(*mb); @@ -138,7 +138,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement #endif stopped = true; threaded.join(); - LOG(MCdebugProgress, thorJob, "Stopped watchdog"); + LOG(MCdebugProgress, "Stopped watchdog"); } } virtual void debugRequest(MemoryBuffer &msg, const char *request) const override @@ -173,7 +173,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement // IThreaded virtual void threadmain() override { - LOG(MCthorDetailedDebugInfo, thorJob, "Watchdog: thread running"); + LOG(MCthorDetailedDebugInfo, "Watchdog: thread running"); gatherAndSend(); // send initial data assertex(HEARTBEAT_INTERVAL>=8); unsigned count = HEARTBEAT_INTERVAL+getRandom()%8-4; diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index 896a6b042a7..e93c69d96d8 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -119,7 +119,7 @@ static bool RegisterSelf(SocketEndpoint &masterEp) { StringBuffer slfStr; StringBuffer masterStr; - LOG(MCdebugProgress, thorJob, "registering %s - master %s",slfEp.getEndpointHostText(slfStr).str(),masterEp.getEndpointHostText(masterStr).str()); + LOG(MCdebugProgress, "registering %s - master %s",slfEp.getEndpointHostText(slfStr).str(),masterEp.getEndpointHostText(masterStr).str()); try { SocketEndpoint ep = masterEp; @@ -234,11 +234,11 @@ static bool RegisterSelf(SocketEndpoint &masterEp) OERRLOG("Failed to connect to all nodes"); else PROGLOG("verified mp connection to rest of cluster"); - LOG(MCdebugProgress, thorJob, "registered %s",slfStr.str()); + LOG(MCdebugProgress, "registered %s",slfStr.str()); } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"slave registration error"); + FLLOG(MCexception(e), e,"slave registration error"); e->Release(); return false; } @@ -258,7 +258,7 @@ bool UnregisterSelf(IException *e) StringBuffer slfStr; slfEp.getEndpointHostText(slfStr); - LOG(MCdebugProgress, thorJob, "Unregistering slave : %s", slfStr.str()); + LOG(MCdebugProgress, "Unregistering slave : %s", slfStr.str()); try { CMessageBuffer msg; @@ -266,16 +266,16 @@ bool UnregisterSelf(IException *e) serializeException(e, msg); // NB: allows exception to be NULL if (!queryWorldCommunicator().send(msg, masterNode, MPTAG_THORREGISTRATION, 60*1000)) { - LOG(MCerror, thorJob, "Failed to unregister slave : %s", slfStr.str()); + LOG(MCerror, "Failed to unregister slave : %s", slfStr.str()); return false; } - LOG(MCdebugProgress, thorJob, "Unregistered slave : %s", slfStr.str()); + LOG(MCdebugProgress, "Unregistered slave : %s", slfStr.str()); isRegistered = false; return true; } catch (IException *e) { if (!jobListenerStopped) - FLLOG(MCexception(e), thorJob, e,"slave unregistration error"); + FLLOG(MCexception(e), e,"slave unregistration error"); e->Release(); } return false; @@ -284,9 +284,9 @@ bool UnregisterSelf(IException *e) bool ControlHandler(ahType type) { if (ahInterrupt == type) - LOG(MCdebugProgress, thorJob, "CTRL-C detected"); + LOG(MCdebugProgress, "CTRL-C detected"); else if (!jobListenerStopped) - LOG(MCdebugProgress, thorJob, "SIGTERM detected"); + LOG(MCdebugProgress, "SIGTERM detected"); bool unregOK = false; if (!jobListenerStopped) { @@ -330,7 +330,7 @@ ILogMsgHandler *startSlaveLog() queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler()); #endif - LOG(MCdebugProgress, thorJob, "Opened log file %s", lf->queryLogFileSpec()); + LOG(MCdebugProgress, "Opened log file %s", lf->queryLogFileSpec()); } else { @@ -340,13 +340,12 @@ ILogMsgHandler *startSlaveLog() if (getComponentConfigSP()->getProp("@workunit", wuid)) { LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); setDefaultJobId(thorJobId); } } //setupContainerizedStorageLocations(); - LOG(MCdebugProgress, thorJob, "Build %s", hpccBuildInfo.buildTag); + LOG(MCdebugProgress, "Build %s", hpccBuildInfo.buildTag); return logHandler; } @@ -427,7 +426,7 @@ int main( int argc, const char *argv[] ) #ifdef USE_MP_LOG startLogMsgParentReceiver(); - LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp)); + LOG(MCdebugProgress, "MPServer started on port %d", getFixedPort(TPORT_mp)); #endif SocketEndpoint masterEp(master); @@ -456,7 +455,7 @@ int main( int argc, const char *argv[] ) if (err) { IException *e = makeErrnoExceptionV(-1, "Failed to change dir to '%s'", thorPath.str()); - FLLOG(MCexception(e), thorJob, e); + FLLOG(MCexception(e), e); throw e; } @@ -507,16 +506,16 @@ int main( int argc, const char *argv[] ) useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead")); - LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR); + LOG(MCdebugProgress, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR); #ifdef _WIN32 ULARGE_INTEGER userfree; ULARGE_INTEGER total; ULARGE_INTEGER free; if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) { unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart); - LOG(MCdebugInfo, thorJob, "Total disk space = %" I64F "d k", total.QuadPart/1000); - LOG(MCdebugInfo, thorJob, "Free disk space = %" I64F "d k", free.QuadPart/1000); - LOG(MCdebugInfo, thorJob, "%d%% disk free\n",pc); + LOG(MCdebugInfo, "Total disk space = %" I64F "d k", total.QuadPart/1000); + LOG(MCdebugInfo, "Free disk space = %" I64F "d k", free.QuadPart/1000); + LOG(MCdebugInfo, "%d%% disk free\n",pc); } #endif @@ -581,12 +580,12 @@ int main( int argc, const char *argv[] ) slaveMain(jobListenerStopped, slaveLogHandler); } - LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK"); + LOG(MCdebugProgress, "ThorSlave terminated OK"); } catch (IException *e) { if (!jobListenerStopped) - FLLOG(MCexception(e), thorJob, e,"ThorSlave"); + FLLOG(MCexception(e), e,"ThorSlave"); unregisterException.setown(e); } #ifndef _CONTAINERIZED diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index 2a3450ad6a5..d64d06da1a2 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -801,7 +801,7 @@ class CSharedWriteAheadBase : public CSimpleInterface, implements ISharedSmartBu { unsigned whichChunk = queryCOutput(output).currentChunkNum - lowestChunk; #ifdef TRACE_WRITEAHEAD - LOG(MCthorDetailedDebugInfo, thorJob, "output=%d, chunk=%d (whichChunk=%d)", output, currentChunkNum, whichChunk); + LOG(MCthorDetailedDebugInfo, "output=%d, chunk=%d (whichChunk=%d)", output, currentChunkNum, whichChunk); #endif rowSet.setown(readRows(output, whichChunk)); assertex(rowSet); @@ -1523,7 +1523,7 @@ class CSharedWriteAheadDisk : public CSharedWriteAheadBase Owned chunk = savedChunks.dequeue(); if (!chunk) break; } - LOG(MCthorDetailedDebugInfo, thorJob, "CSharedWriteAheadDisk: highOffset=%" I64F "d", highOffset); + LOG(MCthorDetailedDebugInfo, "CSharedWriteAheadDisk: highOffset=%" I64F "d", highOffset); } virtual void reset() { diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index ff98bd8c015..0c38400b5b4 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -144,7 +144,7 @@ void ActPrintLogArgs(const CGraphElementBase *container, const ActLogEnum flags, return; // suppress logging child activities unless thorlog_all flag StringBuffer res; ActPrintLogArgsPrep(res, container, flags, format, args); - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void ActPrintLogArgs(const CGraphElementBase *container, IException *e, const ActLogEnum flags, const LogMsgCategory &logCat, const char *format, va_list args) @@ -156,7 +156,7 @@ void ActPrintLogArgs(const CGraphElementBase *container, IException *e, const Ac res.append(" : "); e->errorMessage(res); } - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void ActPrintLogEx(const CGraphElementBase *container, const ActLogEnum flags, const LogMsgCategory &logCat, const char *format, ...) @@ -168,7 +168,7 @@ void ActPrintLogEx(const CGraphElementBase *container, const ActLogEnum flags, c va_start(args, format); ActPrintLogArgsPrep(res, container, flags, format, args); va_end(args); - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void ActPrintLog(const CActivityBase *activity, const char *format, ...) @@ -213,7 +213,7 @@ void GraphPrintLogArgs(CGraphBase *graph, const ActLogEnum flags, const LogMsgCa return; // suppress logging from child graph unless thorlog_all flag StringBuffer res; GraphPrintLogArgsPrep(res, graph, flags, logCat, format, args); - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void GraphPrintLogArgs(CGraphBase *graph, IException *e, const ActLogEnum flags, const LogMsgCategory &logCat, const char *format, va_list args) @@ -227,7 +227,7 @@ void GraphPrintLogArgs(CGraphBase *graph, IException *e, const ActLogEnum flags, res.append(" : "); e->errorMessage(res); } - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void GraphPrintLog(CGraphBase *graph, IException *e, const char *format, ...) @@ -659,12 +659,12 @@ class CTempNameHandler if (file.isFile()==fileBool::foundYes) { if (log) - LOG(MCdebugInfo, thorJob, "Deleting %s", file.queryFilename()); + LOG(MCdebugInfo, "Deleting %s", file.queryFilename()); try { file.remove(); } catch (IException *e) { if (log) - FLLOG(MCwarning, thorJob, e); + FLLOG(MCwarning, e); e->Release(); } } @@ -717,7 +717,7 @@ class CTempNameHandler catch (IException *e) { if (log) - FLLOG(MCwarning, thorJob, e); + FLLOG(MCwarning, e); e->Release(); } subDirPath.clear(); @@ -756,7 +756,7 @@ void GetTempFilePath(StringBuffer &name, const char *suffix) void SetTempDir(const char *rootTempDir, const char *uniqueSubDir, const char *tempPrefix, bool clearDir) { TempNameHandler.setTempDir(rootTempDir, uniqueSubDir, tempPrefix, clearDir); - LOG(MCdebugProgress, thorJob, "temporary rootTempdir: %s, uniqueSubDir: %s, prefix: %s", rootTempDir, uniqueSubDir, tempPrefix); + LOG(MCdebugProgress, "temporary rootTempdir: %s, uniqueSubDir: %s, prefix: %s", rootTempDir, uniqueSubDir, tempPrefix); } void ClearTempDir() @@ -764,7 +764,7 @@ void ClearTempDir() try { TempNameHandler.clear(true); - LOG(MCthorDetailedDebugInfo, thorJob, "temp directory cleared"); + LOG(MCthorDetailedDebugInfo, "temp directory cleared"); } catch (IException *e) { @@ -812,8 +812,6 @@ void loadCmdProp(IPropertyTree *tree, const char *cmdProp) } } -LogMsgJobInfo thorJob(UnknownJob, UnknownUser); // configured at job start (in manager and workers) - void ensureDirectoryForFile(const char *fName) { if (!recursiveCreateDirectoryForFile(fName)) @@ -823,7 +821,7 @@ void ensureDirectoryForFile(const char *fName) // Not recommended to be used from slaves as tend to be one or more trying at same time. void reportExceptionToWorkunit(IConstWorkUnit &workunit,IException *e, ErrorSeverity severity) { - LOG(MCwarning, thorJob, e, "Reporting exception to WU"); + LOG(MCwarning, e, "Reporting exception to WU"); Owned wu = &workunit.lock(); if (wu) { @@ -1160,12 +1158,12 @@ void CFifoFileCache::deleteFile(IFile &ifile) try { if (!ifile.remove()) - FLLOG(MCoperatorWarning, thorJob, "CFifoFileCache: Failed to remove file (missing) : %s", ifile.queryFilename()); + FLLOG(MCoperatorWarning, "CFifoFileCache: Failed to remove file (missing) : %s", ifile.queryFilename()); } catch (IException *e) { StringBuffer s("Failed to remove file: "); - FLLOG(MCoperatorWarning, thorJob, e, s.append(ifile.queryFilename())); + FLLOG(MCoperatorWarning, e, s.append(ifile.queryFilename())); } } diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index f378ab1e604..6ebfd8f0850 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -122,8 +122,6 @@ #define THOR_SEM_RETRY_TIMEOUT 2 // Logging -extern graph_decl LogMsgJobInfo thorJob; - enum ThorExceptionAction { tea_null, tea_warning, tea_abort, tea_shutdown }; enum RegistryCode:unsigned { rc_register, rc_deregister }; @@ -284,7 +282,7 @@ class graph_decl CTimeoutTrigger : public CInterface, implements IThreaded void stop() { running = false; todo.signal(); } void inform(IException *e) { - LOG(MCdebugProgress, thorJob, "INFORM [%s]", description.get()); + LOG(MCdebugProgress, "INFORM [%s]", description.get()); CriticalBlock block(crit); if (exception.get()) e->Release(); @@ -299,7 +297,7 @@ class graph_decl CTimeoutTrigger : public CInterface, implements IThreaded CriticalBlock block(crit); IException *e = exception.getClear(); if (e) - LOG(MCdebugProgress, thorJob, "CLEARING TIMEOUT [%s]", description.get()); + LOG(MCdebugProgress, "CLEARING TIMEOUT [%s]", description.get()); todo.signal(); return e; } diff --git a/tools/swapnode/swapnodelib.cpp b/tools/swapnode/swapnodelib.cpp index 8a8df9a1108..649c6dcfd09 100644 --- a/tools/swapnode/swapnodelib.cpp +++ b/tools/swapnode/swapnodelib.cpp @@ -42,8 +42,6 @@ #define SDS_LOCK_TIMEOUT 30000 #define SWAPNODE_RETRY_TIME (1000 * 60 * 60 * 1) // 1hr -static const LogMsgJobInfo swapnodeJob(UnknownJob, UnknownUser); - static bool ensureThorIsDown(const char *cluster, bool nofail, bool wait) { bool retry = false; @@ -781,7 +779,7 @@ class CAutoSwapNode : public CSwapNode StringBuffer msg; msg.appendf("AUTOSWAPNODE: cluster %s node %d: swapped out %s, swapped in %s", groupName.get(), badrank.item(i4) + 1, from.str(), to.str()); emailSwap(msg.str()); - FLLOG(MCoperatorError, swapnodeJob, "%s", msg.str()); + FLLOG(MCoperatorError, "%s", msg.str()); } } return true; From 9951f0e9b648d49eadc767a039cb4eb140e3e472 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 14 Mar 2024 19:13:40 +0000 Subject: [PATCH 23/23] HPCC-31460 Fix problem introduced when resolving merge conflicts Signed-off-by: Gavin Halliday --- roxie/ccd/ccdsnmp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roxie/ccd/ccdsnmp.cpp b/roxie/ccd/ccdsnmp.cpp index 5fdfbf4f302..169b6a17200 100644 --- a/roxie/ccd/ccdsnmp.cpp +++ b/roxie/ccd/ccdsnmp.cpp @@ -213,7 +213,7 @@ CRoxieMetricsManager::CRoxieMetricsManager() addMetric(dataPacketsReceived); addMetric(flowRequestsSent); addMetric(flowPermitsReceived); - addMetric(dataPacketsSentfalse); + addMetric(dataPacketsSent); } void CRoxieMetricsManager::doAddMetric(RelaxedAtomic &counter, const char *name, bool isMinVal)