From 919059fa29331dcaeec543eb41c337f45aabb208 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 15 Sep 2023 15:19:54 +0100 Subject: [PATCH 1/7] HPCC-30298 Allow tracing detail level to be controlled by tracing flags Signed-off-by: Gavin Halliday --- system/jlib/jlog.cpp | 12 ++++++++++-- system/jlib/jtrace.hpp | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 8e44015fb6d..06fed7f91d3 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -3383,10 +3383,18 @@ TraceFlags loadTraceFlags(const IPropertyTree *ptree, const std::initializer_lis attrName.clear().appendf("_%s", o.name); if (ptree->hasProp(attrName)) { - if (ptree->getPropBool(attrName, false)) + if (o.value <= TraceFlags::LevelMask) + { + dft &= ~TraceFlags::LevelMask; dft |= o.value; + } else - dft &= ~o.value; + { + if (ptree->getPropBool(attrName, false)) + dft |= o.value; + else + dft &= ~o.value; + } } } diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 6aa184bbdc0..68437091182 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -121,6 +121,12 @@ BITMASK_ENUM(TraceFlags); // Feature trace flags for hthor, thor and Roxie engines +// Detail levels +constexpr TraceFlags traceNone = TraceFlags::None; +constexpr TraceFlags traceStandard = TraceFlags::Standard; +constexpr TraceFlags traceDetailed = TraceFlags::Detailed; +constexpr TraceFlags traceMax = TraceFlags::Max; + // Common to several engines constexpr TraceFlags traceHttp = TraceFlags::flag1; constexpr TraceFlags traceSockets = TraceFlags::flag2; @@ -158,6 +164,10 @@ struct TraceOption { const char * name; TraceFlags value; }; constexpr std::initializer_list roxieTraceOptions { + TRACEOPT(traceNone), + TRACEOPT(traceStandard), + TRACEOPT(traceDetailed), + TRACEOPT(traceMax), TRACEOPT(traceHttp), TRACEOPT(traceSockets), TRACEOPT(traceCassandra), From 3444c2ee0577e4426d906084d1b16fa63d25c44f Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 12 Oct 2023 17:40:23 +0100 Subject: [PATCH 2/7] HPCC-30504 Fix scope checking for files in the root of a dropzone Signed-off-by: Gavin Halliday --- esp/smc/SMCLib/TpCommon.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/esp/smc/SMCLib/TpCommon.cpp b/esp/smc/SMCLib/TpCommon.cpp index b1fe756c302..16d29983848 100644 --- a/esp/smc/SMCLib/TpCommon.cpp +++ b/esp/smc/SMCLib/TpCommon.cpp @@ -187,9 +187,6 @@ extern TPWRAPPER_API bool validateDropZonePath(const char* dropZoneName, const c static SecAccessFlags getDropZoneScopePermissions(IEspContext& context, const IPropertyTree* dropZone, const char* dropZonePath) { - if (isEmptyString(dropZonePath)) - throw makeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "getDropZoneScopePermissions(): DropZone path must be specified."); - //If the dropZonePath is an absolute path, change it to a relative path. StringBuffer s; const char* prefix = dropZone->queryProp("@prefix"); @@ -210,9 +207,6 @@ static SecAccessFlags getDropZoneScopePermissions(IEspContext& context, const IP extern TPWRAPPER_API SecAccessFlags getDZPathScopePermissions(IEspContext& context, const char* dropZoneName, const char* dropZonePath, const char* dropZoneHost) { - if (isEmptyString(dropZonePath)) - throw makeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "getDZPathScopePermissions(): DropZone path must be specified."); - Owned dropZone; if (isEmptyString(dropZoneName)) { From 9856c064bec81737713dddba8e8d6fcec72f2868 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 13 Oct 2023 10:11:01 +0100 Subject: [PATCH 3/7] HPCC-30515 Update the version of kubectl installed in the image Signed-off-by: Gavin Halliday --- dockerfiles/platform-build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/platform-build/Dockerfile b/dockerfiles/platform-build/Dockerfile index 4ea4826496d..56de4e60405 100644 --- a/dockerfiles/platform-build/Dockerfile +++ b/dockerfiles/platform-build/Dockerfile @@ -44,7 +44,7 @@ RUN groupadd -g 10001 hpcc RUN useradd -s /bin/bash -r -m -N -c "hpcc runtime User" -u 10000 -g hpcc hpcc RUN passwd -l hpcc -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.18/bin/linux/amd64/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/local/bin +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/local/bin WORKDIR /hpcc-dev RUN mkdir build From 86d2be80add275d8d95bcadde486e323da9269dd Mon Sep 17 00:00:00 2001 From: wangkx Date: Thu, 12 Oct 2023 13:32:32 -0400 Subject: [PATCH 4/7] HPCC-30469 Avoid duplicated PathSepChar in getExternalPath() Also add code to change the path to a relative path if needed. Signed-off-by: wangkx --- dali/base/dautils.cpp | 11 ++++++++++- esp/services/ws_fileio/ws_fileioservice.cpp | 1 + esp/smc/SMCLib/TpCommon.cpp | 10 ++++++++++ system/jlib/jfile.cpp | 18 ++++++++++++++++++ system/jlib/jfile.hpp | 1 + 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp index 3a2328c962a..40816a5d1ed 100644 --- a/dali/base/dautils.cpp +++ b/dali/base/dautils.cpp @@ -1479,7 +1479,16 @@ bool CDfsLogicalFileName::getExternalPath(StringBuffer &dir, StringBuffer &tail, *e = makeStringExceptionV(-1, "plane:: does not support planes with more than one device '%s'", planeName.str()); return false; } - dir.append(plane->queryPrefix()); + const char * prefix = plane->queryPrefix(); + //If the prefix is a PathSepChar, it should not be appended to the dir here because + //a PathSepChar will be appended to the dir inside the expandExternalPath() if the s + //is started with the "::". + //Also a trailing pathsepchar in the prefix should be removed. + if (!isRootDirectory(prefix)) + { + dir.append(prefix); + removeTrailingPathSepChar(dir); + } } } } diff --git a/esp/services/ws_fileio/ws_fileioservice.cpp b/esp/services/ws_fileio/ws_fileioservice.cpp index f99b4df3f67..304adaa908d 100644 --- a/esp/services/ws_fileio/ws_fileioservice.cpp +++ b/esp/services/ws_fileio/ws_fileioservice.cpp @@ -97,6 +97,7 @@ bool CWsFileIOEx::onReadFileData(IEspContext &context, IEspReadFileDataRequest & return true; } + //Despite the "DestRelativePath" saying it's relative for legacy reason, it also supports absolute paths. const char* destRelativePath = req.getDestRelativePath(); if (!destRelativePath || (destRelativePath[0] == 0)) { diff --git a/esp/smc/SMCLib/TpCommon.cpp b/esp/smc/SMCLib/TpCommon.cpp index b1fe756c302..9d470bd7109 100644 --- a/esp/smc/SMCLib/TpCommon.cpp +++ b/esp/smc/SMCLib/TpCommon.cpp @@ -256,6 +256,16 @@ extern TPWRAPPER_API void validateDropZoneAccess(IEspContext& context, const cha if (!isHostInPlane(dropZone, hostReq, true)) throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Host %s is not valid DropZone plane %s", hostReq, targetDZNameOrHost); } + + //If the dropZonePath is an absolute path, change it to a relative path. + if (isAbsolutePath(fileNameWithRelPath)) + { + const char* relativePath = getRelativePath(fileNameWithRelPath, dropZone->queryProp("@prefix")); + if (nullptr == relativePath) + throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid DropZone path %s.", fileNameWithRelPath); + fileNameWithRelPath = relativePath; + } + const char *dropZoneName = dropZone->queryProp("@name"); SecAccessFlags permission = getDZFileScopePermissions(context, dropZoneName, fileNameWithRelPath, hostReq); if (permission < permissionReq) diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index ceac0fb3391..f56e64c1de3 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -5642,6 +5642,24 @@ const char *splitRelativePath(const char *full,const char *basedir,StringBuffer return t; } +const char *getRelativePath(const char *path,const char *leadingPath) +{ + size_t pathLen = strlen(path); + size_t leadingLen = strlen(leadingPath); + if ((pathLen==leadingLen-1)&&isPathSepChar(leadingPath[leadingLen-1])) + --leadingLen; + if (0 == strncmp(path,leadingPath,leadingLen)) + { + const char *rel = path + leadingLen; + if ('\0' == *rel) + return rel; + if (isPathSepChar(*rel)) + return rel+1; + return rel; + } + return nullptr; +} + const char *splitDirMultiTail(const char *multipath,StringBuffer &dir,StringBuffer &tail) { // the first directory is the significant one diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index f391982c4ce..f9e0cae2863 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -605,6 +605,7 @@ extern jlib_decl StringBuffer &makeAbsolutePath(StringBuffer &relpath,bool mustE extern jlib_decl StringBuffer &makeAbsolutePath(const char *relpath, const char *basedir, StringBuffer &out); extern jlib_decl StringBuffer &makePathUniversal(const char *path, StringBuffer &out); extern jlib_decl const char *splitRelativePath(const char *full,const char *basedir,StringBuffer &reldir); // removes basedir if matches, returns tail and relative dir +extern jlib_decl const char *getRelativePath(const char *path, const char *leadingPath); extern jlib_decl const char *splitDirMultiTail(const char *multipath,StringBuffer &dir,StringBuffer &tail); extern jlib_decl StringBuffer &mergeDirMultiTail(const char *dir,const char *tail, StringBuffer &multipath); extern jlib_decl StringBuffer &removeRelativeMultiPath(const char *full,const char *dir,StringBuffer &reltail); // removes dir if matches, returns relative multipath From 255ec2fe4c86a3ac95d4a234f64b7fd97502439f Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 13 Oct 2023 18:01:01 +0100 Subject: [PATCH 5/7] HPCC-30515 Update the version of kubectl installed in the image Signed-off-by: Gavin Halliday --- dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile index 74eea1a1bdf..ff20fac4848 100644 --- a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile +++ b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile @@ -59,7 +59,7 @@ RUN apt-get install -y \ gdb \ nano -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.18/bin/linux/amd64/kubectl && \ +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin From 795fb26b1262dd7814a992807b1d8350eb9457b6 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 13 Oct 2023 18:02:24 +0100 Subject: [PATCH 6/7] HPCC-30515 Update the version of kubectl installed in the image Signed-off-by: Gavin Halliday --- dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile index baf55e7edd8..7f55eb63a7d 100644 --- a/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile +++ b/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile @@ -59,7 +59,7 @@ RUN apt-get install -y \ gdb \ nano -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.18/bin/linux/amd64/kubectl && \ +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin From bd401f53249256d32986c592f0f1165d3100d255 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 12 Oct 2023 18:11:03 +0100 Subject: [PATCH 7/7] HPCC-30505 Ensure locations of missing files are traced in standard logging Signed-off-by: Gavin Halliday --- roxie/ccd/ccdfile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp index e0f861d451c..fbc27e32724 100644 --- a/roxie/ccd/ccdfile.cpp +++ b/roxie/ccd/ccdfile.cpp @@ -1180,7 +1180,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress throw MakeStringException(ROXIE_FILE_ERROR, "Local file %s does not match DFS information", localLocation); else { - if (doTrace(TraceFlags::Always, TraceFlags::Detailed)) + if (doTrace(TraceFlags::Always, TraceFlags::Standard)) { #ifndef _CONTAINERIZED DBGLOG("Failed to open file at any of the following %d local locations:", localLocations.length());