From 5dc7b2305ede4c1796e49be0315b7ef591052506 Mon Sep 17 00:00:00 2001 From: Terrence Asselin Date: Mon, 11 Nov 2024 12:46:13 -0600 Subject: [PATCH 01/13] HPCC-32963 Fix ecl unused-files for bare-metal roxie with TLS and custom port In non-containerized deployments, fix the `ecl roxie unused-files` command to work when roxies are configured for TLS (SSL) and/or they're using a nonstandard port other than 9876. Signed-off-by: Terrence Asselin --- esp/services/ws_dfu/ws_dfuXRefService.cpp | 17 +++++++++++++---- esp/services/ws_dfu/ws_dfuXRefService.hpp | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/esp/services/ws_dfu/ws_dfuXRefService.cpp b/esp/services/ws_dfu/ws_dfuXRefService.cpp index 0bd88094f0f..759df1a7144 100644 --- a/esp/services/ws_dfu/ws_dfuXRefService.cpp +++ b/esp/services/ws_dfu/ws_dfuXRefService.cpp @@ -117,6 +117,10 @@ void CWsDfuXRefEx::init(IPropertyTree *cfg, const char *process, const char *ser throw MakeStringException(-1, "No Dali Connection Active. Please Specify a Dali to connect to in you configuration file"); } +#ifndef _CONTAINERIZED + initBareMetalRoxieTargets(roxieConnMap); +#endif + XRefNodeManager.setown(CreateXRefNodeFactory()); //Start out builder thread...... @@ -750,19 +754,24 @@ void CWsDfuXRefEx::findUnusedFilesWithDetailsInDFS(IEspContext &context, const c void CWsDfuXRefEx::getRoxieFiles(const char *process, bool checkPackageMaps, MapStringTo &usedFileMap) { SocketEndpointArray servers; + Owned controlXrefInfo; #ifdef _CONTAINERIZED StringBuffer epStr; getService(epStr, process, true); SocketEndpoint ep(epStr); servers.append(ep); -#else - getRoxieProcessServers(process, servers); if (!servers.length()) throw MakeStringExceptionDirect(ECLWATCH_INVALID_CLUSTER_INFO, "process cluster, not found."); + Owned sock = ISocket::connect_timeout(servers.item(0), ROXIECONNECTIONTIMEOUT); + controlXrefInfo.setown(sendRoxieControlQuery(sock, "", ROXIECONTROLXREFTIMEOUT)); +#else + ISmartSocketFactory *conn = roxieConnMap.getValue(process); + if (!conn) + throw makeStringExceptionV(ECLWATCH_CANNOT_GET_ENV_INFO, "Connection info for '%s' process cluster not found.", process ? process : "(null)"); + + controlXrefInfo.setown(sendRoxieControlQuery(conn, "", ROXIECONTROLXREFTIMEOUT, ROXIECONNECTIONTIMEOUT)); #endif - Owned sock = ISocket::connect_timeout(servers.item(0), ROXIECONNECTIONTIMEOUT); - Owned controlXrefInfo = sendRoxieControlQuery(sock, "", ROXIECONTROLXREFTIMEOUT); if (!controlXrefInfo) throw MakeStringExceptionDirect(ECLWATCH_INTERNAL_ERROR, "roxie cluster, not responding."); Owned roxieFiles = controlXrefInfo->getElements("//File"); diff --git a/esp/services/ws_dfu/ws_dfuXRefService.hpp b/esp/services/ws_dfu/ws_dfuXRefService.hpp index 9a17c4664a5..ded9f450a97 100644 --- a/esp/services/ws_dfu/ws_dfuXRefService.hpp +++ b/esp/services/ws_dfu/ws_dfuXRefService.hpp @@ -187,6 +187,7 @@ class CWsDfuXRefEx : public CWsDFUXRef { Owned XRefNodeManager; Owned m_XRefbuilder; + MapStringToMyClass roxieConnMap; IXRefFilesNode* getFileNodeInterface(IXRefNode& XRefNode,const char* nodeType); void addXRefNode(const char* name, IPropertyTree* pXRefNodeTree); From 3e17a7aee930713604ac3207201b42f3d3442f59 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 22 Nov 2024 13:59:02 +0000 Subject: [PATCH 02/13] Split off 9.6.68 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 9e12932c99e..a67980dcab3 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.6.67-closedown0 +version: 9.6.69-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.6.67-closedown0 +appVersion: 9.6.69-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index cb069da81dd..4c3f180aa31 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.6.67-closedown0 +helmVersion: 9.6.69-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 91299005514..7457bc8385b 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 6 ) -set ( HPCC_POINT 67 ) +set ( HPCC_POINT 69 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-11-14T17:31:36Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-11-22T13:59:02Z" ) ### From 403568f30ceeba4dd010f3039e2b6034a3d268da Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 22 Nov 2024 14:00:27 +0000 Subject: [PATCH 03/13] Split off 9.4.116 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index d8a791fd4f8..2737b61aa83 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.115-closedown0 +version: 9.4.117-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.115-closedown0 +appVersion: 9.4.117-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 8f760438540..7db1dc26aa4 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1473,7 +1473,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.4.115-closedown0 +helmVersion: 9.4.117-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 0b286b20c44..3b7bb7ca295 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 115 ) +set ( HPCC_POINT 117 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-11-14T17:32:53Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-11-22T14:00:27Z" ) ### From 1abb2c3a27e1059f1787a33558cc27f76800885f Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 22 Nov 2024 14:01:36 +0000 Subject: [PATCH 04/13] Split off 9.2.142 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index a656fa19654..32d69f0d4e5 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.141-closedown0 +version: 9.2.143-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.141-closedown0 +appVersion: 9.2.143-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 32c061ccea7..d3d321d0f19 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.141-closedown0 +helmVersion: 9.2.143-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 5925876533a..c4b5abd9894 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 141 ) +set ( HPCC_POINT 143 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-11-14T17:34:04Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-11-22T14:01:36Z" ) ### From b343c10f22e55048d5805c76aabe5926f06f6a22 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Mon, 25 Nov 2024 16:21:54 +0000 Subject: [PATCH 05/13] HPCC-33028 Win + CMake > v3.28.3 issue REGEX include / exclude getting ignored when declared _after_ the DESTINATION Signed-off-by: Gordon Smith --- cmake_modules/commonSetup.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake_modules/commonSetup.cmake b/cmake_modules/commonSetup.cmake index 1ccf2ee5607..7f3f1304d06 100644 --- a/cmake_modules/commonSetup.cmake +++ b/cmake_modules/commonSetup.cmake @@ -1086,16 +1086,16 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "") list(INSERT ARGS 3 ${ARGV1}_deps) if (WIN32) install(RUNTIME_DEPENDENCY_SET ${ARGV1}_deps - DESTINATION ${EXEC_DIR} PRE_EXCLUDE_REGEXES "api-ms-win-.*\.dll" POST_INCLUDE_REGEXES "^${VCPKG_FILES_DIR}.*" POST_EXCLUDE_REGEXES ".*" + DESTINATION ${EXEC_DIR} ) else() install(RUNTIME_DEPENDENCY_SET ${ARGV1}_deps - DESTINATION ${LIB_DIR} POST_INCLUDE_REGEXES "^${VCPKG_FILES_DIR}\/vcpkg_installed\/.*" POST_EXCLUDE_REGEXES ".*" + DESTINATION ${LIB_DIR} ) endif() endif() From b34fc035e751446bf1ba95a14b59cdb717b07d4a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 25 Nov 2024 17:50:51 +0000 Subject: [PATCH 06/13] HPCC-33029 Change index blocked time to mean the total time to resolve a node Signed-off-by: Gavin Halliday --- system/jhtree/jhtree.cpp | 53 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index 8148c37477c..2dc8856661b 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -81,7 +81,6 @@ std::atomic leafSearchCycles{0}; static cycle_t traceCacheLockingFrequency{0}; static cycle_t traceNodeLoadFrequency{0}; -static cycle_t traceCacheLockingThreshold{0}; static cycle_t traceNodeLoadThreshold{0}; MODULE_INIT(INIT_PRIORITY_JHTREE_JHTREE) @@ -90,7 +89,6 @@ MODULE_INIT(INIT_PRIORITY_JHTREE_JHTREE) fetchThresholdCycles = nanosec_to_cycle(defaultFetchThresholdNs); traceCacheLockingFrequency = millisec_to_cycle(60000); // Report locking delays at most once per minute traceNodeLoadFrequency = millisec_to_cycle(60000); // Report slow loads at most once per minute - traceCacheLockingThreshold = millisec_to_cycle(50); // Report locks that take > 50ms traceNodeLoadThreshold = millisec_to_cycle(20); // Report node loads that take > 5ms return 1; } @@ -2610,8 +2608,6 @@ void setIndexWarningThresholds(IPropertyTree * options) traceCacheLockingFrequency = nanosec_to_cycle(options->getPropInt64("@traceCacheLockingFrequencyNs")); if (options->hasProp("@traceNodeLoadFrequencyNs")) traceNodeLoadFrequency = nanosec_to_cycle(options->getPropInt64("@traceNodeLoadFrequencyNs")); - if (options->hasProp("@traceCacheLockingThresholdNs")) - traceCacheLockingThreshold = nanosec_to_cycle(options->getPropInt64("@traceCacheLockingThresholdNs")); if (options->hasProp("@traceNodeLoadThresholdNs")) traceNodeLoadThreshold = nanosec_to_cycle(options->getPropInt64("@traceNodeLoadThresholdNs")); } @@ -2640,6 +2636,37 @@ void CNodeCache::getCacheInfo(ICacheInfoRecorder &cacheInfo) static std::atomic lastLockingReportCycles{0}; static std::atomic lastLoadReportCycles{0}; +//Keep track of the total time that is taken to resolve a node from the cache - including loading it from disk. +class NodeCacheLookupTimer +{ +public: + inline NodeCacheLookupTimer(IContextLogger * _ctx) : ctx(_ctx) + { + if (ctx) + startCycles = get_cycles_now(); + } + inline ~NodeCacheLookupTimer() + { + if (ctx) + ctx->noteStatistic(StCycleIndexCacheBlockedCycles, get_cycles_now() - startCycles); + } + + //Call this function to avoid the destructor having to call get_cycles_now() + void noteComplete(cycle_t now) + { + if (ctx) + { + ctx->noteStatistic(StCycleIndexCacheBlockedCycles, now - startCycles); + // prevent double recording the time - compiler will likely optimize away the destructor + ctx = nullptr; + } + } + +private: + IContextLogger * ctx; + cycle_t startCycles = 0; +}; + const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsigned iD, offset_t pos, NodeType type, IContextLogger *ctx, bool isTLK) { // MORE - could probably be improved - I think having the cache template separate is not helping us here @@ -2647,6 +2674,9 @@ const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsign if (!pos) return NULL; + //Time how long it takes to resolve an item in the node cache, and record it as the blocked time. + NodeCacheLookupTimer lookupTimer(ctx); + // No benefit in caching the following, especially since they will evict useful pages if ((type == NodeMeta) || (type == NodeBloom)) return keyIndex->loadNode(nullptr, pos, nullptr); @@ -2718,12 +2748,13 @@ const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsign if (ctx) ctx->noteStatistic(addStatId[cacheType], 1); } + cycle_t startCycles = get_cycles_now(); + lookupTimer.noteComplete(startCycles); + //The common case is that this flag has already been set (by a previous add). if (likely(ownedCacheEntry->isReady())) return ownedCacheEntry->getNode(); - //Shame that the hash code is recalculated - it might be possible to remove this. - cycle_t startCycles = get_cycles_now(); cycle_t fetchCycles = 0; cycle_t startLoadCycles; @@ -2755,15 +2786,6 @@ const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsign } } cycle_t endLoadCycles = get_cycles_now(); - cycle_t lockingCycles = startLoadCycles - startCycles; - if (lockingCycles > traceCacheLockingThreshold) - { - if ((endLoadCycles - lastLockingReportCycles) >= traceCacheLockingFrequency) - { - lastLockingReportCycles = endLoadCycles; - WARNLOG("CNodeCache::getNode lock(%s) took %lluns", cacheTypeText[cacheType], cycle_to_nanosec(lockingCycles)); - } - } cycle_t actualLoadCycles = endLoadCycles - startLoadCycles; if (actualLoadCycles > traceNodeLoadThreshold) { @@ -2779,7 +2801,6 @@ const CJHTreeNode *CNodeCache::getCachedNode(const INodeLoader *keyIndex, unsign { ctx->noteStatistic(loadStatId[cacheType], endLoadCycles - startCycles); ctx->noteStatistic(readStatId[cacheType], fetchCycles); - ctx->noteStatistic(StCycleIndexCacheBlockedCycles, lockingCycles); if (fetchCycles >= fetchThresholdCycles) { ctx->noteStatistic(fetchStatId[cacheType], 1); From c85f7b0766f15bb514dd3a4596fe86c1165d4f34 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Tue, 26 Nov 2024 16:35:11 +0000 Subject: [PATCH 07/13] HPCC-33036 Bump git-lfs to v3.6.0 Signed-off-by: Gordon Smith --- dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile | 2 +- dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile index c8d29b5210a..76d0b1db8ea 100644 --- a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile +++ b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile @@ -62,7 +62,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/b chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin -RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.5.1_amd64.deb/download && \ +RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.0_amd64.deb/download && \ dpkg -i download && \ rm download diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile index 8166413434b..c7bbe050514 100644 --- a/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile +++ b/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile @@ -62,7 +62,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/b chmod +x ./kubectl && \ mv ./kubectl /usr/local/bin -RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.5.1_amd64.deb/download && \ +RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.0_amd64.deb/download && \ dpkg -i download && \ rm download From 8e7d4515edb605c1862d2b825bc94e93a30dfcd4 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 28 Nov 2024 10:23:32 +0000 Subject: [PATCH 08/13] HPCC-33043 Ensure blob storage files are completely written before being read Signed-off-by: Gavin Halliday --- dali/base/dadfs.cpp | 42 ++++++++++++++++++++++++++++++++++++ helm/hpcc/values.schema.json | 5 +++++ system/jlib/jfile.cpp | 9 +++++++- system/jlib/jfile.hpp | 3 +++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 7b008f187b4..005767ac263 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -3779,6 +3779,47 @@ protected: friend class CDistributedFilePart; clusters.kill(); } + //Ensure that enough time has passed from when the file was last modified for reads to be consistent + //Important for blob storage or remote, geographically synchronized storage + void checkWriteSync() + { + time_t modifiedTime = 0; + time_t now = 0; + + Owned iter = root->getElements("Cluster"); + ForEach(*iter) + { + const char * name = iter->query().queryProp("@name"); + unsigned marginMs = getWriteSyncMarginMs(name); + if (marginMs) + { + if (0 == modifiedTime) + { + CDateTime modified; + if (!getModificationTime(modified)) + return; + modifiedTime = modified.getSimple(); + } + + if (0 == now) + now = time(&now); + + //Round the elapsed time down - so that a change on the last ms of one time period does not count as a whole second of elapsed time + //This could be avoided if the modified time was more granular + unsigned __int64 elapsedMs = (now - modifiedTime) * 1000; + if (elapsedMs >= 1000) + elapsedMs -= 999; + + if (unlikely(elapsedMs < marginMs)) + { + LOG(MCuserProgress, "Delaying access to %s on %s for %ums to ensure write sync", queryLogicalName(), name, (unsigned)(marginMs - elapsedMs)); + MilliSleep(marginMs - elapsedMs); + now = 0; // re-evaluate now - unlikely to actually happen + } + } + } + } + bool hasDirPerPart() const { return FileDescriptorFlags::none != (fileFlags & FileDescriptorFlags::dirperpart); @@ -8299,6 +8340,7 @@ IDistributedFile *CDistributedFileDirectory::dolookup(CDfsLogicalFileName &_logi } CDistributedFile *ret = new CDistributedFile(this,fcl.detach(),*logicalname,accessMode,user); // found ret->setSuperOwnerLock(superOwnerLock.detach()); + ret->checkWriteSync(); return ret; } // now super file diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 312019680bc..1d853f13619 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -603,6 +603,11 @@ "eclwatchVisible": { "type": "boolean" }, + "writeSyncMarginMs": { + "description": "Time that is required to elapse between writing a file and all read copies to be consistently updated", + "type": "integer", + "default": 0 + }, "components": {}, "prefix": {}, "subPath": {}, diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index b5a330dd775..8afe9594806 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7911,7 +7911,8 @@ static const std::array planeAttributeI { PlaneAttrType::integer, 1024, false, "blockedFileIOKB" }, // enum PlaneAttributeType::BlockedSequentialIO {0} { PlaneAttrType::integer, 1024, false, "blockedRandomIOKB" }, // enum PlaneAttributeType::blockedRandomIOKB {1} { PlaneAttrType::boolean, 0, true, "fileSyncWriteClose" }, // enum PlaneAttributeType::fileSyncWriteClose {2} - { PlaneAttrType::boolean, 0, true, "concurrentWriteSupport" } // enum PlaneAttributeType::concurrentWriteSupport {3} + { PlaneAttrType::boolean, 0, true, "concurrentWriteSupport" },// enum PlaneAttributeType::concurrentWriteSupport {3} + { PlaneAttrType::integer, 1, false, "writeSyncMarginMs" }, // enum PlaneAttributeType::WriteSyncMarginMs {4} }}; // {prefix, {key1: value1, key2: value2, ...}} @@ -8089,6 +8090,12 @@ bool getFileSyncWriteCloseEnabled(const char *planeName) return 0 != getPlaneAttributeValue(planeName, FileSyncWriteClose, defaultFileSyncWriteCloseEnabled ? 1 : 0); } +unsigned getWriteSyncMarginMs(const char * planeName) +{ + constexpr unsigned dft = 0; + return (unsigned)getPlaneAttributeValue(planeName, WriteSyncMarginMs, dft); +} + static constexpr bool defaultConcurrentWriteSupport = isContainerized() ? false : true; bool getConcurrentWriteSupported(const char *planeName) { diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index 543fb355e79..53080ff5f35 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -744,12 +744,14 @@ extern jlib_decl IPropertyTreeIterator * getRemoteStoragesIterator(); extern jlib_decl IPropertyTreeIterator * getPlanesIterator(const char * category, const char *name); extern jlib_decl IFileIO *createBlockedIO(IFileIO *base, size32_t blockSize); +//MORE: Should use enum class to avoid potential symbol clashes enum PlaneAttributeType // remember to update planeAttributeInfo in jfile.cpp { BlockedSequentialIO, BlockedRandomIO, FileSyncWriteClose, ConcurrentWriteSupport, + WriteSyncMarginMs, PlaneAttributeCount }; extern jlib_decl const char *getPlaneAttributeString(PlaneAttributeType attr); @@ -761,6 +763,7 @@ extern jlib_decl size32_t getBlockedFileIOSize(const char *planeName, size32_t d extern jlib_decl size32_t getBlockedRandomIOSize(const char *planeName, size32_t defaultSize=0); extern jlib_decl bool getFileSyncWriteCloseEnabled(const char *planeName); extern jlib_decl bool getConcurrentWriteSupported(const char *planeName); +extern jlib_decl unsigned getWriteSyncMarginMs(const char * planeName); //---- Pluggable file type related functions ---------------------------------------------- From 6e37c54423e54f33a95cb8d46eadae75b1161329 Mon Sep 17 00:00:00 2001 From: Attila Vamos Date: Thu, 28 Nov 2024 16:47:22 +0000 Subject: [PATCH 09/13] HPCC-33049 Fix artifact download version Bundles test GH Actions Changes: .github/workflows/bundleTest-thor.md: .github/workflows/bundleTest-thor.yml: Change 'Download Package' 'uses: actions/download-artifact@v3' to 'v4' Signed-off-by: Attila Vamos --- .github/workflows/bundleTest-thor.md | 7 ++++--- .github/workflows/bundleTest-thor.yml | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bundleTest-thor.md b/.github/workflows/bundleTest-thor.md index 8d82b12660e..57768175ed3 100644 --- a/.github/workflows/bundleTest-thor.md +++ b/.github/workflows/bundleTest-thor.md @@ -102,7 +102,7 @@ The steps in the workflow run on the specified operating system, with Ubuntu-22. This step enables us to download the ready-to-install HPCC Platform's artifact built on the latest commit. ```yaml - name: Download Package - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.asset-name }} path: ${{ inputs.asset-name }} @@ -348,11 +348,12 @@ If any logs, ZAP reports, or .trace files are generated, they are uploaded as ar ```yaml - name: ml-thor-test-logs-artifact if: ${{ failure() || cancelled() || env.uploadArtifact == 'true' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ inputs.asset-name }}-bundle-test-logs path: | /home/runner/HPCCSystems-regression/log/* /home/runner/HPCCSystems-regression/zap/* if-no-files-found: ignore - ``` \ No newline at end of file + ``` + diff --git a/.github/workflows/bundleTest-thor.yml b/.github/workflows/bundleTest-thor.yml index f2ac0edad56..5da5906f8b6 100644 --- a/.github/workflows/bundleTest-thor.yml +++ b/.github/workflows/bundleTest-thor.yml @@ -50,7 +50,7 @@ jobs: sudo rm -rf /usr/local/lib/android - name: Download Package - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.asset-name }} path: ${{ inputs.asset-name }} @@ -268,7 +268,7 @@ jobs: - name: ml-thor-test-logs-artifact if: ${{ failure() || cancelled() || env.uploadArtifact == 'true' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ inputs.asset-name }}-bundle-test-logs path: | From b2dcb97acf01ac4a8953ccef4a993d32e343bbee Mon Sep 17 00:00:00 2001 From: Attila Vamos Date: Thu, 28 Nov 2024 17:09:57 +0000 Subject: [PATCH 10/13] HPCC-33050 Fix artifact download and Chrome and ChromeDriver version mismatch of ECL Watch UI test GH Action .github/workflows/test-ui-gh_runner.yml: Change 'Download Support Files' 'uses: actions/download-artifact@v3' to 'v4' Signed-off-by: Attila Vamos --- .github/workflows/test-ui-gh_runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-ui-gh_runner.yml b/.github/workflows/test-ui-gh_runner.yml index 57c86db615f..1343d528fd1 100644 --- a/.github/workflows/test-ui-gh_runner.yml +++ b/.github/workflows/test-ui-gh_runner.yml @@ -78,7 +78,7 @@ jobs: path: ${{ inputs.asset-name }} - name: Download Support Files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.asset-name }}-support-files path: ${{ inputs.asset-name }}-support-files From 93b534acbe484c1a9aa42877ebdff0db1d6a671a Mon Sep 17 00:00:00 2001 From: Michael Gardner Date: Mon, 2 Dec 2024 12:51:46 -0500 Subject: [PATCH 11/13] HPCC-33006 Include 20.04 k8 build logic community/internal Signed-off-by: Michael Gardner --- .github/workflows/build-assets.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 76a50e78684..576d9e8282f 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -91,15 +91,24 @@ jobs: - os: ubuntu-22.04 name: k8s container: true + build-docker-image: true - os: ubuntu-22.04 name: docs documentation: true - os: ubuntu-20.04 + - os: ubuntu-20.04 + name: k8s + container: true - os: rockylinux-8 - os: ubuntu-22.04 name: LN k8s ln: true container: true + build-docker-image: true + - os: ubuntu-20.04 + name: LN k8s + ln: true + container: true - os: ubuntu-22.04 name: LN ln: true @@ -272,7 +281,7 @@ jobs: ${{ needs.preamble.outputs.folder_build }}/*.md5sum - name: Locate k8s deb file (community) - if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation && matrix.build-docker-image }} id: container run: | k8s_pkg_path=$(ls -t ${{ needs.preamble.outputs.folder_build }}/*64_k8s.deb 2>/dev/null | head -1) @@ -282,7 +291,7 @@ jobs: - name: Create Docker Image (community) uses: docker/build-push-action@v5 - if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation }} + if: ${{ !matrix.ln && !matrix.ee && matrix.container && !matrix.documentation && matrix.build-docker-image }} with: builder: ${{ steps.buildx.outputs.name }} file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/platform-core-${{ matrix.os }}/Dockerfile @@ -345,7 +354,7 @@ jobs: cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - name: Upload Assets to Jfrog (debian internal) - if: ${{ matrix.ln && !matrix.container && contains(matrix.os, 'ubuntu') && github.repository_owner == 'hpcc-systems'}} + if: ${{ matrix.ln && contains(matrix.os, 'ubuntu') && github.repository_owner == 'hpcc-systems'}} shell: bash run: | cd ${{ needs.preamble.outputs.folder_build }} @@ -368,7 +377,7 @@ jobs: done - name: Locate k8s deb file (internal) - if: ${{ matrix.ln && matrix.container && !matrix.documentation }} + if: ${{ matrix.ln && matrix.container && !matrix.documentation && matrix.build-docker-image }} id: ln-container run: | k8s_pkg_path=$(ls -t ${{ needs.preamble.outputs.folder_build }}/*64_k8s.deb 2>/dev/null | head -1) @@ -378,7 +387,7 @@ jobs: - name: Create Docker Image (internal) uses: docker/build-push-action@v5 - if: ${{ matrix.ln && matrix.container && !matrix.documentation }} + if: ${{ matrix.ln && matrix.container && !matrix.documentation && matrix.build-docker-image }} with: builder: ${{ steps.buildx.outputs.name }} file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/platform-core-${{ matrix.os }}/Dockerfile @@ -393,7 +402,7 @@ jobs: type=registry,ref=hpccsystems/platform-core-${{ matrix.os }}:${{ needs.preamble.outputs.candidate_base_branch }} - name: JFrog Docker Push and Publish - if: ${{ matrix.ln && matrix.container && !matrix.documentation && github.repository_owner == 'hpcc-systems' }} + if: ${{ matrix.ln && matrix.container && !matrix.documentation && matrix.build-docker-image && github.repository_owner == 'hpcc-systems' }} run: | jf docker push ${{ secrets.JFROG_REGISTRY || 'dummy.io' }}/hpccpl-docker-local/platform-core-ln:${{ needs.preamble.outputs.hpcc_version }} --build-name=platform-core-ln --build-number=${{ needs.preamble.outputs.hpcc_version }} --project=hpccpl jf rt bp platform-core-ln ${{ needs.preamble.outputs.hpcc_version }} --project=hpccpl From 116c785330464c378ca1226e6b198aa216ad5f45 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 3 Dec 2024 12:07:54 +0000 Subject: [PATCH 12/13] HPCC-33064 Check files size consistency before reading Signed-off-by: Gavin Halliday --- dali/base/dafdesc.cpp | 72 +++++++++++++++++++ dali/base/dafdesc.hpp | 3 + ecl/hthor/hthor.cpp | 11 +++ roxie/ccd/ccdfile.cpp | 2 + .../activities/diskread/thdiskreadslave.cpp | 8 +++ 5 files changed, 96 insertions(+) diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index b03967a19c2..da66231570a 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -665,6 +665,10 @@ class CFileDescriptorBase: public CInterface virtual IFileDescriptor &querySelf() = 0; virtual unsigned copyClusterNum(unsigned partidx, unsigned copy,unsigned *replicate=NULL) = 0; + IPropertyTree & queryAttributes() const + { + return *attr; + } }; class CPartDescriptor : implements IPartDescriptor @@ -819,6 +823,74 @@ protected: friend class CFileDescriptor; return ismulti; } + IPropertyTree & queryAttributes() const + { + return *props; + } + + offset_t getFileSize(bool allowphysical,bool forcephysical) + { + offset_t ret = (offset_t)((forcephysical&&allowphysical)?-1:queryAttributes().getPropInt64("@size", -1)); + if (allowphysical&&(ret==(offset_t)-1)) + ret = getSize(true); + return ret; + } + + offset_t getDiskSize(bool allowphysical,bool forcephysical) + { + if (!::isCompressed(parent.queryAttributes())) + return getFileSize(allowphysical, forcephysical); + + if (forcephysical && allowphysical) + return getSize(false); // i.e. only if force, because all compressed should have @compressedSize attribute + + // NB: compressSize is disk size + return queryAttributes().getPropInt64("@compressedSize", -1); + } + + offset_t getSize(bool checkCompressed) + { + offset_t ret = (offset_t)-1; + StringBuffer firstname; + bool compressed = ::isCompressed(parent.queryAttributes()); + unsigned nc=parent.numCopies(partIndex); + for (unsigned copy=0;copy partfile = createIFile(getFilename(copy, rfn)); + if (checkCompressed && compressed) + { + Owned partFileIO = partfile->open(IFOread); + if (partFileIO) + { + Owned compressedIO = createCompressedFileReader(partFileIO); + if (compressedIO) + ret = compressedIO->size(); + else + throw makeStringExceptionV(DFSERR_PhysicalCompressedPartInvalid, "Compressed part is not in the valid format: %s", partfile->queryFilename()); + } + } + else + ret = partfile->size(); + if (ret!=(offset_t)-1) + return ret; + } + catch (IException *e) + { + StringBuffer s("CDistributedFilePart::getSize "); + rfn.getRemotePath(s); + EXCLOG(e, s.str()); + e->Release(); + } + if (copy==0) + rfn.getRemotePath(firstname); + } + throw makeStringExceptionV(DFSERR_CannotFindPartFileSize, "Cannot find physical file size for %s", firstname.str());; + } + + RemoteMultiFilename &getMultiFilename(unsigned copy, RemoteMultiFilename &rmfn) { if (ismulti) { diff --git a/dali/base/dafdesc.hpp b/dali/base/dafdesc.hpp index 535178828e2..377bfd1ab67 100644 --- a/dali/base/dafdesc.hpp +++ b/dali/base/dafdesc.hpp @@ -177,6 +177,9 @@ interface IPartDescriptor: extends IInterface virtual const char *queryOverrideName() = 0; // for non-standard files virtual unsigned copyClusterNum(unsigned copy,unsigned *replicate=NULL)=0; // map copy number to cluster (and optionally replicate number) virtual IReplicatedFile *getReplicatedFile()=0; + + virtual offset_t getFileSize(bool allowphysical,bool forcephysical)=0; // gets the part filesize (NB this will be the *expanded* size) + virtual offset_t getDiskSize(bool allowphysical,bool forcephysical)=0; // gets the part size on disk (NB this will be the compressed size) }; typedef IArrayOf CPartDescriptorArray; typedef IIteratorOf IPartDescriptorIterator; diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index 610c15507cc..06267a4908d 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -8735,6 +8735,17 @@ bool CHThorDiskReadBaseActivity::openNext() { inputfile.setown(createIFile(rfilename)); + if (curPart) + { + offset_t expectedSize = curPart->getDiskSize(false, false); + if (expectedSize != unknownFileSize) + { + offset_t actualSize = inputfile->size(); + if(actualSize != expectedSize) + throw MakeStringException(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", inputfile->queryFilename(), expectedSize, actualSize); + } + } + if (compressed) { Owned eexp; diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp index 9f627e91bee..e9976005597 100644 --- a/roxie/ccd/ccdfile.cpp +++ b/roxie/ccd/ccdfile.cpp @@ -3909,6 +3909,8 @@ class CcdFileTest : public CppUnit::TestFixture virtual const char *queryOverrideName() { UNIMPLEMENTED; } virtual unsigned copyClusterNum(unsigned copy,unsigned *replicate=NULL) { UNIMPLEMENTED; } virtual IReplicatedFile *getReplicatedFile() { UNIMPLEMENTED; } + virtual offset_t getFileSize(bool allowphysical,bool forcephysical) { UNIMPLEMENTED; } + virtual offset_t getDiskSize(bool allowphysical,bool forcephysical) { UNIMPLEMENTED; } }; void testCopy() diff --git a/thorlcr/activities/diskread/thdiskreadslave.cpp b/thorlcr/activities/diskread/thdiskreadslave.cpp index 59d001238a0..c09718a041d 100644 --- a/thorlcr/activities/diskread/thdiskreadslave.cpp +++ b/thorlcr/activities/diskread/thdiskreadslave.cpp @@ -366,6 +366,14 @@ void CDiskRecordPartHandler::open() rwFlags |= DEFAULT_RWFLAGS; + offset_t expectedSize = partDesc->getDiskSize(false, false); + if (expectedSize != unknownFileSize) + { + offset_t actualSize = iFile->size(); + if(actualSize != expectedSize) + throw MakeStringException(0, "File size mismatch: file %s was supposed to be %" I64F "d bytes but appears to be %" I64F "d bytes", iFile->queryFilename(), expectedSize, actualSize); + } + if (compressed) { rwFlags |= rw_compress; From 60d96e6c56bd0c8b19020c7fac8144c23afb1dfc Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 3 Dec 2024 16:43:16 +0000 Subject: [PATCH 13/13] Split off 9.8.44 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index cb470dd97e6..015b53d1aa0 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.8.43-closedown0 +version: 9.8.45-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.8.43-closedown0 +appVersion: 9.8.45-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e480670bb56..45a985387f1 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1477,7 +1477,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.8.43-closedown0 +helmVersion: 9.8.45-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index dd9d4686b9a..37ba4d0519d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 8 ) -set ( HPCC_POINT 43 ) +set ( HPCC_POINT 45 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-11-22T13:57:35Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-12-03T16:43:16Z" ) ###