diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 6fadaf58ca0..687fc079ce7 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -248,7 +248,7 @@ jobs: sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ - cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \ + cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES=OFF -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \ cmake --build /hpcc-dev/build --parallel $(nproc) --target all" docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf" diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 9dc0d1b708c..2b6fa5281f3 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -1256,7 +1256,7 @@ protected: friend class CDistributedFile; return ret; } virtual bool removePhysicalPartFiles(const char *logicalName, IFileDescriptor *fileDesc, IMultiException *mexcept, unsigned numParallelDeletes=0) override; - virtual void setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout); + virtual void setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout); }; @@ -11635,11 +11635,11 @@ void CDistributedFileDirectory::setFileAccessed(CDfsLogicalFileName &dlfn,IUserD checkDfsReplyException(mb); } -void CDistributedFileDirectory::setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout) +void CDistributedFileDirectory::setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout) { CDfsLogicalFileName dlfn; dlfn.set(logicalName); - setFileAccessed(dlfn, nullptr, dt, foreigndali, foreigndalitimeout); + setFileAccessed(dlfn, udesc, dt, foreigndali, foreigndalitimeout); } void CDistributedFileDirectory::setFileProtect(CDfsLogicalFileName &dlfn,IUserDescriptor *user, const char *owner, bool set, const INode *foreigndali,unsigned foreigndalitimeout) diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp index b18ae460bb0..003fca751c9 100644 --- a/dali/base/dadfs.hpp +++ b/dali/base/dadfs.hpp @@ -731,7 +731,7 @@ interface IDistributedFileDirectory: extends IInterface // useful to clearup after temporary unpublished file. virtual bool removePhysicalPartFiles(const char *logicalName, IFileDescriptor *fileDesc, IMultiException *mexcept, unsigned numParallelDeletes=0) = 0; - virtual void setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali=nullptr, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0; + virtual void setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali=nullptr, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0; }; diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml index d1efc80c613..a6cf61bccfb 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml @@ -37,7 +37,7 @@ similarity to actual persons, living or dead, is purely coincidental. - + Containerized HPCC Overview - Starting with version 8.0, the HPCC - Systems® Platform is focusing on containerized - deployments. This is useful for cloud-based deployments (large or small) - or local testing/development deployments. + Since version 8.0, the HPCC Systems® + Platform started focusing significantly on containerized deployments. This + is useful for cloud-based deployments (large or small) or local + testing/development deployments. Docker containers managed by Kubernetes (K8s) is a new target operating environment, alongside continued support for traditional @@ -81,10 +81,10 @@ to run, but a significant change in how components are configured, how and when they start up, and where they store their data. - This book focuses on containerized deployments. The first section is - about using Docker containers and Helm charts locally. Docker and Helm do - a lot of the work for you. The second part uses the same techniques in the - cloud. + This book focuses on these containerized deployments. The first + section is about using Docker containers and Helm charts locally. Docker + and Helm do a lot of the work for you. The second part uses the same + techniques in the cloud. For local small deployments (for development and testing), we suggest using Docker Desktop and Helm. This is useful for learning, @@ -92,8 +92,7 @@ For Cloud deployments, you can use any flavor of Cloud services, if it supports Docker, Kubernetes, and Helm. This book, however, will focus - on Microsoft Azure for Cloud Services. Future versions may include - specifics for other Cloud providers. + on Microsoft Azure for Cloud Services. If you want to manually manage your local or Cloud deployment, you can still use the traditional installers and Configuration Manager, but @@ -106,28 +105,12 @@ be easy for someone familiar with Kubernetes and Helm to install and manage the HPCC Systems platform. - - - Note: - - - The traditional bare-metal version of the HPCC Systems - platform is mature and has been heavily used in commercial - applications for almost two decades and is fully intended for - production use. The containerized version is new and is not yet 100% - ready for production. In addition, aspects of that version could - change without notice. We encourage you to use it and provide - feedback so we can make this version as robust as a bare-metal - installation. - - - - Bare-metal vs Containers - If you are familiar with the HPCC Systems platform, there are a - few fundamental changes to note. + If you are familiar with the traditional bare-metal HPCC Systems + platform deployments, there are a few fundamental changes to + note. Processes and Pods, not Machines diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml index 482a9ad8ebe..55b62e7c113 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml @@ -10,7 +10,9 @@ File.CreateExternalDirectory CreateExternalDirectory - ( location, path ( location, path [ , planename ] ) @@ -24,7 +26,8 @@ location A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planename is + provided. @@ -33,6 +36,14 @@ A null-terminated string containing the directory path to create. + + + planename + + A null-terminated string containing name of the data plane + containing the file. Optional if location is + provided, but planename is preferred. + @@ -44,8 +55,10 @@ Example: - IP := '10.150.254.6'; -path := '/c$/training/import/NewDir'; -STD.File.CreateExternalDirectory(IP,path); + IMPORT STD; +IP := ''; +path := '/var/lib/HPCCSystems/dropzone/advancedtraining/'; +planename := 'mydropzone'; +STD.File.CreateExternalDirectory(IP,path,planename); diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml index 211aedd6bb2..15194c9d374 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml @@ -10,7 +10,9 @@ File.DeleteExternalFile DeleteExternalFile - ( location, path ( location, path [ , planename ] ) @@ -24,7 +26,8 @@ location A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planename is + provided. @@ -33,6 +36,14 @@ A null-terminated string containing the path/name of the file to remove. + + + planename + + A null-terminated string containing name of the data plane + containing the file. Optional if location is + provided, but planename is preferred. + @@ -44,8 +55,10 @@ Example: - IP := '10.150.254.6'; -infile := '/c$/training/import/AdvancedECL/people'; -STD.File.DeleteExternalFile(IP,infile); + IMPORT STD; +IP := ''; +infile := '/var/lib/HPCCSystems/dropzone/originalperson'; +planename := 'mydropzone'; +STD.File.DeleteExternalFile(IP,infile,planename); diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml index 2983b611365..a812219cee0 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml @@ -10,8 +10,10 @@ File.MoveExternalFile MoveExternalFile - ( location, frompath, topath - ) + ( location, frompath, topath [ , planename ] ) @@ -24,7 +26,8 @@ location A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planename is + provided. @@ -40,6 +43,14 @@ A null-terminated string containing the path/name of the target file. + + + planeName + + A null-terminated string containing name of the data plane + containing the file. Optional if location is + provided, but planename is preferred. + @@ -53,8 +64,10 @@ Example: - IP := '10.150.254.6'; -infile := '/c$/training/import/AdvancedECL/people'; -outfile := '/c$/training/import/DFUtest/people'; -STD.File.MoveExternalFile(IP,infile,outfile); + IMPORT STD; +IP := ''; +infile := '/var/lib/HPCCSystems/dropzone/originalperson'; +outfile := '/var/lib/HPCCSystems/dropzone/originalperson_bak'; +planename := 'mydropzone'; +STD.File.MoveExternalFile(IP,infile,outfile,planename); diff --git a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml index 69e390bd2df..add90d0bf22 100644 --- a/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml +++ b/docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml @@ -4,17 +4,19 @@ RemoteDirectory - STD.File.RemoteDirectory + STD.File.RemoteDirectory( + machineIP, dir [ , + mask ][ , recurse + ][ , planeName ] ) + STD.File.RemoteDirectory File.RemoteDirectory RemoteDirectory - ( machineIP, dir [ , mask ][ , recurse ] ) + @@ -27,7 +29,8 @@ machineIP A null-terminated string containing the IP address of the - remote machine. + remote machine. Optional if planeName is + provided. @@ -54,6 +57,14 @@ omitted, the default is FALSE. + + planeName + + A null-terminated string containing name of the data plane + containing the file. Optional if machineIP is + provided, but planeName is preferred. + + Return: @@ -84,7 +95,10 @@ END; Example: - OUTPUT(STD.File.RemoteDirectory('edata12','\in','*.d00')); -OUTPUT(STD.File.RemoteDirectory('10.150.254.6', - '/var/lib/HPCCSystems/hpcc-data/thor/','acc*',TRUE)); + IMPORT STD; +machineIP := ''; +dir := '/var/lib/HPCCSystems/dropzone/training'; +recurse:= FALSE; +planeName := 'mydropzone'; +OUTPUT(STD.File.RemoteDirectory(machineIP,dir,'*.csv',recurse,planeName)); diff --git a/esp/services/ws_dfsservice/ws_dfsservice.cpp b/esp/services/ws_dfsservice/ws_dfsservice.cpp index 866064e5e0d..d79e3afdd7a 100644 --- a/esp/services/ws_dfsservice/ws_dfsservice.cpp +++ b/esp/services/ws_dfsservice/ws_dfsservice.cpp @@ -50,7 +50,7 @@ enum LfnMetaOpts : byte LfnMOptTls = 0x02, }; BITMASK_ENUM(LfnMetaOpts); -static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, LfnMetaOpts opts, IPropertyTree *metaRoot, IPropertyTree *meta) +static void populateLFNMeta(IUserDescriptor *userDesc, const char *logicalName, unsigned __int64 leaseId, LfnMetaOpts opts, IPropertyTree *metaRoot, IPropertyTree *meta) { CDfsLogicalFileName lfn; lfn.set(logicalName); @@ -59,7 +59,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, L assertex(!lfn.isMulti()); // not supported, don't think needs to be/will be. - Owned tree = queryDistributedFileDirectory().getFileTree(logicalName, nullptr); + Owned tree = queryDistributedFileDirectory().getFileTree(logicalName, userDesc); if (!tree) return; if (hasMask(opts, LfnMOptRemap)) @@ -111,7 +111,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, L { IPropertyTree &sub = *(orderedSubFiles[f]); sub.getProp("@name", subname.clear()); - populateLFNMeta(subname, leaseId, opts, metaRoot, fileMeta); + populateLFNMeta(userDesc, subname, leaseId, opts, metaRoot, fileMeta); } } } @@ -182,7 +182,7 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r opts |= LfnMOptTls; Owned responseTree = createPTree(); - populateLFNMeta(logicalName, leaseId, opts, responseTree, responseTree); + populateLFNMeta(userDesc, logicalName, leaseId, opts, responseTree, responseTree); // serialize response MemoryBuffer respMb, compressedRespMb; @@ -198,7 +198,7 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r // Really this should be done at end (or at end as well), but this is same as existing DFS lookup. CDateTime dt; dt.setNow(); - queryDistributedFileDirectory().setFileAccessed(logicalName, dt); + queryDistributedFileDirectory().setFileAccessed(userDesc, logicalName, dt); LOG(MCauditInfo,",FileAccess,EspProcess,READ,%s,%u,%s", logicalName, timeoutSecs, userID.str()); } diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 385eb4bde2e..cc94aa31eff 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -3367,7 +3367,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface */ unsigned ms = timer.elapsedMs(); totalThrottleDelay += ms; - PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %u milliseconds, proceeding as cpu(%u) avoidRename{-1}; +static CriticalSection avoidRenameCS; +static bool isAvoidRenameEnabled() +{ + if (-1 == avoidRename) + { + CriticalBlock b(avoidRenameCS); + if (-1 == avoidRename) + { + avoidRename = getComponentConfigSP()->getPropBool("expert/@avoidRename"); + DBGLOG("FileSystemProperties.canRename = %s", boolToStr(!avoidRename)); // NB: canRename if !avoidRename + } + } + return avoidRename; +} //This implementation should eventually make use of the file hook. const FileSystemProperties & queryFileSystemProperties(const char * filename) { if (isUrl(filename)) return defaultUrlFileSystemProperties; + else if (isAvoidRenameEnabled()) + return linuxFileSystemNoRenameProperties; else return linuxFileSystemProperties; } diff --git a/system/jlib/jthread.cpp b/system/jlib/jthread.cpp index 9bf0dff90fa..8545d2bd641 100644 --- a/system/jlib/jthread.cpp +++ b/system/jlib/jthread.cpp @@ -1002,36 +1002,42 @@ class CThreadPool: public CThreadPoolBase, implements IThreadPool, public CInter PooledThreadHandle _start(void *param,const char *name, bool noBlock, unsigned timeout=0) { CCycleTimer startTimer; - bool timedout = defaultmax && !availsem.wait(noBlock ? 0 : (timeout>0?timeout:delay)); + bool waited = false; + bool timedout = false; + if (defaultmax) + { + waited = !availsem.wait(0); + if (noBlock) + timedout = waited; + else if (waited) + timedout = !availsem.wait(timeout>0?timeout:delay); + } PooledThreadHandle ret; { CriticalBlock block(crit); if (timedout) { - if (!availsem.wait(0)) { // make sure take allocated sem if has become available + if (!availsem.wait(0)) // make sure take allocated sem if has become available + { if (noBlock || timeout > 0) throw MakeStringException(0, "No threads available in pool %s", poolname.get()); IWARNLOG("Pool limit exceeded for %s", poolname.get()); } - else - timedout = false; } if (traceStartDelayPeriod) { ++startsInPeriod; - if (timedout) - { + if (waited) startDelayInPeriod += startTimer.elapsedCycles(); - if (overAllTimer.elapsedCycles() >= queryOneSecCycles()*traceStartDelayPeriod) // check avg. delay per minute - { - double totalDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod)))/1000000; - double avgDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod/startsInPeriod)))/1000000; - unsigned totalElapsedSecs = overAllTimer.elapsedMs()/1000; - PROGLOG("%s: %u threads started in last %u seconds, total delay = %0.2f milliseconds, average delay = %0.2f milliseconds, currently running = %u", poolname.get(), startsInPeriod, totalElapsedSecs, totalDelayMs, avgDelayMs, runningCount()); - startsInPeriod = 0; - startDelayInPeriod = 0; - overAllTimer.reset(); - } + if (overAllTimer.elapsedCycles() >= queryOneSecCycles()*traceStartDelayPeriod) // check avg. delay per minute + { + double totalDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod)))/1000000; + double avgDelayMs = (static_cast(cycle_to_nanosec(startDelayInPeriod/startsInPeriod)))/1000000; + unsigned totalElapsedSecs = overAllTimer.elapsedMs()/1000; + PROGLOG("%s: %u threads started in last %u seconds, total delay = %0.2f milliseconds, average delay = %0.2f milliseconds, currently running = %u", poolname.get(), startsInPeriod, totalElapsedSecs, totalDelayMs, avgDelayMs, runningCount()); + startsInPeriod = 0; + startDelayInPeriod = 0; + overAllTimer.reset(); } } CPooledThreadWrapper &t = allocThread(); diff --git a/thorlcr/activities/thactivityutil.cpp b/thorlcr/activities/thactivityutil.cpp index bb8bbb10279..09989bd3937 100644 --- a/thorlcr/activities/thactivityutil.cpp +++ b/thorlcr/activities/thactivityutil.cpp @@ -733,7 +733,7 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc, partDesc.getFilename(0, rfn); StringBuffer primaryName; rfn.getPath(primaryName); - if (isUrl(primaryName)) + if (isUrl(primaryName) || activity->getOptBool(THOROPT_AVOID_RENAME)) // THOROPT_AVOID_RENAME see HPCC-31559 { twFlags &= ~TW_RenameToPrimary; twFlags |= TW_Direct; diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 07ef3c0c14d..4e3943e13a1 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -1185,7 +1185,7 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, { Owned factory = new CProcessorFactory(*this); processorPool.setown(createThreadPool("KJService processor pool", factory, true, this, keyLookupMaxProcessThreads, 10000)); - processorPool->setStartDelayTracing(60000); + processorPool->setStartDelayTracing(60); } public: IMPLEMENT_IINTERFACE_USING(CSimpleInterfaceOf); diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 21cae128b7f..399233d6221 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -115,6 +115,7 @@ #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel" // The trace SOAP level (default=1) #define THOROPT_SORT_ALGORITHM "sortAlgorithm" // The algorithm used to sort records (quicksort/mergesort) #define THOROPT_COMPRESS_ALLFILES "compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) +#define THOROPT_AVOID_RENAME "avoidRename" // Avoid rename, write directly to target physical filenames (no temp file) #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000 // max of row matches before selfjoin emits warning diff --git a/vcpkg b/vcpkg index 761c26eeb51..229f836da66 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 761c26eeb51bcf9a100e34a2b20a7a4f77382d5c +Subproject commit 229f836da6616f2982ce1a7a5b18cf48f6c27e0c