Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…

…9.6.x

Signed-off-by: Gavin Halliday <[email protected]>

ecl/hqlcpp/hqlecl.cpp

@@ -328,7 +328,7 @@ void HqlDllGenerator::expandCode(StringBuffer & filename, const char * codeTempl
 
     totalGeneratedSize += out->size();
 
-    if (!publishGenerated)
+    if (publishGenerated)
     {
         unsigned minActivity, maxActivity;
         code->getActivityRange(pass, minActivity, maxActivity);

helm/hpcc/values.schema.json

@@ -1866,7 +1866,7 @@
         },
         "fieldTranslationEnabled": {
           "default": "payload",
-          "enum": ["false", "true", "payload"],
+          "enum": ["false", "true", "payload", "payloadRemoveOnly"],
           "description": "Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only"
         },
         "highTimeout": { 

initfiles/bin/init_thorslave.in

@@ -44,24 +44,14 @@ slavename=thorslave_${hpcc_compname}
 stop_slaves()
 {
     local timer=15
-    local isAlive=0
-
-    log "Attempting to kill $slavename with SIGTERM"
-    killall -SIGTERM $slavename > /dev/null 2>&1
-    while [[ $isAlive -eq 0 && $timer -gt 0 ]];do
-        killall -0 $slavename > /dev/null 2>&1
-        isAlive=$?
-        [[ $isAlive -eq 0 ]] && sleep 0.5
-        ((timer--))
+    local _pidarray
+    local _pidname
+
+    _pidarray=($( ls -1 ${PID_DIR} | grep -E "${slavename}_[1-9][0-9]*\.pid" ))
+    for _pidname in ${_pidarray[@]}; do
+        kill_process ${PID_DIR}/$_pidname $slavename $timer
     done
-
-    if [[ $isAlive -eq 0 ]]; then
-        log "Failed to kill slaves with SIGTERM. Sending SIGKILL"
-        killall -SIGKILL $slavename > /dev/null
-    fi
-
-    # need regex here to prevent removing other Thor instance pid files
-    find ${PID_DIR} -maxdepth 1 -type f -regex ".*/${slavename}_[1-9][0-9]*\.pid" -delete > /dev/null 2>&1
+
 }
 
 start_slaves()

plugins/Rembed/CMakeLists.txt

@@ -43,9 +43,9 @@ if(REMBED)
             ./../../system/mp
             ./../../system/jlib)
 
-        set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-error=format-nonliteral")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=format-nonliteral")
-        set_property (SOURCE Rembed.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-format-nonliteral")
+        set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-error=format-nonliteral -Wno-error=format-security -Wno-error=format=")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=format-nonliteral -Wno-error=format-security -Wno-error=format=")
+        set_property (SOURCE Rembed.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-format-nonliteral -Wno-format-security -Wno-error-format=")
         add_definitions(-D_USRDLL -DREMBED_EXPORTS)
         if(RCPP_LIBRARY STREQUAL "")
             add_definitions(-DRCPP_HEADER_ONLY)
@@ -77,4 +77,4 @@ if(PLATFORM OR CLIENTTOOLS_ONLY)
         FILES ${CMAKE_CURRENT_SOURCE_DIR}/R.ecllib
         DESTINATION plugins
         COMPONENT Runtime)
-endif()
+endif()

roxie/ccd/ccdfile.cpp

@@ -2080,6 +2080,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress
         IArrayOf<ILazyFileIO> expired;
         IArrayOf<ILazyFileIO> goers;
         unsigned openLimit = maxFilesOpen[remote];
+        unsigned __int64 nowNs = nsTick();
         {
             CriticalBlock b(crit);
             if (files.ordinality() > openLimit || maxFileAgeNS[remote] != (unsigned __int64) -1)
@@ -2093,7 +2094,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress
                         Owned<ILazyFileIO> f = match;
                         if (f->isOpen() && f->isRemote()==remote && !f->isCopying())
                         {
-                            unsigned __int64 age = nsTick() - f->getLastAccessed();
+                            unsigned __int64 age = nowNs - f->getLastAccessed();
                             if (age > maxFileAgeNS[remote])
                                 expired.append(*f.getClear());
                             else if (files.ordinality() > openLimit)
@@ -2115,7 +2116,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress
                     // NOTE - querySource will cause the file to be opened if not already open
                     // That's OK here, since we know the file is open and remote.
                     // But don't be tempted to move this line outside these if's (eg. to trace the idle case)
-                    unsigned __int64 age = nsTick() - f.getLastAccessed();
+                    unsigned __int64 age = nowNs - f.getLastAccessed();
                     const char *fname = remote ? f.querySource()->queryFilename() : f.queryFilename();
                     DBGLOG("Closing inactive %s file %s (last accessed %" I64F "u ms ago)", remote ? "remote" : "local",  fname, nanoToMilli(age));
                 }

roxie/ccd/ccdmain.cpp

@@ -1250,7 +1250,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
             maxFileAgeNS[true] = milliToNano(temp);
         else
             maxFileAgeNS[true] = (unsigned __int64) -1;
-        maxFilesOpen[false] = topology->getPropInt("@maxLocalFilesOpen", 4000);
+        maxFilesOpen[false] = topology->getPropInt("@maxLocalFilesOpen", 20000);
         maxFilesOpen[true] = topology->getPropInt("@maxRemoteFilesOpen", 1000);
         minFilesOpen[false] = topology->getPropInt("@minLocalFilesOpen", maxFilesOpen[false]/2);
         if (minFilesOpen[false] >= maxFilesOpen[false])

system/jlib/jlzw.cpp

@@ -1985,6 +1985,16 @@ struct WinCompressedFileTrailer
 
 #pragma pack(pop)
 
+static size32_t countZeros(size32_t size, const byte * data)
+{
+    size32_t len;
+    for (len = 0; len < size; len++)
+    {
+        if (data[len])
+            break;
+    }
+    return len;
+}
 
 class CCompressedFile : implements ICompressedFileIO, public CInterface
 {
@@ -2069,7 +2079,7 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
             {
                 size32_t nextSize = expander->expandNext(curblockbuf);
                 if (nextSize == 0)
-                    throwUnexpected(); // Should have failed the outer block test if nextSize is 0
+                    throw makeStringExceptionV(-1, "Unexpected zero length compression block at position %llu", curblockpos);
 
                 curblockpos = nextExpansionPos;
                 nextExpansionPos = nextExpansionPos+nextSize;
@@ -2093,11 +2103,11 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
 
             size32_t r = fileio->read(p,toread,b);
             assertex(r==toread);
-            expand(b,curblockbuf,expsize);
+            expand(b,curblockbuf,expsize,p);
         }
         else { // memory mapped
             assertex((memsize_t)p==p);
-            expand(mmfile->base()+(memsize_t)p,curblockbuf,expsize);
+            expand(mmfile->base()+(memsize_t)p,curblockbuf,expsize,p);
         }
     }
     void checkedwrite(offset_t pos, size32_t len, const void * data) 
@@ -2110,7 +2120,7 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
 
     }
 
-    void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize)
+    void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize, offset_t compressedPos)
     {
         size32_t rs = trailer.recordSize;
         if (rs) { // diff expand
@@ -2133,6 +2143,13 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
         else { // lzw or fastlz or lz4
             assertex(expander.get());
             size32_t exp = expander->expandFirst(expbuf, compbuf);
+            if (exp == 0)
+            {
+                unsigned numZeros = countZeros(trailer.blockSize, (const byte *)compbuf);
+                if (numZeros >= 16)
+                    throw makeStringExceptionV(-1, "Unexpected zero fill in compressed file at position %llu length %u", compressedPos, numZeros);
+            }
+
             startBlockPos = curblockpos;
             nextExpansionPos = startBlockPos + exp;
             fullBlockSize = expsize;

system/jlib/jprop.cpp

@@ -214,6 +214,16 @@ class CPropertiesBase : implements IPROP, public CInterface
         }
         return dft;
     }
+    virtual __int64 getPropInt64(PTYPE propname, int dft) const override
+    {
+        if (propname)
+        {
+            const char *val = queryProp(propname);
+            if (val)
+                return _atoi64(val);
+        }
+        return dft;
+    }
     virtual bool getPropBool(PTYPE propname, bool dft) const override
     {
         if (propname)

system/jlib/jprop.hpp

@@ -57,6 +57,7 @@ interface jlib_decl IPropertiesOf : extends serializable
     virtual void saveFile(const char *filename) const = 0;
     virtual bool removeProp(PTYPE propname) = 0;
     virtual bool getPropBool(PTYPE propname, bool dft=false) const = 0;
+    virtual __int64 getPropInt64(PTYPE propname, int dft=0) const = 0;
 };
 
 #ifdef _MSC_VER

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -492,8 +492,7 @@ class CDiskReadSlaveActivity : public CDiskReadSlaveActivityRecord
                 {
                     eoi = true;
                     StringBuffer s;
-                    e->errorMessage(s);
-                    s.append(" - handling file: ").append(filename.get());
+                    s.append("handling file: ").append(filename.get());
                     IException *e2 = MakeActivityException(&activity, e, "%s", s.str());
                     e->Release();
                     throw e2;
@@ -851,8 +850,7 @@ class CDiskSimplePartHandler : public CDiskRecordPartHandler
             catch (IException *e)
             {
                 StringBuffer s;
-                e->errorMessage(s);
-                s.append(" - handling file: ").append(filename.get());
+                s.append("handling file: ").append(filename.get());
                 IException *e2 = MakeActivityException(&activity, e, "%s", s.str());
                 e->Release();
                 eoi = true;

tools/dumpkey/dumpkey.cpp

@@ -231,17 +231,18 @@ int main(int argc, const char **argv)
                 printf("%s does not appear to be an index file\n", keyName);
                 continue;
             }
-            Owned <IKeyIndex> index;
-            index.setown(createKeyIndex(keyName, 0, false, 0));
+            Owned<IFile> in = createIFile(keyName);
+            Owned<IFileIO> io = in->open(IFOread);
+            if (!io)
+                throw MakeStringException(999, "Failed to open file %s", keyName);
+
+            //read with a buffer size of 4MB - for optimal speed, and minimize azure read costs
+            Owned <IKeyIndex> index(createKeyIndex(keyName, 0, *io, 1, false, 0x400000));
             size32_t key_size = index->keySize();  // NOTE - in variable size case, this may be 32767 + sizeof(offset_t)
             size32_t keyedSize = index->keyedSize();
             unsigned nodeSize = index->getNodeSize();
             if (optFullHeader)
             {
-                Owned<IFile> in = createIFile(keyName);
-                Owned<IFileIO> io = in->open(IFOread);
-                if (!io)
-                    throw MakeStringException(999, "Failed to open file %s", keyName);
                 Owned<CKeyHdr> header = new CKeyHdr;
                 MemoryAttr block(sizeof(KeyHdr));
                 io->read(0, sizeof(KeyHdr), (void *)block.get());
@@ -279,14 +280,14 @@ int main(int argc, const char **argv)
                 }
                 else
                 {
-                    int node = globals->getPropInt("node");
+                    offset_t node = globals->getPropInt64("node");
                     if (node != 0)
                         index->dumpNode(stdout, node * nodeSize, globals->getPropInt("recs", 0), optRaw);
                 }
             }
             else if (globals->hasProp("fpos"))
             {
-                index->dumpNode(stdout, globals->getPropInt("fpos"), globals->getPropInt("recs", 0), optRaw);
+                index->dumpNode(stdout, globals->getPropInt64("fpos"), globals->getPropInt("recs", 0), optRaw);
             }
             else
             {