diff --git a/ecl/hqlcpp/hqlecl.cpp b/ecl/hqlcpp/hqlecl.cpp index 43e0ec5e4f5..e52ac773ca9 100644 --- a/ecl/hqlcpp/hqlecl.cpp +++ b/ecl/hqlcpp/hqlecl.cpp @@ -328,7 +328,7 @@ void HqlDllGenerator::expandCode(StringBuffer & filename, const char * codeTempl totalGeneratedSize += out->size(); - if (!publishGenerated) + if (publishGenerated) { unsigned minActivity, maxActivity; code->getActivityRange(pass, minActivity, maxActivity); diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 2ff9c3825d6..65386a10014 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1866,7 +1866,7 @@ }, "fieldTranslationEnabled": { "default": "payload", - "enum": ["false", "true", "payload"], + "enum": ["false", "true", "payload", "payloadRemoveOnly"], "description": "Enables translation (where possible) of mismatched file layouts on-the-fly. Specify 'payload' to attempt to translate payload fields only" }, "highTimeout": { diff --git a/initfiles/bin/init_thorslave.in b/initfiles/bin/init_thorslave.in index 1affe3ba9c3..1048ce6f9ac 100755 --- a/initfiles/bin/init_thorslave.in +++ b/initfiles/bin/init_thorslave.in @@ -44,24 +44,14 @@ slavename=thorslave_${hpcc_compname} stop_slaves() { local timer=15 - local isAlive=0 - - log "Attempting to kill $slavename with SIGTERM" - killall -SIGTERM $slavename > /dev/null 2>&1 - while [[ $isAlive -eq 0 && $timer -gt 0 ]];do - killall -0 $slavename > /dev/null 2>&1 - isAlive=$? - [[ $isAlive -eq 0 ]] && sleep 0.5 - ((timer--)) + local _pidarray + local _pidname + + _pidarray=($( ls -1 ${PID_DIR} | grep -E "${slavename}_[1-9][0-9]*\.pid" )) + for _pidname in ${_pidarray[@]}; do + kill_process ${PID_DIR}/$_pidname $slavename $timer done - - if [[ $isAlive -eq 0 ]]; then - log "Failed to kill slaves with SIGTERM. Sending SIGKILL" - killall -SIGKILL $slavename > /dev/null - fi - - # need regex here to prevent removing other Thor instance pid files - find ${PID_DIR} -maxdepth 1 -type f -regex ".*/${slavename}_[1-9][0-9]*\.pid" -delete > /dev/null 2>&1 + } start_slaves() diff --git a/plugins/Rembed/CMakeLists.txt b/plugins/Rembed/CMakeLists.txt index 9b44e07f777..03f0c2a9f8b 100644 --- a/plugins/Rembed/CMakeLists.txt +++ b/plugins/Rembed/CMakeLists.txt @@ -43,9 +43,9 @@ if(REMBED) ./../../system/mp ./../../system/jlib) - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-error=format-nonliteral") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=format-nonliteral") - set_property (SOURCE Rembed.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-format-nonliteral") + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-error=format-nonliteral -Wno-error=format-security -Wno-error=format=") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=format-nonliteral -Wno-error=format-security -Wno-error=format=") + set_property (SOURCE Rembed.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-format-nonliteral -Wno-format-security -Wno-error-format=") add_definitions(-D_USRDLL -DREMBED_EXPORTS) if(RCPP_LIBRARY STREQUAL "") add_definitions(-DRCPP_HEADER_ONLY) @@ -77,4 +77,4 @@ if(PLATFORM OR CLIENTTOOLS_ONLY) FILES ${CMAKE_CURRENT_SOURCE_DIR}/R.ecllib DESTINATION plugins COMPONENT Runtime) -endif() +endif() \ No newline at end of file diff --git a/roxie/ccd/ccdfile.cpp b/roxie/ccd/ccdfile.cpp index 7983a8cf724..27fc1daff36 100644 --- a/roxie/ccd/ccdfile.cpp +++ b/roxie/ccd/ccdfile.cpp @@ -2080,6 +2080,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress IArrayOf expired; IArrayOf goers; unsigned openLimit = maxFilesOpen[remote]; + unsigned __int64 nowNs = nsTick(); { CriticalBlock b(crit); if (files.ordinality() > openLimit || maxFileAgeNS[remote] != (unsigned __int64) -1) @@ -2093,7 +2094,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress Owned f = match; if (f->isOpen() && f->isRemote()==remote && !f->isCopying()) { - unsigned __int64 age = nsTick() - f->getLastAccessed(); + unsigned __int64 age = nowNs - f->getLastAccessed(); if (age > maxFileAgeNS[remote]) expired.append(*f.getClear()); else if (files.ordinality() > openLimit) @@ -2115,7 +2116,7 @@ class CRoxieFileCache : implements IRoxieFileCache, implements ICopyFileProgress // NOTE - querySource will cause the file to be opened if not already open // That's OK here, since we know the file is open and remote. // But don't be tempted to move this line outside these if's (eg. to trace the idle case) - unsigned __int64 age = nsTick() - f.getLastAccessed(); + unsigned __int64 age = nowNs - f.getLastAccessed(); const char *fname = remote ? f.querySource()->queryFilename() : f.queryFilename(); DBGLOG("Closing inactive %s file %s (last accessed %" I64F "u ms ago)", remote ? "remote" : "local", fname, nanoToMilli(age)); } diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index dd630ac594d..426e2139f32 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -1250,7 +1250,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) maxFileAgeNS[true] = milliToNano(temp); else maxFileAgeNS[true] = (unsigned __int64) -1; - maxFilesOpen[false] = topology->getPropInt("@maxLocalFilesOpen", 4000); + maxFilesOpen[false] = topology->getPropInt("@maxLocalFilesOpen", 20000); maxFilesOpen[true] = topology->getPropInt("@maxRemoteFilesOpen", 1000); minFilesOpen[false] = topology->getPropInt("@minLocalFilesOpen", maxFilesOpen[false]/2); if (minFilesOpen[false] >= maxFilesOpen[false]) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index e67dac2d838..625e1a1652d 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -1985,6 +1985,16 @@ struct WinCompressedFileTrailer #pragma pack(pop) +static size32_t countZeros(size32_t size, const byte * data) +{ + size32_t len; + for (len = 0; len < size; len++) + { + if (data[len]) + break; + } + return len; +} class CCompressedFile : implements ICompressedFileIO, public CInterface { @@ -2069,7 +2079,7 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface { size32_t nextSize = expander->expandNext(curblockbuf); if (nextSize == 0) - throwUnexpected(); // Should have failed the outer block test if nextSize is 0 + throw makeStringExceptionV(-1, "Unexpected zero length compression block at position %llu", curblockpos); curblockpos = nextExpansionPos; nextExpansionPos = nextExpansionPos+nextSize; @@ -2093,11 +2103,11 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface size32_t r = fileio->read(p,toread,b); assertex(r==toread); - expand(b,curblockbuf,expsize); + expand(b,curblockbuf,expsize,p); } else { // memory mapped assertex((memsize_t)p==p); - expand(mmfile->base()+(memsize_t)p,curblockbuf,expsize); + expand(mmfile->base()+(memsize_t)p,curblockbuf,expsize,p); } } void checkedwrite(offset_t pos, size32_t len, const void * data) @@ -2110,7 +2120,7 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface } - void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize) + void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize, offset_t compressedPos) { size32_t rs = trailer.recordSize; if (rs) { // diff expand @@ -2133,6 +2143,13 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface else { // lzw or fastlz or lz4 assertex(expander.get()); size32_t exp = expander->expandFirst(expbuf, compbuf); + if (exp == 0) + { + unsigned numZeros = countZeros(trailer.blockSize, (const byte *)compbuf); + if (numZeros >= 16) + throw makeStringExceptionV(-1, "Unexpected zero fill in compressed file at position %llu length %u", compressedPos, numZeros); + } + startBlockPos = curblockpos; nextExpansionPos = startBlockPos + exp; fullBlockSize = expsize; diff --git a/system/jlib/jprop.cpp b/system/jlib/jprop.cpp index 22718685946..7297f8883be 100644 --- a/system/jlib/jprop.cpp +++ b/system/jlib/jprop.cpp @@ -214,6 +214,16 @@ class CPropertiesBase : implements IPROP, public CInterface } return dft; } + virtual __int64 getPropInt64(PTYPE propname, int dft) const override + { + if (propname) + { + const char *val = queryProp(propname); + if (val) + return _atoi64(val); + } + return dft; + } virtual bool getPropBool(PTYPE propname, bool dft) const override { if (propname) diff --git a/system/jlib/jprop.hpp b/system/jlib/jprop.hpp index 3dcf8588c62..6b5485df65d 100644 --- a/system/jlib/jprop.hpp +++ b/system/jlib/jprop.hpp @@ -57,6 +57,7 @@ interface jlib_decl IPropertiesOf : extends serializable virtual void saveFile(const char *filename) const = 0; virtual bool removeProp(PTYPE propname) = 0; virtual bool getPropBool(PTYPE propname, bool dft=false) const = 0; + virtual __int64 getPropInt64(PTYPE propname, int dft=0) const = 0; }; #ifdef _MSC_VER diff --git a/thorlcr/activities/diskread/thdiskreadslave.cpp b/thorlcr/activities/diskread/thdiskreadslave.cpp index 3f86495263b..59d001238a0 100644 --- a/thorlcr/activities/diskread/thdiskreadslave.cpp +++ b/thorlcr/activities/diskread/thdiskreadslave.cpp @@ -492,8 +492,7 @@ class CDiskReadSlaveActivity : public CDiskReadSlaveActivityRecord { eoi = true; StringBuffer s; - e->errorMessage(s); - s.append(" - handling file: ").append(filename.get()); + s.append("handling file: ").append(filename.get()); IException *e2 = MakeActivityException(&activity, e, "%s", s.str()); e->Release(); throw e2; @@ -851,8 +850,7 @@ class CDiskSimplePartHandler : public CDiskRecordPartHandler catch (IException *e) { StringBuffer s; - e->errorMessage(s); - s.append(" - handling file: ").append(filename.get()); + s.append("handling file: ").append(filename.get()); IException *e2 = MakeActivityException(&activity, e, "%s", s.str()); e->Release(); eoi = true; diff --git a/tools/dumpkey/dumpkey.cpp b/tools/dumpkey/dumpkey.cpp index 72756b1588d..88ef41f7a31 100644 --- a/tools/dumpkey/dumpkey.cpp +++ b/tools/dumpkey/dumpkey.cpp @@ -231,17 +231,18 @@ int main(int argc, const char **argv) printf("%s does not appear to be an index file\n", keyName); continue; } - Owned index; - index.setown(createKeyIndex(keyName, 0, false, 0)); + Owned in = createIFile(keyName); + Owned io = in->open(IFOread); + if (!io) + throw MakeStringException(999, "Failed to open file %s", keyName); + + //read with a buffer size of 4MB - for optimal speed, and minimize azure read costs + Owned index(createKeyIndex(keyName, 0, *io, 1, false, 0x400000)); size32_t key_size = index->keySize(); // NOTE - in variable size case, this may be 32767 + sizeof(offset_t) size32_t keyedSize = index->keyedSize(); unsigned nodeSize = index->getNodeSize(); if (optFullHeader) { - Owned in = createIFile(keyName); - Owned io = in->open(IFOread); - if (!io) - throw MakeStringException(999, "Failed to open file %s", keyName); Owned header = new CKeyHdr; MemoryAttr block(sizeof(KeyHdr)); io->read(0, sizeof(KeyHdr), (void *)block.get()); @@ -279,14 +280,14 @@ int main(int argc, const char **argv) } else { - int node = globals->getPropInt("node"); + offset_t node = globals->getPropInt64("node"); if (node != 0) index->dumpNode(stdout, node * nodeSize, globals->getPropInt("recs", 0), optRaw); } } else if (globals->hasProp("fpos")) { - index->dumpNode(stdout, globals->getPropInt("fpos"), globals->getPropInt("recs", 0), optRaw); + index->dumpNode(stdout, globals->getPropInt64("fpos"), globals->getPropInt("recs", 0), optRaw); } else {