diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index b1984679559..1138242ffc1 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -408,20 +408,6 @@ jobs: cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DBUILD_LEVEL=ENTERPRISE -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DPLATFORM=ON -DINCLUDE_PLUGINS=ON -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF && \ cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - name: Upload Assets (enterprise) - if: ${{ matrix.ee }} - uses: ncipollo/release-action@v1.12.0 - with: - allowUpdates: true - generateReleaseNotes: false - prerelease: ${{ contains(github.ref, '-rc') }} - owner: ${{ secrets.LNB_ACTOR }} - repo: LN - token: ${{ secrets.LNB_TOKEN }} - tag: ${{ needs.preamble.outputs.internal_tag }} - artifacts: "${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.deb,${{ needs.preamble.outputs.folder_build }}/hpccsystems-platform-enterprise*.rpm" - - # Common --- - name: Cleanup Environment if: always() @@ -687,12 +673,22 @@ jobs: C:\"Program Files (x86)"\"Microsoft SDKs"\ClickOnce\SignTool\signtool.exe sign /debug /f ../../sign/hpcc_code_signing.pfx /p ${{ secrets.SIGNING_CERTIFICATE_PASSPHRASE}} /t http://timestamp.digicert.com /fd SHA256 hpccsystems-eclide*.exe - name: Upload Assets - uses: ncipollo/release-action@v1.12.0 + uses: ncipollo/release-action@v1.14.0 with: allowUpdates: true generateReleaseNotes: false prerelease: ${{ contains(github.ref, '-rc') }} artifacts: "./ECLIDE/build/*.exe" + + - name: Upload Assets to Jfrog (windows) + if: ${{ github.repository_owner == 'hpcc-systems' }} + shell: bash + run: | + cd ./ECLIDE/build + packages=($(ls -1 hpccsystems-*.exe )) + for _package in ${packages[@]}; do + curl -u${{ secrets.JFROG_USERNAME }}:${{ secrets.JFROG_PASSWORD }} "https://${{ secrets.JFROG_REGISTRY }}/hpccpl-windows-local/LN/windows/x86_64/${_package}" -T ${_package} + done - name: Upload error logs if: ${{ failure() || cancelled() }} diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp index 38f29328615..82bfa6b785e 100644 --- a/common/thorhelper/thorcommon.cpp +++ b/common/thorhelper/thorcommon.cpp @@ -1647,7 +1647,10 @@ IExtRowWriter *createRowWriter(IFile *iFile, IRowInterfaces *rowIf, unsigned fla { OwnedIFileIO iFileIO; if (TestRwFlag(flags, rw_compress)) + { + flags &= ~rw_buffered; // if compressed, do not want buffered stream as well iFileIO.setown(createCompressedFileWriter(iFile, rowIf, flags, compressor, compressorBlkSz)); + } else iFileIO.setown(iFile->open((flags & rw_extend)?IFOwrite:IFOcreate)); if (!iFileIO) diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index 000d4a80e81..e7ea8283878 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -2559,10 +2559,16 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo { checkTimeLimitExceeded(&remainingMS); checkRoxieAbortMonitor(master->roxieAbortMonitor); - OwnedSpanScope socketOperationSpan = master->activitySpanScope->createClientSpan("Socket Write"); - setSpanURLAttributes(socketOperationSpan, url); - Owned traceHeaders = ::getClientHeaders(socketOperationSpan); + StringBuffer spanName; + spanName.appendf("%s %s %s:%d", getWsCallTypeName(master->wscType), master->service.str(), url.host.str(), url.port); + OwnedSpanScope requestSpan = master->activitySpanScope->createClientSpan(spanName.str()); + + setSpanURLAttributes(requestSpan, url); + requestSpan->setSpanAttribute("request.type", getWsCallTypeName(master->wscType)); + requestSpan->setSpanAttribute("service.name", master->service.str()); + + Owned traceHeaders = ::getClientHeaders(requestSpan); createHttpRequest(request, url, traceHeaders); socket->write(request.str(), request.length()); @@ -2575,7 +2581,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo bool keepAlive2; StringBuffer contentType; int rval = readHttpResponse(response, socket, keepAlive2, contentType); - socketOperationSpan->setSpanAttribute("http.response.status_code", (int64_t)rval); + requestSpan->setSpanAttribute("http.response.status_code", (int64_t)rval); keepAlive = keepAlive && keepAlive2; if (soapTraceLevel > 4) @@ -2583,22 +2589,22 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (rval != 200) { - socketOperationSpan->setSpanStatusSuccess(false); + requestSpan->setSpanStatusSuccess(false); if (rval == 503) { - socketOperationSpan->recordError(SpanError("Server Too Busy", 1001, true, true)); + requestSpan->recordError(SpanError("Server Too Busy", 1001, true, true)); throw new ReceivedRoxieException(1001, "Server Too Busy"); } StringBuffer text; text.appendf("HTTP error (%d) in processQuery",rval); rtlAddExceptionTag(text, "soapresponse", response.str()); - socketOperationSpan->recordError(SpanError(text.str(), -1, true, true)); + requestSpan->recordError(SpanError(text.str(), -1, true, true)); throw MakeStringExceptionDirect(-1, text.str()); } if (response.length() == 0) { - socketOperationSpan->recordError(SpanError("Zero length response in processQuery", -1, true, true)); + requestSpan->recordError(SpanError("Zero length response in processQuery", -1, true, true)); throw MakeStringException(-1, "Zero length response in processQuery"); } checkTimeLimitExceeded(&remainingMS); @@ -2614,7 +2620,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo persistentHandler->add(socket, &ep, proto); } - socketOperationSpan->setSpanStatusSuccess(true); + requestSpan->setSpanStatusSuccess(true); break; } catch (IReceivedRoxieException *e) diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index f3f25fad57b..d1ea8bd9b02 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -265,34 +265,24 @@ bool importFromFile(const char *path,const char *filename,bool add,StringBuffer //============================================================================= -bool erase(const char *path,bool backup,StringBuffer &out) +bool erase(const char *path, bool backup, StringBuffer &out) { - StringBuffer head; - StringBuffer tmp; - const char *tail=splitpath(path,head,tmp); - Owned conn = querySDS().connect(head.str(),myProcessSession(),RTM_LOCK_WRITE, daliConnectTimeoutMs); - if (!conn) { + Owned conn = querySDS().connect(path, myProcessSession(), RTM_LOCK_WRITE, daliConnectTimeoutMs); + if (!conn) + { out.appendf("Could not connect to %s",path); return false; } - Owned root = conn->getRoot(); - Owned child = root->getPropTree(tail); - if (!child) { - out.appendf("Couldn't find %s/%s",head.str(),tail); - return false; - } - if (backup) { + IPropertyTree *root = conn->queryRoot(); + if (backup) + { StringBuffer bakname; - Owned io = createUniqueFile(NULL,"daliadmin", "bak", bakname); - out.appendf("Saving backup of %s/%s to %s",head.str(),tail,bakname.str()); + Owned io = createUniqueFile(NULL, "daliadmin", "bak", bakname); + out.appendf("Saving backup of %s to %s", path, bakname.str()); Owned fstream = createBufferedIOStream(io); - toXML(child, *fstream); // formatted (default) + toXML(root, *fstream); // formatted (default) } - root->removeTree(child); - child.clear(); - root.clear(); - conn->commit(); - conn->close(); + conn->close(true); return true; } diff --git a/ecl/hqlcpp/hqlres.cpp b/ecl/hqlcpp/hqlres.cpp index f3445589360..87c485a3dca 100644 --- a/ecl/hqlcpp/hqlres.cpp +++ b/ecl/hqlcpp/hqlres.cpp @@ -648,7 +648,8 @@ bool ResourceManager::flush(StringBuffer &filename, const char *basename, bool f } fwrite(s.data.get(), 1, s.data.length(), bin); fclose(bin); - fprintf(f, " .size %s,%u\n", label.str(), (unsigned)s.data.length()); + if (!generateClang) + fprintf(f, " .size %s,%u\n", label.str(), (unsigned)s.data.length()); } fclose(f); #endif diff --git a/ecllibrary/std/DataPatterns/CMakeLists.txt b/ecllibrary/std/DataPatterns/CMakeLists.txt index f7474bec76c..7de94fa1a68 100644 --- a/ecllibrary/std/DataPatterns/CMakeLists.txt +++ b/ecllibrary/std/DataPatterns/CMakeLists.txt @@ -22,6 +22,6 @@ set(SRCS foreach(module ${SRCS}) SIGN_MODULE(${module}) - INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/${module} DESTINATION share/ecllibrary/std/DataPatterns COMPONENT Runtime) + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${module} DESTINATION share/ecllibrary/std/DataPatterns COMPONENT Runtime) endforeach() diff --git a/esp/services/ws_store/espstorelib/daliKVStore.cpp b/esp/services/ws_store/espstorelib/daliKVStore.cpp index b3f1a40419d..cc45f00ce40 100644 --- a/esp/services/ws_store/espstorelib/daliKVStore.cpp +++ b/esp/services/ws_store/espstorelib/daliKVStore.cpp @@ -17,18 +17,29 @@ #include "daliKVStore.hpp" +void getEncodedLowerCaseUserName(StringBuffer & out, ISecUser * username) +{ + StringBuffer userlowercased; + userlowercased.set(username->getName()).toLowerCase(); + + encodePTreeName(out, userlowercased.str()); +} + bool CDALIKVStore::createStore(const char * apptype, const char * storename, const char * description, ISecUser * owner, unsigned int maxvalsize=DALI_KVSTORE_MAXVALSIZE_DEFAULT) { if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore createStore(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + ensureAttachedToDali(); //throws if in offline mode Owned conn = querySDS().connect(DALI_KVSTORE_PATH, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "Unable to connect to DALI KeyValue store root: '%s'", DALI_KVSTORE_PATH); - VStringBuffer xpath("Store[%s='%s'][1]", DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("Store[%s='%s'][1]", DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); { Owned root = conn->getRoot(); if (root->hasProp(xpath.str())) @@ -53,7 +64,7 @@ bool CDALIKVStore::createStore(const char * apptype, const char * storename, con } Owned apptree = createPTree(); - apptree->setProp(DALI_KVSTORE_NAME_ATT, storename); + apptree->setProp(DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); CDateTime dt; dt.setNow(); StringBuffer str; @@ -89,15 +100,21 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore set(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!owner || isEmptyString(owner->getName()))) throw MakeStringException(-1, "DALI Keystore set(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(thenamespace)) throw MakeStringException(-1, "DALI Keystore set(): namespace not provided"); + StringBuffer encodedNameSpace; + encodePTreeName(encodedNameSpace, thenamespace); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore set(): Unable to connect to DALI KeyValue store path '%s'", xpath.str()); //rodrigo, not sure if this is too much info @@ -111,7 +128,7 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(owner->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), owner); Owned ownertree = storetree->getPropTree(xpath.str()); if (!ownertree) @@ -121,18 +138,20 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const dt.setNow(); StringBuffer str; - Owned nstree = ownertree->getPropTree(thenamespace); + Owned nstree = ownertree->getPropTree(encodedNameSpace.str()); if (!nstree) { - nstree.setown(createPTree(thenamespace)); + nstree.setown(createPTree(encodedNameSpace.str())); nstree->setProp(DALI_KVSTORE_CREATEDTIME_ATT,dt.getString(str).str()); } - Owned valuetree = nstree->getPropTree(key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + Owned valuetree = nstree->getPropTree(encodedKey.str()); if (!valuetree) { - nstree->setProp(key, value); - valuetree.setown(nstree->getPropTree(key)); + nstree->setProp(encodedKey.str(), value); + valuetree.setown(nstree->getPropTree(encodedKey.str())); valuetree->setProp(DALI_KVSTORE_CREATEDTIME_ATT,dt.getString(str).str()); valuetree->setProp(DALI_KVSTORE_CREATEDBY_ATT, owner ? owner->getName(): ""); } @@ -143,7 +162,7 @@ bool CDALIKVStore::set(const char * storename, const char * thenamespace, const valuetree->setProp(".", value); } - ownertree->setPropTree(thenamespace, LINK(nstree)); + ownertree->setPropTree(encodedNameSpace.str(), LINK(nstree)); storetree->setPropTree(xpath.str(), LINK(ownertree)); conn->commit(); @@ -156,26 +175,31 @@ IPropertyTree * CDALIKVStore::getAllKeyProperties(const char * storename, const if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!username || isEmptyString(username->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperties(): namespace not provided"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); if (global) xpath.append(DALI_KVSTORE_GLOBAL); else - { - StringBuffer userlowercased; - userlowercased.set(username->getName()).toLowerCase(); - xpath.append(userlowercased); - } + getEncodedLowerCaseUserName(xpath, username); - xpath.appendf("/%s/%s", ns, key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + + xpath.appendf("/%s/%s", encodedNS.str(), encodedKey.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) @@ -189,26 +213,31 @@ bool CDALIKVStore::fetchKeyProperty(StringBuffer & propval , const char * storen if (isEmptyString(storename)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!username || isEmptyString(username->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): Attempting to set non-global entry but owner name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeyProperty(): namespace not provided"); + StringBuffer encodedNamespace; + encodePTreeName(encodedNamespace, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]/", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); if (global) xpath.append(DALI_KVSTORE_GLOBAL); else - { - StringBuffer userlowercased; - userlowercased.set(username->getName()).toLowerCase(); - xpath.append(userlowercased); - } + getEncodedLowerCaseUserName(xpath, username); + + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); - xpath.appendf("/%s/%s", ns, key); + xpath.appendf("/%s/%s", encodedNamespace.str(), encodedKey.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) @@ -216,7 +245,10 @@ bool CDALIKVStore::fetchKeyProperty(StringBuffer & propval , const char * storen Owned keytree = conn->getRoot(); - keytree->getProp(property,propval.clear()); + StringBuffer encodedPropName; + encodePTreeName(encodedPropName, property); + + keytree->getProp(encodedPropName.str(),propval.clear()); return true; } @@ -225,18 +257,27 @@ bool CDALIKVStore::deletekey(const char * storename, const char * thenamespace, if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore deletekey(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!thenamespace || !*thenamespace) throw MakeStringException(-1, "DALI KV Store deletekey(): target namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, thenamespace); + if (!key || !*key) throw MakeStringException(-1, "DALI KV Store deletekey(): target key not provided!"); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore set(): Attempting to set non-global entry but user not provided"); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s'][1]", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore deletekey(): Unable to connect to DALI KeyValue store root path '%s'", DALI_KVSTORE_PATH); @@ -246,9 +287,9 @@ bool CDALIKVStore::deletekey(const char * storename, const char * thenamespace, if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s/%s", thenamespace,key); + xpath.appendf("/%s/%s", encodedNS.str(), encodedKey.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI KV Store deletekey(): Could not find '%s/%s/%s' for user '%s'", storename, thenamespace, key, global ? "GLOBAL USER" : user->getName()); @@ -264,15 +305,21 @@ bool CDALIKVStore::deleteNamespace(const char * storename, const char * thenames if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore deletekey(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore deleteNamespace(): Attempting to fetch non-global keys but user not provided"); if (isEmptyString(thenamespace)) throw MakeStringException(-1, "DALI KV Store deleteNamespace(): target namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, thenamespace); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore deleteNamespace(): Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -282,9 +329,9 @@ bool CDALIKVStore::deleteNamespace(const char * storename, const char * thenames if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", thenamespace); //we're interested in the children of the namespace + xpath.appendf("/%s", encodedNS.str()); //we're interested in the children of the namespace if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI KV Store deleteNamespace(): invalid namespace detected '%s/%s' for user '%s'", storename, thenamespace, global ? "GLOBAL USER" : user->getName()); @@ -300,12 +347,15 @@ bool CDALIKVStore::fetchAllNamespaces(StringArray & namespaces, const char * sto if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces(): Attempting to fetch non-global keys but requester name not provided"); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchAllNamespaces: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -313,16 +363,21 @@ bool CDALIKVStore::fetchAllNamespaces(StringArray & namespaces, const char * sto Owned storetree = conn->getRoot(); if (global) - xpath.setf("%s/*", DALI_KVSTORE_GLOBAL); //we're interested in the children of the namespace + xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.setf("%s/*", user->getName()).toLowerCase(); //we're interested in the children of the namespace + getEncodedLowerCaseUserName(xpath.clear(), user); + + xpath.append("/*"); //we're interested in the children of the namespace StringBuffer name; Owned iter = storetree->getElements(xpath.str()); ForEach(*iter) { iter->query().getName(name.clear()); - namespaces.append(name.str()); + StringBuffer decodedName; + decodePtreeName(decodedName, name.str()); + + namespaces.append(decodedName.str()); } return true; @@ -333,15 +388,21 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchKeySet(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchKeySet(): Attempting to fetch non-global keys but requester name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchKeySet: Namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchKeySet: Unable to connect to DALI KeyValue store path '%s'", DALI_KVSTORE_PATH); @@ -351,9 +412,10 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); + + xpath.appendf("/%s/*", encodedNS.str()); //we're interested in the children of the namespace - xpath.appendf("/%s/*", ns); //we're interested in the children of the namespace if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetchKeySet: invalid namespace '%s' detected!", ns); @@ -362,7 +424,9 @@ bool CDALIKVStore::fetchKeySet(StringArray & keyset, const char * storename, con ForEach(*iter) { iter->query().getName(name.clear()); - keyset.append(name.str()); + StringBuffer decodedName; + decodePtreeName(decodedName, name.str()); + keyset.append(decodedName.str()); } return true; @@ -373,15 +437,21 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetch(): Store name not provided"); - if (!global && (!user || isEmptyString(user->getName()))) + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetch(): Attempting to fetch non-global entry but requester name not provided"); - if (isEmptyString(ns)) - throw MakeStringException(-1, "DALI Keystore fetch: key not provided!"); + if (isEmptyString(ns)) + throw MakeStringException(-1, "DALI Keystore fetch: key not provided!"); + + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetch: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -391,25 +461,23 @@ bool CDALIKVStore::fetch(const char * storename, const char * ns, const char * k if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", ns); + xpath.appendf("/%s", encodedNS.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetch: invalid namespace '%s' detected!", ns); if (key && *key) { - xpath.appendf("/%s", key); + StringBuffer encodedKey; + encodePTreeName(encodedKey, key); + xpath.appendf("/%s", encodedKey.str()); if(!storetree->hasProp(xpath.str())) - { - throw makeStringExceptionV(ECLWATCH_INVALID_QUERY_KEY, "DALI Keystore fetch: invalid key '%s' detected!", key); - } + return false; else - { value.set(storetree->queryProp(xpath.str())); - } - return value.str(); + return true; } else throw makeStringException(-1, "DALI Keystore fetch: Key not provided!"); @@ -422,15 +490,21 @@ IPropertyTree * CDALIKVStore::getAllPairs(const char * storename, const char * n if (!storename || !*storename) throw MakeStringException(-1, "DALI Keystore fetchAll(): Store name not provided"); + StringBuffer encodedStoreName; + encodePTreeName(encodedStoreName, storename); + if (!global && (!user || isEmptyString(user->getName()))) throw MakeStringException(-1, "DALI Keystore fetchAll(): Attempting to fetch non-global entries but requester name not provided"); if (isEmptyString(ns)) throw MakeStringException(-1, "DALI Keystore fetchAll: Namespace not provided!"); + StringBuffer encodedNS; + encodePTreeName(encodedNS, ns); + ensureAttachedToDali(); //throws if in offline mode - VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, storename); + VStringBuffer xpath("%s/Store[%s='%s']", DALI_KVSTORE_PATH, DALI_KVSTORE_NAME_ATT, encodedStoreName.str()); Owned conn = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT_KVSTORE); if (!conn) throw MakeStringException(-1, "DALI Keystore fetchAll: Unable to connect to DALI KeyValue store path '%s'", xpath.str()); @@ -440,9 +514,9 @@ IPropertyTree * CDALIKVStore::getAllPairs(const char * storename, const char * n if (global) xpath.set(DALI_KVSTORE_GLOBAL); else - xpath.set(user->getName()).toLowerCase(); + getEncodedLowerCaseUserName(xpath.clear(), user); - xpath.appendf("/%s", ns); + xpath.appendf("/%s", encodedNS.str()); if(!storetree->hasProp(xpath.str())) throw MakeStringException(-1, "DALI Keystore fetchAll: invalid namespace '%s' detected!", ns); diff --git a/esp/services/ws_store/ws_storeService.cpp b/esp/services/ws_store/ws_storeService.cpp index 75ddde7d6e5..24b9dfea8fb 100644 --- a/esp/services/ws_store/ws_storeService.cpp +++ b/esp/services/ws_store/ws_storeService.cpp @@ -271,6 +271,7 @@ bool CwsstoreEx::onListKeys(IEspContext &context, IEspListKeysRequest &req, IEsp StringArray keys; m_storeProvider->fetchKeySet(keys, storename, ns, secuser.get(), !req.getUserSpecific()); + resp.setKeySet(keys); resp.setNamespace(ns); resp.setStoreName(storename); @@ -312,25 +313,11 @@ bool CwsstoreEx::onFetch(IEspContext &context, IEspFetchRequest &req, IEspFetchR storename = m_defaultStore.get(); } - try - { - m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, secuser.get(), !req.getUserSpecific()); + bool success = m_storeProvider->fetch(storename, req.getNamespace(), req.getKey(), value, secuser.get(), !req.getUserSpecific()); + if (success) resp.setValue(value.str()); - } - catch(IException * e) - { - if (e->errorCode() == ECLWATCH_INVALID_QUERY_KEY) - { - StringBuffer msg; - LOG(MCuserInfo, "WsStore: %s", e->errorMessage(msg).str()); - e->Release(); - return false; - } - else - throw e; - } - return true; + return success; } bool CwsstoreEx::onFetchKeyMetadata(IEspContext &context, IEspFetchKeyMDRequest &req, IEspFetchKeyMDResponse &resp) @@ -385,22 +372,28 @@ bool CwsstoreEx::onFetchAll(IEspContext &context, IEspFetchAllRequest &req, IEsp storename = m_defaultStore.get(); } - Owned nstree = m_storeProvider->getAllPairs(storename, ns, secuser.get(), !req.getUserSpecific()); - IArrayOf pairs; - - Owned iter = nstree->getElements("*"); - ForEach(*iter) { - StringBuffer name; - StringBuffer value; - iter->query().getName(name); - nstree->getProp(name.str(), value); - - Owned kvpair = createKVPair("",""); - kvpair->setKey(name.str()); - kvpair->setValue(value.str()); - pairs.append(*kvpair.getClear()); + Owned nstree = m_storeProvider->getAllPairs(storename, ns, secuser.get(), !req.getUserSpecific()); + + Owned iter = nstree->getElements("*"); + ForEach(*iter) + { + StringBuffer name; + StringBuffer value; + iter->query().getName(name); + nstree->getProp(name.str(), value); + + Owned kvpair = createKVPair("",""); + + //it's possible this has been encoded, so decode it + StringBuffer decoded; + decodePtreeName(decoded, name.str()); + + kvpair->setKey(decoded.str()); + kvpair->setValue(value.str()); + pairs.append(*kvpair.getClear()); + } } resp.setPairs(pairs); diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 14939bd6164..08f5f953c34 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -171,7 +171,7 @@ export const Files: React.FunctionComponent = ({ }, Name: { label: nlsHPCC.LogicalName, - width: 360, + width: 180, formatter: (name, row) => { const file = Get(row.NodeGroup, name, row); if (row.__hpcc_isDir) { diff --git a/helm/hpcc/docs/expert.md b/helm/hpcc/docs/expert.md index a42fc346ba5..5d2249072e2 100644 --- a/helm/hpcc/docs/expert.md +++ b/helm/hpcc/docs/expert.md @@ -13,6 +13,8 @@ global: time: 200 interval: 75 probes: 9 + regex: + cacheSize: 500 ``` NB: Some components (e.g. DfuServer and Thor) also have an 'expert' settings area (see values schema) that can be used for relavent settings @@ -73,6 +75,13 @@ service via remote file reads with the ~remote:: syntax. Setting expert.allowForeign to true, enables foreign access for compatibility with legacy bare-metal environments that have their Dali and Dafilesrv's open. +## regex (cacheSize: unsigned) + +See the regex example above. If set, this should be added at the global level. +The default value is 500. Set to zero to disable the cache. +This value is applied at the process level: Each Thor worker, Roxie process, and hthor worker receives +its own cache. Threads/channels within a process share that process's cache. + # Plane Expert Settings diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 15eb2e607f5..bdfeaff83d4 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -77,6 +77,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali + serviceName: {{ $dali.name | quote }} updateStrategy: type: RollingUpdate rollingUpdate: diff --git a/helm/managed/logging/loki-stack/README.md b/helm/managed/logging/loki-stack/README.md index 6c4714e898c..e50a0437444 100644 --- a/helm/managed/logging/loki-stack/README.md +++ b/helm/managed/logging/loki-stack/README.md @@ -85,27 +85,87 @@ The logAccess feature allows HPCC to query and package relevant logs for various ### Provide target Grafana/Loki access information -HPCC logAccess requires access to the Grafana username/password. Those values must be provided via a secure secret object. +HPCC logAccess requires access to the Grafana username/password credentials. Those values must be provided via a secure secret object. The secret is expected to be in the 'esp' category, and be named 'grafana-logaccess'. The following key-value pairs are required (key names must be spelled exactly as shown here) username - This should contain the Grafana username password - This should contain the Grafana password +#### Create secret using script The included 'create-grafana-logaccess-secret.sh' helper can be used to create the necessary secret. -Example scripted secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys): +Example scripted secret creation command: ``` - create-grafana-logaccess-secret.sh -d HPCC-Platform/helm/managed/logging/loki-stack/secrets-templates/ -n hpcc + create-grafana-logaccess-secret.sh -u admin -p somepass -n hpcc ``` +#### Create secret manually from file Otherwise, users can create the secret manually. -Example manual secret creation command (assuming ./secrets-templates contains a file named exactly as the above keys): +Example manual secret creation command (assuming ./secrets-templates contains files named exactly as the above keys): ``` - kubectl create secret generic grafana-logaccess --from-file=HPCC-Platform/helm/managed//logging/loki-stack/secrets-templates/ -n hpcc + kubectl create secret generic grafana-logaccess --from-file=HPCC-Platform/helm/managed/logging/loki-stack/secrets-templates/ -n hpcc +``` + +#### Create secret manually from manifest +Otherwise, users can create the secret through a manifest file. + +First, base64 encode the credentials: + +``` +echo -n 'admin' | base64 +echo -n 'whatevergrafanapassword' | base64 +``` + +Add the encoded values to the provided manifest file 'grafana-logaccess-secret.yaml' + +``` +apiVersion: v1 +kind: Secret +metadata: + name: grafana-logaccess +type: Opaque +data: + #Base64 encoded username and password for Grafana + #can be encoded using the following command: + # echo -n 'admin' | base64 + username: YWRtaW4= + # echo -n 'whatevergrafanapassword' | base64 + password: d2hhdGV2ZXJncmFmYW5hcGFzc3dvcmQ= +``` + +Then apply the manifest values: + +``` +kubectl apply -f ./grafana-logaccess-secret.yaml --namespace hpcc --server-side +``` + +#### Verify secret + +At this point, confirm the secret has been created with the expected key values: + +``` +kubectl describe secret grafana-logaccess -n hpcc +``` + +The output should be something like this: + +``` +kubectl describe secret grafana-logaccess -n hpcc +Name: grafana-logaccess +Namespace: hpcc +Labels: +Annotations: + +Type: Opaque + +Data +==== +password: 40 bytes +username: 5 bytes ``` ### Configure HPCC logAccess diff --git a/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh b/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh index f4c7efbed09..e1246b1f62e 100755 --- a/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh +++ b/helm/managed/logging/loki-stack/create-grafana-logaccess-secret.sh @@ -1,30 +1,24 @@ #!/bin/bash WORK_DIR=$(dirname $0) -source ${WORK_DIR}/env-loganalytics k8scommand="kubectl" secretname="grafana-logaccess" -secretsdir="${WORK_DIR}/secrets-templates" namespace="default" +username="admin" +password="" usage() { echo "Creates necessary k8s secret used by HPCC's logAccess to access Loki data source through Grafana" echo "> create-grafana-logaccess-secret.sh [Options]" echo "" + echo "Example: create-grafana-logaccess-secret.sh -u admin -p mypassword -n mynamespace" + echo "" echo "Options:" - echo "-d Specifies directory containing required secret values in self named files." - echo " Defaults to /<${secretssubdir}>" + echo "-u Grafana user name (default: admin)" + echo "-p Grafana password (required)" echo "-h Print Usage message" - echo "-n Specifies namespace for secret" - echo "" - echo "Requires directory containing secret values in dedicated files." - echo "Defaults to ${secretssubdir} if not specified via -d option." - echo "" - echo "Expected directory structure:" - echo "${secretsdir}/" - echo " password - Should contain Grafana user name" - echo " username - Should contain Grafana password" + echo "-n Specifies namespace for secret (default: default)" } while [ "$#" -gt 0 ]; do @@ -37,6 +31,12 @@ while [ "$#" -gt 0 ]; do -d) shift secretsdir=$1 ;; + -u) shift + username=$1 + ;; + -p) shift + password=$1 + ;; -n) shift namespace=$1 ;; @@ -44,6 +44,13 @@ while [ "$#" -gt 0 ]; do shift done +if [ -z "${password}" ]; +then + echo "Error: Missing required password!" + echo >&2 + usage + exit 1 +fi echo "Creating '${namespace}/${secretname}' secret." command -v ${k8scommand} >/dev/null 2>&1 || { echo >&2 "Aborting - '${k8scommand}' not found!"; exit 1; } @@ -53,10 +60,11 @@ if [[ $? -eq 0 ]] then echo "WARNING: Target secret '${namespace}/${secretname}' already exists! Delete it and re-run if secret update desired." echo "${errormessage}" + echo "use this command: '${k8scommand} delete secret ${secretname} -n ${namespace}'" exit 1 fi -errormessage=$(${k8scommand} create secret generic ${secretname} --from-file=${secretsdir} -n ${namespace} ) +errormessage=$(${k8scommand} create secret generic ${secretname} --from-literal=username=${username} --from-literal=password=${password} -n ${namespace}) if [[ $? -ne 0 ]] then echo "Error creating: Target secret '${namespace}/${secretname}'!" diff --git a/helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml b/helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml new file mode 100644 index 00000000000..2e1cec96eae --- /dev/null +++ b/helm/managed/logging/loki-stack/grafana-logaccess-secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-logaccess +type: Opaque +data: + #Base64 encoded username and password for Grafana + #can be encoded using the following command: + # echo -n 'admin' | base64 + username: YWRtaW4= + # echo -n 'whatevergrafanapassword' | base64 + password: d2hhdGV2ZXJncmFmYW5hcGFzc3dvcmQ= \ No newline at end of file diff --git a/helm/managed/logging/loki-stack/secrets-templates/password b/helm/managed/logging/loki-stack/secrets-templates/password deleted file mode 100644 index 6b3a9a39380..00000000000 --- a/helm/managed/logging/loki-stack/secrets-templates/password +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/helm/managed/logging/loki-stack/secrets-templates/username b/helm/managed/logging/loki-stack/secrets-templates/username deleted file mode 100644 index f77b00407e0..00000000000 --- a/helm/managed/logging/loki-stack/secrets-templates/username +++ /dev/null @@ -1 +0,0 @@ -admin \ No newline at end of file diff --git a/rtl/include/eclhelper.hpp b/rtl/include/eclhelper.hpp index ef442890c8c..cc0c0c843ba 100644 --- a/rtl/include/eclhelper.hpp +++ b/rtl/include/eclhelper.hpp @@ -46,6 +46,17 @@ typedef unsigned short UChar; #endif #include "rtlconst.hpp" + +#if !defined(ECLRTL_LOCAL) +#ifdef ECLRTL_EXPORTS +#define ECLRTL_API DECL_EXPORT +#else +#define ECLRTL_API DECL_IMPORT +#endif +#else +#define ECLRTL_API +#endif + //Should be incremented whenever the virtuals in the context or a helper are changed, so //that a work unit can't be rerun. Try as hard as possible to retain compatibility. #define ACTIVITY_INTERFACE_VERSION 654 @@ -484,7 +495,7 @@ inline byte getVirtualInitializer(const void * initializer) { return (byte)(mems typedef IThorDiskCallback IVirtualFieldCallback; //Core struct used for representing meta for a field. Effectively used as an interface. -struct RtlFieldInfo +struct ECLRTL_API RtlFieldInfo { constexpr inline RtlFieldInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, unsigned _flags = 0, const char *_initializer = NULL) : name(_name), xpath(_xpath), type(_type), initializer(_initializer), flags(_type->fieldType | _flags) {} diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 78b9813c4a0..17154d4a99c 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -1453,7 +1453,13 @@ static void getMemUsage(unsigned &inuse,unsigned &active,unsigned &total,unsigne bufptr = strchr(bufptr, '\n'); } inuse = total-free-cached; - swapinuse = swaptotal-swapfree-swapcached; + + // not sure if a bug in kernel or container or ... + // but sometimes we see swapfree > 0 when swaptotal == 0 + if ((swapfree + swapcached) >= swaptotal) + swapinuse = 0; + else + swapinuse = swaptotal-swapfree-swapcached; #endif } diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index e25bbac59e7..0d33fa28a1d 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -7961,9 +7961,12 @@ unsigned __int64 getPlaneAttributeValue(const char *planeName, PlaneAttributeTyp CriticalBlock b(planeAttriubuteMapCrit); auto it = planeAttributesMap.find(planeName); if (it != planeAttributesMap.end()) - return it->second[planeAttrType]; - else - return defaultValue; + { + unsigned v = it->second[planeAttrType]; + if (v) // a plane attribute value of 0 is considered as not set + return v; + } + return defaultValue; } size32_t getBlockedFileIOSize(const char *planeName, size32_t defaultSize) diff --git a/system/jlib/jsmartsock.cpp b/system/jlib/jsmartsock.cpp index e2a080e7a51..cc11e40a370 100644 --- a/system/jlib/jsmartsock.cpp +++ b/system/jlib/jsmartsock.cpp @@ -30,6 +30,7 @@ class SmartSocketListParser public: SmartSocketListParser(const char * text) { + assertex(text); fullText = strdup(text); } @@ -235,11 +236,9 @@ CSmartSocketFactory::CSmartSocketFactory(IPropertyTree &service, bool _retry, un dnsInterval=_dnsInterval; retry = _retry; + retryInterval = _retryInterval; if (retry) - { - retryInterval = _retryInterval; this->start(false); - } } CSmartSocketFactory::CSmartSocketFactory(const char *_socklist, IPropertyTree* _tlsConfig, bool _retry, unsigned _retryInterval, unsigned _dnsInterval) @@ -261,11 +260,9 @@ CSmartSocketFactory::CSmartSocketFactory(const char *_socklist, IPropertyTree* _ dnsInterval=_dnsInterval; retry = _retry; + retryInterval = _retryInterval; if (retry) - { - retryInterval = _retryInterval; this->start(false); - } } CSmartSocketFactory::~CSmartSocketFactory() diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h index aa133ad5ba2..9e034906601 100644 --- a/system/jlib/jstatcodes.h +++ b/system/jlib/jstatcodes.h @@ -311,6 +311,7 @@ enum StatisticKind StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax, + StNumParallelExecute, StMax, //For any quantity there is potentially the following variants. diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp index 92f02d1bdec..ac6352ddc80 100644 --- a/system/jlib/jstats.cpp +++ b/system/jlib/jstats.cpp @@ -983,6 +983,7 @@ static const constexpr StatisticMeta statsMetaData[StMax] = { { NUMSTAT(MatchRightRowsMax), "The largest number of right rows in a join group" }, { NUMSTAT(MatchCandidates), "The number of candidate combinations of left and right rows forming join groups" }, { NUMSTAT(MatchCandidatesMax), "The largest number of candidate combinations of left and right rows in a single group" }, + { NUMSTAT(ParallelExecute), "The number of parallel execution paths for this activity" }, }; static MapStringTo statisticNameMap(true); diff --git a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp index 5ada0237838..8e58aa8c596 100644 --- a/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp +++ b/system/logaccess/Grafana/CurlClient/GrafanaCurlClient.cpp @@ -547,6 +547,9 @@ const char * sortByDirection(SortByDirection direction) */ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, const LogAccessConditions & options, StringBuffer & returnbuf, LogAccessLogFormat format) { + if (m_dataSourcesAPIURI.isEmpty()) + throw makeStringExceptionV(-1, "%s: Cannot query because Grafana datasource was not established, check logaccess configuration!", COMPONENT_NAME); + try { resultDetails.totalReceived = 0; @@ -651,7 +654,7 @@ bool GrafanaLogAccessCurlClient::fetchLog(LogQueryResultDetails & resultDetails, encodeURL(fullQuery, logLineParser.str()); fullQuery.appendf("&start=%s000000000", std::to_string(trange.getStartt().getSimple()).c_str()); - if (trange.getEndt().isNull() != -1) //aka 'to' has been initialized + if (!trange.getEndt().isNull()) //aka 'to' has been initialized { fullQuery.appendf("&end=%s000000000", std::to_string(trange.getEndt().getSimple()).c_str()); } @@ -729,12 +732,30 @@ GrafanaLogAccessCurlClient::GrafanaLogAccessCurlClient(IPropertyTree & logAccess m_grafanaPassword.set(logAccessPluginConfig.queryProp("connection/@password")); } - //this is very important, without this, we can't target the correct datasource - fetchDatasourceByName(m_targetDataSource.name.get()); + try + { + //this is very important, without this, we can't target the correct datasource + fetchDatasourceByName(m_targetDataSource.name.get()); + } + catch(IException * e) + { + StringBuffer description; + OERRLOG("%s: Exception fetching Loki/Grafana datasource!!: (%d) - %s", COMPONENT_NAME, e->errorCode(), e->errorMessage(description).str()); + e->Release(); + } - std::string availableLabels; - fetchLabels(availableLabels); - DBGLOG("%s: Available labels on target loki/grafana: %s", COMPONENT_NAME, availableLabels.c_str()); + try + { + std::string availableLabels; + fetchLabels(availableLabels); + DBGLOG("%s: Available labels on target loki/grafana: %s", COMPONENT_NAME, availableLabels.c_str()); + } + catch(IException * e) + { + StringBuffer description; + OERRLOG("%s: Exception fetching available labels: (%d) - %s", COMPONENT_NAME, e->errorCode(), e->errorMessage(description).str()); + e->Release(); + } m_expectedLogFormat = defaultExpectedLogFormat; if (logAccessPluginConfig.hasProp("logFormat/@type")) diff --git a/testing/unittests/jstreamtests.cpp b/testing/unittests/jstreamtests.cpp index 7e13a495e04..dda4523835d 100644 --- a/testing/unittests/jstreamtests.cpp +++ b/testing/unittests/jstreamtests.cpp @@ -510,7 +510,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture public: CDataProvider & dataProvider; IBufferedSerialOutputStream * out; - offset_t totalSent; + offset_t totalSent = 0; unsigned numRows; }; diff --git a/thorlcr/activities/loop/thloop.cpp b/thorlcr/activities/loop/thloop.cpp index ebdde73be06..b32da6e09ab 100644 --- a/thorlcr/activities/loop/thloop.cpp +++ b/thorlcr/activities/loop/thloop.cpp @@ -334,7 +334,7 @@ class CLocalResultActivityMasterBase : public CMasterActivity Owned inputRowIf; public: - CLocalResultActivityMasterBase(CMasterGraphElement *info) : CMasterActivity(info) + CLocalResultActivityMasterBase(CMasterGraphElement *info) : CMasterActivity(info, spillingActivityStatistics) { } virtual void init() override diff --git a/thorlcr/activities/loop/thloopslave.cpp b/thorlcr/activities/loop/thloopslave.cpp index 722358f0e4d..ea7d149ff89 100644 --- a/thorlcr/activities/loop/thloopslave.cpp +++ b/thorlcr/activities/loop/thloopslave.cpp @@ -643,7 +643,7 @@ class CLocalResultSpillActivity : public CSlaveActivity } public: - CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container) + CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container, spillingActivityStatistics) { helper = (IHThorLocalResultSpillArg *)queryHelper(); appendOutputLinked(this); @@ -708,7 +708,7 @@ class CLocalResultSpillActivity : public CSlaveActivity class CLocalResultWriteActivityBase : public ProcessSlaveActivity { public: - CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container) + CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container, spillingActivityStatistics) { } virtual IThorResult *createResult() = 0; diff --git a/thorlcr/activities/nsplitter/thnsplitterslave.cpp b/thorlcr/activities/nsplitter/thnsplitterslave.cpp index ca9ccd0d6df..5331fedf845 100644 --- a/thorlcr/activities/nsplitter/thnsplitterslave.cpp +++ b/thorlcr/activities/nsplitter/thnsplitterslave.cpp @@ -152,7 +152,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf } } public: - NSplitterSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, nsplitterActivityStatistics), writer(*this) + NSplitterSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, spillingActivityStatistics), writer(*this) { numOutputs = container.getOutputs(); connectedOutputSet.setown(createBitSet()); diff --git a/thorlcr/graph/thgraph.hpp b/thorlcr/graph/thgraph.hpp index b0adfbade47..b1045fda17a 100644 --- a/thorlcr/graph/thgraph.hpp +++ b/thorlcr/graph/thgraph.hpp @@ -630,7 +630,7 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults CChildGraphTable childGraphsTable; CGraphStubArrayCopy orderedChildGraphs; Owned tmpHandler; - + Owned tempFileSizeTracker; void clean(); protected: @@ -805,7 +805,24 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults virtual void end(); virtual void abort(IException *e) override; virtual IThorGraphResults *createThorGraphResults(unsigned num); - + CFileSizeTracker * queryTempFileSizeTracker() + { + if (!tempFileSizeTracker) + tempFileSizeTracker.setown(new CFileSizeTracker); + return tempFileSizeTracker; + } + offset_t queryPeakTempSize() + { + if (tempFileSizeTracker) + return tempFileSizeTracker->queryPeakSize(); + return 0; + } + offset_t queryActiveTempSize() + { + if (tempFileSizeTracker) + return tempFileSizeTracker->queryActiveSize(); + return 0; + } // IExceptionHandler virtual bool fireException(IException *e); @@ -1175,7 +1192,7 @@ class graph_decl CActivityBase : implements CInterfaceOf, im CFileSizeTracker * queryTempFileSizeTracker() { if (!tempFileSizeTracker) - tempFileSizeTracker.setown(new CFileSizeTracker); + tempFileSizeTracker.setown(new CFileSizeTracker(queryGraph().queryParent()->queryTempFileSizeTracker())); return tempFileSizeTracker; } offset_t queryActiveTempSize() const diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 14c8d7ea6d0..e39c45909d5 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1323,20 +1323,7 @@ bool CSlaveGraph::serializeStats(MemoryBuffer &mb) if (tempHandler) stats.mergeStatistic(StSizeGraphSpill, sizeGraphSpill); - // calculate peak spill size - if (started&&initialized) - { - offset_t activeTempSize = 0; - Owned iter = getConnectedIterator(); - ForEach (*iter) - { - CGraphElementBase &element = iter->query(); - CSlaveActivity &activity = (CSlaveActivity &)*element.queryActivity(); - activeTempSize += activity.queryActiveTempSize(); - } - if (activeTempSize > peakTempSize) - peakTempSize = activeTempSize; - } + offset_t peakTempSize = queryPeakTempSize(); if (peakTempSize) stats.mergeStatistic(StSizePeakTempDisk, peakTempSize); if (peakTempSize + sizeGraphSpill) diff --git a/thorlcr/graph/thgraphslave.hpp b/thorlcr/graph/thgraphslave.hpp index f6b73c912c1..95edc9ea4d0 100644 --- a/thorlcr/graph/thgraphslave.hpp +++ b/thorlcr/graph/thgraphslave.hpp @@ -471,6 +471,7 @@ class graphslave_decl CThorStrandProcessor : public CInterfaceOf strands; @@ -496,7 +497,12 @@ class graphslave_decl CThorStrandedActivity : public CSlaveActivity virtual CThorStrandProcessor *createStrandSourceProcessor(bool inputOrdered) = 0; inline unsigned numStrands() const { return strands.ordinality(); } - +// CSlaveActivity + virtual void gatherActiveStats(CRuntimeStatisticCollection &activeStats) const override + { + PARENT::gatherActiveStats(activeStats); + activeStats.addStatistic(StNumParallelExecute, numStrands()); + } // IThorDataLink virtual IStrandJunction *getOutputStreams(CActivityBase &_ctx, unsigned idx, PointerArrayOf &streams, const CThorStrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override; virtual unsigned __int64 queryTotalCycles() const override; @@ -522,7 +528,6 @@ class graphslave_decl CSlaveGraph : public CGraphBase bool doneInit = false; std::atomic_bool progressActive; ProcessInfo processStartInfo; - offset_t peakTempSize = 0; public: diff --git a/thorlcr/master/thactivitymaster.cpp b/thorlcr/master/thactivitymaster.cpp index f80ee0bd258..86210a47c87 100644 --- a/thorlcr/master/thactivitymaster.cpp +++ b/thorlcr/master/thactivitymaster.cpp @@ -210,7 +210,7 @@ class CGenericMasterGraphElement : public CMasterGraphElement ret = new CMasterActivity(this); break; case TAKsplit: - ret = new CMasterActivity(this, nsplitterActivityStatistics); + ret = new CMasterActivity(this, spillingActivityStatistics); break; case TAKsoap_rowdataset: case TAKsoap_rowaction: diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 55afb16704a..7e58fa50ea1 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -257,8 +257,8 @@ class CWriteIntercept : public CSimpleInterface } output->flush(); offset_t end = output->getPosition(); + dataFile->noteSize(output->getStatistic(StSizeDiskWrite)); output.clear(); - dataFile->noteSize(end); writeidxofs(end); if (idxFileIO) { diff --git a/thorlcr/thorutil/thmem.cpp b/thorlcr/thorutil/thmem.cpp index 299f1e3118a..6f2c7e4fcfd 100644 --- a/thorlcr/thorutil/thmem.cpp +++ b/thorlcr/thorutil/thmem.cpp @@ -336,6 +336,11 @@ class CSharedSpillableRowSet : public CSpillableStreamBase block.clearCB = true; assertex(((offset_t)-1) != outputOffset); unsigned rwFlags = DEFAULT_RWFLAGS | mapESRToRWFlags(owner->emptyRowSemantics); + if (owner->spillCompInfo) + { + rwFlags |= rw_compress; + rwFlags |= owner->spillCompInfo; + } spillStream.setown(::createRowStreamEx(&(owner->spillFile->queryIFile()), owner->rowIf, outputOffset, (offset_t)-1, (unsigned __int64)-1, rwFlags)); owner->rows.unregisterWriteCallback(*this); // no longer needed ret = spillStream->nextRow(); @@ -376,9 +381,10 @@ class CSharedSpillableRowSet : public CSpillableStreamBase }; public: - CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority) + CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillCompInfo, unsigned _spillPriority) : CSpillableStreamBase(_activity, inRows, _rowIf, _emptyRowSemantics, _spillPriority) { + spillCompInfo = _spillCompInfo; } IRowStream *createRowStream() { @@ -387,6 +393,11 @@ class CSharedSpillableRowSet : public CSpillableStreamBase { block.clearCB = true; unsigned rwFlags = DEFAULT_RWFLAGS | mapESRToRWFlags(emptyRowSemantics); + if (spillCompInfo) + { + rwFlags |= rw_compress; + rwFlags |= spillCompInfo; + } return ::createRowStream(&spillFile->queryIFile(), rowIf, rwFlags); } rowidx_t toRead = rows.numCommitted(); @@ -1380,10 +1391,7 @@ rowidx_t CThorSpillableRowArray::save(CFileOwner &iFileOwner, unsigned _spillCom rowidx_t n = numCommitted(); if (0 == n) return 0; - ActPrintLog(&activity, "%s: CThorSpillableRowArray::save (skipNulls=%s, emptyRowSemantics=%u) max rows = %" RIPF "u", _tracingPrefix, boolToStr(skipNulls), emptyRowSemantics, n); - - if (_spillCompInfo) - assertex(0 == writeCallbacks.ordinality()); // incompatible + ActPrintLog(&activity, "%s: CThorSpillableRowArray::save (skipNulls=%s, emptyRowSemantics=%u) max rows = %" RIPF "u", _tracingPrefix, boolToStr(skipNulls), emptyRowSemantics, n); unsigned rwFlags = DEFAULT_RWFLAGS; if (_spillCompInfo) @@ -1773,7 +1781,7 @@ class CThorRowCollectorBase : public CSpillable * because roxiemem's background thread may be blocked on that lock, and calling roxiemem::addRowBuffer here would cause deadlock */ activateSharedCallback = true; - spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, emptyRowSemantics, spillPriority)); + spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, emptyRowSemantics, spillCompInfo, spillPriority)); spillableRowSet->setTracingPrefix(tracingPrefix); } } diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 09b3fc233af..0781cee53d4 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -75,15 +75,14 @@ static Owned ClusterMPAllocator; // stat. mappings shared between master and slave activities const StatisticsMapping spillStatistics({StTimeSpillElapsed, StTimeSortElapsed, StNumSpills, StSizeSpillFile, StSizePeakTempDisk}); const StatisticsMapping soapcallStatistics({StTimeSoapcall}); -const StatisticsMapping basicActivityStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked}); +const StatisticsMapping basicActivityStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked, StNumParallelExecute}); const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics); -const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, basicActivityStatistics); const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jhtreeCacheStatistics); const StatisticsMapping indexReadActivityStatistics({StNumRowsProcessed}, indexReadFileStatistics, basicActivityStatistics); const StatisticsMapping indexWriteActivityStatistics({StPerReplicated, StNumLeafCacheAdds, StNumNodeCacheAdds, StNumBlobCacheAdds }, basicActivityStatistics, diskWriteRemoteStatistics); const StatisticsMapping keyedJoinActivityStatistics({ StNumIndexAccepted, StNumPreFiltered, StNumDiskSeeks, StNumDiskAccepted, StNumDiskRejected}, basicActivityStatistics, indexReadFileStatistics); -const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics); const StatisticsMapping commonJoinActivityStatistics({StNumMatchLeftRowsMax, StNumMatchRightRowsMax, StNumMatchCandidates, StNumMatchCandidatesMax}, basicActivityStatistics); +const StatisticsMapping hashJoinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics); const StatisticsMapping allJoinActivityStatistics({}, commonJoinActivityStatistics); const StatisticsMapping lookupJoinActivityStatistics({StNumSmartJoinSlavesDegradedToStd, StNumSmartJoinDegradedToLocal}, spillStatistics, commonJoinActivityStatistics); const StatisticsMapping joinActivityStatistics({StNumLeftRows, StNumRightRows}, commonJoinActivityStatistics, spillStatistics); @@ -96,8 +95,9 @@ const StatisticsMapping indexDistribActivityStatistics({}, basicActivityStatisti const StatisticsMapping soapcallActivityStatistics({}, basicActivityStatistics, soapcallStatistics); const StatisticsMapping hashDedupActivityStatistics({}, spillStatistics, diskWriteRemoteStatistics, basicActivityStatistics); const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics); -const StatisticsMapping nsplitterActivityStatistics({}, spillStatistics, basicActivityStatistics); +const StatisticsMapping spillingActivityStatistics({}, spillStatistics, basicActivityStatistics); const StatisticsMapping spillingWriteAheadStatistics({}, spillStatistics); +const StatisticsMapping loopActivityStatistics({StNumIterations}, spillingActivityStatistics); MODULE_INIT(INIT_PRIORITY_STANDARD) { diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index cf0b92bb4b0..fc45b30e472 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -167,7 +167,7 @@ extern graph_decl const StatisticsMapping soapcallActivityStatistics; extern graph_decl const StatisticsMapping indexReadFileStatistics; extern graph_decl const StatisticsMapping hashDedupActivityStatistics; extern graph_decl const StatisticsMapping hashDistribActivityStatistics; -extern graph_decl const StatisticsMapping nsplitterActivityStatistics; +extern graph_decl const StatisticsMapping spillingActivityStatistics; extern graph_decl const StatisticsMapping spillingWriteAheadStatistics; class BooleanOnOff @@ -326,19 +326,29 @@ class CFileSizeTracker: public CInterface { RelaxedAtomic activeSize{0}; RelaxedAtomic peakSize{0}; + CFileSizeTracker * parentFileSizeTracker; public: + CFileSizeTracker(CFileSizeTracker *parent=nullptr): parentFileSizeTracker(parent) + { + } void growSize(offset_t size) { if (size) { offset_t newActiveSize = activeSize.add_fetch(size); peakSize.store_max(newActiveSize); + if (parentFileSizeTracker) + parentFileSizeTracker->growSize(size); } } void shrinkSize(offset_t size) { if (size) + { activeSize.fetch_sub(size); + if (parentFileSizeTracker) + parentFileSizeTracker->shrinkSize(size); + } } offset_t queryActiveSize() const {