diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 992737387f8..d4281be7525 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -436,11 +436,6 @@ jobs: strategy: matrix: include: - - os: "windows-2022" - mono: "" - sudo: "" - cmake_config_options: '-T host=x64 -A Win32 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' - cmake_build_options: "--config Release --parallel $NUMBER_OF_PROCESSORS" - os: "windows-2022" mono: "" sudo: "" diff --git a/CMakeLists.txt b/CMakeLists.txt index 621f77deaaa..663d17c0648 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ if ( PLUGIN ) HPCC_ADD_SUBDIRECTORY (system/security/securesocket) HPCC_ADD_SUBDIRECTORY (system/security/zcrypt) HPCC_ADD_SUBDIRECTORY (dali/base) + HPCC_ADD_SUBDIRECTORY (plugins/eclblas "ECLBLAS") HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED") HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED") HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED") diff --git a/CMakePresets.json b/CMakePresets.json.sample similarity index 100% rename from CMakePresets.json rename to CMakePresets.json.sample diff --git a/common/dllserver/dllserver.cpp b/common/dllserver/dllserver.cpp index 028ee42c324..46cef2609f4 100644 --- a/common/dllserver/dllserver.cpp +++ b/common/dllserver/dllserver.cpp @@ -632,7 +632,7 @@ ILoadedDllEntry * DllServerBase::doLoadDll(const char * name, DllLocationType ty location->getDllFilename(rfile); StringBuffer x; rfile.getPath(x); - LOG(MCdebugInfo, unknownJob, "Loading dll (%s) from location %s", name, x.str()); + LOG(MCdebugInfo, "Loading dll (%s) from location %s", name, x.str()); return createDllEntry(x.str(), false, NULL, resourcesOnly); } diff --git a/common/environment/environment.cpp b/common/environment/environment.cpp index d3d33cf2a2e..53ef6a8e44d 100644 --- a/common/environment/environment.cpp +++ b/common/environment/environment.cpp @@ -1877,7 +1877,7 @@ bool CLocalEnvironment::getRunInfo(IStringVal & path, IStringVal & dir, const ch Owned machine = getMachineByAddress(machineaddr); if (!machine) { - LOG(MCdebugInfo, unknownJob, "Unable to find machine for %s", machineaddr); + LOG(MCdebugInfo, "Unable to find machine for %s", machineaddr); return false; } @@ -1885,7 +1885,7 @@ bool CLocalEnvironment::getRunInfo(IStringVal & path, IStringVal & dir, const ch Owned domain = machine->getDomain(); if (!domain) { - LOG(MCdebugInfo, unknownJob, "Unable to find domain for %s", machineaddr); + LOG(MCdebugInfo, "Unable to find domain for %s", machineaddr); return false; } @@ -1896,7 +1896,7 @@ bool CLocalEnvironment::getRunInfo(IStringVal & path, IStringVal & dir, const ch Owned instance = getInstance(tag, version, targetdomain); if (!instance) { - LOG(MCdebugInfo, unknownJob, "Unable to find process %s for domain %s", tag, targetdomain.get()); + LOG(MCdebugInfo, "Unable to find process %s for domain %s", tag, targetdomain.get()); return false; } return instance->getRunInfo(path,dir,defprogname); @@ -1944,7 +1944,7 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * ne unsigned dropzonePathLen = _MAX_PATH + 1; #ifdef _DEBUG - LOG(MCdebugInfo, unknownJob, "Netaddress: '%s', targetFilePath: '%s'", netaddress, targetFilePath); + LOG(MCdebugInfo, "Netaddress: '%s', targetFilePath: '%s'", netaddress, targetFilePath); #endif // Check the directory path first @@ -1973,7 +1973,7 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * ne #ifdef _DEBUG StringBuffer serverIpString; serverIP.getHostText(serverIpString); - LOG(MCdebugInfo, unknownJob, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str()); + LOG(MCdebugInfo, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str()); #endif if (strisame(netaddress, dropzoneServer) || targetIp.ipequals(serverIP)) { @@ -2551,7 +2551,7 @@ unsigned getAccessibleServiceURLList(const char *serviceType, std::vector espBindingIter = espProcessIter->query().getElements("EspBinding"); ForEach(*espBindingIter) { - xpath.setf("Software/EspService[@name=\"%s\"]/Properties/@type", espBindingIter->query().queryProp("@service")); + xpath.setf("Software/EspService[@name=\"%s\"]/Properties/@type", espBindingIter->query().queryProp("@service")); if (strisame(env->queryProp(xpath), serviceType)) { diff --git a/common/remote/rmtspawn.cpp b/common/remote/rmtspawn.cpp index 64f213b1fcb..fb029ab023c 100644 --- a/common/remote/rmtspawn.cpp +++ b/common/remote/rmtspawn.cpp @@ -137,7 +137,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint if (abort && abort->abortRequested()) { - LOG(MCdetailDebugInfo, unknownJob, "Action aborted before connecting to slave (%3d)", replyTag); + LOG(MCdetailDebugInfo, "Action aborted before connecting to slave (%3d)", replyTag); return NULL; } @@ -169,7 +169,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint unsigned attempts = 20; SocketEndpoint connectEP(childEP); connectEP.port = port; - LOG(MCdetailDebugInfo, unknownJob, "Start connect to correct slave (%3d)", replyTag); + LOG(MCdetailDebugInfo, "Start connect to correct slave (%3d)", replyTag); IException * error = NULL; ISocket * result = NULL; while (!result && attempts) @@ -178,7 +178,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint { StringBuffer tmp; connectEP.getEndpointHostText(tmp); - LOG(MCdetailDebugInfo, unknownJob, "Try to connect to slave %s",tmp.str()); + LOG(MCdetailDebugInfo, "Try to connect to slave %s",tmp.str()); Owned socket = ISocket::connect_wait(connectEP,MASTER_CONNECT_SLAVE_TIMEOUT); if (socket) { @@ -202,7 +202,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint if (connected) { assertex(slaveTag == replyTag); - LOG(MCdetailDebugInfo, unknownJob, "Connected to correct slave (%3d)", replyTag); + LOG(MCdetailDebugInfo, "Connected to correct slave (%3d)", replyTag); result = socket.getClear(); break; } @@ -218,9 +218,9 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint break; } if (slaveKind != kind) - LOG(MCdetailDebugInfo, unknownJob, "Connected to wrong kind of slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); + LOG(MCdetailDebugInfo, "Connected to wrong kind of slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); else - LOG(MCdetailDebugInfo, unknownJob, "Failed to connect to correct slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); + LOG(MCdetailDebugInfo, "Failed to connect to correct slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag); //Wrong slave listening, need to leave time for the other, don't count as an attempt MilliSleep(fastRand() % 5000 + 5000); @@ -230,7 +230,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint StringBuffer s; s.appendf("Retry after exception talking to slave (%d): ",replyTag); e->errorMessage(s); - LOG(MCdetailDebugInfo, unknownJob, "%s", s.str()); + LOG(MCdetailDebugInfo, "%s", s.str()); e->Release(); //Probably another element just connected, and the listening socket has just been killed. //So try again. Wait just long enough to give another thread a chance. @@ -241,7 +241,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint catch (IException * e) { StringBuffer s; - LOG(MCdetailDebugInfo, unknownJob, e, s.appendf("Failed to connect to slave (%d) (try again): ", replyTag).str()); + LOG(MCdetailDebugInfo, e, s.appendf("Failed to connect to slave (%d) (try again): ", replyTag).str()); e->Release(); // No socket listening or contention - try again fairly soon MilliSleep(fastRand()%400+100); @@ -288,7 +288,7 @@ bool CRemoteParentInfo::processCommandLine(int argc, const char * * argv, String void CRemoteParentInfo::log() { StringBuffer temp; - LOG(MCdebugProgress, unknownJob, "Starting remote slave. Master=%s reply=%d port=%d", parent.getEndpointHostText(temp).str(), replyTag, port); + LOG(MCdebugProgress, "Starting remote slave. Master=%s reply=%d port=%d", parent.getEndpointHostText(temp).str(), replyTag, port); } bool CRemoteParentInfo::sendReply(unsigned version) @@ -300,7 +300,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) { try { - LOG(MCdetailDebugInfo, unknownJob, "Ready to listen. reply=%d port=%d", replyTag, port); + LOG(MCdetailDebugInfo, "Ready to listen. reply=%d port=%d", replyTag, port); Owned listen = ISocket::create(port, 1); if (listen) { @@ -313,11 +313,11 @@ bool CRemoteParentInfo::sendReply(unsigned version) { try { - LOG(MCdetailDebugInfo, unknownJob, "Ready to accept connection. reply=%d", replyTag); + LOG(MCdetailDebugInfo, "Ready to accept connection. reply=%d", replyTag); if (!listen->wait_read(SLAVE_LISTEN_FOR_MASTER_TIMEOUT)) { - LOG(MCdetailDebugInfo, unknownJob, "Gave up waiting for a connection. reply=%d", replyTag); + LOG(MCdetailDebugInfo, "Gave up waiting for a connection. reply=%d", replyTag); return false; } @@ -333,7 +333,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) buffer.read(connectTag); masterIP.getHostText(masterIPtext.clear()); - LOG(MCdetailDebugInfo, unknownJob, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str()); + LOG(MCdetailDebugInfo, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str()); same = (kind == connectKind) && masterIP.ipequals(parent) && (connectTag == replyTag); } @@ -344,7 +344,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) //can remove when all .exes have new code. if (connectKind != kind) { - LOG(MCdetailDebugInfo, unknownJob, "Connection for wrong slave kind (%u vs %u)- ignore", connectKind, kind); + LOG(MCdetailDebugInfo, "Connection for wrong slave kind (%u vs %u)- ignore", connectKind, kind); } } @@ -358,12 +358,12 @@ bool CRemoteParentInfo::sendReply(unsigned version) if (same) { socket.setown(connect.getClear()); - LOG(MCdetailDebugInfo, unknownJob, "Connection matched - continue...."); + LOG(MCdetailDebugInfo, "Connection matched - continue...."); return true; } if ((connectKind == kind) && (version != connectVersion)) { - LOG(MCdebugInfo, unknownJob, "Version mismatch - terminating slave process expected %d got %d", version, connectVersion); + LOG(MCdebugInfo, "Version mismatch - terminating slave process expected %d got %d", version, connectVersion); return false; } } @@ -419,7 +419,7 @@ void CRemoteSlave::run(int argc, const char * * argv) logFile.append(".log"); attachStandardFileLogMsgMonitor(logFile.str(), 0, MSGFIELD_STANDARD, MSGAUD_all, MSGCLS_all, TopDetail, LOGFORMAT_table, true, true); queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler()); // no point logging output to screen if run remote! - LOG(MCdebugProgress, unknownJob, "Starting %s %s %s %s %s %s %s",slaveName.get(),(argc>1)?argv[1]:"",(argc>2)?argv[2]:"",(argc>3)?argv[3]:"",(argc>4)?argv[4]:"",(argc>5)?argv[5]:"",(argc>6)?argv[6]:""); + LOG(MCdebugProgress, "Starting %s %s %s %s %s %s %s",slaveName.get(),(argc>1)?argv[1]:"",(argc>2)?argv[2]:"",(argc>3)?argv[3]:"",(argc>4)?argv[4]:"",(argc>5)?argv[5]:"",(argc>6)?argv[6]:""); #else setupContainerizedLogMsgHandler(); #endif @@ -485,7 +485,7 @@ void CRemoteSlave::run(int argc, const char * * argv) } catch (RELEASE_CATCH_ALL) { - LOG(MCwarning, unknownJob, "Server seems to have crashed - close done gracefully"); + LOG(MCwarning, "Server seems to have crashed - close done gracefully"); error.setown(MakeStringException(999, "Server seems to have crashed - close done gracefully")); } @@ -495,25 +495,25 @@ void CRemoteSlave::run(int argc, const char * * argv) msg.append(results); catchWriteBuffer(masterSocket, msg); - LOG(MCdebugProgress, unknownJob, "Results sent from slave %d", info.replyTag); + LOG(MCdebugProgress, "Results sent from slave %d", info.replyTag); //Acknowledgement before closing down... msg.clear(); if (catchReadBuffer(masterSocket, msg, RMTTIME_RESPONSE_MASTER)) { - LOG(MCdebugProgress, unknownJob, "Terminate acknowledgement received from master for slave %d", info.replyTag); + LOG(MCdebugProgress, "Terminate acknowledgement received from master for slave %d", info.replyTag); msg.read(ok); assertex(ok); } else - LOG(MCdebugProgress, unknownJob, "No terminate acknowledgement received from master for slave %d", info.replyTag); + LOG(MCdebugProgress, "No terminate acknowledgement received from master for slave %d", info.replyTag); if (error) break; timeOut = 24*60*60*1000; } while (stayAlive); - LOG(MCdebugProgress, unknownJob, "Terminate acknowledgement received from master for slave %d", info.replyTag); + LOG(MCdebugProgress, "Terminate acknowledgement received from master for slave %d", info.replyTag); } } catch (IException * e) @@ -522,7 +522,7 @@ void CRemoteSlave::run(int argc, const char * * argv) e->Release(); } } - LOG(MCdebugProgress, unknownJob, "Stopping %s", slaveName.get()); + LOG(MCdebugProgress, "Stopping %s", slaveName.get()); } @@ -570,7 +570,7 @@ void checkForRemoteAbort(ISocket * socket) bool sendRemoteAbort(ISocket * socket) { - LOG(MCdebugInfo, unknownJob, "Send abort to remote slave (%d)", isAborting()); + LOG(MCdebugInfo, "Send abort to remote slave (%d)", isAborting()); MemoryBuffer msg; msg.append(true); diff --git a/common/thorhelper/roxiehelper.ipp b/common/thorhelper/roxiehelper.ipp index 33eaa5f662c..398c9c3902c 100644 --- a/common/thorhelper/roxiehelper.ipp +++ b/common/thorhelper/roxiehelper.ipp @@ -61,12 +61,12 @@ interface IRoxieContextLogger : extends IContextLogger { // Override base interface with versions that add prefix // We could consider moving some or all of these down into IContextLogger - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { StringBuffer text, prefix; getLogPrefix(prefix); text.valist_appendf(format, args); - CTXLOGa(LOG_TRACING, cat, job, code, prefix.str(), text.str()); + CTXLOGa(LOG_TRACING, cat, code, prefix.str(), text.str()); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -77,7 +77,7 @@ interface IRoxieContextLogger : extends IContextLogger virtual StringBuffer &getLogPrefix(StringBuffer &ret) const = 0; virtual bool isIntercepted() const = 0; - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const = 0; + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const = 0; void CTXLOGae(IException *E, const char *file, unsigned line, const char *prefix, const char *format, ...) const __attribute__((format(printf, 6, 7))); virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const = 0; virtual void CTXLOGl(LogItem *) const = 0; diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp index 2c3a63452e1..3381bc08316 100644 --- a/common/thorhelper/thorcommon.hpp +++ b/common/thorhelper/thorcommon.hpp @@ -398,14 +398,14 @@ class CStatsContextLogger : public CSimpleInterfaceOf Owned activeSpan = getNullSpan(); mutable CRuntimeStatisticCollection stats; public: - CStatsContextLogger(const CRuntimeStatisticCollection &_mapping, const LogMsgJobInfo & _job=unknownJob) : job(_job), stats(_mapping) {} + CStatsContextLogger(const CRuntimeStatisticCollection &_mapping) : stats(_mapping) {} void reset() { stats.reset(); } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { - VALOG(cat, job, code, format, args); + VALOG(cat, code, format, args); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -419,7 +419,7 @@ class CStatsContextLogger : public CSimpleInterfaceOf E->errorMessage(ss.append(": ")); if (format) ss.append(": ").valist_appendf(format, args); - LOG(MCoperatorProgress, queryJob(), "%s", ss.str()); + LOG(MCoperatorProgress, "%s", ss.str()); } virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const override { @@ -485,7 +485,6 @@ class CStatsContextLogger : public CSimpleInterfaceOf { previous.updateDelta(to, stats); } - virtual const LogMsgJobInfo & queryJob() const override { return job; } }; extern THORHELPER_API bool isActivitySink(ThorActivityKind kind); diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index c8707a01ad6..50d71b2bbcc 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -1950,7 +1950,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (httpheaders && *httpheaders) { if (soapTraceLevel > 6 || master->logXML) - master->logctx.mCTXLOG("%s: Adding HTTP Headers(%s)", master->wscCallTypeText(), httpheaders); + master->logctx.mCTXLOG("%s: Adding HTTP Headers(%s)", master->wscCallTypeText(), httpheaders); request.append(httpheaders); } diff --git a/common/thorhelper/thortparse.cpp b/common/thorhelper/thortparse.cpp index 8271bd476c4..b13f368274f 100644 --- a/common/thorhelper/thortparse.cpp +++ b/common/thorhelper/thortparse.cpp @@ -574,7 +574,7 @@ bool LRActiveState::mergePackedNode(unsigned stateId, StackElement * next, bool { cur->shifted.setown(curSymbol->createMerged(nextSymbol)); #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Nodes Merged: %p = %p, %p", cur->shifted.get(), curSymbol, nextSymbol); + LOG(MCdebugProgress, "Nodes Merged: %p = %p, %p", cur->shifted.get(), curSymbol, nextSymbol); #endif return true; } @@ -704,11 +704,11 @@ StackElement * LRParser::createState(StackElement * prev, state_id nextState, Gr newState = new StackElement(shifted, nextState, prev, this); } #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "%p: Push state %d symbol %d[%p] previous: %d[%p]", newState, nextState, shifted ? shifted->id : -1, shifted, prev ? prev->state : -1, prev); + LOG(MCdebugProgress, "%p: Push state %d symbol %d[%p] previous: %d[%p]", newState, nextState, shifted ? shifted->id : -1, shifted, prev ? prev->state : -1, prev); StringBuffer s; newState->getDebugText(s); s.newline(); - LOG(MCdebugProgress, unknownJob, s.str()); + LOG(MCdebugProgress, s.str()); #endif return newState; } @@ -769,7 +769,7 @@ void LRParser::doReductions(GrammarSymbol * next, bool singleToken) void LRParser::process(GrammarSymbol * next, bool singleToken) { #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Process token '%.*s' %d at position %d", next->queryEndPtr()-next->queryStartPtr(), next->queryStartPtr(), next->id, next->queryStartPtr()-rowState.inputText); + LOG(MCdebugProgress, "Process token '%.*s' %d at position %d", next->queryEndPtr()-next->queryStartPtr(), next->queryStartPtr(), next->id, next->queryStartPtr()-rowState.inputText); #endif doReductions(next, singleToken); selectEndPosition((size32_t)(next->queryEndPtr()-rowState.inputText)); @@ -791,7 +791,7 @@ void LRParser::expandReduction(StackElement & element, LRProduction * production StringBuffer s; for (unsigned i = 0; i < production->getNumSymbols(); i++) s.appendf("%p ", reducedArgs[i]); - LOG(MCdebugProgress, unknownJob, "Reduce by production %d new element %p[%s]", reduced->id, reduced.get(), s.str()); + LOG(MCdebugProgress, "Reduce by production %d new element %p[%s]", reduced->id, reduced.get(), s.str()); #endif //MORE: Some kind of recursion checking needed? if (activeOutput->okToAddReduction(nextState)) @@ -848,7 +848,7 @@ void LRParser::doShifts(LRActiveState * active, GrammarSymbol * next) if (nextState != NO_STATE) { #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Shift to state %d", nextState); + LOG(MCdebugProgress, "Shift to state %d", nextState); #endif activeOutput->addElementOwn(createState(&cur, nextState, next), chooseBest); } @@ -859,7 +859,7 @@ void LRParser::doShifts(LRActiveState * active, GrammarSymbol * next) GrammarSymbol * sym = accept->shifted; accepted.append(*LINK(sym)); #ifdef TRACING - LOG(MCdebugProgress, unknownJob, "Accepted %p[%p]", accept, sym); + LOG(MCdebugProgress, "Accepted %p[%p]", accept, sym); #endif } } diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index e5c391e9731..ec3b3590e70 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -9569,7 +9569,7 @@ void CLocalWorkUnit::deleteTempFiles(const char *graph, bool deleteOwned, bool d if(needDelete) { const char *name = file.queryProp("@name"); - LOG(MCdebugProgress, unknownJob, "Removing workunit file %s from DFS", name); + LOG(MCdebugProgress, "Removing workunit file %s from DFS", name); queryDistributedFileDirectory().removeEntry(name, queryUserDescriptor()); toRemove.append(file); } @@ -9705,7 +9705,7 @@ void CLocalWorkUnit::releaseFile(const char *fileName) if (!name.isEmpty()&&(1 == usageCount)) { if (queryDistributedFileDirectory().removeEntry(fileName, queryUserDescriptor())) - LOG(MCdebugProgress, unknownJob, "Removed (released) file %s from DFS", name.get()); + LOG(MCdebugProgress, "Removed (released) file %s from DFS", name.get()); } } if (!more) diff --git a/dali/base/dacoven.cpp b/dali/base/dacoven.cpp index 30cd69fa001..081f5f2d73c 100644 --- a/dali/base/dacoven.cpp +++ b/dali/base/dacoven.cpp @@ -254,7 +254,7 @@ class CCovenBase: implements ICoven, public CInterface if (server) { _ServerVersion.set(ServerVersion); - LOG(MCdebugProgress, unknownJob, "Server Version = %s, required minimum client version %s", ServerVersion, MinClientVersion); + LOG(MCdebugProgress, "Server Version = %s, required minimum client version %s", ServerVersion, MinClientVersion); } else { diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 887cafe07f3..1ff1c0fee5d 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -13767,12 +13767,12 @@ bool CDistributedFileDirectory::removePhysicalPartFiles(const char *logicalName, { unsigned start = msTick(); if (!partfile->remove()&&(copy==0)&&!islazy) // only warn about missing primary files - LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); + LOG(MCwarning, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); else { unsigned t = msTick()-start; if (t>5*1000) - LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); + LOG(MCwarning, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } } catch (IException *e) diff --git a/dali/base/danqs.cpp b/dali/base/danqs.cpp index bf8bd7885fe..d22cf9a4fac 100644 --- a/dali/base/danqs.cpp +++ b/dali/base/danqs.cpp @@ -646,7 +646,7 @@ class CDaliNamedQueueServer: public IDaliServer, public Thread, implements IConn catch (IException *e) { // should just be aborted exceptions, but remove subscription whatever the error. - LOG(MCdebugInfo, unknownJob, e, "Named Queue Server - MQR_ADD_QUEUE notify"); + LOG(MCdebugInfo, e, "Named Queue Server - MQR_ADD_QUEUE notify"); e->Release(); stubs.remove(i,true); toremove.append(stub); diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index dcbc2bd7aa4..5e5907cfd5a 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -1219,7 +1219,7 @@ class CDeltaWriter : implements IThreaded { exception.setown(e); StringBuffer err("Saving external (backup): "); - LOG(MCoperatorError, unknownJob, e, err.append(rL).str()); + LOG(MCoperatorError, e, err.append(rL).str()); } if (!exception.get()) break; @@ -1252,7 +1252,7 @@ class CDeltaWriter : implements IThreaded { exception.setown(e); StringBuffer err("Removing external (backup): "); - LOG(MCoperatorWarning, unknownJob, e, err.append(rL).str()); + LOG(MCoperatorWarning, e, err.append(rL).str()); } if (!exception.get()) break; @@ -1580,7 +1580,7 @@ class CLegacyBinaryFileExternal : public CExternalFile, implements IExternalHand if (*_name) s.append("in property ").append(_name); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); if (withValue) { StringBuffer str("EXTERNAL BINARY FILE: \""); @@ -1662,7 +1662,7 @@ class CXMLFileExternal : public CExternalFile, implements IExternalHandler if (name && *name) s.append("in property ").append(name); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); StringBuffer str("EXTERNAL XML FILE: \""); str.append(filename.str()).append("\" MISSING"); tree.setown(createPTree(owner.queryName())); @@ -2190,7 +2190,7 @@ void CBinaryFileExternal::readValue(const char *name, MemoryBuffer &mb) { StringBuffer s("Missing external file "); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); StringBuffer str("EXTERNAL BINARY FILE: \""); str.append(filename.str()).append("\" MISSING"); CPTValue v(str.length()+1, str.str(), false); @@ -2236,7 +2236,7 @@ void CBinaryFileExternal::read(const char *name, IPropertyTree &owner, MemoryBuf if (*_name) s.append("in property ").append(_name); Owned e = MakeSDSException(SDSExcpt_MissingExternalFile, "%s", filename.str()); - LOG(MCoperatorWarning, unknownJob, e, s.str()); + LOG(MCoperatorWarning, e, s.str()); StringBuffer str("EXTERNAL BINARY FILE: \""); str.append(filename.str()).append("\" MISSING"); CPTValue v(str.length()+1, str.str(), false); @@ -2468,7 +2468,7 @@ CServerConnection::~CServerConnection() void CServerConnection::aborted(SessionId id) { - LOG(MCdebugInfo, unknownJob, "CServerConnection: connection aborted (%" I64F "x) sessId=%" I64F "x",connectionId, id); + LOG(MCdebugInfo, "CServerConnection: connection aborted (%" I64F "x) sessId=%" I64F "x",connectionId, id); #if 0 // JCSMORE - think this is ok, but concerned about deadlock, change later. Owned lockBlock = new CLCLockBlock(((CCovenSDSManager &)manager).dataRWLock, false, readWriteTimeout, __FILE__, __LINE__); SDSManager->disconnect(connectionId, false); @@ -2685,7 +2685,7 @@ class CServerRemoteTree : public CRemoteTreeBase try { SDSManager->deleteExternal(index); } catch (IException *e) { - LOG(MCoperatorWarning, unknownJob, e, StringBuffer("Deleting external reference for ").append(queryName()).str()); + LOG(MCoperatorWarning, e, StringBuffer("Deleting external reference for ").append(queryName()).str()); e->Release(); } } @@ -4811,7 +4811,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) mb.append(e->errorMessage(s).str()); StringBuffer clientUrl("EXCEPTION in reply to client "); mb.getSender().getEndpointHostText(clientUrl); - LOG(MCoperatorError, unknownJob, e); + LOG(MCoperatorError, e); } try { CheckTime block10("DAMP_REQUEST reply"); @@ -4837,7 +4837,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) try { coven.reply(mb); - LOG(MCdebugInfo, unknownJob, "Failed to reply, but succeeded sending initial reply error to client"); + LOG(MCdebugInfo, "Failed to reply, but succeeded sending initial reply error to client"); } catch (IException *e) { @@ -5047,7 +5047,7 @@ IPropertyTree *loadStore(const char *storeFilename, unsigned edition, IPTreeMake catch (DALI_CATCHALL) { IException *e = MakeStringException(0, "Unknown exception - loading store file : %s", storeFilename); - LOG(MCoperatorDisaster, unknownJob, e, ""); + LOG(MCoperatorDisaster, e, ""); if (!logErrorsOnly) throw; e->Release(); @@ -5715,7 +5715,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface } virtual void saveStore(IPropertyTree *root, unsigned *_newEdition, bool currentEdition=false) { - LOG(MCdebugProgress, unknownJob, "Saving store"); + LOG(MCdebugProgress, "Saving store"); refreshStoreInfo(); @@ -5798,7 +5798,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface try { renameDelta(edition, newEdition, remoteBackupLocation); } catch (IException *e) { - LOG(MCoperatorError, unknownJob, e, "Failure handling backup"); + LOG(MCoperatorError, e, "Failure handling backup"); e->Release(); } } @@ -5828,7 +5828,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface catch (IException *e) { StringBuffer s; - LOG(MCoperatorError, unknownJob, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation)); + LOG(MCoperatorError, e, s.append("Failure to backup dali to remote location: ").append(remoteBackupLocation)); e->Release(); } @@ -5836,7 +5836,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface *_newEdition = newEdition; done = true; - LOG(MCdebugProgress, unknownJob, "Store saved"); + LOG(MCdebugProgress, "Store saved"); } catch (IException *e) { @@ -6247,7 +6247,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) if (!root) { StringBuffer s(storeName); - LOG(MCdebugInfo, unknownJob, "Store %d does not exist, creating new store", iStoreHelper->queryCurrentEdition()); + LOG(MCdebugInfo, "Store %d does not exist, creating new store", iStoreHelper->queryCurrentEdition()); root = new CServerRemoteTree("SDS"); } bool errors; @@ -6260,12 +6260,12 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) if (deltaE.get()) throw LINK(deltaE); } - LOG(MCdebugInfo, unknownJob, "store and deltas loaded"); + LOG(MCdebugInfo, "store and deltas loaded"); const char *environment = config.queryProp("@environment"); if (environment && *environment) { - LOG(MCdebugInfo, unknownJob, "loading external Environment from: %s", environment); + LOG(MCdebugInfo, "loading external Environment from: %s", environment); Owned envFile = createIFile(environment); if (!envFile->exists()) throw MakeStringException(0, "'%s' does not exist", environment); @@ -6559,14 +6559,14 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) unsigned items = treeMaker.convertQueue.ordinality(); if (items) { - LOG(MCdebugInfo, unknownJob, "Converting %d items larger than threshold size %d, to external definitions", items, externalSizeThreshold); + LOG(MCdebugInfo, "Converting %d items larger than threshold size %d, to external definitions", items, externalSizeThreshold); ForEachItemIn(i, treeMaker.convertQueue) SDSManager->writeExternal(treeMaker.convertQueue.item(i), true); saveNeeded = true; } if (saveNeeded) { - LOG(MCdebugInfo, unknownJob, "Saving converted store"); + LOG(MCdebugInfo, "Saving converted store"); SDSManager->saveStore(); } } @@ -6591,7 +6591,7 @@ void CCovenSDSManager::loadStore(const char *storeName, const bool *abort) if (remoteBackupLocation.length()) { try { validateBackup(); } - catch (IException *e) { LOG(MCoperatorError, unknownJob, e, "Validating backup"); e->Release(); } + catch (IException *e) { LOG(MCoperatorError, e, "Validating backup"); e->Release(); } StringBuffer deltaFilename(dataPath); iStoreHelper->getCurrentDeltaFilename(deltaFilename); @@ -8785,7 +8785,7 @@ bool CCovenSDSManager::fireException(IException *e) { if (handled) { - LOG(MCoperatorDisaster, unknownJob, e, "FATAL, too many exceptions"); + LOG(MCoperatorDisaster, e, "FATAL, too many exceptions"); return false; // did not successfully handle. } IERRLOG(e, "Exception while restarting or shutting down"); @@ -8822,7 +8822,7 @@ bool CCovenSDSManager::fireException(IException *e) } manager.unhandledThread.clear(); } - catch (IException *_e) { LOG(MCoperatorError, unknownJob, _e, "Exception while restarting or shutting down"); _e->Release(); } + catch (IException *_e) { LOG(MCoperatorError, _e, "Exception while restarting or shutting down"); _e->Release(); } catch (DALI_CATCHALL) { IERRLOG("Unknown exception while restarting or shutting down"); } if (!restart) { @@ -8914,7 +8914,7 @@ class CDaliSDSServer: implements IDaliServer, public CInterface try { manager->loadStore(NULL, &cancelLoad); } catch (IException *) { - LOG(MCdebugInfo, unknownJob, "Failed to load main store"); + LOG(MCdebugInfo, "Failed to load main store"); throw; } storeLoaded = true; diff --git a/dali/base/dasess.cpp b/dali/base/dasess.cpp index e356731ea86..8b40da715f8 100644 --- a/dali/base/dasess.cpp +++ b/dali/base/dasess.cpp @@ -1191,7 +1191,7 @@ class CLdapWorkItem : public Thread ret = ldapconn->getPermissions(key,obj,udesc,flags); } catch(IException *e) { - LOG(MCoperatorError, unknownJob, e, "CLdapWorkItem"); + LOG(MCoperatorError, e, "CLdapWorkItem"); e->Release(); } ready.signal(); diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 5e4ae47c28f..160c51cf07e 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -453,7 +453,7 @@ class CDFUengine: public CInterface, implements IDFUengine const char * pfilePath = filePath.str(); #ifdef _DEBUG - LOG(MCdebugInfo, unknownJob, "File path is '%s'", filePath.str()); + LOG(MCdebugInfo, "File path is '%s'", filePath.str()); #endif const char pathSep = filename.getPathSeparator(); @@ -488,7 +488,7 @@ class CDFUengine: public CInterface, implements IDFUengine if (env->isDropZoneRestrictionEnabled()) throwError2(DFTERR_NoMatchingDropzonePath, netaddress.str(), pfilePath); else - LOG(MCdebugInfo, unknownJob, "No matching drop zone path on '%s' to file path: '%s'", netaddress.str(), pfilePath); + LOG(MCdebugInfo, "No matching drop zone path on '%s' to file path: '%s'", netaddress.str(), pfilePath); } #ifdef _DEBUG else @@ -496,7 +496,7 @@ class CDFUengine: public CInterface, implements IDFUengine SCMStringBuffer dropZoneName; dropZone->getName(dropZoneName); - LOG(MCdebugInfo, unknownJob, "Drop zone path '%s' is %svisible in ECLWatch." + LOG(MCdebugInfo, "Drop zone path '%s' is %svisible in ECLWatch." , dropZoneName.str() , (dropZone->isECLWatchVisible() ? "" : "not ") ); @@ -1212,6 +1212,8 @@ class CDFUengine: public CInterface, implements IDFUengine OWARNLOG("DFURUN: Workunit %s not found",dfuwuid); return DFUstate_unknown; } + + JobNameScope jobName(dfuwuid); if (dfuServerName.length()) wu->setDFUServerName(dfuServerName.str()); StringBuffer logname; diff --git a/dali/dfu/dfuserver.cpp b/dali/dfu/dfuserver.cpp index dee00b31dd7..8777cf8e4af 100644 --- a/dali/dfu/dfuserver.cpp +++ b/dali/dfu/dfuserver.cpp @@ -292,7 +292,7 @@ int main(int argc, const char *argv[]) engine->joinListeners(); if (replserver.get()) replserver->stopServer(); - LOG(MCprogress, unknownJob, "Exiting"); + LOG(MCprogress, "Exiting"); } } diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp index 9a37925a2f0..e065ef5d43f 100644 --- a/dali/dfu/dfuwu.cpp +++ b/dali/dfu/dfuwu.cpp @@ -377,9 +377,14 @@ static StringBuffer &getSubTaskParentXPath(StringBuffer &wuRoot, const char *wui return wuRoot.append('/').append(wuid); } +static bool isSubTaskWuid(const char *subtaskWuid) +{ + return (!isEmptyString(subtaskWuid) && 'P'==*subtaskWuid && strchr(subtaskWuid, 'T')!=nullptr); +} + static bool notePublisherSubTaskState(const char *subtaskWuid, DFUstate state) { - if (!subtaskWuid || 'P'!=*subtaskWuid || !strchr(subtaskWuid, 'T')) + if (!isSubTaskWuid(subtaskWuid)) return false; switch (state) { @@ -699,6 +704,7 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress queryRoot()->removeProp("@kbpersecave"); queryRoot()->removeProp("@kbpersec"); queryRoot()->removeProp("@slavesdone"); + queryRoot()->removeProp("@noted"); parent->commit(); } void setDone(const char * timeTaken, unsigned kbPerSec, bool set100pc) @@ -718,6 +724,8 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress { CriticalBlock block(parent->crit); CDateTime dt; + bool noted = queryRoot()->getPropBool("@noted", false); + bool subtask = isSubTaskWuid(parent->queryId()); switch (state) { case DFUstate_started: dt.setNow(); @@ -737,13 +745,17 @@ class CDFUprogress: public CLinkedDFUWUchild, implements IDFUprogress state = DFUstate_aborted; dt.setNow(); setTimeStopped(dt); + if (subtask && !noted) + queryRoot()->setPropBool("@noted", true); break; } StringBuffer s; encodeDFUstate(state,s); queryRoot()->setProp("@state",s.str()); + parent->commit(); - notePublisherSubTaskState(parent->queryId(), state); + if (subtask && !noted) + notePublisherSubTaskState(parent->queryId(), state); } void setTimeStarted(const CDateTime &val) { diff --git a/dali/ft/daft.cpp b/dali/ft/daft.cpp index e2e1cd5d1b8..051375099ac 100644 --- a/dali/ft/daft.cpp +++ b/dali/ft/daft.cpp @@ -183,7 +183,7 @@ offset_t CDistributedFileSystem::getSize(IDistributedFile * file, bool forceget, lock.queryAttributes().setPropInt64("@size", totalSize); } } - //LOG(MCdebugInfo(1000), unknownJob, "DFS: getSize(%s)=%" I64F "d", file->queryLogicalName(), totalSize); + //LOG(MCdebugInfo(1000), "DFS: getSize(%s)=%" I64F "d", file->queryLogicalName(), totalSize); return totalSize; } @@ -241,7 +241,7 @@ offset_t CDistributedFileSystem::getSize(IDistributedFilePart * part, bool force } } - //LOG(MCdebugInfo(2000), unknownJob, "DFS: getSize(%s)=%" I64F "d", part->queryPartName(), size); + //LOG(MCdebugInfo(2000), "DFS: getSize(%s)=%" I64F "d", part->queryPartName(), size); return size; } diff --git a/dali/ft/daftformat.cpp b/dali/ft/daftformat.cpp index ecd55b91eaf..6cc8230ae56 100644 --- a/dali/ft/daftformat.cpp +++ b/dali/ft/daftformat.cpp @@ -105,7 +105,7 @@ void CPartitioner::commonCalcPartitions() if (endOffset == totalSize) lastSplit = numParts-1; if (lastSplit >= numParts) lastSplit = numParts-1; // very rare with variable length records, last file is very small or copying a couple of records 50 ways. - //LOG(MCdebugInfo, unknownJob, "commonCalcPartitions: partSize:%lld, endOffset: %lld, firstSplit: %d, lastSplit: %d ",partSize ,endOffset, firstSplit, lastSplit); + //LOG(MCdebugInfo, "commonCalcPartitions: partSize:%lld, endOffset: %lld, firstSplit: %d, lastSplit: %d ",partSize ,endOffset, firstSplit, lastSplit); JSON_DBGLOG("commonCalcPartitions: partSize:%lld, endOffset: %lld, firstSplit: %d, lastSplit: %d ",partSize ,endOffset, firstSplit, lastSplit); if (!partSeparator.isEmpty() && appendingContent) //appending to existing content, add a separator if necessary @@ -215,7 +215,7 @@ void CPartitioner::getRecordStructure(StringBuffer & _recordStructure) CSimpleFixedPartitioner::CSimpleFixedPartitioner(unsigned _recordSize, bool _noTranslation) { - LOG(MCdebugProgressDetail, unknownJob, "CSimpleFixedPartitioner::CSimpleFixedPartitioner( _recordSize:%d, _noTranslation:%d)", _recordSize, _noTranslation); + LOG(MCdebugProgressDetail, "CSimpleFixedPartitioner::CSimpleFixedPartitioner( _recordSize:%d, _noTranslation:%d)", _recordSize, _noTranslation); recordSize = _recordSize; noTranslation = _noTranslation; } @@ -241,7 +241,7 @@ void CSimpleFixedPartitioner::setPartitionRange(offset_t _totalSize, offset_t _t CSimpleBlockedPartitioner::CSimpleBlockedPartitioner(bool _noTranslation) : CSimpleFixedPartitioner(EFX_BLOCK_SIZE, _noTranslation) { - LOG(MCdebugProgressDetail, unknownJob, "CSimpleBlockedPartitioner::CSimpleBlockedPartitioner( _noTranslation:%d)", _noTranslation); + LOG(MCdebugProgressDetail, "CSimpleBlockedPartitioner::CSimpleBlockedPartitioner( _noTranslation:%d)", _noTranslation); } //---------------------------------------------------------------------------- @@ -298,7 +298,7 @@ void CInputBasePartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor { // Display progress oldInputOffset = nextInputOffset; - LOG(MCdebugProgressDetail, unknownJob, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, (double)100.0*(double)nextInputOffset/(double)splitOffset); + LOG(MCdebugProgressDetail, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, (double)100.0*(double)nextInputOffset/(double)splitOffset); } inputOffset = nextInputOffset; @@ -315,7 +315,7 @@ void CInputBasePartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor if (size > bufferSize) { - LOG(MCdebugProgressDetail, unknownJob, "Split record size %d (0x%08x) is larger than the buffer size: %d", size, size, bufferSize); + LOG(MCdebugProgressDetail, "Split record size %d (0x%08x) is larger than the buffer size: %d", size, size, bufferSize); throwError2(DFTERR_WrongSplitRecordSize, size, size); } @@ -347,7 +347,7 @@ void CInputBasePartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor cursor.inputOffset = inputOffset; cursor.nextInputOffset = nextInputOffset; - LOG(MCdebugProgressDetail, unknownJob, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, 100.0); + LOG(MCdebugProgressDetail, "findSplitPoint(splitOffset:%" I64F "d) progress: %3.0f%% done.", splitOffset, 100.0); } @@ -466,7 +466,7 @@ unsigned CInputBasePartitioner::transformBlock(offset_t endOffset, TransformCurs CFixedPartitioner::CFixedPartitioner(size32_t _recordSize) : CInputBasePartitioner(0, _recordSize) { recordSize = _recordSize; - LOG(MCdebugProgressDetail, unknownJob, "CFixedPartitioner::CFixedPartitioner( recordSize:%d)", recordSize); + LOG(MCdebugProgressDetail, "CFixedPartitioner::CFixedPartitioner( recordSize:%d)", recordSize); } size32_t CFixedPartitioner::getSplitRecordSize(const byte * record, unsigned maxToRead, bool processFullBuffer) @@ -484,7 +484,7 @@ size32_t CFixedPartitioner::getTransformRecordSize(const byte * record, unsigned CBlockedPartitioner::CBlockedPartitioner() : CFixedPartitioner(EFX_BLOCK_SIZE) { - LOG(MCdebugProgressDetail, unknownJob, "CBlockedPartitioner::CBlockedPartitioner()"); + LOG(MCdebugProgressDetail, "CBlockedPartitioner::CBlockedPartitioner()"); } @@ -498,7 +498,7 @@ void CBlockedPartitioner::setTarget(IOutputProcessor * _target) CVariablePartitioner::CVariablePartitioner(bool _bigendian) : CInputBasePartitioner(sizeof(varLenType), EXPECTED_VARIABLE_LENGTH) { - LOG(MCdebugProgressDetail, unknownJob, "CVariablePartitioner::CVariablePartitioner(_bigendian:%d)", _bigendian); + LOG(MCdebugProgressDetail, "CVariablePartitioner::CVariablePartitioner(_bigendian:%d)", _bigendian); assertex(sizeof(varLenType) == 4); bigendian = _bigendian; } @@ -539,7 +539,7 @@ void CVariablePartitioner::setTarget(IOutputProcessor * _target) CRECFMvbPartitioner::CRECFMvbPartitioner(bool blocked) : CInputBasePartitioner(blocked?BDW_SIZE:RDW_SIZE, EXPECTED_VARIABLE_LENGTH) { - LOG(MCdebugProgressDetail, unknownJob, "CRECFMvbPartitioner::CRECFMvbPartitioner(blocked:%d)", blocked); + LOG(MCdebugProgressDetail, "CRECFMvbPartitioner::CRECFMvbPartitioner(blocked:%d)", blocked); isBlocked = blocked; } @@ -553,7 +553,7 @@ size32_t CRECFMvbPartitioner::getRecordSize(const byte * record, unsigned maxToR if (rest) { - LOG(MCdebugProgressDetail, unknownJob, "Wrong RECFMv RDW info: size:%d (0x%04x) rest:%d (0x%04x) at pos :%d", recordsize ,recordsize, rest, rest, unsigned (record - bufferBase()) ); + LOG(MCdebugProgressDetail, "Wrong RECFMv RDW info: size:%d (0x%04x) rest:%d (0x%04x) at pos :%d", recordsize ,recordsize, rest, rest, unsigned (record - bufferBase()) ); throwError1(DFTERR_WrongRECFMvRecordDescriptorWord, rest); } @@ -613,7 +613,7 @@ size32_t CRECFMvbPartitioner::getTransformRecordSize(const byte * record, unsign unsigned CRECFMvbPartitioner::transformBlock(offset_t endOffset, TransformCursor & cursor) { - LOG(MCdebugProgressDetail, unknownJob, "CRECFMvbPartitioner::transformBlock(offset_t endOffset: %" I64F "d (0x%016" I64F "x), TransformCursor & cursor)", endOffset ,endOffset); + LOG(MCdebugProgressDetail, "CRECFMvbPartitioner::transformBlock(offset_t endOffset: %" I64F "d (0x%016" I64F "x), TransformCursor & cursor)", endOffset ,endOffset); const byte *buffer = bufferBase(); offset_t startOffset = cursor.inputOffset; offset_t inputOffset = startOffset; @@ -680,11 +680,11 @@ unsigned CRECFMvbPartitioner::transformBlock(offset_t endOffset, TransformCursor CCsvPartitioner::CCsvPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize) { - LOG(MCdebugProgressDetail, unknownJob, "CCsvPartitioner::CCsvPartitioner(_format :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); - LOG(MCdebugProgressDetail, unknownJob, " separator :'%s'", _format.separate.get()); - LOG(MCdebugProgressDetail, unknownJob, " quote :'%s'", _format.quote.get()); - LOG(MCdebugProgressDetail, unknownJob, " terminator:'%s'", _format.terminate.get()); - LOG(MCdebugProgressDetail, unknownJob, " escape :'%s'", _format.escape.get()); + LOG(MCdebugProgressDetail, "CCsvPartitioner::CCsvPartitioner(_format :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); + LOG(MCdebugProgressDetail, " separator :'%s'", _format.separate.get()); + LOG(MCdebugProgressDetail, " quote :'%s'", _format.quote.get()); + LOG(MCdebugProgressDetail, " terminator:'%s'", _format.terminate.get()); + LOG(MCdebugProgressDetail, " escape :'%s'", _format.escape.get()); maxElementLength = 1; format.set(_format); @@ -700,7 +700,7 @@ CCsvPartitioner::CCsvPartitioner(const FileFormat & _format) : CInputBasePartiti if (escape && *escape) { if (quote && (*escape == *quote)) - LOG(MCdebugProgressDetail, unknownJob, "The quote ('%s') and the escape ('%s') are same, ignore escape.", quote, escape); + LOG(MCdebugProgressDetail, "The quote ('%s') and the escape ('%s') are same, ignore escape.", quote, escape); else addActionList(matcher, escape, ESCAPE, &maxElementLength); } @@ -744,14 +744,14 @@ void CCsvPartitioner::getRecordStructure(StringBuffer & _recordStructure) if (size > bufferSize) { - LOG(MCdebugProgressDetail, unknownJob, "First record size %d (0x%08x) is larger than the buffer size: %d. Please check the separator and terminator parameters.", size, size, bufferSize); + LOG(MCdebugProgressDetail, "First record size %d (0x%08x) is larger than the buffer size: %d. Please check the separator and terminator parameters.", size, size, bufferSize); throwError2(DFTERR_WrongSplitRecordSize, size, size); } else if (isFirstRow) { // Possibly reached the end of the file but the processing is still in first row // so no terminator detected - LOG(MCdebugProgressDetail, unknownJob, "%d (0x%08x) bytes processed, but there is no terminator found. Please check the separator and terminator parameters.", size, size); + LOG(MCdebugProgressDetail, "%d (0x%08x) bytes processed, but there is no terminator found. Please check the separator and terminator parameters.", size, size); throwError1(DFTERR_EndOfCsvRecordNotFound, size); } } @@ -970,7 +970,7 @@ size32_t CCsvPartitioner::getSplitRecordSize(const byte * start, unsigned maxToR throwError(DFTERR_EndOfRecordNotFound); - LOG(MCdebugProgress, unknownJob, "CSV splitRecordSize(%d) at end of file", (unsigned) (end - start)); + LOG(MCdebugProgress, "CSV splitRecordSize(%d) at end of file", (unsigned) (end - start)); if (++numOfBufferOverrun > maxNumberOfBufferOverrun) throwError1(DFTERR_EndOfCsvRecordNotFound, numOfProcessedBytes); @@ -995,7 +995,7 @@ void CCsvPartitioner::setTarget(IOutputProcessor * _target) CCsvQuickPartitioner::CCsvQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CCsvPartitioner(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CCsvQuickPartitioner::CCsvQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); + LOG(MCdebugProgressDetail, "CCsvQuickPartitioner::CCsvQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); noTranslation = _noTranslation; } @@ -1050,13 +1050,13 @@ void CCsvQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor if (ensureSize == format.maxRecordSize) throw; e->Release(); - LOG(MCdebugProgress, unknownJob, "Failed to find split after reading %d", ensureSize); + LOG(MCdebugProgress, "Failed to find split after reading %d", ensureSize); ensureSize += blockSize; if (ensureSize > format.maxRecordSize) ensureSize = format.maxRecordSize; } } - LOG(MCdebugProgress, unknownJob, "Found split after reading %d", ensureSize); + LOG(MCdebugProgress, "Found split after reading %d", ensureSize); } } } @@ -1088,10 +1088,10 @@ void CCsvQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor CUtfPartitioner::CUtfPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize) { - LOG(MCdebugProgressDetail, unknownJob, "CUtfPartitioner::CUtfPartitioner(_format.type :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); - LOG(MCdebugProgressDetail, unknownJob, " separator :'%s'", _format.separate.get()); - LOG(MCdebugProgressDetail, unknownJob, " quote :'%s'", _format.quote.get()); - LOG(MCdebugProgressDetail, unknownJob, " terminator:'%s'", _format.terminate.get()); + LOG(MCdebugProgressDetail, "CUtfPartitioner::CUtfPartitioner(_format.type :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize); + LOG(MCdebugProgressDetail, " separator :'%s'", _format.separate.get()); + LOG(MCdebugProgressDetail, " quote :'%s'", _format.quote.get()); + LOG(MCdebugProgressDetail, " terminator:'%s'", _format.terminate.get()); maxElementLength = 1; format.set(_format); @@ -1311,7 +1311,7 @@ size32_t CUtfPartitioner::getSplitRecordSize(const byte * start, unsigned maxToR numOfProcessedBytes += (unsigned)(end - start); - LOG(MCdebugProgress, unknownJob, "UTF splitRecordSize(%d) at end of file", (unsigned) (end - start)); + LOG(MCdebugProgress, "UTF splitRecordSize(%d) at end of file", (unsigned) (end - start)); if (++numOfBufferOverrun > maxNumberOfBufferOverrun) throwError1(DFTERR_EndOfUtfRecordNotFound, numOfProcessedBytes); @@ -1332,7 +1332,7 @@ void CUtfPartitioner::setTarget(IOutputProcessor * _target) CUtfQuickPartitioner::CUtfQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CUtfPartitioner(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CUtfQuickPartitioner::CUtfQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); + LOG(MCdebugProgressDetail, "CUtfQuickPartitioner::CUtfQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); noTranslation = _noTranslation; } @@ -1389,13 +1389,13 @@ void CUtfQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor if (ensureSize == format.maxRecordSize) throw; e->Release(); - LOG(MCdebugProgress, unknownJob, "Failed to find split after reading %d", ensureSize); + LOG(MCdebugProgress, "Failed to find split after reading %d", ensureSize); ensureSize += blockSize; if (ensureSize > format.maxRecordSize) ensureSize = format.maxRecordSize; } } - LOG(MCdebugProgress, unknownJob, "Found split after reading %d", ensureSize); + LOG(MCdebugProgress, "Found split after reading %d", ensureSize); } } } @@ -1463,7 +1463,7 @@ size32_t BufferedDirectReader::ensure(size32_t required) XmlSplitter::XmlSplitter(const FileFormat & format) { - LOG(MCdebugProgressDetail, unknownJob, "XmlSplitter::XmlSplitter(_format.type :'%s', format.rowTag:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); + LOG(MCdebugProgressDetail, "XmlSplitter::XmlSplitter(_format.type :'%s', format.rowTag:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); maxElementLength = 1; utfFormat = getUtfFormatType(format.type); StringBuffer openTag, closeTag, endTag, endCloseTag; @@ -1686,7 +1686,7 @@ offset_t XmlSplitter::getFooterLength(BufferedDirectReader & reader, offset_t si CJsonInputPartitioner::CJsonInputPartitioner(const FileFormat & _format) { - LOG(MCdebugProgressDetail, unknownJob, "CJsonInputPartitioner::CJsonInputPartitioner(format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), _format.getUnitSize()); + LOG(MCdebugProgressDetail, "CJsonInputPartitioner::CJsonInputPartitioner(format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), _format.getUnitSize()); format.set(_format); CriticalBlock block(openfilecachesect); @@ -1750,7 +1750,7 @@ CJsonPartitioner::CJsonPartitioner(const FileFormat & _format) : CJsonInputParti CXmlPartitioner::CXmlPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize), splitter(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlPartitioner::CXmlPartitioner(_format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), format.getUnitSize()); + LOG(MCdebugProgressDetail, "CXmlPartitioner::CXmlPartitioner(_format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), format.getUnitSize()); format.set(_format); unitSize = format.getUnitSize(); utfFormat = getUtfFormatType(format.type); @@ -1782,7 +1782,7 @@ void CXmlPartitioner::setTarget(IOutputProcessor * _target) CXmlQuickPartitioner::CXmlQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CXmlPartitioner(_format) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::CXmlQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); + LOG(MCdebugProgressDetail, "CXmlQuickPartitioner::CXmlQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation); noTranslation = _noTranslation; } @@ -1792,7 +1792,7 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor numInBuffer = bufferOffset = 0; if (splitOffset != 0) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::findSplitPoint(splitOffset:%" I64F "d)", splitOffset); + LOG(MCdebugProgressDetail, "CXmlQuickPartitioner::findSplitPoint(splitOffset:%" I64F "d)", splitOffset); unsigned delta = (unsigned)(splitOffset & (unitSize-1)); if (delta) splitOffset += (unitSize - delta); @@ -1818,16 +1818,16 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor } if (sizeAvailable >= format.maxRecordSize) { - LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::findSplitPoint: record size (>%d bytes) is larger than expected maxRecordSize (%d bytes) [and blockSize (%d bytes)]", sizeRecord, format.maxRecordSize, blockSize); + LOG(MCdebugProgressDetail, "CXmlQuickPartitioner::findSplitPoint: record size (>%d bytes) is larger than expected maxRecordSize (%d bytes) [and blockSize (%d bytes)]", sizeRecord, format.maxRecordSize, blockSize); throwError3(DFTERR_EndOfXmlRecordNotFound, splitOffset+bufferOffset, sizeRecord, format.maxRecordSize); } - LOG(MCdebugProgress, unknownJob, "Failed to find split after reading %d", ensureSize); + LOG(MCdebugProgress, "Failed to find split after reading %d", ensureSize); ensureSize += blockSize; if (ensureSize > format.maxRecordSize) ensureSize = format.maxRecordSize; ensureBuffered(ensureSize); } - LOG(MCdebugProgress, unknownJob, "Found split after reading %d", ensureSize); + LOG(MCdebugProgress, "Found split after reading %d", ensureSize); } else if (splitOffset - thisOffset < thisSize) throwError2(DFTERR_UnexpectedReadFailure, fullPath.get(), splitOffset-thisOffset+thisHeaderSize); @@ -1842,7 +1842,7 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor CRemotePartitioner::CRemotePartitioner(FileSprayer &_sprayer, const SocketEndpoint & _ep, const FileFormat & _srcFormat, const FileFormat & _tgtFormat, const char * _slave, const char *_wuid) : wuid(_wuid), sprayer(_sprayer) { - LOG(MCdebugProgressDetail, unknownJob, "CRemotePartitioner::CRemotePartitioner(_srcFormat.type :'%s', _tgtFormat.type:'%s', _slave:'%s', _wuid:'%s')", _srcFormat.getFileFormatTypeString(), _tgtFormat.getFileFormatTypeString(), _slave, _wuid); + LOG(MCdebugProgressDetail, "CRemotePartitioner::CRemotePartitioner(_srcFormat.type :'%s', _tgtFormat.type:'%s', _slave:'%s', _wuid:'%s')", _srcFormat.getFileFormatTypeString(), _tgtFormat.getFileFormatTypeString(), _slave, _wuid); ep.set(_ep); srcFormat.set(_srcFormat); tgtFormat.set(_tgtFormat); @@ -1887,7 +1887,6 @@ void CRemotePartitioner::callRemote() HANDLE localFtSlaveHandle = 0; // used only if ftslave is launched on this host try { - LogMsgJobInfo job(unknownJob); StringBuffer url, tmp; ep.getEndpointHostText(url); @@ -1931,7 +1930,7 @@ void CRemotePartitioner::callRemote() sendDaFsFtSlaveCmd(socket, msg); } - LOG(MCdebugProgressDetail, job, "Remote partition part %s[%d]", url.str(), whichInput); + LOG(MCdebugProgressDetail, "Remote partition part %s[%d]", url.str(), whichInput); msg.clear(); if (!catchReadBuffer(socket, msg, FTTIME_PARTITION)) @@ -1952,7 +1951,7 @@ void CRemotePartitioner::callRemote() catchWriteBuffer(socket, msg); } - LOG(MCdebugProgressDetail, job, "Remote partition calculated %s[%d] ok(%d)", url.str(), whichInput, ok); + LOG(MCdebugProgressDetail, "Remote partition calculated %s[%d] ok(%d)", url.str(), whichInput, ok); } catch (IException * e) { @@ -2300,7 +2299,7 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil { IFormatProcessor * partitioner; bool sameFormats = srcFormat.equals(tgtFormat); - LOG(MCdebugProgressDetail, unknownJob, "createFormatProcessor(srcFormat:'%s', tgtFormat:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); + LOG(MCdebugProgressDetail, "createFormatProcessor(srcFormat:'%s', tgtFormat:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); switch (srcFormat.type) { case FFTfixed: @@ -2360,7 +2359,7 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil IOutputProcessor * createOutputProcessor(const FileFormat & format) { - LOG(MCdebugProgressDetail, unknownJob, "createOutputProcessor(format.type:'%s')", format.getFileFormatTypeString()); + LOG(MCdebugProgressDetail, "createOutputProcessor(format.type:'%s')", format.getFileFormatTypeString()); switch (format.type) { case FFTfixed: @@ -2382,7 +2381,7 @@ IOutputProcessor * createOutputProcessor(const FileFormat & format) IFormatPartitioner * createFormatPartitioner(FileSprayer &sprayer, const SocketEndpoint & ep, const FileFormat & srcFormat, const FileFormat & tgtFormat, bool calcOutput, const char * slave, const char *wuid) { bool sameFormats = sameEncoding(srcFormat, tgtFormat); - LOG(MCdebugProgressDetail, unknownJob, "createFormatProcessor(srcFormat.type:'%s', tgtFormat.type:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); + LOG(MCdebugProgressDetail, "createFormatProcessor(srcFormat.type:'%s', tgtFormat.type:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats); if (sameFormats) { switch (srcFormat.type) diff --git a/dali/ft/daftformat.ipp b/dali/ft/daftformat.ipp index 34c2e85328e..3db0465ee24 100644 --- a/dali/ft/daftformat.ipp +++ b/dali/ft/daftformat.ipp @@ -379,7 +379,7 @@ public: JsonSplitter(const FileFormat & format, IFileIOStream &stream) : headerLength(0), pathPos(0), tangent(0), rowDepth(0), rowStart((offset_t)-1), rowEnd(0), footerLength((offset_t)-1), newRowSet(true), hasRootArray(false) { - LOG(MCdebugProgressDetail, unknownJob, "JsonSplitter::JsonSplitter(format.type :'%s', rowPath:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); + LOG(MCdebugProgressDetail, "JsonSplitter::JsonSplitter(format.type :'%s', rowPath:'%s')", format.getFileFormatTypeString(), format.rowTag.get()); size = stream.size(); const char *rowPath = format.rowTag; diff --git a/dali/ft/daftprogress.cpp b/dali/ft/daftprogress.cpp index 5070d8614a0..a83a81ec1a1 100644 --- a/dali/ft/daftprogress.cpp +++ b/dali/ft/daftprogress.cpp @@ -136,11 +136,11 @@ void DemoProgress::displayProgress(unsigned percentDone, unsigned secsLeft, cons unsigned kbPerSecondAve, unsigned kbPerSecondRate, unsigned numNodes) { - LOG(MCdebugProgress, unknownJob, "Progress: %d%% done, %s left. (%" I64F "d/%" I64F "d%s @Ave(%dKB/s) Rate(%dKB/s) [%d/%d]", + LOG(MCdebugProgress, "Progress: %d%% done, %s left. (%" I64F "d/%" I64F "d%s @Ave(%dKB/s) Rate(%dKB/s) [%d/%d]", percentDone, timeLeft, scaledDone, scaledTotal, scale, kbPerSecondAve, kbPerSecondRate, numNodes, totalNodes); } void DemoProgress::displaySummary(const char * timeTaken, unsigned kbPerSecond) { - LOG(MCdebugProgress, unknownJob, "Summary: Total time taken %s, Average transfer %dKB/sec", timeTaken, kbPerSecond); + LOG(MCdebugProgress, "Summary: Total time taken %s, Average transfer %dKB/sec", timeTaken, kbPerSecond); } diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 267457c6e2f..885dc2b2168 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -207,7 +207,6 @@ FileTransferThread::FileTransferThread(FileSprayer & _sprayer, byte _action, con // progressInfo = _progressInfo; sem = NULL; ok = false; - job = unknownJob; allDone = false; started = false; } @@ -424,7 +423,7 @@ bool FileTransferThread::launchFtSlaveCmd() newProgress.deserializeExtra(msg, 2); sprayer.updateProgress(newProgress); - LOG(MCdebugProgress(10000), job, "Update %s: %d %" I64F "d->%" I64F "d", url.str(), newProgress.whichPartition, newProgress.inputLength, newProgress.outputLength); + LOG(MCdebugProgress(10000), "Update %s: %d %" I64F "d->%" I64F "d", url.str(), newProgress.whichPartition, newProgress.inputLength, newProgress.outputLength); if (isAborting()) { if (!sendRemoteAbort(socket)) @@ -434,7 +433,7 @@ bool FileTransferThread::launchFtSlaveCmd() msg.read(ok); setErrorOwn(deserializeException(msg)); - LOG(MCdebugProgressDetail, job, "Finished generating part %s [%p] ok(%d) error(%d)", url.str(), this, (int)ok, (int)(error!=NULL)); + LOG(MCdebugProgressDetail, "Finished generating part %s [%p] ok(%d) error(%d)", url.str(), this, (int)ok, (int)(error!=NULL)); // if communicating with ftslave, the process has a final ack wait if (sprayer.useFtSlave) @@ -463,7 +462,7 @@ bool FileTransferThread::launchFtSlaveCmd() bool FileTransferThread::performTransfer() { - LOG(MCdebugProgress, job, "Transferring part %s [%p]", url.str(), this); + LOG(MCdebugProgress, "Transferring part %s [%p]", url.str(), this); started = true; allDone = true; if (sprayer.isSafeMode || action == FTactionpush) @@ -493,19 +492,19 @@ bool FileTransferThread::performTransfer() if (allDone) { - LOG(MCdebugInfo, job, "Creation of part %s already completed", url.str()); + LOG(MCdebugInfo, "Creation of part %s already completed", url.str()); return true; } if (partition.empty()) { - LOG(MCdebugInfo, job, "No elements to transfer for this slave"); + LOG(MCdebugInfo, "No elements to transfer for this slave"); return true; } - LOG(MCdebugProgressDetail, job, "Start generate part %s [%p]", url.str(), this); + LOG(MCdebugProgressDetail, "Start generate part %s [%p]", url.str(), this); bool ok = launchFtSlaveCmd(); - LOG(MCdebugProgressDetail, job, "Stopped generate part %s [%p]", url.str(), this); + LOG(MCdebugProgressDetail, "Stopped generate part %s [%p]", url.str(), this); allDone = true; return ok; @@ -530,7 +529,7 @@ bool FileTransferThread::transferAndSignal() } catch (IException * e) { - FLLOG(MCexception(e), job, e, "Transferring files"); + FLLOG(MCexception(e), e, "Transferring files"); setErrorOwn(e); } } @@ -674,7 +673,6 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR allowRecovery = !options->getPropBool(ANnoRecover) && !querySplitPrefix(); isRecovering = allowRecovery && progressTree->hasProp(ANhasProgress); isSafeMode = options->getPropBool(ANsafe); - job = unknownJob; progressReport = NULL; abortChecker = NULL; sizeToBeRead = 0; @@ -692,7 +690,7 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR copyCompressed = false; transferBufferSize = options->getPropInt(ANtransferBufferSize); if (transferBufferSize) - LOG(MCdebugProgressDetail, job, "Using transfer buffer size %d", transferBufferSize); + LOG(MCdebugProgressDetail, "Using transfer buffer size %d", transferBufferSize); else // zero is default transferBufferSize = DEFAULT_STD_BUFFER_SIZE; progressDone = false; @@ -710,7 +708,7 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR fileUmask = (int)strtol(umaskStr, &eptr, 8); if (errno || *eptr != '\0') { - LOG(MCdebugInfo, job, "Invalid umask value <%s> ignored", umaskStr); + LOG(MCdebugInfo, "Invalid umask value <%s> ignored", umaskStr); fileUmask = -1; } else @@ -763,7 +761,7 @@ void FileSprayer::addEmptyFilesToPartition(unsigned from, unsigned to) { for (unsigned i = from; i < to ; i++) { - LOG(MCdebugProgressDetail, job, "Insert a dummy entry for target %d", i); + LOG(MCdebugProgressDetail, "Insert a dummy entry for target %d", i); PartitionPoint & next = createLiteral(0, NULL, 0); next.whichOutput = i; partition.append(next); @@ -791,7 +789,7 @@ void FileSprayer::afterTransfer() { if (calcInputCRC()) { - LOG(MCdebugProgressDetail, job, "Checking input CRCs"); + LOG(MCdebugProgressDetail, "Checking input CRCs"); CRC32Merger partCRC; unsigned startCurSource = 0; @@ -802,7 +800,7 @@ void FileSprayer::afterTransfer() if (!curProgress.hasInputCRC) { - LOG(MCdebugProgressDetail, job, "Could not calculate input CRCs - cannot check"); + LOG(MCdebugProgressDetail, "Could not calculate input CRCs - cannot check"); break; } partCRC.addChildCRC(curProgress.inputLength, curProgress.inputCRC, false); @@ -969,9 +967,9 @@ void FileSprayer::beforeTransfer() } } catch (IException *e) { - FLLOG(MCexception(e), job, e, "Creating Directory"); + FLLOG(MCexception(e), e, "Creating Directory"); e->Release(); - LOG(MCdebugInfo, job, "Ignoring create directory error"); + LOG(MCdebugInfo, "Ignoring create directory error"); } // If pushing files, and not recovering, then need to delete the target files, because the slaves might be writing in any order @@ -1041,7 +1039,7 @@ void FileSprayer::beforeTransfer() if (machine->getOS() == MachineOsW2K) { throttleNicSpeed = machine->getNicSpeedMbitSec(); - LOG(MCdebugInfo, job, "Throttle target speed to %dMbit/sec", throttleNicSpeed); + LOG(MCdebugInfo, "Throttle target speed to %dMbit/sec", throttleNicSpeed); } } } @@ -1082,7 +1080,7 @@ bool FileSprayer::calcInputCRC() void FileSprayer::calculateOne2OnePartition() { - LOG(MCdebugProgressDetail, job, "Setting up one2One partition"); + LOG(MCdebugProgressDetail, "Setting up one2One partition"); if (sources.ordinality() != targets.ordinality()) throwError(DFTERR_ReplicateNumPartsDiffer); if (!srcFormat.equals(tgtFormat)) @@ -1153,7 +1151,7 @@ void FileSprayer::calculateSplitPrefixPartition(const char * splitPrefix) if (!srcFormat.equals(tgtFormat)) throwError(DFTERR_SplitPrefixSameFormat); - LOG(MCdebugProgressDetail, job, "Setting up split prefix partition"); + LOG(MCdebugProgressDetail, "Setting up split prefix partition"); Owned target = &targets.popGet(); // remove target, add lots of new ones RemoteFilename blobTarget; @@ -1190,7 +1188,7 @@ void FileSprayer::calculateSplitPrefixPartition(const char * splitPrefix) void FileSprayer::calculateMany2OnePartition() { - LOG(MCdebugProgressDetail, job, "Setting up many2one partition"); + LOG(MCdebugProgressDetail, "Setting up many2one partition"); const char *partSeparator = srcFormat.getPartSeparatorString(); offset_t partSeparatorLength = ( partSeparator == nullptr ? 0 : strlen(partSeparator)); offset_t lastContentLength = 0; @@ -1222,7 +1220,7 @@ void FileSprayer::calculateMany2OnePartition() void FileSprayer::calculateNoSplitPartition() { - LOG(MCdebugProgressDetail, job, "Setting up no split partition"); + LOG(MCdebugProgressDetail, "Setting up no split partition"); if (!usePullOperation() && !srcFormat.equals(tgtFormat)) throwError(DFTERR_NoSplitPushChangeFormat); @@ -1285,7 +1283,7 @@ void FileSprayer::calculateNoSplitPartition() void FileSprayer::calculateSprayPartition() { - LOG(MCdebugProgressDetail, job, "Calculating N:M partition"); + LOG(MCdebugProgressDetail, "Calculating N:M partition"); bool calcOutput = needToCalcOutput(); FormatPartitionerArray partitioners; @@ -1347,7 +1345,7 @@ IFormatPartitioner * FileSprayer::createPartitioner(aindex_t index, bool calcOut StringBuffer remoteFilename; FilePartInfo & cur = sources.item(index); cur.filename.getRemotePath(remoteFilename.clear()); - LOG(MCdebugInfoDetail, job, "Partition %d(%s)", index, remoteFilename.str()); + LOG(MCdebugInfoDetail, "Partition %d(%s)", index, remoteFilename.str()); srcFormat.quotedTerminator = options->getPropBool("@quotedTerminator", true); const SocketEndpoint & ep = cur.filename.queryEndpoint(); @@ -1379,7 +1377,7 @@ void FileSprayer::examineCsvStructure() storeCsvRecordStructure(*partitioner); } else - LOG(MCdebugInfoDetail, job, "No source CSV file to examine."); + LOG(MCdebugInfoDetail, "No source CSV file to examine."); } void FileSprayer::calculateOutputOffsets() @@ -1670,7 +1668,7 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize) FilePartInfo & cur = sources.item(idx); StringBuffer s; cur.filename.getPath(s); - LOG(MCdebugInfo, job, "Examine header of file %s", s.str()); + LOG(MCdebugInfo, "Examine header of file %s", s.str()); Owned file = createIFile(cur.filename); Owned io = file->open(IFOread); @@ -1928,14 +1926,14 @@ void FileSprayer::displayPartition() #ifdef _DEBUG if ((partition.item(idx).whichInput >= 0) && (partition.item(idx).whichInput < sources.ordinality()) ) - LOG(MCdebugInfoDetail, unknownJob, + LOG(MCdebugInfoDetail, " Header size: %" I64F "u, XML header size: %" I64F "u, XML footer size: %" I64F "u", sources.item(partition.item(idx).whichInput).headerSize, sources.item(partition.item(idx).whichInput).xmlHeaderLength, sources.item(partition.item(idx).whichInput).xmlFooterLength ); else - LOG(MCdebugInfoDetail, unknownJob," No source file for this partition"); + LOG(MCdebugInfoDetail, " No source file for this partition"); #endif } } @@ -1963,7 +1961,7 @@ void FileSprayer::gatherFileSizes(bool errorIfMissing) { FilePartInfoArray fileSizeQueue; - LOG(MCdebugProgress, job, "Start gathering file sizes..."); + LOG(MCdebugProgress, "Start gathering file sizes..."); ForEachItemIn(idx, sources) { @@ -1973,7 +1971,7 @@ void FileSprayer::gatherFileSizes(bool errorIfMissing) } gatherFileSizes(fileSizeQueue, errorIfMissing); - LOG(MCdebugProgress, job, "Finished gathering file sizes..."); + LOG(MCdebugProgress, "Finished gathering file sizes..."); } void FileSprayer::afterGatherFileSizes() @@ -1985,7 +1983,7 @@ void FileSprayer::afterGatherFileSizes() { FilePartInfo & cur = sources.item(idx2); - LOG(MCdebugProgress, job, "%9u:%s (size: %llu bytes)", + LOG(MCdebugProgress, "%9u:%s (size: %llu bytes)", idx2, cur.filename.getTail(tailStr.clear()).str(), cur.size ); cur.offset = totalSize; @@ -1999,8 +1997,8 @@ void FileSprayer::afterGatherFileSizes() throwError2(DFTERR_InputIsInvalidMultiple, cur.filename.getRemotePath(s).str(), srcFormat.getUnitSize()); } } - LOG(MCdebugProgress, job, "----------------------------------------------"); - LOG(MCdebugProgress, job, "All together: %llu bytes in %u file(s)", totalSize, sources.ordinality()); + LOG(MCdebugProgress, "----------------------------------------------"); + LOG(MCdebugProgress, "All together: %llu bytes in %u file(s)", totalSize, sources.ordinality()); } } @@ -2016,7 +2014,7 @@ void FileSprayer::gatherFileSizes(FilePartInfoArray & fileSizeQueue, bool errorI unsigned numThreads = (unsigned)sqrt((float)fileSizeQueue.ordinality()); if (numThreads>20) numThreads = 20; - LOG(MCdebugProgress, job, "Gathering %d file sizes on %d threads", fileSizeQueue.ordinality(), numThreads); + LOG(MCdebugProgress, "Gathering %d file sizes on %d threads", fileSizeQueue.ordinality(), numThreads); unsigned idx; for (idx = 0; idx < numThreads; idx++) threads.append(*new FileSizeThread(fileSizeQueue, fileSizeCS, compressedInput&&!copyCompressed, errorIfMissing)); @@ -2098,7 +2096,7 @@ void FileSprayer::gatherMissingSourceTarget(IFileDescriptor * source) } else if (secondarySize != -1) { - LOG(MCwarning, unknownJob, "Replicate - primary and secondary copies have different sizes (%" I64F "d v %" I64F "d) for part %u", primarySize, secondarySize, idx); + LOG(MCwarning, "Replicate - primary and secondary copies have different sizes (%" I64F "d v %" I64F "d) for part %u", primarySize, secondarySize, idx); continue; // ignore copy } } @@ -2106,7 +2104,7 @@ void FileSprayer::gatherMissingSourceTarget(IFileDescriptor * source) { if (primarySize == -1) { - LOG(MCwarning, unknownJob, "Replicate - neither primary or secondary copies exist for part %u", idx); + LOG(MCwarning, "Replicate - neither primary or secondary copies exist for part %u", idx); primarySize = 0; // to stop later failure to gather the file size } continue; // ignore copy @@ -2482,7 +2480,7 @@ void FileSprayer::insertHeaders() } addPrefix(filePrefix.length(), filePrefix.toByteArray(), idx, partitionWork); } - LOG(MCdebugProgress, job, "Publish headers"); + LOG(MCdebugProgress, "Publish headers"); partition.swapWith(partitionWork); } } @@ -2519,7 +2517,7 @@ void FileSprayer::calcNumConcurrentTransfers() numConcurrentTransfers = numTransferParts/3; if (numConcurrentTransfers > numTransferParts) numConcurrentTransfers = numTransferParts; - LOG(MCdebugInfo, job, "numTransferParts: %u, numConcurrentTransfers: %u", numTransferParts, numConcurrentTransfers); + LOG(MCdebugInfo, "numTransferParts: %u, numConcurrentTransfers: %u", numTransferParts, numConcurrentTransfers); } void FileSprayer::performTransfer() @@ -2541,7 +2539,7 @@ void FileSprayer::performTransfer() progressReport->setRange(getSizeReadAlready(), sizeToBeRead, transferSlaves.ordinality()); - LOG(MCdebugInfo, job, "Begin to transfer parts (%d threads)\n", numConcurrentTransfers); + LOG(MCdebugInfo, "Begin to transfer parts (%d threads)\n", numConcurrentTransfers); //Throttle maximum number of concurrent transfers by starting n threads, and //then waiting for one to complete before going on to the next @@ -2588,7 +2586,7 @@ void FileSprayer::performTransfer() void FileSprayer::pullParts() { bool needCalcCRC = calcCRC(); - LOG(MCdebugInfoDetail, job, "Calculate CRC = %d", needCalcCRC); + LOG(MCdebugInfoDetail, "Calculate CRC = %d", needCalcCRC); ForEachItemIn(idx, targets) { FileTransferThread & next = * new FileTransferThread(*this, FTactionpull, targets.item(idx).filename.queryEndpoint(), needCalcCRC, wuid); @@ -2610,7 +2608,7 @@ void FileSprayer::pullParts() void FileSprayer::pushWholeParts() { bool needCalcCRC = calcCRC(); - LOG(MCdebugInfoDetail, job, "Calculate CRC = %d", needCalcCRC); + LOG(MCdebugInfoDetail, "Calculate CRC = %d", needCalcCRC); //Create a slave for each of the target files, but execute it on the node corresponding to the first source file //For container mode this will need to execute on this node, or on a load balanced service ForEachItemIn(idx, targets) @@ -2662,7 +2660,7 @@ void FileSprayer::pushParts() void FileSprayer::transferUsingAPI(IAPICopyClient * copyClient) { - LOG(MCdebugInfo, job, "Transfer files using api: %s", copyClient->name()); + LOG(MCdebugInfo, "Transfer files using api: %s", copyClient->name()); OwnedPointerArrayOf apiClients; unsigned failCount = 0; @@ -2773,7 +2771,7 @@ void FileSprayer::transferUsingAPI(IAPICopyClient * copyClient) void FileSprayer::removeSource() { - LOG(MCwarning, job, "Source file removal not yet implemented"); + LOG(MCwarning, "Source file removal not yet implemented"); } bool FileSprayer::restorePartition() @@ -2940,7 +2938,7 @@ void FileSprayer::setSource(IFileDescriptor * source, unsigned copy, unsigned mi } if (sources.ordinality() == 0) - LOG(MCuserWarning, unknownJob, "The wildcarded source did not match any filenames"); + LOG(MCuserWarning, "The wildcarded source did not match any filenames"); // throwError(DFTERR_NoFilesMatchWildcard); //Now get the size of the files directly (to check they exist). If they don't exist then switch to the backup instead. @@ -3005,7 +3003,7 @@ void FileSprayer::setTarget(IDistributedFile * target) compressOutput = target->isCompressed(); - LOG(MCdebugInfo, unknownJob, "FileSprayer::setTarget: compressedInput:%s, compressOutput:%s", + LOG(MCdebugInfo, "FileSprayer::setTarget: compressedInput:%s, compressOutput:%s", boolToStr(compressedInput), boolToStr(compressOutput)); @@ -3119,7 +3117,7 @@ void FileSprayer::waitForTransferSem(Semaphore & sem) unsigned timeSinceProgress = msTick() - lastProgressTick; if (timeSinceProgress > EXPECTED_RESPONSE_TIME) { - LOG(MCwarning, unknownJob, "No response from any slaves in last %d seconds.", timeSinceProgress/1000); + LOG(MCwarning, "No response from any slaves in last %d seconds.", timeSinceProgress/1000); CriticalBlock block(soFarCrit); StringBuffer list; @@ -3158,7 +3156,7 @@ bool FileSprayer::isAborting() { if (abortChecker->abortRequested()) { - LOG(MCdebugInfo, unknownJob, "Abort requested via callback"); + LOG(MCdebugInfo, "Abort requested via callback"); aborting = true; } lastAbortCheckTick = nowTick; @@ -3258,15 +3256,15 @@ void FileSprayer::spray() progressTree->setPropBool("@noFileMatch", true); } - LOG(MCdebugInfo, job, "compressedInput:%d, compressOutput:%d", compressedInput, compressOutput); - LOG(MCdebugInfo, job, "noCommon:%s", boolToStr(options->getPropBool(ANnocommon))); - LOG(MCdebugInfo, job, "maxConnections option:%d", options->getPropInt(ANmaxConnections)); + LOG(MCdebugInfo, "compressedInput:%d, compressOutput:%d", compressedInput, compressOutput); + LOG(MCdebugInfo, "noCommon:%s", boolToStr(options->getPropBool(ANnocommon))); + LOG(MCdebugInfo, "maxConnections option:%d", options->getPropInt(ANmaxConnections)); LocalAbortHandler localHandler(daftAbortHandler); if (allowRecovery && progressTree->getPropBool(ANcomplete)) { - LOG(MCdebugInfo, job, "Command completed successfully in previous invocation"); + LOG(MCdebugInfo, "Command completed successfully in previous invocation"); return; } @@ -3315,11 +3313,11 @@ void FileSprayer::spray() if (restorePartition()) { - LOG(MCdebugProgress, job, "Partition restored from recovery information"); + LOG(MCdebugProgress, "Partition restored from recovery information"); } else { - LOG(MCdebugProgress, job, "Calculate partition information"); + LOG(MCdebugProgress, "Calculate partition information"); if (replicate || copySource) calculateOne2OnePartition(); else if (!allowSplit()) @@ -3337,7 +3335,7 @@ void FileSprayer::spray() assignPartitionFilenames(); // assign source filenames - used in insertHeaders.. if (!replicate && !copySource) { - LOG(MCdebugProgress, job, "Insert headers"); + LOG(MCdebugProgress, "Insert headers"); insertHeaders(); } addEmptyFilesToPartition(); @@ -3395,7 +3393,7 @@ void FileSprayer::spray() copyEventText.append(distributedTarget->queryLogicalName()); //MORE: use new interface to send 'file copied' event - //LOG(MCevent, unknownJob, EVENT_FILECOPIED, copyEventText.str()); + //LOG(MCevent, EVENT_FILECOPIED, copyEventText.str()); cleanupRecovery(); } @@ -3961,7 +3959,7 @@ bool FileSprayer::calcUsePull() const if (allowRecovery && progressTree->hasProp(ANpull)) { bool usePull = progressTree->getPropBool(ANpull); - LOG(MCdebugInfo, job, "Pull = %d from recovery", (int)usePull); + LOG(MCdebugInfo, "Pull = %d from recovery", (int)usePull); return usePull; } @@ -3970,12 +3968,12 @@ bool FileSprayer::calcUsePull() const if (options->getPropBool(ANpull, false)) { - LOG(MCdebugInfo, job, "Use pull since explicitly specified"); + LOG(MCdebugInfo, "Use pull since explicitly specified"); return true; } if (options->getPropBool(ANpush, false)) { - LOG(MCdebugInfo, job, "Use push since explicitly specified"); + LOG(MCdebugInfo, "Use push since explicitly specified"); return false; } @@ -3985,7 +3983,7 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; sources.item(idx2).filename.queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use pull operation because %s cannot push", s.str()); + LOG(MCdebugInfo, "Use pull operation because %s cannot push", s.str()); return true; } } @@ -3993,7 +3991,7 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; sources.item(0).filename.queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use pull operation because %s doesn't appear to have an ftslave", s.str()); + LOG(MCdebugInfo, "Use pull operation because %s doesn't appear to have an ftslave", s.str()); return true; } @@ -4003,7 +4001,7 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; targets.item(idx).queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use push operation because %s cannot pull", s.str()); + LOG(MCdebugInfo, "Use push operation because %s cannot pull", s.str()); return false; } } @@ -4012,18 +4010,18 @@ bool FileSprayer::calcUsePull() const { StringBuffer s; targets.item(0).queryIP().getHostText(s); - LOG(MCdebugInfo, job, "Use push operation because %s doesn't appear to have an ftslave", s.str()); + LOG(MCdebugInfo, "Use push operation because %s doesn't appear to have an ftslave", s.str()); return false; } //Use push if going to a single node. if ((targets.ordinality() == 1) && (sources.ordinality() > 1)) { - LOG(MCdebugInfo, job, "Use push operation because going to a single node from many"); + LOG(MCdebugInfo, "Use push operation because going to a single node from many"); return false; } - LOG(MCdebugInfo, job, "Use pull operation as default"); + LOG(MCdebugInfo, "Use pull operation as default"); return true; } diff --git a/dali/ft/filecopy.ipp b/dali/ft/filecopy.ipp index 36d6f24cda5..e6005431304 100644 --- a/dali/ft/filecopy.ipp +++ b/dali/ft/filecopy.ipp @@ -88,7 +88,6 @@ protected: Semaphore * sem; byte action; bool calcCRC; - LogMsgJobInfo job; bool allDone; bool started; StringAttr wuid; @@ -318,7 +317,6 @@ protected: OutputProgressArray progress; IDaftProgress * progressReport; IAbortRequestCallback * abortChecker; - LogMsgJobInfo job; offset_t totalSize; unsigned __int64 sizeToBeRead; bool replicate; diff --git a/dali/ft/ftbase.cpp b/dali/ft/ftbase.cpp index 8417af2f355..47cd917ddb9 100644 --- a/dali/ft/ftbase.cpp +++ b/dali/ft/ftbase.cpp @@ -64,7 +64,7 @@ void renameDfuTempToFinal(const RemoteFilename & realname) EXCLOG(e, "Failed to rename target file"); StringBuffer oldName; realname.getPath(oldName); - LOG(MCdebugInfoDetail, unknownJob, "Error: Rename %s->%s failed - tring to delete target and rename again", oldName.str(), newTailname.str()); + LOG(MCdebugInfoDetail, "Error: Rename %s->%s failed - tring to delete target and rename again", oldName.str(), newTailname.str()); e->Release(); OwnedIFile old = createIFile(realname); old->remove(); @@ -124,7 +124,7 @@ void PartitionPoint::deserialize(MemoryBuffer & in) void PartitionPoint::display() { StringBuffer fulli, fullo; - LOG(MCdebugInfoDetail, unknownJob, + LOG(MCdebugInfoDetail, "Partition %s{%d}[%" I64F "d size %" I64F "d]->%s{%d}[%" I64F "d size %" I64F "d]", inputName.getPath(fulli).str(), whichInput, inputOffset, inputLength, outputName.getPath(fullo).str(), whichOutput, outputOffset, outputLength); @@ -581,7 +581,7 @@ MemoryBuffer & OutputProgress::deserializeExtra(MemoryBuffer & in, unsigned vers static const char * const statusText[] = {"Init","Active","Copied","Renamed"}; void OutputProgress::trace() { - LOG(MCdebugInfoDetail, unknownJob, "Chunk %d status: %s input length: %" I64F "d[CRC:%x] -> output length:%" I64F "d[CRC:%x]", whichPartition, statusText[status], inputLength, inputCRC, outputLength, outputCRC); + LOG(MCdebugInfoDetail, "Chunk %d status: %s input length: %" I64F "d[CRC:%x] -> output length:%" I64F "d[CRC:%x]", whichPartition, statusText[status], inputLength, inputCRC, outputLength, outputCRC); } MemoryBuffer & OutputProgress::serializeCore(MemoryBuffer & out) @@ -666,7 +666,7 @@ void OutputProgress::save(IPropertyTree * tree) void displayProgress(OutputProgressArray & progress) { - LOG(MCdebugInfoDetail, unknownJob, "Progress:"); + LOG(MCdebugInfoDetail, "Progress:"); ForEachItemIn(idx, progress) progress.item(idx).trace(); } @@ -675,7 +675,7 @@ void displayProgress(OutputProgressArray & progress) void displayPartition(PartitionPointArray & partition) { - LOG(MCdebugInfoDetail, unknownJob, "Partition:"); + LOG(MCdebugInfoDetail, "Partition:"); ForEachItemIn(idx, partition) partition.item(idx).display(); } @@ -747,7 +747,7 @@ size32_t CrcIOStream::write(size32_t len, const void * data) static int breakCount; bool daftAbortHandler() { - LOG(MCprogress, unknownJob, "Aborting..."); + LOG(MCprogress, "Aborting..."); // hit ^C 3 times to really stop it... if (breakCount++ >= 2) { diff --git a/dali/ft/ftslavelib.cpp b/dali/ft/ftslavelib.cpp index 6e17857a4f3..7ebf3cbe59b 100644 --- a/dali/ft/ftslavelib.cpp +++ b/dali/ft/ftslavelib.cpp @@ -89,7 +89,7 @@ bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryB StringBuffer text; fullPath.getRemotePath(text); - LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str()); + LOG(MCdebugProgress, "Process partition %d(%s)", whichInput, text.str()); Owned processor = createFormatProcessor(srcFormat, tgtFormat, true); Owned target = createOutputProcessor(tgtFormat); processor->setTarget(target); diff --git a/dali/ft/fttransform.cpp b/dali/ft/fttransform.cpp index ee3ab53a3ee..4e8ca2fdec8 100644 --- a/dali/ft/fttransform.cpp +++ b/dali/ft/fttransform.cpp @@ -538,7 +538,7 @@ void TransferServer::appendTransformed(unsigned chunkIndex, ITransformer * input offset_t outputOffset = out->tell(); offset_t inputOffset = input->tell(); if (totalLengthToRead) - LOG(MCdebugProgress, unknownJob, "Progress: %d%% done. [%" I64F "u]", (unsigned)(totalLengthRead*100/totalLengthToRead), (unsigned __int64)totalLengthRead); + LOG(MCdebugProgress, "Progress: %d%% done. [%" I64F "u]", (unsigned)(totalLengthRead*100/totalLengthToRead), (unsigned __int64)totalLengthRead); curProgress.status = (gotLength == 0) ? OutputProgress::StatusCopied : OutputProgress::StatusActive; curProgress.inputLength = input->tell()-startInputOffset; @@ -570,7 +570,7 @@ void TransferServer::appendTransformed(unsigned chunkIndex, ITransformer * input #ifdef _WIN32 if (gpfFrequency && ((fastRand() % gpfFrequency) == 0)) { - LOG(MCdebugInfo, unknownJob, "About to crash...."); + LOG(MCdebugInfo, "About to crash...."); *(char *)0 = 0; } #endif @@ -611,15 +611,15 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action) if (action == FTactionpull) { partition.item(0).outputName.getPath(localFilename); - LOG(MCdebugProgress, unknownJob, "Process Pull Command: %s", localFilename.str()); + LOG(MCdebugProgress, "Process Pull Command: %s", localFilename.str()); } else { partition.item(0).inputName.getPath(localFilename); - LOG(MCdebugProgress, unknownJob, "Process Push Command: %s", localFilename.str()); + LOG(MCdebugProgress, "Process Push Command: %s", localFilename.str()); } - LOG(MCdebugProgress, unknownJob, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency); - LOG(MCdebugProgress, unknownJob, "copySourceTimeStamp(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", copySourceTimeStamp, mirror, isSafeMode, calcInputCRC, calcOutputCRC); + LOG(MCdebugProgress, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency); + LOG(MCdebugProgress, "copySourceTimeStamp(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", copySourceTimeStamp, mirror, isSafeMode, calcInputCRC, calcOutputCRC); displayPartition(partition); @@ -658,7 +658,7 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action) progress.item(i2).deserializeExtra(msg, 2); } - LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize); + LOG(MCdebugProgress, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize); PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0); PROGLOG("encrypt(%d), decrypt(%d)", encryptKey.isEmpty()?0:1, decryptKey.isEmpty()?0:1); if (fileUmask != -1) @@ -691,7 +691,7 @@ void TransferServer::transferChunk(unsigned chunkIndex) StringBuffer targetPath; curPartition.outputName.getPath(targetPath); - LOG(MCdebugProgress, unknownJob, "Begin to transfer chunk %d (offset: %" I64F "d, size: %" I64F "d) to target:'%s' (offset: %" I64F "d, size: %" I64F "d) ", + LOG(MCdebugProgress, "Begin to transfer chunk %d (offset: %" I64F "d, size: %" I64F "d) to target:'%s' (offset: %" I64F "d, size: %" I64F "d) ", chunkIndex, curPartition.inputOffset, curPartition.inputLength, targetPath.str(), curPartition.outputOffset, curPartition.outputLength); const unsigned __int64 startOutOffset = out->tell(); if (startOutOffset != curPartition.outputOffset+curProgress.outputLength) @@ -779,7 +779,7 @@ bool TransferServer::pull() unsigned __int64 progressOffset = curOutputOffset + curProgress.outputLength; if (progressOffset > size) { - LOG(MCwarning, unknownJob, "Recovery information seems to be invalid (%" I64F "d %" I64F "d) start copying from the beginning", + LOG(MCwarning, "Recovery information seems to be invalid (%" I64F "d %" I64F "d) start copying from the beginning", size, progressOffset); //reset any remaining partitions... for (i = start; i < numPartitions; i++) @@ -796,7 +796,7 @@ bool TransferServer::pull() StringBuffer localFilename; localTempFilename.getPath(localFilename); - LOG(MCdebugProgress, unknownJob, "Continue pulling to file: %s from recovery position", localFilename.str()); + LOG(MCdebugProgress, "Continue pulling to file: %s from recovery position", localFilename.str()); start = i; goto processedProgress; // break out of both loops } @@ -835,7 +835,7 @@ bool TransferServer::pull() curOutput = curPartition.whichOutput; if (curProgress.status == OutputProgress::StatusRenamed) { - LOG(MCdebugProgress, unknownJob, "Renamed file found - must be CRC recovery"); + LOG(MCdebugProgress, "Renamed file found - must be CRC recovery"); idx = queryLastOutput(curOutput); continue; } @@ -868,7 +868,7 @@ bool TransferServer::pull() outio.setown(createCompressedFileWriter(outio, false, 0, true, compressor, COMPRESS_METHOD_LZ4)); } - LOG(MCdebugProgress, unknownJob, "Start pulling to file: %s", localFilename.str()); + LOG(MCdebugProgress, "Start pulling to file: %s", localFilename.str()); //Find the last partition entry that refers to the same file. if (!compressOutput && fsProperties.preExtendOutput) @@ -881,7 +881,7 @@ bool TransferServer::pull() stat_type prevWrites = outio->getStatistic(StNumDiskWrites); outio->write(lastOffset-sizeof(null),sizeof(null),&null); curProgress.numWrites += (outio->getStatistic(StNumDiskWrites)-prevWrites); - LOG(MCdebugProgress, unknownJob, "Extend length of target file to %" I64F "d", lastOffset); + LOG(MCdebugProgress, "Extend length of target file to %" I64F "d", lastOffset); } } diff --git a/dali/sasha/sacoalescer.cpp b/dali/sasha/sacoalescer.cpp index 7341411429a..95b885a025f 100644 --- a/dali/sasha/sacoalescer.cpp +++ b/dali/sasha/sacoalescer.cpp @@ -159,7 +159,7 @@ void coalesceDatastore(IPropertyTree *coalesceProps, bool force) } catch (IException *e) { - LOG(MCoperatorError, unknownJob, e, "COALESCER: Unexpected exception, coalesce component halted"); + LOG(MCoperatorError, e, "COALESCER: Unexpected exception, coalesce component halted"); throw; } } diff --git a/dali/sasha/saserver.cpp b/dali/sasha/saserver.cpp index 8dcc464b59d..16f808542cd 100644 --- a/dali/sasha/saserver.cpp +++ b/dali/sasha/saserver.cpp @@ -84,7 +84,7 @@ static void stopServer() ForEachItemInRev(i,servers) { ISashaServer &server=servers.item(i); - LOG(MCprogress, unknownJob, "Stopping %d",i); + LOG(MCprogress, "Stopping %d",i); server.stop(); } ForEachItemInRev(j,servers) @@ -96,10 +96,10 @@ static void stopServer() static bool actionOnAbort() { - LOG(MCprogress, unknownJob, "Stop signalled"); + LOG(MCprogress, "Stop signalled"); if (stopped) { - LOG(MCprogress, unknownJob, "Previously marked stopped. Killing process.."); + LOG(MCprogress, "Previously marked stopped. Killing process.."); queryLogMsgManager()->flushQueue(10*1000); #ifdef _WIN32 TerminateProcess(GetCurrentProcess(), 1); @@ -118,8 +118,8 @@ static bool actionOnAbort() void requestStop(IException *e) { if (e) - LOG(MCoperatorError, unknownJob, e, "SASERVER: Unexpected exception, saserver terminating"); - LOG(MCprogress, unknownJob, "Stop requested"); + LOG(MCoperatorError, e, "SASERVER: Unexpected exception, saserver terminating"); + LOG(MCprogress, "Stop requested"); stopSem.signal(); } diff --git a/dali/sasha/saverify.cpp b/dali/sasha/saverify.cpp index 774fc4c7855..53131d7b571 100644 --- a/dali/sasha/saverify.cpp +++ b/dali/sasha/saverify.cpp @@ -543,9 +543,9 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread ep.getEndpointHostText(epstr); CriticalBlock block(sect); if (failurelimit) { - LOG(MCoperatorError, unknownJob,"DAFSMON: dafilesrv on %s cannot be contacted",epstr.str()); + LOG(MCoperatorError, "DAFSMON: dafilesrv on %s cannot be contacted",epstr.str()); if (--failurelimit==0) - LOG(MCoperatorError, unknownJob,"DAFSMON: monitoring suspended for cluster %s (too many failures)",clustername); + LOG(MCoperatorError, "DAFSMON: monitoring suspended for cluster %s (too many failures)",clustername); } } } @@ -557,7 +557,7 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread ep.getEndpointHostText(epstr); StringBuffer s; s.appendf("DAFSMON: dafilesrv %s",epstr.str()); - LOG(MCoperatorError, unknownJob, e, s.str()); + LOG(MCoperatorError, e, s.str()); e->Release(); return; } diff --git a/dali/server/daserver.cpp b/dali/server/daserver.cpp index d02451e3ca6..3e4aac3c583 100644 --- a/dali/server/daserver.cpp +++ b/dali/server/daserver.cpp @@ -95,13 +95,13 @@ static void stopServer() ForEachItemInRev(h,servers) { IDaliServer &server=servers.item(h); - LOG(MCprogress, unknownJob, "Suspending %d",h); + LOG(MCprogress, "Suspending %d",h); server.suspend(); } ForEachItemInRev(i,servers) { IDaliServer &server=servers.item(i); - LOG(MCprogress, unknownJob, "Stopping %d",i); + LOG(MCprogress, "Stopping %d",i); server.stop(); } closeCoven(); @@ -649,7 +649,7 @@ int main(int argc, const char* argv[]) catch (IException *e) { StringBuffer s("Failure whilst preparing dali backup location: "); - LOG(MCoperatorError, unknownJob, e, s.append(mirrorPath).append(". Backup disabled").str()); + LOG(MCoperatorError, e, s.append(mirrorPath).append(". Backup disabled").str()); serverConfig->removeProp("SDS/@remoteBackupLocation"); e->Release(); } diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index 6d4f793b19d..4fd82e96076 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -128,7 +128,7 @@ static const char * getResultText(StringBuffer & s, const char * stepname, unsig void logGetResult(const char * name, const char * stepname, unsigned sequence) { - LOG(MCgetresult, unknownJob, "getResult%s(%s,%d)", name, nullText(stepname), sequence); + LOG(MCgetresult, "getResult%s(%s,%d)", name, nullText(stepname), sequence); } //======================================================================================= @@ -725,7 +725,7 @@ IConstWUResult *EclAgent::getResultForGet(const char *name, unsigned sequence) IConstWUResult *EclAgent::getExternalResult(const char * wuid, const char *name, unsigned sequence) { - LOG(MCsetresult, unknownJob, "EclAgent::getExternalResult(wuid:'%s',name='%s',sequence:%d)", nullText(wuid), nullText(name), sequence); + LOG(MCsetresult, "EclAgent::getExternalResult(wuid:'%s',name='%s',sequence:%d)", nullText(wuid), nullText(name), sequence); Owned factory = getWorkUnitFactory(); Owned externalWU = factory->openWorkUnit(wuid); if (externalWU) @@ -774,7 +774,7 @@ void EclAgent::outputFormattedResult(const char * name, unsigned sequence, bool void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val, unsigned size) { - LOG(MCsetresult, unknownJob, "setResultInt(%s,%d,%" I64F "d)", nullText(name), sequence, val); + LOG(MCsetresult, "setResultInt(%s,%d,%" I64F "d)", nullText(name), sequence, val); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -798,7 +798,7 @@ void EclAgent::setResultInt(const char * name, unsigned sequence, __int64 val, u void EclAgent::setResultUInt(const char * name, unsigned sequence, unsigned __int64 val, unsigned size) { - LOG(MCsetresult, unknownJob, "setResultUInt(%s,%d,%" I64F "u)", nullText(name), sequence, val); + LOG(MCsetresult, "setResultUInt(%s,%d,%" I64F "u)", nullText(name), sequence, val); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -823,7 +823,7 @@ void EclAgent::setResultUInt(const char * name, unsigned sequence, unsigned __in void EclAgent::setResultReal(const char *name, unsigned sequence, double val) { // Still a bit of a mess - variables vs results - LOG(MCsetresult, unknownJob, "setResultReal(%s,%d,%6f)", nullText(name), sequence, val); + LOG(MCsetresult, "setResultReal(%s,%d,%6f)", nullText(name), sequence, val); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1123,7 +1123,7 @@ void EclAgent::setResultData(const char * stepname, unsigned sequence, int len, void EclAgent::doSetResultString(type_t type, const char *name, unsigned sequence, int len, const char *val) { - LOG(MCsetresult, unknownJob, "setResultString(%s,%d,(%d bytes))", nullText(name), sequence, len); + LOG(MCsetresult, "setResultString(%s,%d,(%d bytes))", nullText(name), sequence, len); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1148,7 +1148,7 @@ void EclAgent::doSetResultString(type_t type, const char *name, unsigned sequenc //used to output a row void EclAgent::setResultRaw(const char * name, unsigned sequence, int len, const void *val) { - LOG(MCsetresult, unknownJob, "setResultRaw(%s,%d,(%d bytes))", nullText(name), sequence, len); + LOG(MCsetresult, "setResultRaw(%s,%d,(%d bytes))", nullText(name), sequence, len); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1172,7 +1172,7 @@ void EclAgent::setResultRaw(const char * name, unsigned sequence, int len, const void EclAgent::setResultSet(const char * name, unsigned sequence, bool isAll, size32_t len, const void *val, ISetToXmlTransformer *xform) { - LOG(MCsetresult, unknownJob, "setResultSet(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultSet(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1232,7 +1232,7 @@ void EclAgent::setResultSet(const char * name, unsigned sequence, bool isAll, si void EclAgent::setResultUnicode(const char * name, unsigned sequence, int len, UChar const * val) { - LOG(MCsetresult, unknownJob, "setResultUnicode(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultUnicode(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); @@ -1260,7 +1260,7 @@ void EclAgent::setResultUnicode(const char * name, unsigned sequence, int len, U void EclAgent::setResultBool(const char *name, unsigned sequence, bool val) { - LOG(MCsetresult, unknownJob, "setResultBool(%s,%d,%s)", nullText(name), sequence, val ? "true" : "false"); + LOG(MCsetresult, "setResultBool(%s,%d,%s)", nullText(name), sequence, val ? "true" : "false"); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); @@ -1285,7 +1285,7 @@ void EclAgent::setResultBool(const char *name, unsigned sequence, bool val) void EclAgent::setResultDecimal(const char *name, unsigned sequence, int len, int precision, bool isSigned, const void *val) { - LOG(MCsetresult, unknownJob, "setResultDecimal(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultDecimal(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned r = updateWorkUnitResult(w, name, sequence); if (r) @@ -1314,7 +1314,7 @@ void EclAgent::setResultDecimal(const char *name, unsigned sequence, int len, in void EclAgent::setResultDataset(const char * name, unsigned sequence, size32_t len, const void *val, unsigned numRows, bool extend) { - LOG(MCsetresult, unknownJob, "setResultDataset(%s,%d)", nullText(name), sequence); + LOG(MCsetresult, "setResultDataset(%s,%d)", nullText(name), sequence); WorkunitUpdate w = updateWorkUnit(); Owned result = updateWorkUnitResult(w, name, sequence); if (!result) @@ -1866,11 +1866,11 @@ void EclAgent::doProcess() CCycleTimer elapsedTimer; try { - LOG(MCrunlock, unknownJob, "Waiting for workunit lock"); + LOG(MCrunlock, "Waiting for workunit lock"); unsigned eclccCodeVersion; { WorkunitUpdate w = updateWorkUnit(); - LOG(MCrunlock, unknownJob, "Obtained workunit lock"); + LOG(MCrunlock, "Obtained workunit lock"); if (w->hasDebugValue("traceLevel")) traceLevel = w->getDebugValueInt("traceLevel", 10); w->setTracingValue("EclAgentBuild", hpccBuildInfo.buildTag); @@ -2266,9 +2266,9 @@ void EclAgent::runProcess(IEclProcess *process) debugContext->checkBreakpoint(DebugStateFinished, NULL, NULL); debugContext->debugTerminate(); } - LOG(MCrunlock, unknownJob, "Releasing persist read locks"); + LOG(MCrunlock, "Releasing persist read locks"); persistReadLocks.kill(); - LOG(MCrunlock, unknownJob, "Released persist read locks"); + LOG(MCrunlock, "Released persist read locks"); } unsigned EclAgent::getWorkflowIdDeprecated() @@ -2301,7 +2301,7 @@ IRemoteConnection *EclAgentWorkflowMachine::startPersist(const char * logicalNam { persistLock = persistCache.getValue(logicalName); persistCache.setValue(logicalName, NULL); - LOG(MCrunlock, unknownJob, "Decached persist read lock for %s", logicalName); + LOG(MCrunlock, "Decached persist read lock for %s", logicalName); } else persistLock = agent.startPersist(logicalName); @@ -2383,7 +2383,7 @@ void EclAgentWorkflowMachine::prelockPersists() char const * name = names.item(idx); Owned persistLock = agent.startPersist(name); persistCache.setValue(name, persistLock); - LOG(MCrunlock, unknownJob, "Cached persist read lock for %s", name); + LOG(MCrunlock, "Cached persist read lock for %s", name); } persistsPrelocked = true; } @@ -2424,16 +2424,16 @@ void EclAgentWorkflowMachine::obtainRunlock() { StringBuffer xpath; xpath.append("/WorkUnitRunLocks/").append(agent.wuid.get()); - LOG(MCrunlock, unknownJob, "Waiting for run lock"); + LOG(MCrunlock, "Waiting for run lock"); runlock.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_CREATE | RTM_LOCK_WRITE | RTM_DELETE_ON_DISCONNECT, INFINITE)); - LOG(MCrunlock, unknownJob, "Obtained run lock"); + LOG(MCrunlock, "Obtained run lock"); if(!runlock) agent.fail(0, "EclAgent could not get a lock to run the workunit"); } void EclAgentWorkflowMachine::releaseRunlock() { - LOG(MCrunlock, unknownJob, "Releasing run lock"); + LOG(MCrunlock, "Releasing run lock"); if(runlock && queryDaliServerVersion().compare("1.3") < 0) runlock->close(true); runlock.clear(); @@ -2448,7 +2448,7 @@ IRemoteConnection *EclAgentWorkflowMachine::obtainCriticalLock(const char *name) void EclAgentWorkflowMachine::releaseCriticalLock(IRemoteConnection *criticalLock) { - LOG(MCrunlock, unknownJob, "Releasing critical lock"); + LOG(MCrunlock, "Releasing critical lock"); if(criticalLock && queryDaliServerVersion().compare("1.3") < 0) criticalLock->close(true); } @@ -2546,7 +2546,7 @@ void EclAgentWorkflowMachine::doExecutePersistItem(IRuntimeWorkflowItem & item) { persistLock.set(persistCache.getValue(logicalName)); persistCache.setValue(logicalName, NULL); - LOG(MCrunlock, unknownJob, "Decached persist read lock for %s", logicalName); + LOG(MCrunlock, "Decached persist read lock for %s", logicalName); } else persistLock.setown(agent.startPersist(logicalName)); @@ -2804,7 +2804,7 @@ bool EclAgent::checkPersistUptoDate(IRuntimeWorkflowItem & item, const char * lo bool EclAgent::changePersistLockMode(IRemoteConnection *persistLock, unsigned mode, const char * name, bool repeat) { - LOG(MCrunlock, unknownJob, "Waiting to change persist lock to %s for %s", (mode == RTM_LOCK_WRITE) ? "write" : "read", name); + LOG(MCrunlock, "Waiting to change persist lock to %s for %s", (mode == RTM_LOCK_WRITE) ? "write" : "read", name); //When converting a read lock to a write lock so the persist can be rebuilt hold onto the lock as short as //possible. Otherwise lots of workunits each trying to convert read locks to write locks will mean //that the read lock is never released by all the workunits at the same time, so no workunit can progress. @@ -2850,7 +2850,7 @@ IRemoteConnection *EclAgent::getPersistReadLock(const char * logicalName) for (const char * cur = name;*cur;cur++) xpath.append(isalnum(*cur) ? *cur : '_'); - LOG(MCrunlock, unknownJob, "Waiting for persist read lock for %s", name); + LOG(MCrunlock, "Waiting for persist read lock for %s", name); Owned persistLock; for (;;) { @@ -2972,7 +2972,7 @@ bool EclAgent::alreadyLockedPersist(const char * persistName) void EclAgent::finishPersist(const char * persistName, IRemoteConnection *persistLock) { - LOG(MCrunlock, unknownJob, "Finished persists - add to read lock list"); + LOG(MCrunlock, "Finished persists - add to read lock list"); processedPersists.append(persistName); persistReadLocks.append(*persistLock); } @@ -3289,7 +3289,7 @@ void EclAgent::reportProgress(const char *progress, unsigned flags) if (progress) { // MORE - think about how to best do this - LOG(MCdebugProgress, unknownJob,"%s", progress); + LOG(MCdebugProgress, "%s", progress); // WorkunitUpdate wu = updateWorkUnit(); // wu->reportProgress(progress, flags); } diff --git a/ecl/eclcc/eclcc.cpp b/ecl/eclcc/eclcc.cpp index eda80d7d39e..ece9f7717d7 100644 --- a/ecl/eclcc/eclcc.cpp +++ b/ecl/eclcc/eclcc.cpp @@ -1022,7 +1022,7 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char void EclCC::getComplexity(IWorkUnit *wu, IHqlExpression * query, IErrorReceiver & errs) { double complexity = getECLcomplexity(query, &errs, wu, optTargetClusterType); - LOG(MCoperatorProgress, unknownJob, "Complexity = %g", complexity); + LOG(MCoperatorProgress, "Complexity = %g", complexity); } //========================================================================================= @@ -2317,7 +2317,7 @@ void EclCC::traceError(char const * format, ...) UERRLOG("", encoded.str()); } else - VALOG(MCuserError, unknownJob, format, args); + VALOG(MCuserError, format, args); va_end(args); } diff --git a/ecl/hql/hqlexpr.cpp b/ecl/hql/hqlexpr.cpp index 7b86bb23ab3..90e2ef1d91a 100644 --- a/ecl/hql/hqlexpr.cpp +++ b/ecl/hql/hqlexpr.cpp @@ -15469,7 +15469,7 @@ extern HQL_API IHqlExpression * getFileViewerRecord(IHqlExpression * record, boo } catch (IException * e) { - LOG(MCwarning, unknownJob, e); + LOG(MCwarning, e); e->Release(); } return NULL; diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp index 2b8a188b6bc..56139d12077 100644 --- a/ecl/hqlcpp/hqlhtcpp.cpp +++ b/ecl/hqlcpp/hqlhtcpp.cpp @@ -19562,11 +19562,11 @@ static void logECL(const LogMsgCategory & category, size32_t len, const char * e unsigned size = next-ecl; if (ecl[size-1] == '\r') size--; - LOG(category, unknownJob, "%.*s", size, ecl); + LOG(category, "%.*s", size, ecl); len -= (next+1-ecl); ecl = next+1; } - LOG(category, unknownJob, "%s", ecl); + LOG(category, "%s", ecl); } @@ -19580,7 +19580,7 @@ void HqlCppTranslator::traceExpression(const char * title, IHqlExpression * expr if (!queryTrace(title)) return; - LOG(MCdebugInfo, unknownJob, "Tracing expressions: %s", title); + LOG(MCdebugInfo, "Tracing expressions: %s", title); if (options.traceIR) { EclIR::dbglogIR(expr); @@ -19607,7 +19607,7 @@ void HqlCppTranslator::traceExpressions(const char * title, WorkflowArray & work if (!queryTrace(title)) return; - LOG(MCdebugInfo, unknownJob, "Tracing expressions: %s", title); + LOG(MCdebugInfo, "Tracing expressions: %s", title); ForEachItemIn(idx1, workflow) { WorkflowItem & cur = workflow.item(idx1); @@ -19615,7 +19615,7 @@ void HqlCppTranslator::traceExpressions(const char * title, WorkflowArray & work if (compound) { - LOG(MCdebugInfo, unknownJob, "%s: #%d: id[%d]", title, idx1, cur.queryWfid()); + LOG(MCdebugInfo, "%s: #%d: id[%d]", title, idx1, cur.queryWfid()); if (options.traceIR) { diff --git a/ecl/hthor/hthorkey.cpp b/ecl/hthor/hthorkey.cpp index 241641bd0f6..7fb6c7952ef 100644 --- a/ecl/hthor/hthorkey.cpp +++ b/ecl/hthor/hthorkey.cpp @@ -681,7 +681,7 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist //MORE: We could introduce a more efficient way of checking this that does not create a translator Owned actualTranslator = createRecordTranslator(expectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true)); - DBGLOG("Record layout translator created for %s", f->queryLogicalName()); + DBGLOG("Record layout translator created for %s", f->queryLogicalName()); actualTranslator->describe(); if (actualTranslator->keyedTranslated()) throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName()); @@ -2458,7 +2458,7 @@ class CHThorFlatFetchActivity : public CHThorFetchActivityBase, public IFlatFetc if (actualDiskMeta) { translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true))); - DBGLOG("Record layout translator created for %s", f->queryLogicalName()); + DBGLOG("Record layout translator created for %s", f->queryLogicalName()); translator->describe(); if (translator->canTranslate()) { diff --git a/ecl/wutest/wujobqtest.cpp b/ecl/wutest/wujobqtest.cpp index 276f8d0f07f..40f2e679f5b 100644 --- a/ecl/wutest/wujobqtest.cpp +++ b/ecl/wutest/wujobqtest.cpp @@ -155,11 +155,11 @@ void testEnqueue(unsigned nthreads,const char *qname) { unsigned start = msTick(); if (!partfile->remove()&&(copy==0)) // only warn about missing primary files - LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); + LOG(MCwarning, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); else { unsigned t = msTick()-start; if (t>5*1000) - LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); + LOG(MCwarning, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } } diff --git a/esp/bindings/SOAP/Platform/soapparam.hpp b/esp/bindings/SOAP/Platform/soapparam.hpp index be999d8e10a..a0f1a0f6036 100644 --- a/esp/bindings/SOAP/Platform/soapparam.hpp +++ b/esp/bindings/SOAP/Platform/soapparam.hpp @@ -1079,32 +1079,6 @@ class SoapEnumParamNew UNIMPLEMENTED; } - void getXsdDefinition_(IEspContext &context, IHttpMessage *request, StringBuffer &schema, BoolHash &added, const char* descriptions[]) - { - if (added.getValue(typeName_)) - return; - - added.setValue(typeName_, 1); - - schema.appendf("", typeName_.get()); - - if ( descriptions && ((context.queryOptions()&ESPCTX_NO_ANNOTATION) == 0) ) - { - schema.append(""); - for (unsigned i=0; i", enumstrings.item(i), desc); - } - schema.append(""); - } - - schema.appendf("", baseType_.get()); - for (unsigned i=0; i", enumstrings.item(i)); - schema.append(""); - } }; template diff --git a/esp/bindings/SOAP/Platform/soapservice.cpp b/esp/bindings/SOAP/Platform/soapservice.cpp index cee7fda5e52..4a58cd3c51e 100644 --- a/esp/bindings/SOAP/Platform/soapservice.cpp +++ b/esp/bindings/SOAP/Platform/soapservice.cpp @@ -195,7 +195,7 @@ int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp) rpc_call->preunmarshall(xpp.get()); rpc_call->unmarshall(xpp.get(), multipart.get()); const char* userId = ctx->queryUserId(); - DBGLOG("JSON method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", + DBGLOG("JSON method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown"); ctx->setHTTPMethod("JSON"); } @@ -243,12 +243,12 @@ int CSoapService::processRequest(ISoapMessage &req, ISoapMessage& resp) } catch (...) { response.set_status(SOAP_CLIENT_ERROR); response.set_err("Unknown error when parsing soap body XML"); - IERRLOG("SOAP request from %s@%s. Unknown error when parsing: %s", (userId&&*userId)?userId:"unknown", + IERRLOG("SOAP request from %s@%s. Unknown error when parsing: %s", (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown", requeststr.str()); return 0; } - DBGLOG("SOAP method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", + DBGLOG("SOAP method <%s> from %s@%s.", rpc_call->get_name(), (userId&&*userId)?userId:"unknown", (peerStr.length()>0)?peerStr.str():"unknown"); ctx->setHTTPMethod("SOAP"); } diff --git a/esp/bindings/http/platform/httpbinding.cpp b/esp/bindings/http/platform/httpbinding.cpp index cabdfd0630d..2d1c646f39f 100644 --- a/esp/bindings/http/platform/httpbinding.cpp +++ b/esp/bindings/http/platform/httpbinding.cpp @@ -50,6 +50,7 @@ #include "esdl_def_helper.hpp" + #define FILE_UPLOAD "FileUploadAccess" #define DEFAULT_HTTP_PORT 80 @@ -1453,14 +1454,7 @@ void EspHttpBinding::getSoapMessage(StringBuffer& soapmsg, IEspContext& ctx, CHt StringBuffer req, tag, schema, filtered; msg->marshall(req, NULL); - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(ctx, request, serviceQName, methodQName, getVersion(ctx), false, false, schema); - } - else - { - getSchema(schema,ctx,request,serviceQName,methodQName,false); - } + getServiceSchema(ctx, request, serviceQName, methodQName, getVersion(ctx), false, false, schema); getXMLMessageTag(ctx, true, methodQName, tag); filterXmlBySchema(req,schema,tag.str(),filtered); @@ -1756,88 +1750,10 @@ int EspHttpBinding::onGetVersion(IEspContext &context, CHttpRequest* request, CH bool EspHttpBinding::getSchema(StringBuffer& schema, IEspContext &ctx, CHttpRequest* req, const char *service, const char *method,bool standalone) { - StringBuffer serviceQName; - StringBuffer methodQName; - - if (!qualifyServiceName(ctx, service, method, serviceQName, &methodQName)) - return false; - - const char *sqName = serviceQName.str(); - const char *mqName = methodQName.str(); - - Owned namespaces = createPTree(); - appendSchemaNamespaces(namespaces, ctx, req, service, method); - Owned nsiter = namespaces->getElements("namespace"); + throw MakeStringException(-1, "getSchema deprecated in EspHttpBinding, use getServiceSchema instead"); +} - StringBuffer nstr; - generateNamespace(ctx, req, sqName, mqName, nstr); - schema.appendf("query(); - schema.appendf(" xmlns:%s=\"%s\"", ns.queryProp("@nsvar"), ns.queryProp("@ns")); - } - schema.append(">\n"); - ForEach(*nsiter) - { - IPropertyTree &ns = nsiter->query(); - if (ns.hasProp("@import")) - schema.appendf("", ns.queryProp("@ns"), ns.queryProp("@location")); - } - - - schema.append( - "" - "" - "" - "" - "" - "" - "" - "\n" - "" - "" - "" - "" - "" - "\n" - "\n" - ); - - if (ctx.queryOptions()&ESPCTX_WSDL_EXT) - { - schema.append( - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "" - "\n" - ); - } - bool mda=(req->queryParameters()->getPropInt("mda")!=0); - getXsdDefinition(ctx, req, schema, sqName, mqName, mda); - schema.append("\n"); - schema.append(""); - return true; -} void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* request, const char *serviceQName, const char *methodQName, double version, bool isWsdl, bool addAnnotations, StringBuffer &schema) @@ -1848,6 +1764,12 @@ void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* reque Owned esdlDef = createEsdlDefinition(nullptr, nullptr); Owned defHelper = createEsdlDefinitionHelper(); + StringBuffer xmlFilename; + if (!getServiceXmlFilename(xmlFilename)) + { + throw MakeStringException(-1, "Unable to get service XML filename"); + } + StringBuffer nstr; generateEsdlNamespace(context, request, serviceQName, methodQName, nstr); @@ -1875,7 +1797,7 @@ void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* reque } std::string xmlFilePath(getCFD()); - xmlFilePath.append("esdl_files").append(PATHSEPSTR).append(serviceXmlFilename.str()); + xmlFilePath.append("esdl_files").append(PATHSEPSTR).append(xmlFilename.str()); esdlDef->addDefinitionsFromFile(xmlFilePath.c_str()); unsigned optFlags{DEPFLAG_COLLAPSE|DEPFLAG_ARRAYOF}; Owned structs = esdlDef->getDependencies(serviceQName, methodQName, ";", version, opts, optFlags ); @@ -1889,32 +1811,12 @@ void EspHttpBinding::getServiceSchema(IEspContext& context, CHttpRequest* reque int EspHttpBinding::onGetXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method) { - int rc; - if (!serviceXmlFilename.isEmpty()) - { - rc = getServiceWsdlOrXsd(context, request, response, service, method, false); - } - else - { - rc = getWsdlOrXsd(context,request,response,service,method,false); - } - - return rc; + return getServiceWsdlOrXsd(context, request, response, service, method, false); } int EspHttpBinding::onGetWsdl(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method) { - int rc; - if (!serviceXmlFilename.isEmpty()) - { - rc = getServiceWsdlOrXsd(context, request, response, service, method, true); - } - else - { - rc = getWsdlOrXsd(context,request,response,service,method,true); - } - - return rc; + return getServiceWsdlOrXsd(context, request, response, service, method, true); } int EspHttpBinding::getServiceWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl) @@ -1952,81 +1854,6 @@ int EspHttpBinding::getServiceWsdlOrXsd(IEspContext &context, CHttpRequest* requ return 0; } -int EspHttpBinding::getWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl) -{ - bool mda=(request->queryParameters()->getPropInt("mda")!=0); - try - { - StringBuffer serviceQName; - StringBuffer methodQName; - - if (!qualifyServiceName(context, service, method, serviceQName, &methodQName)) - { - return onGetNotFound(context, request, response, service); - } - else - { - const char *sqName = serviceQName.str(); - const char *mqName = methodQName.str(); - StringBuffer ns; - generateNamespace(context, request, serviceQName.str(), methodQName.str(), ns); - - StringBuffer content(""); - if (context.queryRequestParameters()->hasProp("display")) - content.append(""); - else if (isWsdl && context.queryRequestParameters()->hasProp("wsdlviewer")) - content.append(""); - if (isWsdl) - { - content.appendf("", ns.str(), ns.str()); - content.append(""); - } - - getSchema(content,context,request,service,method,!isWsdl); - - if (isWsdl) - { - content.append(""); - - getWsdlMessages(context, request, content, sqName, mqName, mda); - getWsdlPorts(context, request, content, sqName, mqName, mda); - getWsdlBindings(context, request, content, sqName, mqName, mda); - - StringBuffer location(m_wsdlAddress.str()); - if (request->queryParameters()->hasProp("wsdl_destination_path")) - location.append(request->queryParameters()->queryProp("wsdl_destination_path")); - else - location.append('/').append(sqName).appendf("?ver_=%g", context.getClientVersion()); - - if (request->queryParameters()->hasProp("encode_results")) - { - const char *encval = request->queryParameters()->queryProp("encode_results"); - location.append("&").appendf("encode_=%s", (encval && *encval) ? encval : "1"); - } - - content.appendf("", sqName); - content.appendf("", sqName, sqName); - content.appendf("", location.str()); - content.append(""); - content.append(""); - content.append(""); - } - - response->setContent(content.length(), content.str()); - response->setContentType(HTTP_TYPE_APPLICATION_XML_UTF8); - response->setStatus(HTTP_STATUS_OK); - } - } - catch (IException *e) - { - return onGetException(context, request, response, *e); - } - - response->send(); - return 0; -} static void genSampleXml(StringStack& parent, IXmlType* type, StringBuffer& out, const char* tag, const char* ns=NULL) { @@ -2114,14 +1941,7 @@ void EspHttpBinding::generateSampleXml(bool isRequest, IEspContext &context, CHt return; } - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); - } - else - { - getSchema(schemaXml,context,request,serv,method,false); - } + getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); Owned schema; IXmlType* type = nullptr; @@ -2195,14 +2015,7 @@ void EspHttpBinding::generateSampleJson(bool isRequest, IEspContext &context, CH return; } - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); - } - else - { - getSchema(schemaXml, context, request, serv, method, false); - } + getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schemaXml); Owned schema; IXmlType* type = nullptr; @@ -2677,14 +2490,7 @@ int EspHttpBinding::onGetXForm(IEspContext &context, CHttpRequest* request, CHtt double ver = getVersion(context); StringBuffer schema; - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, schema); - } - else - { - getSchema(schema,context,request,serv,method,false); - } + getServiceSchema(context, request, serviceQName, methodQName, ver, false, true, schema); StringBuffer page; IXslProcessor* xslp = getXmlLibXslProcessor(); @@ -2988,14 +2794,7 @@ void EspHttpBinding::validateResponse(IEspContext& context, CHttpRequest* reques } // schema - if (!serviceXmlFilename.isEmpty()) - { - getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, xsd); - } - else - { - getSchema(xsd,context,request,service,method,false); - } + getServiceSchema(context, request, serviceQName, methodQName, getVersion(context), false, false, xsd); // validation if (getEspLogLevel()>LogMax) @@ -3030,6 +2829,7 @@ void EspHttpBinding::validateResponse(IEspContext& context, CHttpRequest* reques unsigned len = temp.length(); // This has to be done before temp.detach is called! content.setBuffer(len, temp.detach(), true); } + void EspHttpBinding::sortResponse(IEspContext& context, CHttpRequest* request, MemoryBuffer& content, const char *serviceName, const char* methodName) { diff --git a/esp/bindings/http/platform/httpbinding.hpp b/esp/bindings/http/platform/httpbinding.hpp index 1dd35a2c6dc..fa94e6146df 100644 --- a/esp/bindings/http/platform/httpbinding.hpp +++ b/esp/bindings/http/platform/httpbinding.hpp @@ -25,6 +25,7 @@ #include "bindutil.hpp" #include "seclib.hpp" + class CMethodInfo : public CInterface { public: @@ -112,6 +113,7 @@ interface IEspWsdlSections // virtual MethodInfoArray & queryQualifiedNames(IEspContext& ctx)=0; virtual int getQualifiedNames(IEspContext& ctx, MethodInfoArray & methods)=0; virtual int getXsdDefinition(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; + virtual int getServiceXmlFilename(StringBuffer &filename)=0; virtual int getWsdlMessages(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; virtual int getWsdlPorts(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; virtual int getWsdlBindings(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0; @@ -192,7 +194,6 @@ class esp_http_decl EspHttpBinding : Owned corsHelper; - void getXMLMessageTag(IEspContext& ctx, bool isRequest, const char *method, StringBuffer& tag); protected: @@ -201,7 +202,6 @@ class esp_http_decl EspHttpBinding : bool m_includeJsonTest; StringBuffer m_challenge_realm; StringAttr m_defaultSvcVersion; - StringBuffer serviceXmlFilename; public: EspHttpBinding(IPropertyTree* cfg, const char *bindname=NULL, const char *procname=NULL); @@ -348,6 +348,7 @@ class esp_http_decl EspHttpBinding : // MethodInfoArray &queryQualifiedNames(IEspContext& ctx) { m_methods.popAll(); getQualifiedNames(ctx,m_methods); return m_methods;}; int getXsdDefinition(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda){return 0;}; + int getServiceXmlFilename(StringBuffer &filename) {return 0;}; int getWsdlMessages(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda); int getWsdlPorts(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda); int getWsdlBindings(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda); @@ -356,7 +357,6 @@ class esp_http_decl EspHttpBinding : virtual int getMethodHelp(IEspContext &context, const char *serv, const char *method, StringBuffer &page); bool isMethodInService(IEspContext& context, const char *servname, const char *methname); - virtual int getMethodHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &page, bool bIncludeFormTag){return 0;} virtual bool hasSubService(IEspContext &context, const char *name); virtual IRpcRequestBinding *createReqBinding(IEspContext &context, IHttpMessage *ireq, const char *service, const char *method){return NULL;} @@ -445,7 +445,6 @@ class esp_http_decl EspHttpBinding : protected: virtual bool basicAuth(IEspContext* ctx); - int getWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl); int getServiceWsdlOrXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, bool isWsdl); virtual bool getSchema(StringBuffer& schema, IEspContext &ctx, CHttpRequest* req, const char *service, const char *method,bool standalone); virtual void appendSchemaNamespaces(IPropertyTree *namespaces, IEspContext &ctx, CHttpRequest* req, const char *service, const char *method){} diff --git a/esp/esdllib/esdl_def.cpp b/esp/esdllib/esdl_def.cpp index dce6db127f8..62ee2797892 100644 --- a/esp/esdllib/esdl_def.cpp +++ b/esp/esdllib/esdl_def.cpp @@ -1669,7 +1669,7 @@ unsigned EsdlDefinition::walkChildrenDepthFirst( AddedObjs& foundByName, EsdlDef children->next(); } } //else { DBGLOG("%s<%s> no Children", StringBuffer(level*2, " ").str(), esdlObj->queryName()); } - } //else { DBGLOG("%s<%s> is not Struct", StringBuffer(level*2, " ").str(), esdlObj->queryName()); } + } //else { DBGLOG("%s<%s> is not Struct", StringBuffer(level*2, " ").str(), esdlObj->queryName()); } return stateOut; } diff --git a/esp/platform/espcfg.cpp b/esp/platform/espcfg.cpp index 60c495aa97f..911e5e163b9 100644 --- a/esp/platform/espcfg.cpp +++ b/esp/platform/espcfg.cpp @@ -689,7 +689,7 @@ void CEspConfig::loadBinding(binding_cfg &xcfg) { IEspRpcBinding* bind = xproc(xcfg.name.str(), xcfg.type.str(), m_envpt.get(), m_process.str()); if (bind) - LOG(MCoperatorInfo, unknownJob,"Load binding %s (type: %s, process: %s) succeeded", xcfg.name.str(), xcfg.type.str(), m_process.str()); + LOG(MCoperatorInfo, "Load binding %s (type: %s, process: %s) succeeded", xcfg.name.str(), xcfg.type.str(), m_process.str()); else OERRLOG("Failed to load binding %s (type: %s, process: %s)", xcfg.name.str(), xcfg.type.str(), m_process.str()); xcfg.bind.setown(bind); diff --git a/esp/platform/espcontext.cpp b/esp/platform/espcontext.cpp index 3270273fe5b..ea2c90f2d23 100755 --- a/esp/platform/espcontext.cpp +++ b/esp/platform/espcontext.cpp @@ -1029,7 +1029,7 @@ void ESPLOG(LogLevel level, const char* fmt, ...) { va_list args; va_start(args,fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } } @@ -1040,7 +1040,7 @@ void ESPLOG(IEspContext* ctx, LogLevel level, const char* fmt, ...) { va_list args; va_start(args,fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } } diff --git a/esp/platform/espp.hpp b/esp/platform/espp.hpp index 22fc9b742ef..27885454463 100644 --- a/esp/platform/espp.hpp +++ b/esp/platform/espp.hpp @@ -186,7 +186,7 @@ class CEspServer : public CInterface, { va_list args; va_start(args, fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } } diff --git a/esp/scm/ws_smc.ecm b/esp/scm/ws_smc.ecm index 5f2d7cc206f..c8121f80c60 100644 --- a/esp/scm/ws_smc.ecm +++ b/esp/scm/ws_smc.ecm @@ -32,7 +32,7 @@ ESPStruct ActiveWorkunit string Owner; string Jobname; string Server; - strint Instance; + string Instance; string Priority; [min_ver("1.01")] string Extra; [min_ver("1.02")] string GraphName; diff --git a/esp/scm/ws_topology.ecm b/esp/scm/ws_topology.ecm index 5a5d5447759..fafa113e054 100644 --- a/esp/scm/ws_topology.ecm +++ b/esp/scm/ws_topology.ecm @@ -415,14 +415,10 @@ ESPresponse [exceptions_inline,encode(0)] TpClusterInfoResponse }; -ESPenum TpMachineType : int -{ - TPMACHINECLUSTER(0, "TPMACHINECLUSTER") -}; ESPrequest [] TpMachineQueryRequest { //Type of Node the parent is.... - TpMachineType Type; + string Type; string Cluster; string OldIP; //Path to Parent within Dali(Encoded to prevent back-engineering) @@ -437,7 +433,7 @@ ESPresponse [exceptions_inline] TpMachineQueryResponse { bool EnablePreflightInfo; [min_ver("1.15")] bool HasThorSpareProcess; - TpMachineType Type; + string Type; string Cluster; string OldIP; string LogDirectory; diff --git a/esp/services/ws_fileio/ws_fileioservice.cpp b/esp/services/ws_fileio/ws_fileioservice.cpp index 304adaa908d..00335be3c88 100644 --- a/esp/services/ws_fileio/ws_fileioservice.cpp +++ b/esp/services/ws_fileio/ws_fileioservice.cpp @@ -159,13 +159,13 @@ bool CWsFileIOEx::onReadFileData(IEspContext &context, IEspReadFileDataRequest & if (io->read(offset, (int)dataToRead, buf) != dataToRead) { resp.setResult("ReadFileData error."); - LOG(MCprogress, unknownJob, "ReadFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "ReadFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } else { resp.setData(membuf); resp.setResult("ReadFileData done."); - LOG(MCprogress, unknownJob, "ReadFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "ReadFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } return true; @@ -237,12 +237,12 @@ bool CWsFileIOEx::onWriteFileData(IEspContext &context, IEspWriteFileDataRequest if (fileio->write(offset, len, srcdata.readDirect(len)) != len) { resp.setResult("WriteFileData error."); - LOG(MCprogress, unknownJob, "WriteFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "WriteFileData error: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } else { resp.setResult("WriteFileData done."); - LOG(MCprogress, unknownJob, "WriteFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); + LOG(MCprogress, "WriteFileData done: %s: %s %s", context.getUserID(user).str(), server, destRelativePath); } return true; diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp index 9ac375c37ff..4a83c2cc65c 100644 --- a/esp/services/ws_fs/ws_fsService.cpp +++ b/esp/services/ws_fs/ws_fsService.cpp @@ -1904,7 +1904,7 @@ IPropertyTree *CFileSprayEx::getAndValidateDropZone(const char *path, const char if (env->isDropZoneRestrictionEnabled()) throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "DropZone Plane not found for host %s path %s.", host, path); - LOG(MCdebugInfo, unknownJob, "No matching drop zone path on '%s' to file path: '%s'", host, path); + LOG(MCdebugInfo, "No matching drop zone path on '%s' to file path: '%s'", host, path); #endif return nullptr; } diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 309183dfd82..4b6ff05ad2b 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -467,7 +467,7 @@ unsigned WsWuInfo::sendComponentLogContent(IEspContext* context, IRemoteLogAcces if (abortCallback.abortRequested()) { - LOG(MCdebugInfo, unknownJob, "WsWuInfo::sendComponentLogContent(): abort requested via callback"); + LOG(MCdebugInfo, "WsWuInfo::sendComponentLogContent(): abort requested via callback"); break; } } diff --git a/esp/smc/SMCLib/LogicFileWrapper.hpp b/esp/smc/SMCLib/LogicFileWrapper.hpp index 032ae8f969d..3fc70350911 100644 --- a/esp/smc/SMCLib/LogicFileWrapper.hpp +++ b/esp/smc/SMCLib/LogicFileWrapper.hpp @@ -95,7 +95,7 @@ struct DeleteTask: public CInterface, implements ITask { StringBuffer e; e.appendf("Failed to remove file part %s\n",file->queryFilename()); - LOG(MCerror, unknownJob, "%s", e.str()); + LOG(MCerror, "%s", e.str()); errs.append(e); } } diff --git a/esp/smc/SMCLib/TpCommon.cpp b/esp/smc/SMCLib/TpCommon.cpp index ea498ebfcb0..d31c6ecd32f 100644 --- a/esp/smc/SMCLib/TpCommon.cpp +++ b/esp/smc/SMCLib/TpCommon.cpp @@ -90,7 +90,7 @@ extern TPWRAPPER_API void throwOrLogDropZoneLookUpError(int code, char const* fo if (env->isDropZoneRestrictionEnabled()) throw makeStringExceptionVA(code, format, args); - VALOG(MCdebugInfo, unknownJob, format, args); + VALOG(MCdebugInfo, format, args); #endif va_end(args); } diff --git a/esp/smc/SMCLib/TpContainer.cpp b/esp/smc/SMCLib/TpContainer.cpp index ddc310d96d8..84df690672d 100644 --- a/esp/smc/SMCLib/TpContainer.cpp +++ b/esp/smc/SMCLib/TpContainer.cpp @@ -642,6 +642,10 @@ extern TPWRAPPER_API unsigned getContainerWUClusterInfo(CConstWUClusterInfoArray ForEach(*queues) { IPropertyTree& queue = queues->query(); + + // auxillary queues are additional queues and do not have a 1:1 mapping to clusters + if (queue.getPropBool("@isAuxQueue")) + continue; Owned cluster = new CContainerWUClusterInfo(queue.queryProp("@name"), queue.queryProp("@type"), queue.queryProp("@ldapUser"), (unsigned) queue.getPropInt("@width", 1), queue.getPropBool("@queriesOnly")); diff --git a/esp/src/src-react/components/ECLArchiveTree.tsx b/esp/src/src-react/components/ECLArchiveTree.tsx index c39ef9c8f1a..a49d771f6a3 100644 --- a/esp/src/src-react/components/ECLArchiveTree.tsx +++ b/esp/src/src-react/components/ECLArchiveTree.tsx @@ -57,7 +57,7 @@ export const ECLArchiveTree: React.FunctionComponent = ({ fileTimePct: isAttribute(modAttr) && Math.round((archive?.sourcePathTime(modAttr.sourcePath) / archive?.timeTotalExecute) * 100), }); }); - setFlatTreeItems(flatTreeItems); + setFlatTreeItems(flatTreeItems.sort((a, b) => a.value.toString().localeCompare(b.value.toString(), undefined, { sensitivity: "base" }))); }, [archive, archive?.modAttrs, archive?.timeTotalExecute]); const onClick = React.useCallback(evt => { @@ -69,28 +69,29 @@ export const ECLArchiveTree: React.FunctionComponent = ({ }, [archive, setSelectedItem]); const { ...treeProps } = flatTree.getTreeProps(); - return - { - Array.from(flatTree.items(), flatTreeItem => { - console.log(flatTreeItem.getTreeItemProps()); - const { fileTimePct, content, ...treeItemProps } = flatTreeItem.getTreeItemProps(); - return - attrId.startsWith(content)) ? : : - selectedAttrIDs.some(attrId => attrId.startsWith(content)) ? : ) : - selectedAttrIDs.some(attrId => attrId === flatTreeItem.value) ? - : - - } - aside={} - > - {content} - - ; - }) - } - ; + return
+ + { + Array.from(flatTree.items(), flatTreeItem => { + const { fileTimePct, content, ...treeItemProps } = flatTreeItem.getTreeItemProps(); + return + attrId.startsWith(content)) ? : : + selectedAttrIDs.some(attrId => attrId.startsWith(content)) ? : ) : + selectedAttrIDs.some(attrId => attrId === flatTreeItem.value) ? + : + + } + aside={} + > + {content} + + ; + }) + } + +
; }; diff --git a/esp/src/src-react/components/Menu.tsx b/esp/src/src-react/components/Menu.tsx index 382df996894..1e34430c628 100644 --- a/esp/src/src-react/components/Menu.tsx +++ b/esp/src/src-react/components/Menu.tsx @@ -162,6 +162,7 @@ const subMenuItems: SubMenuItems = { { headerText: nlsHPCC.Security + " (L)", itemKey: "/topology/security" }, { headerText: nlsHPCC.DESDL + " (L)", itemKey: "/topology/desdl" }, { headerText: nlsHPCC.DaliAdmin, itemKey: "/topology/daliadmin" }, + { headerText: nlsHPCC.Sasha, itemKey: "/topology/sasha" }, ], "operations": [ { headerText: nlsHPCC.Topology + " (L)", itemKey: "/operations" }, diff --git a/esp/src/src-react/components/Sasha.tsx b/esp/src/src-react/components/Sasha.tsx new file mode 100644 index 00000000000..65ef5af0886 --- /dev/null +++ b/esp/src/src-react/components/Sasha.tsx @@ -0,0 +1,96 @@ +import * as React from "react"; +import { Dropdown, TextField, PrimaryButton } from "@fluentui/react"; +import nlsHPCC from "src/nlsHPCC"; + +interface SashaProps {} + +export const Sasha: React.FunctionComponent = ({ }) => { + const [selectedOption, setSelectedOption] = React.useState(""); + const [wuid, setWuid] = React.useState(""); + const [result, setResult] = React.useState(""); + + const handleOptionChange = (event: React.FormEvent, option: any) => { + setSelectedOption(option.key); + }; + + const handleSubmit = (event: React.FormEvent) => { + event.preventDefault(); + // Perform action based on selected option + switch (selectedOption) { + case "getVersion": + // Implement getVersion function call + break; + case "getLastServerMessage": + // Implement getLastServerMessage function call + break; + case "restoreECLWorkUnit": + // Implement restoreECLWorkUnit function call + break; + case "restoreDFUWorkUnit": + // Implement restoreDFUWorkUnit function call + break; + case "archiveECLWorkUnit": + // Implement archiveECLWorkUnit function call + break; + case "archiveDFUWorkUnit": + // Implement archiveDFUWorkUnit function call + break; + case "backupECLWorkUnit": + // Implement backupECLWorkUnit function call + break; + case "backupDFUWorkUnit": + // Implement backupDFUWorkUnit function call + break; + default: + console.log("Invalid option selected"); + } + // Reset form + setSelectedOption(""); + setWuid(""); + setResult(""); + }; + + // Conditional rendering for default value + const defaultValue = result ? null :
{nlsHPCC.noDataMessage}
; + + return ( +
+
+ + {["restoreECLWorkUnit", "restoreDFUWorkUnit", "archiveECLWorkUnit", "archiveDFUWorkUnit", "backupECLWorkUnit", "backupDFUWorkUnit"].includes(selectedOption) && ( +
+ , newValue?: string) => setWuid(newValue || "")} + styles={{ fieldGroup: { width: 400 } }} + /> +
+ )} + {nlsHPCC.Submit} + {/* Render defaultValue when result is empty */} + {defaultValue} + {/* Render result when available */} + {result &&
{nlsHPCC.Results}: {result}
} + +
+ ); +}; + +export default Sasha; diff --git a/esp/src/src-react/components/forms/ZAPDialog.tsx b/esp/src/src-react/components/forms/ZAPDialog.tsx index 3bfa9367b20..750e997efba 100644 --- a/esp/src/src-react/components/forms/ZAPDialog.tsx +++ b/esp/src/src-react/components/forms/ZAPDialog.tsx @@ -192,14 +192,14 @@ export const ZAPDialog: React.FunctionComponent = ({ if (key === "AbsoluteTimeRange") { const startDate = logFilter.AbsoluteTimeRange.StartDate ? new Date(logFilter.AbsoluteTimeRange.StartDate).toISOString() : ""; const endDate = logFilter.AbsoluteTimeRange.EndDate ? new Date(logFilter.AbsoluteTimeRange.EndDate).toISOString() : ""; - formData.append("LogFilter.AbsoluteTimeRange.StartDate", startDate); - formData.append("LogFilter.AbsoluteTimeRange.EndDate", endDate); + formData.append("LogFilter_AbsoluteTimeRange_StartDate", startDate); + formData.append("LogFilter_AbsoluteTimeRange_EndDate", endDate); } else { - formData.append(`LogFilter.${key}`, logFilter[key]); + formData.append(`LogFilter_${key}`, logFilter[key]); } } - fetch("/WsWorkunits/WUCreateZAPInfo", { + fetch("/WsWorkunits/WUCreateAndDownloadZAPInfo", { method: "POST", body: formData }) @@ -630,7 +630,7 @@ export const ZAPDialog: React.FunctionComponent = ({ = ({ = ({ { key: "0", text: "ASC" }, { key: "1", text: "DESC" }, ]} - defaultSelectedKey="1" + defaultSelectedKey={"1"} + selectedKey={value} onRenderLabel={(props: CustomLabelProps) => } + onChange={(evt, option) => { + onChange(option.key); + }} /> } /> = ({ { key: LogType.Progress, text: "Progress" }, { key: LogType.Metric, text: "Metric" }, ]} - defaultSelectedKey="ALL" + selectedKey={value} onRenderLabel={(props: CustomLabelProps) => } + onChange={(evt, option) => { + onChange(option.key); + }} /> } /> diff --git a/esp/src/src-react/routes.tsx b/esp/src/src-react/routes.tsx index d5e36af6746..c1cf36bef86 100644 --- a/esp/src/src-react/routes.tsx +++ b/esp/src/src-react/routes.tsx @@ -390,7 +390,17 @@ export const routes: RoutesEx = [ }, ] }, - ] + { + path: "/sasha", + children: [ + { + path: "", action: (ctx, params) => import("./components/Sasha").then(_ => { + return <_.Sasha />; + }) + }, + ] + }, + ] }, { mainNav: ["operations"], diff --git a/esp/src/src/nls/hpcc.ts b/esp/src/src/nls/hpcc.ts index 2e01673fba0..df59de5add6 100644 --- a/esp/src/src/nls/hpcc.ts +++ b/esp/src/src/nls/hpcc.ts @@ -58,6 +58,8 @@ export = { AppendCluster: "Append Cluster", Apply: "Apply", AreYouSureYouWantToResetTheme: "Are you sure you want to reset to the default theme?", + ArchiveECLWorkunit: "Archive ECL Workunit", + ArchiveDFUWorkunit: "Archive DFU Workunit", ArchivedOnly: "Archived Only", ArchivedWarning: "Warning: please specify a small date range. If not, it may take some time to retrieve the workunits and the browser may be timed out.", Attach: "Attach", @@ -72,6 +74,8 @@ export = { AutoRefreshIncrement: "Auto Refresh Increment", AutoRefreshEvery: "Auto refresh every x minutes", Back: "Back", + BackupECLWorkunit: "Backup ECL Workunit", + BackupDFUWorkunit: "Backup DFU Workunit", BannerColor: "Banner Color", BannerColorTooltip: "Change the background color of the top navigation", BannerMessage: "Banner Message", @@ -372,10 +376,12 @@ export = { GetDFSCSV: "DFS CSV", GetDFSMap: "DFS Map", GetDFSParents: "DFS Parents", + GetLastServerMessage: "Get Last Server Message", GetLogicalFile: "Logical File", GetLogicalFilePart: "Logical File Part", GetProtectedList: "Protected List", GetValue: "Value", + GetVersion: "Get Version", GetPart: "Get Part", GetSoftwareInformation: "Get Software Information", Graph: "Graph", @@ -802,6 +808,8 @@ export = { Restarted: "Restarted", Restarts: "Restarts", Restore: "Restore", + RestoreECLWorkunit: "Restore ECL Workunit", + RestoreDFUWorkunit: "Restore DFU Workunit", Resubmit: "Resubmit", ResubmitTooltip: "Resubmit existing workunit", Resubmitted: "Resubmitted", @@ -821,6 +829,7 @@ export = { Sample: "Sample", SampleRequest: "Sample Request", SampleResponse: "Sample Response", + Sasha: "Sasha", Save: "Save", Scope: "Scope", SearchResults: "Search Results", @@ -830,6 +839,7 @@ export = { SecurityMessageHTML: "Only view HTML from trusted users. This workunit was created by \"{__placeholder__}\". \nRender HTML?", SeeConfigurationManager: "See Configuration Manager", SelectA: "Select a", + SelectAnOption: "Select an option", Selected: "Selected", SelectValue: "Select a value", SelectEllipsis: "Select...", diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 02cef6b2998..560685e03f2 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -2002,11 +2002,13 @@ class CRemoteXmlReadActivity : public CRemoteMarkupReadActivity public: CRemoteXmlReadActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc, TAKxmlread) { - xpath.set("/Dataset/"); if (customRowTag.isEmpty()) // no override fileDesc->queryProperties().getProp("@rowTag", xpath); else + { + xpath.set("/Dataset/"); xpath.append(customRowTag); + } } }; @@ -3879,7 +3881,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface file->setShareMode((IFSHmode)sMode); } if (TF_TRACE_PRE_IO) - PROGLOG("before open file '%s', (%d,%d,%d,%d,0%o)",name->text.get(),(int)mode,(int)share,extraFlags,sMode,cFlags); + PROGLOG("before open file '%s', (%d,%d,%d,%d,0%o)",name->text.get(),(int)mode,(int)share,extraFlags,sMode,cFlags); Owned fileio; #ifndef _WIN32 @@ -3941,9 +3943,9 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface if (TF_TRACE) { #ifndef _WIN32 - PROGLOG("open file '%s', (%d,%d) handle = %d",name->text.get(),(int)mode,(int)share,handle); + PROGLOG("open file '%s', (%d,%d) handle = %d",name->text.get(),(int)mode,(int)share,handle); #else - PROGLOG("open file '%s', (%d,%d) handle = %d, retries = %u, time(ms) = %u",name->text.get(),(int)mode,(int)share,handle,retries,timer.elapsedMs()); + PROGLOG("open file '%s', (%d,%d) handle = %d, retries = %u, time(ms) = %u",name->text.get(),(int)mode,(int)share,handle,retries,timer.elapsedMs()); #endif } return true; @@ -3956,7 +3958,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface IFileIO *fileio; checkFileIOHandle(reply, handle, fileio, true); if (TF_TRACE) - PROGLOG("close file, handle = %d",handle); + PROGLOG("close file, handle = %d",handle); reply.append(RFEnoerror); return true; } @@ -3976,13 +3978,13 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface size32_t numRead; unsigned posOfLength = reply.length(); if (TF_TRACE_PRE_IO) - PROGLOG("before read file, handle = %d, toread = %d",handle,len); + PROGLOG("before read file, handle = %d, toread = %d",handle,len); reply.reserve(sizeof(numRead)); void *data = reply.reserve(len); numRead = fileio->read(pos,len,data); stats.addRead(len); if (TF_TRACE) - PROGLOG("read file, handle = %d, pos = %" I64F "d, toread = %d, read = %d",handle,pos,len,numRead); + PROGLOG("read file, handle = %d, pos = %" I64F "d, toread = %d, read = %d",handle,pos,len,numRead); reply.setLength(posOfLength + sizeof(numRead) + numRead); reply.writeEndianDirect(posOfLength,sizeof(numRead),&numRead); } @@ -3996,7 +3998,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface __int64 size = fileio->size(); reply.append((unsigned)RFEnoerror).append(size); if (TF_TRACE) - PROGLOG("size file, handle = %d, size = %" I64F "d",handle,size); + PROGLOG("size file, handle = %d, size = %" I64F "d",handle,size); } void cmdSetSize(MemoryBuffer & msg, MemoryBuffer & reply) @@ -4006,7 +4008,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface msg.read(handle).read(size); IFileIO *fileio; if (TF_TRACE) - PROGLOG("set size file, handle = %d, size = %" I64F "d",handle,size); + PROGLOG("set size file, handle = %d, size = %" I64F "d",handle,size); checkFileIOHandle(reply, handle, fileio); fileio->setSize(size); reply.append((unsigned)RFEnoerror); @@ -4022,11 +4024,11 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface checkFileIOHandle(reply, handle, fileio); const byte *data = (const byte *)msg.readDirect(len); if (TF_TRACE_PRE_IO) - PROGLOG("before write file, handle = %d, towrite = %d",handle,len); + PROGLOG("before write file, handle = %d, towrite = %d",handle,len); size32_t numWritten = fileio->write(pos,len,data); stats.addWrite(numWritten); if (TF_TRACE) - PROGLOG("write file, handle = %d, towrite = %d, written = %d",handle,len,numWritten); + PROGLOG("write file, handle = %d, towrite = %d, written = %d",handle,len,numWritten); reply.append((unsigned)RFEnoerror).append(numWritten); } @@ -4035,7 +4037,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("exists, '%s'",name.get()); + PROGLOG("exists, '%s'",name.get()); Owned file=createIFile(name); bool e = file->exists(); reply.append((unsigned)RFEnoerror).append(e); @@ -4046,7 +4048,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("remove, '%s'",name.get()); + PROGLOG("remove, '%s'",name.get()); Owned file=createIFile(name); bool e = file->remove(); reply.append((unsigned)RFEnoerror).append(e); @@ -4075,7 +4077,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr toname; msg.read(toname); if (TF_TRACE) - PROGLOG("rename, '%s' to '%s'",fromname.get(),toname.get()); + PROGLOG("rename, '%s' to '%s'",fromname.get(),toname.get()); Owned file=createIFile(fromname); file->rename(toname); reply.append((unsigned)RFEnoerror); @@ -4088,7 +4090,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr toname; msg.read(toname); if (TF_TRACE) - PROGLOG("move, '%s' to '%s'",fromname.get(),toname.get()); + PROGLOG("move, '%s' to '%s'",fromname.get(),toname.get()); Owned file=createIFile(fromname); file->move(toname); reply.append((unsigned)RFEnoerror); @@ -4101,7 +4103,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr toname; msg.read(toname); if (TF_TRACE) - PROGLOG("copy, '%s' to '%s'",fromname.get(),toname.get()); + PROGLOG("copy, '%s' to '%s'",fromname.get(),toname.get()); copyFile(toname, fromname); reply.append((unsigned)RFEnoerror); } @@ -4120,7 +4122,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface __int64 written = fileio->appendFile(file,pos,len); stats.addWrite(written); if (TF_TRACE) - PROGLOG("append file, handle = %d, file=%s, pos = %" I64F "d len = %" I64F "d written = %" I64F "d",handle,srcname.get(),pos,len,written); + PROGLOG("append file, handle = %d, file=%s, pos = %" I64F "d len = %" I64F "d written = %" I64F "d",handle,srcname.get(),pos,len,written); reply.append((unsigned)RFEnoerror).append(written); } @@ -4129,7 +4131,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("isFile, '%s'",name.get()); + PROGLOG("isFile, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = (unsigned)file->isFile(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4140,7 +4142,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("isDir, '%s'",name.get()); + PROGLOG("isDir, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = (unsigned)file->isDirectory(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4151,7 +4153,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("isReadOnly, '%s'",name.get()); + PROGLOG("isReadOnly, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = (unsigned)file->isReadOnly(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4164,7 +4166,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface msg.read(name).read(set); if (TF_TRACE) - PROGLOG("setReadOnly, '%s' %d",name.get(),(int)set); + PROGLOG("setReadOnly, '%s' %d",name.get(),(int)set); Owned file=createIFile(name); file->setReadOnly(set); reply.append((unsigned)RFEnoerror); @@ -4176,7 +4178,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface unsigned fPerms; msg.read(name).read(fPerms); if (TF_TRACE) - PROGLOG("setFilePerms, '%s' 0%o",name.get(),fPerms); + PROGLOG("setFilePerms, '%s' 0%o",name.get(),fPerms); Owned file=createIFile(name); file->setFilePermissions(fPerms); reply.append((unsigned)RFEnoerror); @@ -4187,7 +4189,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("getTime, '%s'",name.get()); + PROGLOG("getTime, '%s'",name.get()); Owned file=createIFile(name); CDateTime createTime; CDateTime modifiedTime; @@ -4223,7 +4225,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface accessedTime.deserialize(msg); if (TF_TRACE) - PROGLOG("setTime, '%s'",name.get()); + PROGLOG("setTime, '%s'",name.get()); Owned file=createIFile(name); bool ret = file->setTime(creategot?&createTime:NULL,modifiedgot?&modifiedTime:NULL,accessedgot?&accessedTime:NULL); @@ -4235,7 +4237,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("CreateDir, '%s'",name.get()); + PROGLOG("CreateDir, '%s'",name.get()); Owned dir=createIFile(name); bool ret = dir->createDirectory(); reply.append((unsigned)RFEnoerror).append(ret); @@ -4256,7 +4258,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface client.opendir.clear(); } if (TF_TRACE) - PROGLOG("GetDir, '%s', '%s', stream='%u'",name.get(),mask.get(),stream); + PROGLOG("GetDir, '%s', '%s', stream='%u'",name.get(),mask.get(),stream); if (!stream && !containsFileWildcard(mask)) { // if no streaming, and mask contains no wildcard, it is much more efficient to get the info without a directory iterator! @@ -4331,7 +4333,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface prev.setown(createRemoteDirectorIterator(ep, name, msg)); } if (TF_TRACE) - PROGLOG("MonitorDir, '%s' '%s'",name.get(),mask.get()); + PROGLOG("MonitorDir, '%s' '%s'",name.get(),mask.get()); Owned dir=createIFile(name); Owned iter=dir->monitorDirectory(prev,mask.length()?mask.get():NULL,sub,includedir,checkinterval,timeout); reply.append((unsigned)RFEnoerror); @@ -4524,7 +4526,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface StringAttr name; msg.read(name); if (TF_TRACE) - PROGLOG("getCRC, '%s'",name.get()); + PROGLOG("getCRC, '%s'",name.get()); Owned file=createIFile(name); unsigned ret = file->getCRC(); reply.append((unsigned)RFEnoerror).append(ret); @@ -5103,7 +5105,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface msg.append(results); catchWriteBuffer(client.socket, msg); - LOG(MCdebugProgress, unknownJob, "Results sent from slave: %s", client.peerName.str()); + LOG(MCdebugProgress, "Results sent from slave: %s", client.peerName.str()); } void formatException(MemoryBuffer &reply, IException *e, RemoteFileCommandType cmd, bool testSocketFlag, unsigned _dfsErrorCode, CRemoteClientHandler *client) diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 6f2af7f2f47..c15efedd7e5 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1164,6 +1164,7 @@ Generate instance queue names {{- else -}} {{- $newQueueItem := deepCopy $queueItem -}} {{- $_ := set $newQueueItem "name" $auxQueueName -}} + {{- $_ := set $newQueueItem "isAuxQueue" true -}} {{- $_ := set $auxThorQueues $auxQueueName $newQueueItem -}} {{- end -}} {{- end -}} diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index c2fe3f389f8..45f3abd4458 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -2795,7 +2795,7 @@ static IPropertyTree *checkPlaneOrHost(const char *planeName,const char *host,co if (env->isDropZoneRestrictionEnabled()) throw makeStringExceptionV(-1,"DropZone Plane not found for host %s path %s.",host,path); - LOG(MCdebugInfo, unknownJob, "No matching drop zone path on '%s' to file path: '%s'",host,path); + LOG(MCdebugInfo, "No matching drop zone path on '%s' to file path: '%s'",host,path); #endif return nullptr; diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 9ea5ccb8378..86a64edd0f3 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -623,15 +623,15 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface writer.outputEndArray("Log"); } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { StringBuffer text, prefix; getLogPrefix(prefix); text.valist_appendf(format, args); - CTXLOGa(LOG_TRACING, cat, job, code, prefix.str(), text.str()); + CTXLOGa(LOG_TRACING, cat, code, prefix.str(), text.str()); } - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { LogContextScope ls(nullptr); if (category == LOG_TRACING) @@ -659,7 +659,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface { text.append(": ").valist_appendf(format, args); } - LOG(MCoperatorProgress, unknownJob, "[%s] %s", prefix, text.str()); + LOG(MCoperatorProgress, "[%s] %s", prefix, text.str()); if (intercept) { CriticalBlock b(crit); diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 68169884ae0..110b8c0f12a 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -1332,9 +1332,9 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext { globalStats.recordStatistics(progress, false); } - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { - logctx.CTXLOGa(category, cat, job, code, prefix, text); + logctx.CTXLOGa(category, cat, code, prefix, text); } virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index a6524342480..69b231fca0a 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -1197,6 +1197,8 @@ class RoxieWorkUnitWorker : public RoxieQueryWorker daliHelper.setown(connectToDali()); wu.setown(daliHelper->attachWorkunit(wuid.get())); } + + JobNameScope jobName(wuid); Owned logctx = new StringContextLogger(wuid.get()); Owned traceHeaders = extractTraceDebugOptions(wu); diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index a396eca03cc..37c3496e307 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -1739,7 +1739,11 @@ class CRoxieWorker : public CInterface, implements IPooledThread Owned next = queue->dequeue(); if (next) { + JobNameScope jobName; logctx.set(next); + const char * wuid = logctx.queryWuid(); + if (wuid) + jobName.set(wuid); logctx.setStatistic(StTimeAgentQueue, nsTick()-next->queryEnqueuedTimeStamp()); #ifdef NEW_IBYTI logctx.setStatistic(StTimeIBYTIDelay, next->queryIBYTIDelayTime()); diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index 555d8ac29ae..d151b9f1969 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -240,13 +240,13 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { return ctx->collectingDetailedStatistics(); } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { - ctx->CTXLOGva(cat, job, code, format, args); + ctx->CTXLOGva(cat, code, format, args); } - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { - ctx->CTXLOGa(category, cat, job, code, prefix, text); + ctx->CTXLOGa(category, cat, code, prefix, text); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -1331,14 +1331,14 @@ class CRoxieServerActivity : implements CInterfaceOf, impl } // MORE - most of this is copied from ccd.hpp - can't we refactor? - virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *prefix, const char *text) const override + virtual void CTXLOGa(TracingCategory category, const LogMsgCategory & cat, LogMsgCode code, const char *prefix, const char *text) const override { if (ctx) - ctx->CTXLOGa(category, cat, job, code, prefix, text); + ctx->CTXLOGa(category, cat, code, prefix, text); else { LogContextScope ls(nullptr); - LOG(cat, job, code, "[%s] %s", prefix, text); + LOG(cat, code, "[%s] %s", prefix, text); } } @@ -1361,7 +1361,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { ss.append(": ").valist_appendf(format, args); } - LOG(MCoperatorProgress, unknownJob, "%s", ss.str()); + LOG(MCoperatorProgress, "%s", ss.str()); } } diff --git a/roxie/ccd/ccdsnmp.cpp b/roxie/ccd/ccdsnmp.cpp index a4b913c994e..169b6a17200 100644 --- a/roxie/ccd/ccdsnmp.cpp +++ b/roxie/ccd/ccdsnmp.cpp @@ -29,15 +29,13 @@ #include "roxiemem.hpp" #include "udplib.hpp" -#define DEFAULT_PULSE_INTERVAL 30 - RoxieQueryStats unknownQueryStats; RoxieQueryStats loQueryStats; RoxieQueryStats hiQueryStats; RoxieQueryStats slaQueryStats; RoxieQueryStats combinedQueryStats; -#define addMetric(a, b) doAddMetric(a, #a, b) +#define addMetric(a) doAddMetric(a, #a) interface INamedMetric : extends IInterface { @@ -46,11 +44,6 @@ interface INamedMetric : extends IInterface virtual void resetValue() = 0; }; -interface ITimerCallback : extends IInterface -{ - virtual void onTimer() = 0; -}; - class RelaxedAtomicMetric : implements INamedMetric, public CInterface { RelaxedAtomic &counter; @@ -123,101 +116,6 @@ class UserMetric : implements INamedMetric, public CInterface }; -class TickProvider : public Thread -{ - IArrayOf listeners; - CriticalSection crit; - Semaphore stopped; - - void doTicks() - { - CriticalBlock c(crit); - ForEachItemIn(idx, listeners) - { - listeners.item(idx).onTimer(); - } - } - -public: - TickProvider() : Thread("TickProvider") - { - } - int run() - { - for (;;) - { - if (stopped.wait(10000)) - break; - doTicks(); - } - return 0; - } - - void addListener(ITimerCallback *l) - { - CriticalBlock c(crit); - listeners.append(*LINK(l)); - } - - void stop() - { - stopped.signal(); - join(); - } -}; - -class IntervalMetric : implements INamedMetric, implements ITimerCallback, public CInterface -{ - Linked base; - CriticalSection crit; - unsigned lastSnapshotTime; - long lastSnapshotValue; - unsigned minInterval; - long value; - - void takeSnapshot() - { - CriticalBlock c(crit); - unsigned now = msTick(); - unsigned period = now - lastSnapshotTime; - if (period >= minInterval) - { - long newValue = base->getValue(); - value = ((newValue - lastSnapshotValue) * 1000) / period; - lastSnapshotTime = now; - lastSnapshotValue = newValue; - } - } - -public: - IMPLEMENT_IINTERFACE; - IntervalMetric(INamedMetric *_base, unsigned _minInterval=1000) : base(_base), minInterval(_minInterval) - { - lastSnapshotTime = msTick(); - lastSnapshotValue = 0; - value = 0; - } - virtual void onTimer() - { - takeSnapshot(); - } - virtual long getValue() - { - takeSnapshot(); - return value; - } - virtual bool isCumulative() { return false; } - - virtual void resetValue() - { - CriticalBlock c(crit); - lastSnapshotTime = msTick(); - lastSnapshotValue = 0; - value = 0; - } - -}; - class UnsignedRatioMetric : implements INamedMetric, public CInterface { RelaxedAtomic &counter; @@ -252,31 +150,27 @@ class CRoxieMetricsManager : implements IRoxieMetricsManager, public CInterface ~CRoxieMetricsManager(); virtual long getValue(const char * name); - void dumpMetrics(); StringBuffer &getMetrics(StringBuffer &xml); void resetMetrics(); - void doAddMetric(RelaxedAtomic &counter, const char *name, unsigned interval, bool isMinVal = false); - void doAddMetric(INamedMetric *n, const char *name, unsigned interval); - void doAddMetric(AccessorFunction function, const char *name, unsigned interval); + void doAddMetric(RelaxedAtomic &counter, const char *name, bool isMinVal = false); + void doAddMetric(INamedMetric *n, const char *name); + void doAddMetric(AccessorFunction function, const char *name); void addRatioMetric(RelaxedAtomic &counter, const char *name, RelaxedAtomic &elapsed); void addUserMetric(const char *name, const char *regex); private: MapStringToMyClassViaBase metricMap; - bool started; - - TickProvider ticker; }; -void RoxieQueryStats::addMetrics(CRoxieMetricsManager *snmpManager, const char *prefix, unsigned interval) +void RoxieQueryStats::addMetrics(CRoxieMetricsManager *snmpManager, const char *prefix) { StringBuffer name; - snmpManager->doAddMetric(count, name.clear().append(prefix).append("QueryCount"), interval, false); - snmpManager->doAddMetric(failedCount, name.clear().append(prefix).append("QueryFailed"), interval, false); - snmpManager->doAddMetric(active, name.clear().append(prefix).append("QueryActive"), 0, false); - snmpManager->doAddMetric(maxTime, name.clear().append(prefix).append("QueryMaxTime"), 0, false); - snmpManager->doAddMetric(minTime, name.clear().append(prefix).append("QueryMinTime"), 0, true); + snmpManager->doAddMetric(count, name.clear().append(prefix).append("QueryCount"), false); + snmpManager->doAddMetric(failedCount, name.clear().append(prefix).append("QueryFailed"), false); + snmpManager->doAddMetric(active, name.clear().append(prefix).append("QueryActive"), false); + snmpManager->doAddMetric(maxTime, name.clear().append(prefix).append("QueryMaxTime"), false); + snmpManager->doAddMetric(minTime, name.clear().append(prefix).append("QueryMinTime"), true); snmpManager->addRatioMetric(count, name.clear().append(prefix).append("QueryAverageTime"), totalTime); } @@ -287,77 +181,65 @@ using roxiemem::getDataBuffersActive; CRoxieMetricsManager::CRoxieMetricsManager() { - started = false; - unknownQueryStats.addMetrics(this, "unknown", 1000); - loQueryStats.addMetrics(this, "lo", 1000); - hiQueryStats.addMetrics(this, "hi", 1000); - slaQueryStats.addMetrics(this, "sla", 1000); - combinedQueryStats.addMetrics(this, "all", 1000); - addMetric(restarts, 0); - - addMetric(nodesLoaded, 1000); - - addMetric(blobCacheHits, 1000); - addMetric(blobCacheAdds, 1000); - addMetric(blobCacheDups, 1000); - addMetric(leafCacheHits, 1000); - addMetric(leafCacheAdds, 1000); - addMetric(leafCacheDups, 1000); - addMetric(nodeCacheHits, 1000); - addMetric(nodeCacheAdds, 1000); - addMetric(nodeCacheDups, 1000); - - addMetric(unwantedDiscarded, 1000); - - addMetric(getHeapAllocated, 0); - addMetric(getHeapPercentAllocated, 0); - addMetric(getDataBufferPages, 0); - addMetric(getDataBuffersActive, 0); + unknownQueryStats.addMetrics(this, "unknown"); + loQueryStats.addMetrics(this, "lo"); + hiQueryStats.addMetrics(this, "hi"); + slaQueryStats.addMetrics(this, "sla"); + combinedQueryStats.addMetrics(this, "all"); + addMetric(restarts); + + addMetric(nodesLoaded); + + addMetric(blobCacheHits); + addMetric(blobCacheAdds); + addMetric(blobCacheDups); + addMetric(leafCacheHits); + addMetric(leafCacheAdds); + addMetric(leafCacheDups); + addMetric(nodeCacheHits); + addMetric(nodeCacheAdds); + addMetric(nodeCacheDups); + + addMetric(unwantedDiscarded); + + addMetric(getHeapAllocated); + addMetric(getHeapPercentAllocated); + addMetric(getDataBufferPages); + addMetric(getDataBuffersActive); - addMetric(packetsResent, 1000); - addMetric(flowPermitsSent, 1000); - addMetric(flowRequestsReceived, 1000); - addMetric(dataPacketsReceived, 1000); - addMetric(flowRequestsSent, 1000); - addMetric(flowPermitsReceived, 1000); - addMetric(dataPacketsSent, 1000); - ticker.start(false); + addMetric(packetsResent); + addMetric(flowPermitsSent); + addMetric(flowRequestsReceived); + addMetric(dataPacketsReceived); + addMetric(flowRequestsSent); + addMetric(flowPermitsReceived); + addMetric(dataPacketsSent); } -void CRoxieMetricsManager::doAddMetric(RelaxedAtomic &counter, const char *name, unsigned interval, bool isMinVal) +void CRoxieMetricsManager::doAddMetric(RelaxedAtomic &counter, const char *name, bool isMinVal) { - doAddMetric(new RelaxedAtomicMetric(counter, interval != 0, isMinVal), name, interval); + doAddMetric(new RelaxedAtomicMetric(counter, false, isMinVal), name); } -void CRoxieMetricsManager::doAddMetric(INamedMetric *n, const char *name, unsigned interval) +void CRoxieMetricsManager::doAddMetric(INamedMetric *n, const char *name) { - if (interval) - { - StringBuffer fname(name); - fname.append("/s"); - IntervalMetric *im = new IntervalMetric(n, interval); - ticker.addListener(im); - metricMap.setValue(fname.str(), im); - im->Release(); - } metricMap.setValue(name, n); n->Release(); } -void CRoxieMetricsManager::doAddMetric(AccessorFunction function, const char *name, unsigned interval) +void CRoxieMetricsManager::doAddMetric(AccessorFunction function, const char *name) { - assertex(interval==0); - doAddMetric(new FunctionMetric(function), name, interval); + doAddMetric(new FunctionMetric(function), name); } void CRoxieMetricsManager::addRatioMetric(RelaxedAtomic &counter, const char *name, RelaxedAtomic &elapsed) { - doAddMetric(new UnsignedRatioMetric(counter, elapsed), name, 0); + doAddMetric(new UnsignedRatioMetric(counter, elapsed), name); } void CRoxieMetricsManager::addUserMetric(const char *name, const char *regex) { - doAddMetric(new UserMetric(name, regex), name, 0); + doAddMetric(new UserMetric(name, regex), name); } long CRoxieMetricsManager::getValue(const char * name) @@ -375,25 +257,6 @@ long CRoxieMetricsManager::getValue(const char * name) CRoxieMetricsManager::~CRoxieMetricsManager() { - ticker.stop(); - if (started) - dumpMetrics(); -} - -void CRoxieMetricsManager::dumpMetrics() -{ - HashIterator metrics(metricMap); - ForEach(metrics) - { - IMapping &cur = metrics.query(); - INamedMetric *m = (INamedMetric *) *metricMap.mapToValue(&cur); - if (m->isCumulative()) - { - const char *name = (const char *) cur.getKey(); - long val = m->getValue(); - DBGLOG("TOTALS: %s = %ld", name, val); - } - } } StringBuffer &CRoxieMetricsManager::getMetrics(StringBuffer &xml) diff --git a/roxie/ccd/ccdsnmp.hpp b/roxie/ccd/ccdsnmp.hpp index 81ffaaa9ede..49a46db233f 100644 --- a/roxie/ccd/ccdsnmp.hpp +++ b/roxie/ccd/ccdsnmp.hpp @@ -64,7 +64,7 @@ class RoxieQueryStats active--; } - void addMetrics(CRoxieMetricsManager *mgr, const char *prefix, unsigned interval); + void addMetrics(CRoxieMetricsManager *mgr, const char *prefix); }; interface IQueryStatsAggregator : public IInterface @@ -86,7 +86,6 @@ extern RoxieQueryStats combinedQueryStats; interface IRoxieMetricsManager : extends IInterface { - virtual void dumpMetrics() = 0; virtual void addUserMetric(const char *name, const char *regex) = 0; virtual StringBuffer &getMetrics(StringBuffer &) = 0; virtual void resetMetrics() = 0; diff --git a/rtl/eclrtl/eclrtl.cpp b/rtl/eclrtl/eclrtl.cpp index b5d73720409..1b8a9f1d313 100644 --- a/rtl/eclrtl/eclrtl.cpp +++ b/rtl/eclrtl/eclrtl.cpp @@ -4211,7 +4211,7 @@ ECLRTL_API unsigned rtlSleep(unsigned delay) ECLRTL_API unsigned rtlDisplay(unsigned len, const char * src) { - LOG(MCprogress, unknownJob, "%.*s", len, src); + LOG(MCprogress, "%.*s", len, src); return 0; } diff --git a/rtl/eclrtl/rtlbcdtest.cpp b/rtl/eclrtl/rtlbcdtest.cpp index ff9f8684522..ffdf112a3bb 100644 --- a/rtl/eclrtl/rtlbcdtest.cpp +++ b/rtl/eclrtl/rtlbcdtest.cpp @@ -36,7 +36,7 @@ static bool check(bool condition, const char *fmt, ...) { va_list args; va_start(args, fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); } return condition; @@ -50,7 +50,7 @@ static void cppunit_assert(bool condition, const char *fmt, ...) { va_list args; va_start(args, fmt); - VALOG(MCdebugInfo, unknownJob, fmt, args); + VALOG(MCdebugInfo, fmt, args); va_end(args); CPPUNIT_ASSERT(!"Please refer to the errors above"); } diff --git a/system/jlib/jcontainerized.cpp b/system/jlib/jcontainerized.cpp index e80154ed7e6..5cbde1439c5 100644 --- a/system/jlib/jcontainerized.cpp +++ b/system/jlib/jcontainerized.cpp @@ -319,13 +319,13 @@ void runKubectlCommand(const char *title, const char *cmd, const char *input, St output = &_output; unsigned ret = runExternalCommand(title, *output, error, cmd, input, ".", nullptr); if (output->length()) - MLOG(MCdebugInfo, unknownJob, "%s: ret=%u, stdout=%s", cmd, ret, output->trimRight().str()); + MLOG(MCdebugInfo, "%s: ret=%u, stdout=%s", cmd, ret, output->trimRight().str()); if (error.length()) - MLOG(MCdebugError, unknownJob, "%s: ret=%u, stderr=%s", cmd, ret, error.trimRight().str()); + MLOG(MCdebugError, "%s: ret=%u, stderr=%s", cmd, ret, error.trimRight().str()); if (ret) { if (input) - MLOG(MCdebugError, unknownJob, "Using input %s", input); + MLOG(MCdebugError, "Using input %s", input); throw makeStringExceptionV(0, "Failed to run %s: error %u: %s", cmd, ret, error.str()); } } diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 2ca2586b909..78b9813c4a0 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -2642,7 +2642,7 @@ class CUdpStatsReporter if (drops > ret->drops) { if (!firstTime) - LOG(MCoperatorError, unknownJob, "DROPPED UDP PACKETS: port %d rx_queue=%u (peak %u) drops=%u (total %i)", port, queue, ret->rx_queue, drops-ret->drops, drops); + LOG(MCoperatorError, "DROPPED UDP PACKETS: port %d rx_queue=%u (peak %u) drops=%u (total %i)", port, queue, ret->rx_queue, drops-ret->drops, drops); ret->drops = drops; } } @@ -2709,10 +2709,10 @@ class CSnmpStatsReporter { firstCall = false; if (errors) - LOG(MCoperatorWarning, unknownJob, "UDP Initial InError total: %u", errors); + LOG(MCoperatorWarning, "UDP Initial InError total: %u", errors); } else if (errors > prevErrors) - LOG(MCoperatorError, unknownJob, "UDP InErrors: %u (total %u)", errors-prevErrors, errors); + LOG(MCoperatorError, "UDP InErrors: %u (total %u)", errors-prevErrors, errors); prevErrors = errors; } } @@ -3001,7 +3001,7 @@ static class CMemoryUsageReporter: public Thread udpStats.reportUdpInfo(0); } if(traceMode&&str.length()) { - MLOG(MCdebugInfo, unknownJob, "SYS: %s", str.str()); + MLOG(MCdebugInfo, "SYS: %s", str.str()); #ifndef _WIN32 if (traceMode&PerfMonExtended) { diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 417fcff3404..360db1cf1b1 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -2287,7 +2287,7 @@ void CCheckingFileIO::report(const char * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugError, unknownJob, format, args); + VALOG(MCdebugError, format, args); va_end(args); if (!traced) { diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 66d94c8135b..177e206825a 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -293,19 +293,9 @@ void LogMsgJobInfo::deserialize(MemoryBuffer & in) //-------------------------------------------------------------------------------------------------------------------- -const LogMsgJobInfo unknownJob(UnknownJob, UnknownUser); - -static thread_local LogMsgJobInfo tempJobInfo; -const LogMsgJobInfo & checkDefaultJobInfo(const LogMsgJobInfo & _jobInfo) +static const LogMsgJobInfo queryDefaultJobInfo() { - if (&_jobInfo != &unknownJob) - return _jobInfo; - - //Using a thread local as a temporary is a short term change - //the next step is to remove the parameter and always return a stack object - LogMsgJobInfo & result = tempJobInfo; - result.setJobID(queryThreadedJobId()); - return result; + return LogMsgJobInfo(queryThreadedJobId(), UnknownUser); } @@ -315,21 +305,21 @@ LogMsg::LogMsg(LogMsgJobId id, const char *job) : category(MSGAUD_programmer, jo text.append(job); } -LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session) - : category(_cat), sysInfo(_id, port, session), jobInfo(checkDefaultJobInfo(_jobInfo)), msgCode(_code), remoteFlag(false) +LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session) + : category(_cat), sysInfo(_id, port, session), jobInfo(queryDefaultJobInfo()), msgCode(_code), remoteFlag(false) { text.append(_text); } -LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session) - : category(_cat), sysInfo(_id, port, session), jobInfo(checkDefaultJobInfo(_jobInfo)), msgCode(_code), remoteFlag(false) +LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session) + : category(_cat), sysInfo(_id, port, session), jobInfo(queryDefaultJobInfo()), msgCode(_code), remoteFlag(false) { text.append(sz, _text); } -LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * format, va_list args, +LogMsg::LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * format, va_list args, unsigned port, LogMsgSessionId session) - : category(_cat), sysInfo(_id, port, session), jobInfo(checkDefaultJobInfo(_jobInfo)), msgCode(_code), remoteFlag(false) + : category(_cat), sysInfo(_id, port, session), jobInfo(queryDefaultJobInfo()), msgCode(_code), remoteFlag(false) { text.valist_appendf(format, args); } @@ -1259,7 +1249,7 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const char * format, .. buff.append(file).append("(").append(line).append(") : ").append(format); va_list args; va_start(args, format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); va_end(args); } @@ -1267,7 +1257,7 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, const char * format, { StringBuffer buff; buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); } void LogMsgPrepender::report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) @@ -1276,7 +1266,7 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, LogMsgCode code, const buff.append(file).append("(").append(line).append(") : ").append(format); va_list args; va_start(args, format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); va_end(args); } @@ -1284,7 +1274,7 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, LogMsgCode code, con { StringBuffer buff; buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args); + queryLogMsgManager()->report_va(cat, buff.str(), args); } void LogMsgPrepender::report(const LogMsgCategory & cat, const IException * exception, const char * prefix) @@ -1293,55 +1283,12 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const IException * exce buff.append(file).append("(").append(line).append(") : "); if(prefix) buff.append(prefix).append(" : "); exception->errorMessage(buff); - queryLogMsgManager()->report(cat, unknownJob, exception->errorCode(), "%s", buff.str()); -} - -void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - va_list args; - va_start(args, format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); - va_end(args); -} - -void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); -} - -void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - va_list args; - va_start(args, format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); - va_end(args); -} - -void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) -{ - StringBuffer buff; - buff.append(file).append("(").append(line).append(") : ").append(format); - queryLogMsgManager()->report_va(cat, job, buff.str(), args); -} - -void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * exception, const char * prefix) -{ - StringBuffer txt; - if (prefix) - txt.append(prefix).append(" : "); - exception->errorMessage(txt); - queryLogMsgManager()->report(cat, job, exception->errorCode(), "%s(%d) : %s", file, line, txt.str()); + queryLogMsgManager()->report(cat, exception->errorCode(), "%s", buff.str()); } IException * LogMsgPrepender::report(IException * e, const char * prefix, LogMsgClass cls) { - report(MCexception(e, cls), unknownJob, e, prefix); + report(MCexception(e, cls), e, prefix); return e; } @@ -1542,14 +1489,14 @@ void CLogMsgManager::report(const LogMsgCategory & cat, const char * format, ... if(rejectsCategory(cat)) return; va_list args; va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), unknownJob, NoLogMsgCode, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, format, args, port, session)); va_end(args); } void CLogMsgManager::report_va(const LogMsgCategory & cat, const char * format, va_list args) { if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), unknownJob, NoLogMsgCode, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, format, args, port, session)); } void CLogMsgManager::report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) @@ -1557,17 +1504,17 @@ void CLogMsgManager::report(const LogMsgCategory & cat, LogMsgCode code, const c if(rejectsCategory(cat)) return; va_list args; va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), unknownJob, code, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), code, format, args, port, session)); va_end(args); } void CLogMsgManager::report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) { if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), unknownJob, code, format, args, port, session)); + pushMsg(new LogMsg(cat, getNextID(), code, format, args, port, session)); } -void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg) +void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const char * msg) { if(rejectsCategory(cat)) return; const char *cursor = msg; @@ -1578,17 +1525,17 @@ void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const LogMsgJobI { case '\0': if (cursor != lineStart || cursor==msg) - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); return; case '\r': // NB: \r or \r\n translated into newline - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); if ('\n' == *(cursor+1)) cursor++; lineStart = cursor+1; break; case '\n': - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); + pushMsg(new LogMsg(cat, getNextID(), NoLogMsgCode, (int)(cursor-lineStart), lineStart, port, session)); lineStart = cursor+1; break; } @@ -1596,12 +1543,12 @@ void CLogMsgManager::mreport_direct(const LogMsgCategory & cat, const LogMsgJobI } } -void CLogMsgManager::mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) +void CLogMsgManager::mreport_va(const LogMsgCategory & cat, const char * format, va_list args) { if(rejectsCategory(cat)) return; StringBuffer log; log.limited_valist_appendf(1024*1024, format, args); - mreport_direct(cat, job, log); + mreport_direct(cat, log); } void CLogMsgManager::report(const LogMsgCategory & cat, const IException * exception, const char * prefix) @@ -1610,46 +1557,7 @@ void CLogMsgManager::report(const LogMsgCategory & cat, const IException * excep StringBuffer buff; if(prefix) buff.append(prefix).append(" : "); exception->errorMessage(buff); - pushMsg(new LogMsg(cat, getNextID(), unknownJob, exception->errorCode(), buff.str(), port, session)); -} - -void CLogMsgManager::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - if(rejectsCategory(cat)) return; - va_list args; - va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, format, args, port, session)); - va_end(args); -} - -void CLogMsgManager::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), job, NoLogMsgCode, format, args, port, session)); -} - -void CLogMsgManager::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) -{ - if(rejectsCategory(cat)) return; - va_list args; - va_start(args, format); - pushMsg(new LogMsg(cat, getNextID(), job, code, format, args, port, session)); - va_end(args); -} - -void CLogMsgManager::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) -{ - if(rejectsCategory(cat)) return; - pushMsg(new LogMsg(cat, getNextID(), job, code, format, args, port, session)); -} - -void CLogMsgManager::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * exception, const char * prefix) -{ - if(rejectsCategory(cat)) return; - StringBuffer buff; - if(prefix) buff.append(prefix).append(" : "); - exception->errorMessage(buff); - pushMsg(new LogMsg(cat, getNextID(), job, exception->errorCode(), buff.str(), port, session)); + pushMsg(new LogMsg(cat, getNextID(), exception->errorCode(), buff.str(), port, session)); } void CLogMsgManager::pushMsg(LogMsg * _msg) @@ -2295,13 +2203,8 @@ class CNullManager : implements ILogMsgManager virtual void report(const LogMsgCategory & cat, LogMsgCode code , const char * format, ...) override {} virtual void report_va(const LogMsgCategory & cat, LogMsgCode code , const char * format, va_list args) override {} virtual void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL) override {} - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) override {} - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) override {} - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) override {} - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, va_list args) override {} - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL) override {} - virtual void mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg) override {} - virtual void mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) override {} + virtual void mreport_direct(const LogMsgCategory & cat, const char * msg) override {} + virtual void mreport_va(const LogMsgCategory & cat, const char * format, va_list args) override {} virtual void report(const LogMsg & msg) const override {} virtual LogMsgId getNextID() override { return 0; } virtual bool rejectsCategory(const LogMsgCategory & cat) const override { return true; } @@ -2751,7 +2654,7 @@ void IContextLogger::CTXLOG(const char *format, ...) const { va_list args; va_start(args, format); - CTXLOGva(MCdebugInfo, queryJob(), NoLogMsgCode, format, args); + CTXLOGva(MCdebugInfo, NoLogMsgCode, format, args); va_end(args); } @@ -2812,9 +2715,9 @@ class DummyLogCtx : implements IContextLogger virtual void Link() const {} virtual bool Release() const { return false; } - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,5,0))) + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const override __attribute__((format(printf,4,0))) { - VALOG(cat, job, code, format, args); + VALOG(cat, code, format, args); } virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) { @@ -2828,7 +2731,7 @@ class DummyLogCtx : implements IContextLogger E->errorMessage(ss.append(": ")); if (format) ss.append(": ").valist_appendf(format, args); - LOG(MCoperatorProgress, queryJob(), "%s", ss.str()); + LOG(MCoperatorProgress, "%s", ss.str()); } virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const { @@ -3336,24 +3239,27 @@ void setDefaultJobName(const char * name) } -JobNameTranslator::JobNameTranslator(const char * name) +JobNameScope::JobNameScope(const char * name) { set(name); } -void JobNameTranslator::clear() +void JobNameScope::clear() { - if (id) + if (id != UnknownJob) { theManager->removeJobId(id); - id = 0; + id = prevId; + setDefaultJobId(prevId); } } -void JobNameTranslator::set(const char * name) +void JobNameScope::set(const char * name) { clear(); id = theManager->addJobId(name); + prevId = queryThreadedJobId(); + setDefaultJobId(id); } //--------------------------------------------------------------------------------------------------------------------- @@ -3389,65 +3295,39 @@ void ctxlogReport(const LogMsgCategory & cat, const char * format, ...) { va_list args; va_start(args, format); - ctxlogReportVA(cat, unknownJob, NoLogMsgCode, format, args); + ctxlogReportVA(cat, NoLogMsgCode, format, args); va_end(args); } void ctxlogReportVA(const LogMsgCategory & cat, const char * format, va_list args) { - ctxlogReportVA(cat, unknownJob, NoLogMsgCode, format, args); + ctxlogReportVA(cat, NoLogMsgCode, format, args); } void ctxlogReport(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) { va_list args; va_start(args, format); - ctxlogReportVA(cat, unknownJob, code, format, args); + ctxlogReportVA(cat, code, format, args); va_end(args); } void ctxlogReportVA(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) -{ - ctxlogReportVA(cat, unknownJob, code, format, args); -} -void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * prefix) -{ - ctxlogReport(cat, unknownJob, e, prefix); -} -void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - va_list args; - va_start(args, format); - ctxlogReportVA(cat, job, NoLogMsgCode, format, args); - va_end(args); -} -void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - ctxlogReportVA(cat, job, NoLogMsgCode, format, args); -} -void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) -{ - va_list args; - va_start(args, format); - ctxlogReportVA(cat, job, code, format, args); - va_end(args); -} -void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) { if (queryThreadedContextLogger()) { LogContextScope ls(nullptr); - ls.prev->CTXLOGva(cat, job, code, format, args); + ls.prev->CTXLOGva(cat, code, format, args); } else - queryLogMsgManager()->report_va(cat, job, code, format, args); + queryLogMsgManager()->report_va(cat, code, format, args); } -void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix) +void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * prefix) { StringBuffer buff; e->errorMessage(buff); - ctxlogReport(cat, job, e->errorCode(), "%s%s%s", prefix ? prefix : "", prefix ? prefix : " : ", buff.str()); + ctxlogReport(cat, e->errorCode(), "%s%s%s", prefix ? prefix : "", prefix ? prefix : " : ", buff.str()); } IException * ctxlogReport(IException * e, const char * prefix, LogMsgClass cls) { - ctxlogReport(MCexception(e, cls), unknownJob, e, prefix); + ctxlogReport(MCexception(e, cls), e, prefix); return e; } diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index 4a609630b7e..c79d2d56eba 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -593,10 +593,10 @@ class jlib_decl LogMsg : public CInterface public: LogMsg() : category(), sysInfo(), jobInfo(), remoteFlag(false) {} LogMsg(LogMsgJobId id, const char *job); // Used for tracking job ids - LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session); - LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session); - LogMsg(const LogMsgCategory & _cat, LogMsgId _id, const LogMsgJobInfo & _jobInfo, LogMsgCode _code, const char * format, va_list args, - unsigned port, LogMsgSessionId session) __attribute__((format(printf,6, 0))); + LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * _text, unsigned port, LogMsgSessionId session); + LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, size32_t sz, const char * _text, unsigned port, LogMsgSessionId session); + LogMsg(const LogMsgCategory & _cat, LogMsgId _id, LogMsgCode _code, const char * format, va_list args, + unsigned port, LogMsgSessionId session) __attribute__((format(printf,5, 0))); StringBuffer & toStringPlain(StringBuffer & out, unsigned fields) const; StringBuffer & toStringXML(StringBuffer & out, unsigned fields) const; StringBuffer & toStringJSON(StringBuffer & out, unsigned fields) const; @@ -722,13 +722,8 @@ interface jlib_decl ILogMsgManager : public ILogMsgListener virtual void report(const LogMsgCategory & cat, LogMsgCode code , const char * format, ...) __attribute__((format(printf, 4, 5))) = 0; virtual void report_va(const LogMsgCategory & cat, LogMsgCode code , const char * format, va_list args) = 0; virtual void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL) = 0; - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 4, 5))) = 0; - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) = 0; - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) __attribute__((format(printf, 5, 6))) = 0; - virtual void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, va_list args) = 0; - virtual void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL) = 0; - virtual void mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg) = 0; - virtual void mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) = 0; + virtual void mreport_direct(const LogMsgCategory & cat, const char * msg) = 0; + virtual void mreport_va(const LogMsgCategory & cat, const char * format, va_list args) = 0; virtual void report(const LogMsg & msg) const = 0; virtual LogMsgId getNextID() = 0; virtual bool rejectsCategory(const LogMsgCategory & cat) const = 0; @@ -751,12 +746,7 @@ class jlib_decl LogMsgPrepender void report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 4, 5))); void report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args); void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 4, 5))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 5, 6))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL); - IException * report(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), unknownJob, handy for EXCLOG + IException * report(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), handy for EXCLOG private: char const * file; unsigned line; @@ -870,8 +860,6 @@ inline LogMsgCategory MCexception(IException * e, LogMsgClass cls = MSGCLS_error #define MCwarning MCuserWarning #define MCprogress MCuserProgress -extern jlib_decl const LogMsgJobInfo unknownJob; - // Function to return manager, standard handler and the reporters, and the handler's message fields extern jlib_decl ILogMsgManager * queryLogMsgManager(); @@ -883,19 +871,20 @@ extern jlib_decl void setupContainerizedLogMsgHandler(); extern jlib_decl void setDefaultJobName(const char *id); //A class for mapping job names to ids -class jlib_decl JobNameTranslator +class jlib_decl JobNameScope { public: - JobNameTranslator() = default; - JobNameTranslator(const char * name); - ~JobNameTranslator() { clear(); } + JobNameScope() = default; + JobNameScope(const char * name); + ~JobNameScope() { clear(); } void clear(); void set(const char * name); LogMsgJobId queryId() const { return id; } protected: - LogMsgJobId id = 0; + LogMsgJobId id = UnknownJob; + LogMsgJobId prevId = UnknownJob; }; // Macros to make logging as simple as possible @@ -926,30 +915,10 @@ inline void LOG(const LogMsgCategory & cat, LogMsgCode code , const char * forma LOGMSGREPORTER->report_va(cat, code , format, args); va_end(args); } -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 3, 4))); -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - va_list args; - va_start(args, format); - LOGMSGREPORTER->report_va(cat, job, format, args); - va_end(args); -} -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) __attribute__((format(printf, 4, 5))); -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, ...) -{ - va_list args; - va_start(args, format); - LOGMSGREPORTER->report_va(cat, job, code , format, args); - va_end(args); -} inline void LOG(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL) { LOGMSGREPORTER->report(cat, e, prefix); } -inline void LOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL) -{ - LOGMSGREPORTER->report(cat, job, e, prefix); -} inline void VALOG(const LogMsgCategory & cat, const char * format, va_list args) { LOGMSGREPORTER->report_va(cat, format, args); @@ -958,14 +927,6 @@ inline void VALOG(const LogMsgCategory & cat, LogMsgCode code , const char * for { LOGMSGREPORTER->report_va(cat, code , format, args); } -inline void VALOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) -{ - LOGMSGREPORTER->report_va(cat, job, format, args); -} -inline void VALOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code , const char * format, va_list args) -{ - LOGMSGREPORTER->report_va(cat, job, code , format, args); -} #else #define LOG LOGMSGREPORTER->report #define VALOG LOGMSGREPORTER->report_va @@ -976,12 +937,7 @@ extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, const char * fo extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 3, 4))); extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf, 3, 0))); extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL); -extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf, 3, 4))); -extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) __attribute__((format(printf, 3, 0))); -extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 4, 5))); -extern jlib_decl void ctxlogReportVA(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf, 4, 0))); -extern jlib_decl void ctxlogReport(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL); -extern jlib_decl IException * ctxlogReport(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), unknownJob, handy for EXCLOG +extern jlib_decl IException * ctxlogReport(IException * e, const char * prefix = NULL, LogMsgClass cls = MSGCLS_error); // uses MCexception(e, cls), handy for EXCLOG #define LOG ::ctxlogReport #define VALOG ::ctxlogReportVA @@ -992,7 +948,7 @@ inline void DBGLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugInfo, unknownJob, format, args); + VALOG(MCdebugInfo, format, args); va_end(args); } @@ -1001,7 +957,7 @@ inline void DISLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorDisaster, unknownJob, format, args); + VALOG(MCoperatorDisaster, format, args); va_end(args); } @@ -1010,7 +966,7 @@ inline void UERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserError, unknownJob, format, args); + VALOG(MCuserError, format, args); va_end(args); } @@ -1022,7 +978,7 @@ inline void OERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorError, unknownJob, format, args); + VALOG(MCoperatorError, format, args); va_end(args); } @@ -1031,7 +987,7 @@ inline void IERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugError, unknownJob, format, args); + VALOG(MCdebugError, format, args); va_end(args); } @@ -1040,7 +996,7 @@ inline void AERRLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCauditError, unknownJob, format, args); + VALOG(MCauditError, format, args); va_end(args); } @@ -1050,7 +1006,7 @@ inline void UWARNLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserWarning, unknownJob, format, args); + VALOG(MCuserWarning, format, args); va_end(args); } @@ -1062,7 +1018,7 @@ inline void OWARNLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorWarning, unknownJob, format, args); + VALOG(MCoperatorWarning, format, args); va_end(args); } @@ -1071,7 +1027,7 @@ inline void IWARNLOG(char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugWarning, unknownJob, format, args); + VALOG(MCdebugWarning, format, args); va_end(args); } @@ -1080,28 +1036,20 @@ inline void PROGLOG(const char * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserProgress, unknownJob, format, args); + VALOG(MCuserProgress, format, args); va_end(args); } inline void MLOG(const LogMsgCategory & cat, const char * msg) { - LOGMSGREPORTER->mreport_direct(cat, unknownJob, msg); -} - -inline void MLOG(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) -{ - va_list args; - va_start(args, format); - LOGMSGREPORTER->mreport_va(cat, job, format, args); - va_end(args); + LOGMSGREPORTER->mreport_direct(cat, msg); } inline void MLOG(const LogMsgCategory & cat, const char * format, ...) { va_list args; va_start(args, format); - LOGMSGREPORTER->mreport_va(cat, unknownJob, format, args); + LOGMSGREPORTER->mreport_va(cat, format, args); va_end(args); } @@ -1110,7 +1058,7 @@ inline void MLOG(const char * format, ...) { va_list args; va_start(args, format); - LOGMSGREPORTER->mreport_va(MCuserProgress, unknownJob, format, args); + LOGMSGREPORTER->mreport_va(MCuserProgress, format, args); va_end(args); } @@ -1119,7 +1067,7 @@ inline void DBGLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugInfo, unknownJob, code, format, args); + VALOG(MCdebugInfo, code, format, args); va_end(args); } @@ -1128,7 +1076,7 @@ inline void DISLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorDisaster, unknownJob, code, format, args); + VALOG(MCoperatorDisaster, code, format, args); va_end(args); } @@ -1137,7 +1085,7 @@ inline void UWARNLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserWarning, unknownJob, code, format, args); + VALOG(MCuserWarning, code, format, args); va_end(args); } @@ -1146,7 +1094,7 @@ inline void OWARNLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorWarning, unknownJob, code, format, args); + VALOG(MCoperatorWarning, code, format, args); va_end(args); } @@ -1155,7 +1103,7 @@ inline void IWARNLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugWarning, unknownJob, code, format, args); + VALOG(MCdebugWarning, code, format, args); va_end(args); } @@ -1179,7 +1127,7 @@ inline void OERRLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCoperatorError, unknownJob, code, format, args); + VALOG(MCoperatorError, code, format, args); va_end(args); } @@ -1188,7 +1136,7 @@ inline void IERRLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCdebugError, unknownJob, code, format, args); + VALOG(MCdebugError, code, format, args); va_end(args); } @@ -1197,7 +1145,7 @@ inline void UERRLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserError, unknownJob, code, format, args); + VALOG(MCuserError, code, format, args); va_end(args); } @@ -1206,7 +1154,7 @@ inline void PROGLOG(LogMsgCode code, char const * format, ...) { va_list args; va_start(args, format); - VALOG(MCuserProgress, unknownJob, code, format, args); + VALOG(MCuserProgress, code, format, args); va_end(args); } @@ -1274,7 +1222,7 @@ interface jlib_decl IContextLogger : extends IInterface { virtual void CTXLOG(const char *format, ...) const __attribute__((format(printf, 2, 3))); virtual void mCTXLOG(const char *format, ...) const __attribute__((format(printf, 2, 3))); - virtual void CTXLOGva(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char *format, va_list args) const __attribute__((format(printf,5,0))) = 0; + virtual void CTXLOGva(const LogMsgCategory & cat, LogMsgCode code, const char *format, va_list args) const __attribute__((format(printf,4,0))) = 0; void logOperatorException(IException *E, const char *file, unsigned line, const char *format, ...) const __attribute__((format(printf, 5, 6))); virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) = 0; virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const = 0; @@ -1293,7 +1241,6 @@ interface jlib_decl IContextLogger : extends IInterface virtual void setSpanAttribute(const char *name, const char *value) const = 0; virtual void setSpanAttribute(const char *name, __uint64 value) const = 0; virtual void recordStatistics(IStatisticGatherer &progress) const = 0; - virtual const LogMsgJobInfo & queryJob() const { return unknownJob; } }; using ContextSpanScope = ContextSpanScopeImp; diff --git a/system/jlib/jlog.ipp b/system/jlib/jlog.ipp index 361d823cc2c..31a6f05dfeb 100644 --- a/system/jlib/jlog.ipp +++ b/system/jlib/jlog.ipp @@ -740,16 +740,11 @@ public: bool flushQueue(unsigned timeout) { if(processor) return processor->flush(timeout); else return true; } void report(const LogMsgCategory & cat, const char * format, ...) __attribute__((format(printf,3,4))); void report_va(const LogMsgCategory & cat, const char * format, va_list args) __attribute__((format(printf,3,0))); - void mreport_direct(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * msg); - void mreport_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) __attribute__((format(printf,4,0))); + void mreport_direct(const LogMsgCategory & cat, const char * msg); + void mreport_va(const LogMsgCategory & cat, const char * format, va_list args) __attribute__((format(printf,3,0))); void report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf,4,5))); void report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf,4,0))); void report(const LogMsgCategory & cat, const IException * e, const char * prefix = NULL); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...) __attribute__((format(printf,4,5))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args) __attribute__((format(printf,4,0))); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...) __attribute__((format(printf,5,6))); - void report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args) __attribute__((format(printf,5,0))); - void report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * e, const char * prefix = NULL); void report(const LogMsg & msg) const { if(prefilter.includeCategory(msg.queryCategory())) doReport(msg); } bool addMonitor(ILogMsgHandler * handler, ILogMsgFilter * filter); bool addMonitorOwn(ILogMsgHandler * handler, ILogMsgFilter * filter); @@ -814,7 +809,7 @@ private: class DropLogMsg : public LogMsg { public: - DropLogMsg(CLogMsgManager * owner, LogMsgId id, unsigned _count) : LogMsg(dropWarningCategory, id, unknownJob, NoLogMsgCode, "MISSING LOG MESSAGES: ", owner->port, owner->session), count(_count) + DropLogMsg(CLogMsgManager * owner, LogMsgId id, unsigned _count) : LogMsg(dropWarningCategory, id, NoLogMsgCode, "MISSING LOG MESSAGES: ", owner->port, owner->session), count(_count) { text.append("message queue length exceeded, dropped ").append(count).append(" messages"); } diff --git a/system/jlib/jmisc.cpp b/system/jlib/jmisc.cpp index 7f0d55d5e74..9d1fbf5d504 100644 --- a/system/jlib/jmisc.cpp +++ b/system/jlib/jmisc.cpp @@ -251,7 +251,7 @@ jlib_decl void PrintMemoryStatusLog() #ifdef _WIN32 MEMORYSTATUS mS; GlobalMemoryStatus(&mS); - LOG(MCdebugInfo, unknownJob, "Available Physical Memory = %dK", (unsigned)(mS.dwAvailPhys/1024)); + LOG(MCdebugInfo, "Available Physical Memory = %dK", (unsigned)(mS.dwAvailPhys/1024)); #ifdef FRAGMENTATION_CHECK // see if fragmented size32_t sz = MAX_TRY_SIZE; @@ -267,7 +267,7 @@ jlib_decl void PrintMemoryStatusLog() } sz *= 2; if ((szendSpan(); + span.clear(); + } +} + +OwnedSpanScope::~OwnedSpanScope() +{ + clear(); +} + +//--------------------------------------------------------------------------------------------------------------------- + MODULE_INIT(INIT_PRIORITY_STANDARD) { return true; @@ -1475,6 +1512,11 @@ ISpan * getNullSpan() return nullSpan.getLink(); } +ISpan * queryNullSpan() +{ + return nullSpan; +} + void initTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig) { theTraceManager.query([=] () { return new CTraceManager(componentName, componentConfig, globalConfig); }); diff --git a/system/jlib/jtrace.hpp b/system/jlib/jtrace.hpp index 5ada18fc74b..fbdb6cc40ce 100644 --- a/system/jlib/jtrace.hpp +++ b/system/jlib/jtrace.hpp @@ -134,16 +134,24 @@ interface ISpan : extends IInterface virtual const char* queryLocalId() const = 0; }; -class OwnedSpanScope : public Owned +class jlib_decl OwnedSpanScope { public: - inline OwnedSpanScope() { } - inline OwnedSpanScope(ISpan * _ptr) : Owned(_ptr) { } - ~OwnedSpanScope() - { - if (get()) - get()->endSpan(); - } + OwnedSpanScope() = default; + OwnedSpanScope(ISpan * _ptr); + ~OwnedSpanScope(); + + inline ISpan * operator -> () const { return span; } + inline operator ISpan *() const { return span; } + + void clear(); + ISpan * query() const { return span; } + void set(ISpan * _span); + void setown(ISpan * _span); + +private: + Owned span; + ISpan * prevSpan = nullptr; }; extern jlib_decl IProperties * getClientHeaders(const ISpan * span); @@ -180,6 +188,7 @@ interface ITraceManager : extends IInterface virtual bool isTracingEnabled() const = 0; }; +extern jlib_decl ISpan * queryNullSpan(); extern jlib_decl ISpan * getNullSpan(); extern jlib_decl void initTraceManager(const char * componentName, const IPropertyTree * componentConfig, const IPropertyTree * globalConfig); extern jlib_decl ITraceManager & queryTraceManager(); diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index b592cf3e070..d2e4704765f 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -755,7 +755,7 @@ class CMPNotifyClosedThread: public Thread } } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP writepacket"); + FLLOG(MCoperatorWarning, e,"MP writepacket"); e->Release(); } } @@ -969,7 +969,7 @@ protected: friend class CMPPacketReader; { StringBuffer str; #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connecting to %s role: %" I64F "u", remoteep.getEndpointHostText(str).str(), parent->getRole()); + LOG(MCdebugInfo, "MP: connecting to %s role: %" I64F "u", remoteep.getEndpointHostText(str).str(), parent->getRole()); #endif if (((int)tm.timeout)<0) remaining = CONNECT_TIMEOUT; @@ -1005,7 +1005,7 @@ protected: friend class CMPPacketReader; newsock->set_keep_alive(true); #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: connect after socket connect, retrycount = %d", retrycount); + LOG(MCdebugInfo, "MP: connect after socket connect, retrycount = %d", retrycount); #endif SocketEndpoint hostep; @@ -1024,13 +1024,13 @@ protected: friend class CMPPacketReader; connectHdr.id[0].getEndpointHostText(tmp1); tmp1.append(' '); connectHdr.id[1].getEndpointHostText(tmp1); - LOG(MCdebugInfo, unknownJob, "MP: connect after socket write %s",tmp1.str()); + LOG(MCdebugInfo, "MP: connect after socket write %s",tmp1.str()); #endif size32_t rd = 0; #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connect after socket write, waiting for read"); + LOG(MCdebugInfo, "MP: connect after socket write, waiting for read"); #endif // Wait for connection reply but also check for A<->B deadlock (where both processes are here @@ -1110,7 +1110,7 @@ protected: friend class CMPPacketReader; // if other side closes, connect again if (e->errorCode() == JSOCKERR_graceful_close) { - LOG(MCdebugInfo, unknownJob, "MP: Retrying (other side closed connection, probably due to clash)"); + LOG(MCdebugInfo, "MP: Retrying (other side closed connection, probably due to clash)"); e->Release(); break; } @@ -1118,7 +1118,7 @@ protected: friend class CMPPacketReader; e->Release(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); + LOG(MCdebugInfo, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); #endif } else @@ -1145,7 +1145,7 @@ protected: friend class CMPPacketReader; size32_t replyVal; memcpy(&replyVal, (size32_t *)replyBuf, sizeof(size32_t)); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connect after socket read replyVal=%u, sizeof(connectHdr)=%lu", replyVal, sizeof(connectHdr)); + LOG(MCdebugInfo, "MP: connect after socket read replyVal=%u, sizeof(connectHdr)=%lu", replyVal, sizeof(connectHdr)); #endif if (replyVal > sizeof(ConnectHdr)) { @@ -1191,7 +1191,7 @@ protected: friend class CMPPacketReader; if (attachSocket(newsock,remoteep,hostep,true,NULL,addrval)) { #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connected to %s",str.str()); + LOG(MCdebugInfo, "MP: connected to %s",str.str()); #endif lastxfer = msTick(); closed = false; @@ -1222,7 +1222,7 @@ protected: friend class CMPPacketReader; #ifdef _TRACE StringBuffer str; str.clear(); - LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); + LOG(MCdebugInfo, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); #endif } @@ -1276,7 +1276,7 @@ protected: friend class CMPPacketReader; CriticalBlock block(connectsect); if (closed) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "WritePacket closed on entry"); + LOG(MCdebugInfo, "WritePacket closed on entry"); PrintStackReport(); #endif if (!checkReconnect(tm)) @@ -1285,7 +1285,7 @@ protected: friend class CMPPacketReader; if (!channelsock) { if (!connect(tm)) { #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "WritePacket connect failed"); + LOG(MCdebugInfo, "WritePacket connect failed"); #endif return false; } @@ -1311,7 +1311,7 @@ protected: friend class CMPPacketReader; #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "WritePacket(target=%s,(%d,%d,%d))",remoteep.getEndpointHostText(ep1).str(),hdrsize,hdr2size,bodysize); + LOG(MCdebugInfo, "WritePacket(target=%s,(%d,%d,%d))",remoteep.getEndpointHostText(ep1).str(),hdrsize,hdr2size,bodysize); unsigned t2 = msTick(); #endif unsigned n = 0; @@ -1330,17 +1330,17 @@ protected: friend class CMPPacketReader; sizes[n++] = bodysize; } if (!dest) { - LOG(MCdebugInfo, unknownJob, "MP Warning: WritePacket unexpected NULL socket"); + LOG(MCdebugInfo, "MP Warning: WritePacket unexpected NULL socket"); return false; } dest->write_multiple(n,bufs,sizes); lastxfer = msTick(); #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "WritePacket(timewaiting=%d,timesending=%d)",t2-t1,lastxfer-t2); + LOG(MCdebugInfo, "WritePacket(timewaiting=%d,timesending=%d)",t2-t1,lastxfer-t2); #endif } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP writepacket"); + FLLOG(MCoperatorWarning, e,"MP writepacket"); closeSocket(false, true); throw; } @@ -1383,11 +1383,11 @@ protected: friend class CMPPacketReader; return false; } if (tm.timedout()) { - LOG(MCdebugInfo, unknownJob, "MP: verify, ping failed to %s",ep.str()); + LOG(MCdebugInfo, "MP: verify, ping failed to %s",ep.str()); closeSocket(); return false; } - LOG(MCdebugInfo, unknownJob, "MP: verify, ping failed to %s, retrying",ep.str()); + LOG(MCdebugInfo, "MP: verify, ping failed to %s, retrying",ep.str()); unsigned remaining; if (!pingtm.timedout(&remaining)&&remaining) Sleep(remaining); @@ -1501,7 +1501,7 @@ class UserPacketHandler // default #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, "MP: send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif return channel->writepacket(&hdr,sizeof(hdr),mb.toByteArray(),mb.length(),tm); } @@ -1565,7 +1565,7 @@ class MultiPacketHandler // TAG_SYS_MULTI otherMhdr->getDetails(errorMsg).newline(); } msg.getDetails(errorMsg); - LOG(MCerror, unknownJob, "MultiPacketHandler: protocol error (%d) %s", code, errorMsg.str()); + LOG(MCerror, "MultiPacketHandler: protocol error (%d) %s", code, errorMsg.str()); } lastErrMs = ms; } @@ -1633,7 +1633,7 @@ class MultiPacketHandler // TAG_SYS_MULTI #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: multi-send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, "MP: multi-send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif PacketHeader outhdr; outhdr = hdr; @@ -1651,13 +1651,13 @@ class MultiPacketHandler // TAG_SYS_MULTI if (i+1==mhdr.numparts) mhdr.size = mhdr.total-mhdr.ofs; #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: multi-send block=%d, num blocks=%d, ofs=%d, size=%d",i,mhdr.numparts,mhdr.ofs,mhdr.size); + LOG(MCdebugInfo, "MP: multi-send block=%d, num blocks=%d, ofs=%d, size=%d",i,mhdr.numparts,mhdr.ofs,mhdr.size); #endif outhdr.initseq(); outhdr.size = sizeof(outhdr)+sizeof(mhdr)+mhdr.size; if (!channel->writepacket(&outhdr,sizeof(outhdr),&mhdr,sizeof(mhdr),p,mhdr.size,tm)) { #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: multi-send failed"); + LOG(MCdebugInfo, "MP: multi-send failed"); #endif return false; } @@ -1746,7 +1746,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface if (pc) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPPacketReader::notifySelected() about to close socket, mode = 0x%x", selected); + LOG(MCdebugInfo, "CMPPacketReader::notifySelected() about to close socket, mode = 0x%x", selected); #endif pc->closeSocket(false, true); } @@ -1766,7 +1766,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface // assumes packet header will arrive in one go if (sizeavailread(&hdr,sizeof(hdr),sizeof(hdr),szread,60); // I don't *really* want to block here but not much else can do @@ -1787,7 +1787,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: ReadPacket(sender=%s,target=%s,tag=%d,replytag=%d,size=%d)",hdr.sender.getEndpointHostText(ep1).str(),hdr.target.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, "MP: ReadPacket(sender=%s,target=%s,tag=%d,replytag=%d,size=%d)",hdr.sender.getEndpointHostText(ep1).str(),hdr.target.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif remaining = hdr.size-sizeof(hdr); activemsg = new CMessageBuffer(remaining); // will get from low level IO at some stage @@ -1807,7 +1807,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface if (remaining==0) { // we have the packet so process #ifdef _FULLTRACE - LOG(MCdebugInfo, unknownJob, "MP: ReadPacket(timetaken = %d,select iterations=%d)",msTick()-parent->startxfer,parent->numiter); + LOG(MCdebugInfo, "MP: ReadPacket(timetaken = %d,select iterations=%d)",msTick()-parent->startxfer,parent->numiter); #endif do { switch (activemsg->getTag()) { @@ -1843,7 +1843,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface } catch (IException *e) { if (e->errorCode()!=JSOCKERR_graceful_close) - FLLOG(MCoperatorWarning, unknownJob, e,"MP(Packet Reader)"); + FLLOG(MCoperatorWarning, e,"MP(Packet Reader)"); e->Release(); } // error here, so close socket (ignore error as may be closed already) @@ -1939,30 +1939,30 @@ bool CMPChannel::attachSocket(ISocket *newsock,const SocketEndpoint &_remoteep,c StringBuffer ep2; _localep.getEndpointHostText(ep1); _remoteep.getEndpointHostText(ep2); - LOG(MCdebugInfo, unknownJob, "MP: Possible clash between %s->%s %d(%d)",ep1.str(),ep2.str(),(int)ismaster,(int)master); + LOG(MCdebugInfo, "MP: Possible clash between %s->%s %d(%d)",ep1.str(),ep2.str(),(int)ismaster,(int)master); try { if (ismaster!=master) { if (ismaster) { - LOG(MCdebugInfo, unknownJob, "MP: resolving socket attach clash (master)"); + LOG(MCdebugInfo, "MP: resolving socket attach clash (master)"); return false; } else { Sleep(50); // give the other side some time to close CTimeMon tm(10000); if (verifyConnection(tm,false)) { - LOG(MCdebugInfo, unknownJob, "MP: resolving socket attach clash (verified)"); + LOG(MCdebugInfo, "MP: resolving socket attach clash (verified)"); return false; } } } } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP attachsocket(1)"); + FLLOG(MCoperatorWarning, e,"MP attachsocket(1)"); e->Release(); } try { - LOG(MCdebugInfo, unknownJob, "Message Passing - removing stale socket to %s",ep2.str()); + LOG(MCdebugInfo, "Message Passing - removing stale socket to %s",ep2.str()); CriticalUnblock unblock(connectsect); closeSocket(true, true); #ifdef REFUSE_STALE_CONNECTION @@ -1972,7 +1972,7 @@ bool CMPChannel::attachSocket(ISocket *newsock,const SocketEndpoint &_remoteep,c Sleep(100); // pause to allow close socket triggers to run } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP attachsocket(2)"); + FLLOG(MCoperatorWarning, e,"MP attachsocket(2)"); e->Release(); } } @@ -2010,7 +2010,7 @@ bool CMPChannel::send(MemoryBuffer &mb, mptag_t tag, mptag_t replytag, CTimeMon if (closed||(reply&&!isConnected())) // flag error if has been disconnected { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPChannel::send closed on entry %d",(int)closed); + LOG(MCdebugInfo, "CMPChannel::send closed on entry %d",(int)closed); PrintStackReport(); #endif if (!checkReconnect(tm)) @@ -2087,7 +2087,7 @@ bool CMPChannel::sendPing(CTimeMon &tm) ret = parent->pingpackethandler->send(this,hdr,tm)&&!tm.timedout(&remaining); } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP ping(1)"); + FLLOG(MCoperatorWarning, e,"MP ping(1)"); e->Release(); } sendmutex.unlock(); @@ -2117,7 +2117,7 @@ bool CMPChannel::sendPingReply(unsigned timeout,bool identifyself) ret = parent->pingreplypackethandler->send(this,hdr,mb,mon); } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP ping reply(1)"); + FLLOG(MCoperatorWarning, e,"MP ping reply(1)"); e->Release(); ret = false; } @@ -2225,7 +2225,7 @@ CMPConnectThread::CMPConnectThread(CMPServer *_parent, unsigned port, bool _list listensock = NULL; // delay create till running parent->setPort(port); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP Connect Thread Init Port = %d", port); + LOG(MCdebugInfo, "MP Connect Thread Init Port = %d", port); #endif running = false; @@ -2415,7 +2415,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, // not sure how to get here as this is not one of the possible outcomes of above: rd == 0 or rd == sizeof(id) or an exception StringBuffer errMsg("MP Connect Thread: invalid number of connection bytes serialized from "); peerEp.getEndpointHostText(errMsg); - FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str()); + FLLOG(MCoperatorWarning, "%s", errMsg.str()); sock->close(); return false; // did not timeout } @@ -2472,7 +2472,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, // JCSMORE, I think _remoteep really must/should match a IP of this local host errMsg.append("MP Connect Thread: invalid remote and/or host ep serialized from "); peerEp.getEndpointHostText(errMsg); - FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str()); + FLLOG(MCoperatorWarning, "%s", errMsg.str()); } else if (parent->mpTraceLevel >= MPVerboseMsgThreshold) { @@ -2504,7 +2504,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, #ifdef _TRACE StringBuffer str1; StringBuffer str2; - LOG(MCdebugInfo, unknownJob, "MP Connect Thread: connected to %s",_remoteep.getEndpointHostText(str1).str()); + LOG(MCdebugInfo, "MP Connect Thread: connected to %s",_remoteep.getEndpointHostText(str1).str()); #endif } #ifdef _FULLTRACE @@ -2513,7 +2513,7 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, } catch (IException *e) { - FLLOG(MCoperatorWarning, unknownJob, e,"MP Connect Thread: Failed to make connection(1)"); + FLLOG(MCoperatorWarning, e,"MP Connect Thread: Failed to make connection(1)"); sock->close(); e->Release(); } @@ -2523,8 +2523,9 @@ bool CMPConnectThread::handleAcceptedSocket(ISocket *_sock, unsigned timeoutMs, int CMPConnectThread::run() { #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Connect Thread Starting - accept loop"); + LOG(MCdebugInfo, "MP: Connect Thread Starting - accept loop"); #endif + Owned exception; while (running) { Owned sock; @@ -2535,8 +2536,7 @@ int CMPConnectThread::run() } catch (IException *e) { - LOG(MCdebugInfo, unknownJob, e,"MP accept failed"); - throw; // error handling TBD + exception.setown(e); } if (sock) { @@ -2552,11 +2552,23 @@ int CMPConnectThread::run() else { if (running) - LOG(MCdebugInfo, unknownJob, "MP Connect Thread accept returned NULL"); + { + if (exception) + { + constexpr unsigned sleepSecs = 5; + // Log and pause for a few seconds, because accept loop may have failed due to handle exhaustion. + VStringBuffer msg("MP accept failed. Accept loop will be paused for %u seconds", sleepSecs); + EXCLOG(exception, msg.str()); + exception.clear(); + MilliSleep(sleepSecs * 1000); + } + else // not sure this can ever happen (no exception, still running, and sock==nullptr) + LOG(MCdebugInfo, "MP Connect Thread accept returned NULL"); + } } } #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP Connect Thread Stopping"); + LOG(MCdebugInfo, "MP Connect Thread Stopping"); #endif return 0; } @@ -2668,7 +2680,7 @@ CMPServer::~CMPServer() StringBuffer buf; getReceiveQueueDetails(buf); if (buf.length()) - LOG(MCdebugInfo, unknownJob, "MP: Orphan check\n%s",buf.str()); + LOG(MCdebugInfo, "MP: Orphan check\n%s",buf.str()); #endif _releaseAll(); selecthandler->stop(true); @@ -2746,7 +2758,7 @@ bool CMPServer::recv(CMessageBuffer &mbuf, const SocketEndpoint *ep, mptag_t tag } if (nfy.aborted) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPserver::recv closed on notify"); + LOG(MCdebugInfo, "CMPserver::recv closed on notify"); PrintStackReport(); #endif IMP_Exception *e=new CMPException(MPERR_link_closed,*nfy.ep); @@ -2832,7 +2844,7 @@ unsigned CMPServer::probe(const SocketEndpoint *ep, mptag_t tag,CTimeMon &tm,Soc } if (nfy.aborted) { #ifdef _TRACELINKCLOSED - LOG(MCdebugInfo, unknownJob, "CMPserver::probe closed on notify"); + LOG(MCdebugInfo, "CMPserver::probe closed on notify"); PrintStackReport(); #endif IMP_Exception *e=new CMPException(MPERR_link_closed,*ep); @@ -2920,7 +2932,7 @@ void CMPServer::notifyClosed(SocketEndpoint &ep, bool trace) if (trace) { StringBuffer url; - LOG(MCdebugInfo, unknownJob, "MP: CMPServer::notifyClosed %s",ep.getEndpointHostText(url).str()); + LOG(MCdebugInfo, "MP: CMPServer::notifyClosed %s",ep.getEndpointHostText(url).str()); PrintStackReport(); } #endif @@ -3621,7 +3633,7 @@ void stopMPServer() { stopLogMsgReceivers(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Stopping MP Server"); + LOG(MCdebugInfo, "MP: Stopping MP Server"); #endif _globalMPServer = globalMPServer; globalMPServer = NULL; @@ -3632,7 +3644,7 @@ void stopMPServer() _globalMPServer->stop(); _globalMPServer->Release(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Stopped MP Server"); + LOG(MCdebugInfo, "MP: Stopped MP Server"); #endif CriticalBlock block(CGlobalMPServer::sect); initMyNode(0); diff --git a/testing/regress/ecl-test-azure.json b/testing/regress/ecl-test-azure.json index c06fdaaf7ab..d57c20465e3 100644 --- a/testing/regress/ecl-test-azure.json +++ b/testing/regress/ecl-test-azure.json @@ -58,7 +58,7 @@ "roxiegzip.ecl:TargetIP=roxie", "roxiewhitespace.ecl:TargetIP=roxie", "roxiepipe.ecl:TargetIP=roxie", - "soaptext1.ecl:TargetIP=roxie" + "soaptext1.ecl:remoteRoxieService=roxie,remoteEspService=eclqueries" ], "engineParams":[ "failOnLeaks", diff --git a/testing/regress/ecl/soaptext1.ecl b/testing/regress/ecl/soaptext1.ecl index 97bfd53c212..2022c95cc23 100644 --- a/testing/regress/ecl/soaptext1.ecl +++ b/testing/regress/ecl/soaptext1.ecl @@ -31,14 +31,18 @@ import ^ as root; //Simplified configuration -roxieIP := '.' : STORED('TargetIP'); -espIP := '.' : STORED('TargetIP'); +string roxieIP := '.' : STORED('remoteRoxieService'); +string espIP := '.' : STORED('remoteEspService'); +boolean defaultEncryptConnection := false : stored('encryptConnection'); +boolean defaultPersistConnection := false : stored('persistConnection'); +boolean defaultCallDirect := false : stored('callDirect'); +boolean defaultCallViaEsp := false : stored('callViaEsp'); //Simplified call options - doesn't provide all combinations that should be tested eventually -boolean encryptConnection := #IFDEFINED(root.encryptConnection, false); -boolean persistConnection := #IFDEFINED(root.persistConnection, false); -boolean callDirect := #IFDEFINED(root.callDirect, false); -boolean callViaEsp := #IFDEFINED(root.callViaEsp, false); +boolean encryptConnection := #IFDEFINED(root.encryptConnection, defaultEncryptConnection); +boolean persistConnection := #IFDEFINED(root.persistConnection, defaultPersistConnection); +boolean callDirect := #IFDEFINED(root.callDirect, defaultCallDirect); +boolean callViaEsp := #IFDEFINED(root.callViaEsp, defaultCallViaEsp); //--- end of version configuration --- diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp index 1ce8551fd34..0886da12708 100644 --- a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp @@ -2487,7 +2487,7 @@ class IndexDistributeSlaveActivity : public HashDistributeSlaveBase } *lookup; public: - IndexDistributeSlaveActivity(CGraphElementBase *container) : PARENT(container, indexDistribActivityStatistics), lookup(NULL), contextLogger(jhtreeCacheStatistics, thorJob), statsUpdater(jhtreeCacheStatistics, *this, contextLogger) + IndexDistributeSlaveActivity(CGraphElementBase *container) : PARENT(container, indexDistribActivityStatistics), lookup(NULL), contextLogger(jhtreeCacheStatistics), statsUpdater(jhtreeCacheStatistics, *this, contextLogger) { } ~IndexDistributeSlaveActivity() @@ -3422,7 +3422,7 @@ bool CBucket::flush(bool critical) { if (clearHashTable(critical)) { - LOG(MCthorDetailedDebugInfo, thorJob, "Flushed%s bucket %d - %d elements", critical?"(critical)":"", queryBucketNumber(), count); + LOG(MCthorDetailedDebugInfo, "Flushed%s bucket %d - %d elements", critical?"(critical)":"", queryBucketNumber(), count); return true; } } diff --git a/thorlcr/activities/indexread/thindexreadslave.cpp b/thorlcr/activities/indexread/thindexreadslave.cpp index dd6cc9257d5..f276bc2051b 100644 --- a/thorlcr/activities/indexread/thindexreadslave.cpp +++ b/thorlcr/activities/indexread/thindexreadslave.cpp @@ -670,11 +670,11 @@ class CIndexReadSlaveBase : public CSlaveActivity { // Only superfiles required multiple context loggers to track sublevel stats for(unsigned i = 0; i < parts; ++i) - contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics, thorJob)); + contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics)); } else { - contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics, thorJob)); + contextLoggers.push_back(new CStatsContextLogger(jhtreeCacheStatistics)); } } } diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp index 8c6f06ced04..e239d91319a 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave-legacy.cpp @@ -1253,7 +1253,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem public: IMPLEMENT_IINTERFACE_USING(CSimpleInterface); - CKeyLocalLookup(CKeyedJoinSlave &_owner, const RtlRecord &_keyRecInfo) : owner(_owner), keyRecInfo(_keyRecInfo), indexReadFieldsRow(_owner.indexInputAllocator), contextLogger(jhtreeCacheStatistics, thorJob) + CKeyLocalLookup(CKeyedJoinSlave &_owner, const RtlRecord &_keyRecInfo) : owner(_owner), keyRecInfo(_keyRecInfo), indexReadFieldsRow(_owner.indexInputAllocator), contextLogger(jhtreeCacheStatistics) { tlkManager.setown(owner.keyHasTlk ? createLocalKeyManager(keyRecInfo, nullptr, &contextLogger, owner.helper->hasNewSegmentMonitors(), false) : nullptr); reset(); diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp index 36b53ed6f1e..d00cd89595b 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp @@ -1030,7 +1030,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem { StringBuffer log; getInfo(log).append(": ").append(msg); - LOG(MCthorDetailedDebugInfo, thorJob, "%s", log.str()); + LOG(MCthorDetailedDebugInfo, "%s", log.str()); } virtual StringBuffer &getInfo(StringBuffer &info) const { @@ -1377,7 +1377,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem else { for (unsigned i=0; i keyManager{nullptr}; public: - CKeyLookupMergeHandler(CKeyedJoinSlave &_activity) : CKeyLookupLocalBase(_activity), contextLogger(jhtreeCacheStatistics, thorJob) + CKeyLookupMergeHandler(CKeyedJoinSlave &_activity) : CKeyLookupLocalBase(_activity), contextLogger(jhtreeCacheStatistics) { limiter = &activity.lookupThreadLimiter; translators.push_back(nullptr); @@ -1646,7 +1646,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem else { for (unsigned i=0; ijhelper->nextRow(); assertex(!row); @@ -1647,7 +1647,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase } for (unsigned i=0;inumworkers;i++) parent->addWork(NULL,NULL); - LOG(MCthorDetailedDebugInfo, thorJob, "CMultiCoreJoinHelper::cReader exit"); + LOG(MCthorDetailedDebugInfo, "CMultiCoreJoinHelper::cReader exit"); return 0; } } reader; @@ -1671,7 +1671,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase } int run() { - LOG(MCthorDetailedDebugInfo, thorJob, "CMultiCoreJoinHelper::cWorker started"); + LOG(MCthorDetailedDebugInfo, "CMultiCoreJoinHelper::cWorker started"); Owned rowIf = parent->activity.getRowInterfaces(); Owned allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RHFpacked|roxiemem::RHFunique)); @@ -1704,7 +1704,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase rowWriter->putRow(NULL); // end-of-stream rowWriter->flush(); stopped = true; // NB: will not get past flush(), until all read - LOG(MCthorDetailedDebugInfo, thorJob, "CMultiCoreJoinHelper::cWorker exit"); + LOG(MCthorDetailedDebugInfo, "CMultiCoreJoinHelper::cWorker exit"); return 0; } bool isStopped() const { return stopped; } @@ -1848,7 +1848,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase } int run() { - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cReader started"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cReader started"); try { const void * row = parent->jhelper->nextRow(); assertex(!row); @@ -1857,7 +1857,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase parent->setException(e,"CMulticoreUnorderedJoinHelper::cReader"); } parent->stopWorkers(); - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cReader exit"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cReader exit"); return 0; } } reader; @@ -1876,7 +1876,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase Owned allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RHFpacked|roxiemem::RHFunique)); Owned rowWriter = parent->multiWriter->getWriter(); - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cWorker started"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cWorker started"); for (;;) { cWorkItem *work = parent->workqueue.dequeue(); @@ -1897,7 +1897,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase break; } } - LOG(MCthorDetailedDebugInfo, thorJob, "CMulticoreUnorderedJoinHelper::cWorker exit"); + LOG(MCthorDetailedDebugInfo, "CMulticoreUnorderedJoinHelper::cWorker exit"); return 0; } } **workers; diff --git a/thorlcr/activities/soapcall/thsoapcallslave.cpp b/thorlcr/activities/soapcall/thsoapcallslave.cpp index cbb89484c42..b0b81f64e16 100644 --- a/thorlcr/activities/soapcall/thsoapcallslave.cpp +++ b/thorlcr/activities/soapcall/thsoapcallslave.cpp @@ -45,7 +45,7 @@ class CWscRowCallSlaveActivity : public CSlaveActivity, implements IWSCRowProvid public: IMPLEMENT_IINTERFACE_USING(CSlaveActivity); - CWscRowCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + CWscRowCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { buildAuthToken(queryJob().queryUserDescriptor(), authToken); setRequireInitData(false); @@ -141,7 +141,7 @@ class SoapDatasetCallSlaveActivity : public CSlaveActivity, implements IWSCRowPr public: IMPLEMENT_IINTERFACE_USING(CSlaveActivity); - SoapDatasetCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + SoapDatasetCallSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { setRequireInitData(false); appendOutputLinked(this); @@ -235,7 +235,7 @@ class SoapRowActionSlaveActivity : public ProcessSlaveActivity, implements IWSCR public: IMPLEMENT_IINTERFACE_USING(PARENT); - SoapRowActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + SoapRowActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { setRequireInitData(false); } @@ -294,7 +294,7 @@ class SoapDatasetActionSlaveActivity : public ProcessSlaveActivity, implements I public: IMPLEMENT_IINTERFACE_USING(PARENT); - SoapDatasetActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics, thorJob), statsUpdater(soapcallStatistics, *this, contextLogger) + SoapDatasetActionSlaveActivity(CGraphElementBase *_container) : ProcessSlaveActivity(_container, soapcallActivityStatistics), contextLogger(soapcallStatistics), statsUpdater(soapcallStatistics, *this, contextLogger) { setRequireInitData(false); } diff --git a/thorlcr/activities/thdiskbase.cpp b/thorlcr/activities/thdiskbase.cpp index 0feb326a37c..a2159f3683f 100644 --- a/thorlcr/activities/thdiskbase.cpp +++ b/thorlcr/activities/thdiskbase.cpp @@ -328,7 +328,7 @@ void CWriteMasterBase::publish() { StringBuffer clusterName; fileDesc->getClusterGroupName(clusterIdx, clusterName, &queryNamedGroupStore()); - LOG(MCthorDetailedDebugInfo, thorJob, "Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str()); + LOG(MCthorDetailedDebugInfo, "Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str()); unsigned p=0; while (pnumParts()) { diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index 056d9f82d58..a95ddc26b91 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -1200,12 +1200,12 @@ void traceMemUsage() { StringBuffer memStatsStr; roxiemem::memstats(memStatsStr); - LOG(MCthorDetailedDebugInfo, thorJob, "Roxiemem stats: %s", memStatsStr.str()); + LOG(MCthorDetailedDebugInfo, "Roxiemem stats: %s", memStatsStr.str()); memsize_t heapUsage = getMapInfo("heap"); if (heapUsage) // if 0, assumed to be unavailable { memsize_t rmtotal = roxiemem::getTotalMemoryLimit(); - LOG(MCthorDetailedDebugInfo, thorJob, "Heap usage (excluding Roxiemem) : %" I64F "d bytes", (unsigned __int64)(heapUsage-rmtotal)); + LOG(MCthorDetailedDebugInfo, "Heap usage (excluding Roxiemem) : %" I64F "d bytes", (unsigned __int64)(heapUsage-rmtotal)); } } @@ -1412,7 +1412,7 @@ void CGraphBase::executeSubGraph(size32_t parentExtractSz, const byte *parentExt { StringBuffer s; toXML(&queryXGMML(), s, 2); - MLOG(MCthorDetailedDebugInfo, thorJob, "Running graph [%s] : %s", isGlobal()?"global":"local", s.str()); + MLOG(MCthorDetailedDebugInfo, "Running graph [%s] : %s", isGlobal()?"global":"local", s.str()); } } if (localResults) @@ -2296,7 +2296,7 @@ IThorGraphResults *CGraphBase::createThorGraphResults(unsigned num) CFileUsageEntry * CGraphTempHandler::registerFile(const char *name, graph_id graphId, unsigned usageCount, bool temp, WUFileKind fileKind, StringArray *clusters) { assertex(temp); - LOG(MCdebugProgress, thorJob, "registerTmpFile name=%s, usageCount=%d", name, usageCount); + LOG(MCdebugProgress, "registerTmpFile name=%s, usageCount=%d", name, usageCount); CriticalBlock b(crit); if (tmpFiles.find(name)) throw MakeThorException(TE_FileAlreadyUsedAsTempFile, "File already used as temp file (%s)", name); @@ -2307,7 +2307,7 @@ CFileUsageEntry * CGraphTempHandler::registerFile(const char *name, graph_id gra void CGraphTempHandler::deregisterFile(const char *name, bool kept) { - LOG(MCdebugProgress, thorJob, "deregisterTmpFile name=%s", name); + LOG(MCdebugProgress, "deregisterTmpFile name=%s", name); CriticalBlock b(crit); CFileUsageEntry *fileUsage = tmpFiles.find(name); if (!fileUsage) @@ -2324,9 +2324,9 @@ void CGraphTempHandler::deregisterFile(const char *name, bool kept) try { if (!removeTemp(name)) - LOG(MCwarning, thorJob, "Failed to delete tmp file : %s (not found)", name); + LOG(MCwarning, "Failed to delete tmp file : %s (not found)", name); } - catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, thorJob, e, s.append(name).str()); } + catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, e, s.append(name).str()); } } else fileUsage->decUsage(); @@ -2343,9 +2343,9 @@ void CGraphTempHandler::clearTemps() try { if (!removeTemp(tmpname)) - LOG(MCwarning, thorJob, "Failed to delete tmp file : %s (not found)", tmpname); + LOG(MCwarning, "Failed to delete tmp file : %s (not found)", tmpname); } - catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, thorJob, e, s.append(tmpname).str()); } + catch (IException *e) { StringBuffer s("Failed to delete tmp file : "); FLLOG(MCwarning, e, s.append(tmpname).str()); } } iter.clear(); tmpFiles.kill(); @@ -2778,7 +2778,7 @@ CActivityBase &CJobBase::queryChannelActivity(unsigned c, graph_id gid, activity void CJobBase::startJob() { - LOG(MCdebugProgress, thorJob, "New Graph started : %s", graphName.get()); + LOG(MCdebugProgress, "New Graph started : %s", graphName.get()); perfmonhook.setown(createThorMemStatsPerfMonHook(*this, getOptInt(THOROPT_MAX_KERNLOG, 3))); setPerformanceMonitorHook(perfmonhook); PrintMemoryStatusLog(); @@ -2825,7 +2825,7 @@ void CJobBase::endJob() jobEnded = true; setPerformanceMonitorHook(nullptr); - LOG(MCdebugProgress, thorJob, "Job ended : %s", graphName.get()); + LOG(MCdebugProgress, "Job ended : %s", graphName.get()); clearKeyStoreCache(true); PrintMemoryStatusLog(); @@ -3018,7 +3018,7 @@ mptag_t CJobChannel::deserializeMPTag(MemoryBuffer &mb) deserializeMPtag(mb, tag); if (TAG_NULL != tag) { - LOG(MCthorDetailedDebugInfo, thorJob, "deserializeMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "deserializeMPTag: tag = %d", (int)tag); jobComm->flush(tag); } return tag; @@ -3057,7 +3057,7 @@ void CJobChannel::clean() if (!REJECTLOG(MCthorDetailedDebugInfo)) { queryRowManager()->reportMemoryUsage(false); - LOG(MCthorDetailedDebugInfo, thorJob, "CJobBase resetting memory manager"); + LOG(MCthorDetailedDebugInfo, "CJobBase resetting memory manager"); } if (graphExecutor) diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 665c8778a39..8867425a8c2 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -78,7 +78,7 @@ class CFatalHandler : public CTimeoutTrigger, implements IFatalHandler virtual bool action() override { StringBuffer s("FAILED TO RECOVER FROM EXCEPTION, STOPPING THOR"); - FLLOG(MCoperatorWarning, thorJob, exception, s.str()); + FLLOG(MCoperatorWarning, exception, s.str()); Owned jobManager = getJobManager(); if (jobManager) { @@ -298,7 +298,7 @@ void CSlaveMessageHandler::threadmain() } case smt_getPhysicalName: { - LOG(MCdebugProgress, thorJob, "getPhysicalName called from node %d", sender-1); + LOG(MCdebugProgress, "getPhysicalName called from node %d", sender-1); StringAttr logicalName; unsigned partNo; bool create; @@ -317,7 +317,7 @@ void CSlaveMessageHandler::threadmain() } case smt_getFileOffset: { - LOG(MCdebugProgress, thorJob, "getFileOffset called from node %d", sender-1); + LOG(MCdebugProgress, "getFileOffset called from node %d", sender-1); StringAttr logicalName; unsigned partNo; msg.read(logicalName); @@ -330,7 +330,7 @@ void CSlaveMessageHandler::threadmain() } case smt_actMsg: { - LOG(MCdebugProgress, thorJob, "smt_actMsg called from node %d", sender-1); + LOG(MCdebugProgress, "smt_actMsg called from node %d", sender-1); graph_id gid; msg.read(gid); activity_id id; @@ -348,7 +348,7 @@ void CSlaveMessageHandler::threadmain() { unsigned slave; msg.read(slave); - LOG(MCdebugProgress, thorJob, "smt_getresult called from slave %d", slave); + LOG(MCdebugProgress, "smt_getresult called from slave %d", slave); graph_id gid; msg.read(gid); activity_id ownerId; @@ -1001,7 +1001,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase return getWorkUnitResult(workunit, name, sequence); } #define PROTECTED_GETRESULT(STEPNAME, SEQUENCE, KIND, KINDTEXT, ACTION) \ - LOG(MCdebugProgress, thorJob, "getResult%s(%s,%d)", KIND, STEPNAME?STEPNAME:"", SEQUENCE); \ + LOG(MCdebugProgress, "getResult%s(%s,%d)", KIND, STEPNAME?STEPNAME:"", SEQUENCE); \ Owned r = getResultForGet(STEPNAME, SEQUENCE); \ try \ { \ @@ -1266,7 +1266,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase { try { - LOG(MCdebugProgress, thorJob, "getExternalResultRaw %s", stepname); + LOG(MCdebugProgress, "getExternalResultRaw %s", stepname); Owned r = getExternalResult(wuid, stepname, sequence); return r->getResultHash(); @@ -1308,7 +1308,7 @@ class CThorCodeContextMaster : public CThorCodeContextBase tgt = NULL; try { - LOG(MCdebugProgress, thorJob, "getExternalResultRaw %s", stepname); + LOG(MCdebugProgress, "getExternalResultRaw %s", stepname); Variable2IDataVal result(&tlen, &tgt); Owned r = getExternalResult(wuid, stepname, sequence); @@ -1586,7 +1586,7 @@ mptag_t CJobMaster::allocateMPTag() { mptag_t tag = allocateClusterMPTag(); queryJobChannel(0).queryJobComm().flush(tag); - LOG(MCthorDetailedDebugInfo, thorJob, "allocateMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "allocateMPTag: tag = %d", (int)tag); return tag; } @@ -1595,7 +1595,7 @@ void CJobMaster::freeMPTag(mptag_t tag) if (TAG_NULL != tag) { freeClusterMPTag(tag); - LOG(MCthorDetailedDebugInfo, thorJob, "freeMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "freeMPTag: tag = %d", (int)tag); queryJobChannel(0).queryJobComm().flush(tag); } } @@ -1869,7 +1869,7 @@ bool CJobMaster::go() Owned factory = getWorkUnitFactory(); if (factory->isAborting(wu.queryWuid())) { - LOG(MCwarning, thorJob, "ABORT detected from user"); + LOG(MCwarning, "ABORT detected from user"); unsigned code = TE_WorkUnitAborting; // default if (job.getOptBool("dumpInfoOnUserAbort", false)) @@ -2089,13 +2089,13 @@ bool CJobMaster::fireException(IException *e) { case tea_warning: { - LOG(MCwarning, thorJob, e); + LOG(MCwarning, e); reportExceptionToWorkunitCheckIgnore(*workunit, e); break; } default: { - LOG(MCerror, thorJob, e); + LOG(MCerror, e); queryJobManager().replyException(*this, e); fatalHandler->inform(LINK(e)); try { abort(e); } @@ -2341,13 +2341,13 @@ bool CMasterGraph::fireException(IException *e) { case tea_warning: { - LOG(MCwarning, thorJob, e); + LOG(MCwarning, e); reportExceptionToWorkunitCheckIgnore(job.queryWorkUnit(), e); break; } default: { - LOG(MCerror, thorJob, e); + LOG(MCerror, e); if (NULL != fatalHandler) fatalHandler->inform(LINK(e)); if (owner) diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 4c339769d5f..6c1d9868eba 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1804,7 +1804,7 @@ void CJobSlave::reportGraphEnd(graph_id gid) getSocketStatistics(stats); StringBuffer s; getSocketStatisticsString(stats,s); - MLOG(MCthorDetailedDebugInfo, thorJob, "Graph[%" GIDPF "u] - Socket statistics : %s\n", gid, s.str()); + MLOG(MCthorDetailedDebugInfo, "Graph[%" GIDPF "u] - Socket statistics : %s\n", gid, s.str()); } resetSocketStatistics(); } @@ -1850,7 +1850,7 @@ mptag_t CJobSlave::deserializeMPTag(MemoryBuffer &mb) deserializeMPtag(mb, tag); if (TAG_NULL != tag) { - LOG(MCthorDetailedDebugInfo, thorJob, "CJobSlave::deserializeMPTag: tag = %d", (int)tag); + LOG(MCthorDetailedDebugInfo, "CJobSlave::deserializeMPTag: tag = %d", (int)tag); for (unsigned c=0; cmarkdead = true; - LOG(MCdebugProgress, thorJob, "Watchdog : Marking Machine as Down! [%s]", epstr.str()); + LOG(MCdebugProgress, "Watchdog : Marking Machine as Down! [%s]", epstr.str()); //removeSlave(mstate->ep); // more TBD } } @@ -202,7 +202,7 @@ void CMasterWatchdog::stopReading() void CMasterWatchdog::threadmain() { - LOG(MCdebugProgress, thorJob, "Started watchdog"); + LOG(MCdebugProgress, "Started watchdog"); unsigned lastbeat=msTick(); unsigned lastcheck=lastbeat; @@ -234,7 +234,7 @@ void CMasterWatchdog::threadmain() { StringBuffer epstr; hb.sender.getEndpointHostText(epstr); - LOG(MCdebugProgress, thorJob, "Watchdog : Unknown Machine! [%s]", epstr.str()); //TBD + LOG(MCdebugProgress, "Watchdog : Unknown Machine! [%s]", epstr.str()); //TBD } } unsigned now=msTick(); @@ -253,7 +253,7 @@ void CMasterWatchdog::threadmain() { if (MPERR_link_closed != e->errorCode()) { - FLLOG(MCexception(e), thorJob, e,"Watchdog Server Exception"); + FLLOG(MCexception(e), e,"Watchdog Server Exception"); e->Release(); } else @@ -266,7 +266,7 @@ void CMasterWatchdog::threadmain() } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"Watchdog Server Exception"); + FLLOG(MCexception(e), e,"Watchdog Server Exception"); e->Release(); // NB: it is important to continue with master watchdog, to continue to consume packets from workers } diff --git a/thorlcr/master/thactivitymaster.cpp b/thorlcr/master/thactivitymaster.cpp index b7ca5b248f8..82ca2d647a9 100644 --- a/thorlcr/master/thactivitymaster.cpp +++ b/thorlcr/master/thactivitymaster.cpp @@ -701,7 +701,7 @@ void checkFormatCrc(CActivityBase *activity, IDistributedFile *file, unsigned ex Owned e = MakeActivityException(activity, TE_FormatCrcMismatch, "%s: Layout does not match published layout. %s", kindStr.str(), fileStr.str()); if (index && !f->queryAttributes().hasProp("_record_layout")) // Cannot verify if _true_ crc mismatch if soft layout missing anymore - LOG(MCwarning, thorJob, e); + LOG(MCwarning, e); else { if (activity->queryContainer().queryJob().getWorkUnitValueBool("skipFileFormatCrcCheck", false)) diff --git a/thorlcr/master/thdemonserver.cpp b/thorlcr/master/thdemonserver.cpp index 5b4094c3315..2087ff76098 100644 --- a/thorlcr/master/thdemonserver.cpp +++ b/thorlcr/master/thdemonserver.cpp @@ -76,7 +76,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer catch (IException *e) { StringBuffer s; - LOG(MCwarning, thorJob, "Failed to update progress information: %s", e->errorMessage(s).str()); + LOG(MCwarning, "Failed to update progress information: %s", e->errorMessage(s).str()); e->Release(); } } @@ -101,7 +101,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer const cost_type totalCost = workunitCost + sgCost + graph.getDiskAccessCost(); if (costLimit>0 && totalCost > costLimit) { - LOG(MCwarning, thorJob, "ABORT job cost exceeds limit"); + LOG(MCwarning, "ABORT job cost exceeds limit"); graph.fireException(MakeThorException(TE_CostExceeded, "Job cost exceeds limit")); } } @@ -166,7 +166,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer catch (IException *E) { StringBuffer s; - LOG(MCwarning, thorJob, "Failed to update progress information: %s", E->errorMessage(s).str()); + LOG(MCwarning, "Failed to update progress information: %s", E->errorMessage(s).str()); E->Release(); } } @@ -194,7 +194,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer catch (IException *e) { StringBuffer s; - LOG(MCwarning, thorJob, "Failed to update progress information: %s", e->errorMessage(s).str()); + LOG(MCwarning, "Failed to update progress information: %s", e->errorMessage(s).str()); e->Release(); } } @@ -215,7 +215,7 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer if (0 == activeGraphs.ordinality()) { StringBuffer urlStr; - LOG(MCdebugProgress, thorJob, "heartbeat packet received with no active graphs"); + LOG(MCdebugProgress, "heartbeat packet received with no active graphs"); return; } size32_t compressedProgressSz = progressMb.remaining(); @@ -233,12 +233,12 @@ class DeMonServer : public CSimpleInterface, implements IDeMonServer ForEachItemIn(g, activeGraphs) if (activeGraphs.item(g).queryGraphId() == graphId) graph = (CMasterGraph *)&activeGraphs.item(g); if (!graph) { - LOG(MCdebugProgress, thorJob, "heartbeat received from unknown graph %" GIDPF "d", graphId); + LOG(MCdebugProgress, "heartbeat received from unknown graph %" GIDPF "d", graphId); break; } if (!graph->deserializeStats(slave, uncompressedMb)) { - LOG(MCdebugProgress, thorJob, "heartbeat error in graph %" GIDPF "d", graphId); + LOG(MCdebugProgress, "heartbeat error in graph %" GIDPF "d", graphId); break; } } diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 1b6ff4ca6a6..2aa393c5d5c 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -265,7 +265,7 @@ class CJobManager : public CSimpleInterface, implements IJobManager, implements } else if (strncmp(command,"quit", 4) == 0) { - LOG(MCwarning, thorJob, "ABORT detected from user during debug session"); + LOG(MCwarning, "ABORT detected from user during debug session"); Owned e = MakeThorException(TE_WorkUnitAborting, "User signalled abort during debug session"); job->fireException(e); response.appendf(""); @@ -384,7 +384,7 @@ void CJobManager::stop() { if (!stopped) { - LOG(MCdebugProgress, thorJob, "Stopping jobManager"); + LOG(MCdebugProgress, "Stopping jobManager"); stopped = true; if (jobq) { @@ -578,7 +578,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char void CJobManager::run() { - LOG(MCdebugProgress, thorJob, "Listening for graph"); + LOG(MCdebugProgress, "Listening for graph"); setWuid(NULL); #ifndef _CONTAINERIZED @@ -671,7 +671,7 @@ void CJobManager::run() if (exclusiveLockName.length()) { if (globals->getPropBool("@multiThorPriorityLock")) - FLLOG(MCoperatorWarning, thorJob, "multiThorPriorityLock cannot be used in conjunction with multiThorExclusionLockName"); + FLLOG(MCoperatorWarning, "multiThorPriorityLock cannot be used in conjunction with multiThorExclusionLockName"); else { PROGLOG("Multi-Thor exclusive lock defined: %s", exclusiveLockName.str()); @@ -836,7 +836,7 @@ void CJobManager::run() } catch (IException *e) { - FLLOG(MCoperatorWarning, thorJob, e, "CJobManager::run"); + FLLOG(MCoperatorWarning, e, "CJobManager::run"); continue; } Owned factory; @@ -887,11 +887,9 @@ bool CJobManager::doit(IConstWorkUnit *workunit, const char *graphName, const So StringAttr wuid(workunit->queryWuid()); StringAttr user(workunit->queryUser()); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + JobNameScope activeJobName(wuid); - LOG(MCdebugInfo, thorJob, "Processing wuid=%s, graph=%s from agent: %s", wuid.str(), graphName, agentep.getEndpointHostText(s).str()); + LOG(MCdebugInfo, "Processing wuid=%s, graph=%s from agent: %s", wuid.str(), graphName, agentep.getEndpointHostText(s).str()); LOG(MCauditInfo,",Progress,Thor,Start,%s,%s,%s,%s,%s,%s", queryServerStatus().queryProperties()->queryProp("@thorname"), wuid.str(), @@ -914,10 +912,6 @@ bool CJobManager::doit(IConstWorkUnit *workunit, const char *graphName, const So queryServerStatus().queryProperties()->queryProp("@nodeGroup"), queryServerStatus().queryProperties()->queryProp("@queue")); - thorJob.setJobID(UnknownJob); - setDefaultJobId(UnknownJob); - queryLogMsgManager()->removeJobId(thorJobId); - if (e.get()) throw e.getClear(); return allDone; } @@ -942,12 +936,12 @@ void CJobManager::setWuid(const char *wuid, const char *cluster) } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e, "WARNING: Failed to set wuid in SDS:"); + FLLOG(MCexception(e), e, "WARNING: Failed to set wuid in SDS:"); e->Release(); } catch (CATCHALL) { - FLLOG(MCerror, thorJob, "WARNING: Failed to set wuid in SDS: Unknown error"); + FLLOG(MCerror, "WARNING: Failed to set wuid in SDS: Unknown error"); } } @@ -1089,7 +1083,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e, "Failed to write query dll - ignoring!"); + FLLOG(MCexception(e), e, "Failed to write query dll - ignoring!"); e->Release(); } sendSo = getExpertOptBool("dllsToSlaves", true); @@ -1213,7 +1207,7 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob) } EXCLOG(e,"abortThor"); } - LOG(MCdebugProgress, thorJob, "abortThor called"); + LOG(MCdebugProgress, "abortThor called"); if (jM) jM->stop(); if (thorQueue) @@ -1225,7 +1219,7 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob) if (2 > aborting && abortCurrentJob) { aborting = 2; - LOG(MCdebugProgress, thorJob, "aborting any current active job"); + LOG(MCdebugProgress, "aborting any current active job"); if (jM) { if (!e) @@ -1237,7 +1231,7 @@ void abortThor(IException *e, unsigned errCode, bool abortCurrentJob) } if (errCode == TEC_Clean) { - LOG(MCdebugProgress, thorJob, "Removing sentinel upon normal shutdown"); + LOG(MCdebugProgress, "Removing sentinel upon normal shutdown"); Owned sentinelFile = createSentinelTarget(); removeSentinelFile(sentinelFile); } @@ -1425,6 +1419,8 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam while (true) { + JobNameScope activeJobName(currentWuid.str()); + PROGLOG("Executing: wuid=%s, graph=%s", currentWuid.str(), currentGraphName.str()); { @@ -1469,10 +1465,8 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam { if (!streq(currentWuid, wuid)) { - queryLogMsgManager()->removeJobId(thorJob.queryJobID()); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + activeJobName.set(wuid); + // perhaps slightly overkill, but avoid checking/locking wuid to add pod info. // if this instance has already done so. auto it = publishedPodWuids.find(wuid.str()); @@ -1521,7 +1515,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"ThorMaster"); + FLLOG(MCexception(e), e,"ThorMaster"); e->Release(); } if (multiThorMemoryThreshold) diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 748c6011a08..8cebd0ca7b7 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -556,7 +556,7 @@ bool checkClusterRelicateDAFS(IGroup &grp) ep.port = 0; StringBuffer ips; ep.getHostText(ips); - FLLOG(MCoperatorError, thorJob, "VALIDATE FAILED(%d) %s : %s",failedcodes.item(i),ips.str(),failedmessages.item(i)); + FLLOG(MCoperatorError, "VALIDATE FAILED(%d) %s : %s",failedcodes.item(i),ips.str(),failedmessages.item(i)); } PROGLOG("Cluster replicate nodes check completed in %dms",msTick()-start); return (failures.ordinality()==0); @@ -579,7 +579,7 @@ bool ControlHandler(ahType type) { if (firstCtrlC) { - LOG(MCdebugProgress, thorJob, "CTRL-C detected"); + LOG(MCdebugProgress, "CTRL-C detected"); firstCtrlC = false; { Owned registry = CRegistryServer::getRegistryServer(); @@ -590,7 +590,7 @@ bool ControlHandler(ahType type) } else { - LOG(MCdebugProgress, thorJob, "2nd CTRL-C detected - terminating process"); + LOG(MCdebugProgress, "2nd CTRL-C detected - terminating process"); if (auditStartLogged) { @@ -607,7 +607,7 @@ bool ControlHandler(ahType type) // ahTerminate else { - LOG(MCdebugProgress, thorJob, "SIGTERM detected, shutting down"); + LOG(MCdebugProgress, "SIGTERM detected, shutting down"); Owned registry = CRegistryServer::getRegistryServer(); if (registry) registry->stop(); @@ -653,7 +653,7 @@ int main( int argc, const char *argv[] ) StringBuffer daliServer; if (!globals->getProp("@daliServers", daliServer)) { - LOG(MCerror, thorJob, "No Dali server list specified in THOR.XML (daliServers=iport,iport...)\n"); + LOG(MCerror, "No Dali server list specified in THOR.XML (daliServers=iport,iport...)\n"); return 0; // no recycle } @@ -712,14 +712,14 @@ int main( int argc, const char *argv[] ) queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler()); #endif - LOG(MCdebugProgress, thorJob, "Opened log file %s", logUrl.str()); + LOG(MCdebugProgress, "Opened log file %s", logUrl.str()); } #else setupContainerizedLogMsgHandler(); logHandler = queryStderrLogMsgHandler(); logUrl.set("stderr"); #endif - LOG(MCdebugProgress, thorJob, "Build %s", hpccBuildInfo.buildTag); + LOG(MCdebugProgress, "Build %s", hpccBuildInfo.buildTag); Owned serverGroup = createIGroupRetry(daliServer.str(), DALI_SERVER_PORT); @@ -728,7 +728,7 @@ int main( int argc, const char *argv[] ) { try { - LOG(MCdebugProgress, thorJob, "calling initClientProcess %d", thorEp.port); + LOG(MCdebugProgress, "calling initClientProcess %d", thorEp.port); initClientProcess(serverGroup, DCR_ThorMaster, thorEp.port, nullptr, nullptr, MP_WAIT_FOREVER, true); if (0 == thorEp.port) thorEp.port = queryMyNode()->endpoint().port; @@ -742,11 +742,11 @@ int main( int argc, const char *argv[] ) { if ((e->errorCode()!=JSOCKERR_port_in_use)) throw; - FLLOG(MCexception(e), thorJob, e,"InitClientProcess"); + FLLOG(MCexception(e), e,"InitClientProcess"); if (retry++>10) throw; e->Release(); - LOG(MCdebugProgress, thorJob, "Retrying"); + LOG(MCdebugProgress, "Retrying"); Sleep(retry*2000); } } @@ -950,7 +950,7 @@ int main( int argc, const char *argv[] ) } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"ThorMaster"); + FLLOG(MCexception(e), e,"ThorMaster"); e->Release(); return -1; } @@ -995,12 +995,11 @@ int main( int argc, const char *argv[] ) if (isContainerized()) { saveWuidToFile(workunit); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(workunit); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + JobNameScope activeJobName(workunit); + StringBuffer thorEpStr; - LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); - LOG(MCdebugProgress, thorJob, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); + LOG(MCdebugProgress, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); + LOG(MCdebugProgress, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); unsigned numWorkersPerPod = 1; if (!globals->hasProp("@numWorkers")) @@ -1045,8 +1044,8 @@ int main( int argc, const char *argv[] ) else { StringBuffer thorEpStr; - LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); - LOG(MCdebugProgress, thorJob, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); + LOG(MCdebugProgress, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); + LOG(MCdebugProgress, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); unsigned localThorPortInc = globals->getPropInt("@localThorPortInc", DEFAULT_SLAVEPORTINC); unsigned slaveBasePort = globals->getPropInt("@slaveport", DEFAULT_THORSLAVEPORT); Owned rawGroup = getClusterNodeGroup(thorname, "ThorCluster"); @@ -1069,7 +1068,7 @@ int main( int argc, const char *argv[] ) // bare-metal - check health of dafilesrv's on the Thor cluster. if (globals->getPropBool("@replicateOutputs")&&globals->getPropBool("@validateDAFS",true)&&!checkClusterRelicateDAFS(queryNodeGroup())) { - FLLOG(MCoperatorError, thorJob, "ERROR: Validate failure(s) detected, exiting Thor"); + FLLOG(MCoperatorError, "ERROR: Validate failure(s) detected, exiting Thor"); return globals->getPropBool("@validateDAFSretCode"); // default is no recycle! } } @@ -1141,11 +1140,11 @@ int main( int argc, const char *argv[] ) // NB: workunit/graphName only set in one-shot mode (if isCloud()) thorMain(logHandler, workunit, graphName); LOG(MCauditInfo, ",Progress,Thor,Terminate,%s,%s,%s",thorname,nodeGroup.str(),queueName.str()); - LOG(MCdebugProgress, thorJob, "ThorMaster terminated OK"); + LOG(MCdebugProgress, "ThorMaster terminated OK"); } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"ThorMaster"); + FLLOG(MCexception(e), e,"ThorMaster"); exception.setown(e); } if (isContainerized()) diff --git a/thorlcr/mfilemanager/thmfilemanager.cpp b/thorlcr/mfilemanager/thmfilemanager.cpp index b26528dce45..f16549d4769 100644 --- a/thorlcr/mfilemanager/thmfilemanager.cpp +++ b/thorlcr/mfilemanager/thmfilemanager.cpp @@ -388,7 +388,7 @@ class CFileManager : public CSimpleInterface, implements IThorFileManager bool extend = 0 != (helperFlags&TDWextend); bool jobTemp = 0 != (helperFlags&TDXjobtemp); - LOG(MCdebugInfo, thorJob, "createLogicalFile ( %s )", logicalName); + LOG(MCdebugInfo, "createLogicalFile ( %s )", logicalName); Owned efile; CDfsLogicalFileName dlfn; diff --git a/thorlcr/msort/tsortl.cpp b/thorlcr/msort/tsortl.cpp index 00d33a9aeb2..75a70955103 100644 --- a/thorlcr/msort/tsortl.cpp +++ b/thorlcr/msort/tsortl.cpp @@ -119,11 +119,11 @@ class CSocketRowStream: public CSimpleInterface, implements IRowStream if (dsz.eos()) { inbuf.clear(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.nextRow recv (%d,%x)",id,(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.nextRow recv (%d,%x)",id,(unsigned)(memsize_t)socket.get()); #endif size32_t sz = socket->receive_block_size(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.nextRow(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.nextRow(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); #endif if (sz==0) { // eof so terminate (no need to confirm) @@ -135,7 +135,7 @@ class CSocketRowStream: public CSimpleInterface, implements IRowStream socket->receive_block(buf,sz); assertex(!dsz.eos()); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.nextRow got (%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.nextRow got (%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),sz); #endif } RtlDynamicRowBuilder rowBuilder(allocator); @@ -151,12 +151,12 @@ class CSocketRowStream: public CSimpleInterface, implements IRowStream stopped = true; try { #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.stop(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.stop(%x)",(unsigned)(memsize_t)socket.get()); #endif bool eof = true; socket->write(&eof,sizeof(eof)); // confirm stop #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowStream.stopped(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowStream.stopped(%x)",(unsigned)(memsize_t)socket.get()); #endif } catch (IException *e) { @@ -203,7 +203,7 @@ class CSocketRowWriter: public CSimpleInterface, implements ISocketRowWriter preallocated = bufsize+initSize; #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter(%d,%x) preallocated = %d",id,(unsigned)(memsize_t)socket.get(),preallocated); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter(%d,%x) preallocated = %d",id,(unsigned)(memsize_t)socket.get(),preallocated); #endif } @@ -224,11 +224,11 @@ class CSocketRowWriter: public CSimpleInterface, implements ISocketRowWriter flush(); try { #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter.stop(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter.stop(%x)",(unsigned)(memsize_t)socket.get()); #endif socket->send_block(NULL,0); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter.stopped(%x)",(unsigned)(memsize_t)socket.get()); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter.stopped(%x)",(unsigned)(memsize_t)socket.get()); #endif } catch (IJSOCK_Exception *e) { // already gone! @@ -257,11 +257,11 @@ class CSocketRowWriter: public CSimpleInterface, implements ISocketRowWriter { size32_t l = outbuf.length(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter.flush(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),l); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter.flush(%d,%x,%d)",id,(unsigned)(memsize_t)socket.get(),l); #endif if (l) { if (!socket->send_block(outbuf.bufferBase(),l)) { - LOG(MCthorDetailedDebugInfo, thorJob, "CSocketRowWriter remote stop"); + LOG(MCthorDetailedDebugInfo, "CSocketRowWriter remote stop"); stopped = true; } pos += l; @@ -289,7 +289,7 @@ IRowStream *ConnectMergeRead(unsigned id, IThorRowInterfaces *rowif,SocketEndpoi #ifdef _FULL_TRACE StringBuffer s; nodeaddr.getEndpointHostText(s); - LOG(MCthorDetailedDebugInfo, thorJob, "ConnectMergeRead(%d,%s,%x,%" RCPF "d,%" RCPF "u)",id,s.str(),(unsigned)(memsize_t)socket.get(),startrec,numrecs); + LOG(MCthorDetailedDebugInfo, "ConnectMergeRead(%d,%s,%x,%" RCPF "d,%" RCPF "u)",id,s.str(),(unsigned)(memsize_t)socket.get(),startrec,numrecs); #endif socket->write(&hdr,sizeof(hdr)); return new CSocketRowStream(id,rowif->queryRowAllocator(),rowif->queryRowDeserializer(),socket); @@ -337,7 +337,7 @@ ISocketRowWriter *ConnectMergeWrite(IThorRowInterfaces *rowif,ISocket *socket,si #ifdef _FULL_TRACE char name[100]; int port = socket->peer_name(name,sizeof(name)); - LOG(MCthorDetailedDebugInfo, thorJob, "ConnectMergeWrite(%d,%s:%d,%x,%" RCPF "d,%" RCPF "u)",hdr.id,name,port,(unsigned)(memsize_t)socket,startrec,numrecs); + LOG(MCthorDetailedDebugInfo, "ConnectMergeWrite(%d,%s:%d,%x,%" RCPF "d,%" RCPF "u)",hdr.id,name,port,(unsigned)(memsize_t)socket,startrec,numrecs); #endif return new CSocketRowWriter(hdr.id,rowif,socket,bufsize); } diff --git a/thorlcr/msort/tsortmp.cpp b/thorlcr/msort/tsortmp.cpp index 957ec55b238..5e6c52d53c3 100644 --- a/thorlcr/msort/tsortmp.cpp +++ b/thorlcr/msort/tsortmp.cpp @@ -290,7 +290,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag rank_t sender; comm->recv(mb,0,tag,&sender); // NB only recv from master if (mb.length()==0) { - LOG(MCthorDetailedDebugInfo, thorJob, "Stopping SortSlaveMP::marshall"); + LOG(MCthorDetailedDebugInfo, "Stopping SortSlaveMP::marshall"); return false; } byte fn; @@ -301,7 +301,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag mbout.append(okout); #ifdef FULLTRACE StringBuffer tmp1; - LOG(MCthorDetailedDebugInfo, thorJob, ">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getEndpointHostText(tmp1).str(),tag,mb.getReplyTag()); + LOG(MCthorDetailedDebugInfo, ">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getEndpointHostText(tmp1).str(),tag,mb.getReplyTag()); #endif bool replydone = false; Owned err; @@ -531,7 +531,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag if (!replydone) { #ifdef FULLTRACE StringBuffer tmp1; - LOG(MCthorDetailedDebugInfo, thorJob, "reply(mbout); } diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 266d3a4d740..aef0ab2625b 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -106,7 +106,7 @@ class CWriteIntercept : public CSimpleInterface { StringBuffer err; err.append("Cannot create ").append(idxFile->queryFilename()); - LOG(MCerror, thorJob, "%s", err.str()); + LOG(MCerror, "%s", err.str()); throw MakeActivityException(&activity, -1, "%s", err.str()); } idxFileStream.setown(createBufferedIOStream(idxFileIO,0x100000)); @@ -1155,10 +1155,10 @@ class CThorSorter : public CSimpleInterface, implements IThorSorter, implements startmergesem.signal(); ActPrintLog(activity, "StartMiniSort output started"); traceWait("finishedmergesem(2)",finishedmergesem); - LOG(MCthorDetailedDebugInfo, thorJob, "StartMiniSort output done"); + LOG(MCthorDetailedDebugInfo, "StartMiniSort output done"); merger.clear(); intercept.clear(); - LOG(MCthorDetailedDebugInfo, thorJob, "StartMiniSort exit"); + LOG(MCthorDetailedDebugInfo, "StartMiniSort exit"); } virtual void Close() { diff --git a/thorlcr/msort/tsorts1.cpp b/thorlcr/msort/tsorts1.cpp index d5353f973f5..c0ab5c1df1c 100644 --- a/thorlcr/msort/tsorts1.cpp +++ b/thorlcr/msort/tsorts1.cpp @@ -60,7 +60,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream endpoint = targetep; char url[100]; targetep.getEndpointHostText(url,sizeof(url)); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) %s, pos=%" RCPF "d len=%" RCPF "u",streamno,url,startrec,numrecs); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: Stream(%u) %s, pos=%" RCPF "d len=%" RCPF "u",streamno,url,startrec,numrecs); SocketEndpoint mergeep = targetep; mergeep.port+=SOCKETSERVERINC; @@ -86,7 +86,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream stream = ConnectMergeRead(streamno,rowif,mergeep,startrec,numrecs,socket); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) connected to %s",streamno,url); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: Stream(%u) connected to %s",streamno,url); } virtual ~CMergeReadStream() { @@ -108,7 +108,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream #ifdef _FULL_TRACE char url[100]; endpoint.getEndpointHostText(url,sizeof(url)); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: EOS for %s",url); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: EOS for %s",url); #endif eos(); } @@ -121,7 +121,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream #ifdef _FULL_TRACE char url[100]; endpoint.getEndpointHostText(url,sizeof(url)); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: stop for %s",url); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: stop for %s",url); #endif stream->stop(); eos(); @@ -163,12 +163,12 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify ~CSortMerge() { #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "~CSortMerge in"); + LOG(MCthorDetailedDebugInfo, "~CSortMerge in"); #endif if (started) closedown(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "~CSortMerge out"); + LOG(MCthorDetailedDebugInfo, "~CSortMerge out"); #endif } void init() @@ -178,7 +178,7 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify char name[64]; int port = socket->peer_name(name,sizeof(name)); url.append(name).append(':').append(port); - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge WRITE: start %s, pos=%" RCPF "d, len=%" RCPF "d",url.str(),poscount,numrecs); + LOG(MCthorDetailedDebugInfo, "SORT Merge WRITE: start %s, pos=%" RCPF "d, len=%" RCPF "d",url.str(),poscount,numrecs); rowcount_t pos=poscount; try { @@ -228,7 +228,7 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify char peer[16]; if (socket) { socket->peer_name(peer,sizeof(peer)-1); - LOG(MCthorDetailedDebugInfo, thorJob, "waitdone %s",peer); + LOG(MCthorDetailedDebugInfo, "waitdone %s",peer); } else peer[0] = 0; @@ -237,7 +237,7 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify if (exception) throw exception.getClear(); if (peer[0]) - LOG(MCthorDetailedDebugInfo, thorJob, "waitdone exit"); + LOG(MCthorDetailedDebugInfo, "waitdone exit"); } bool notifySelected(ISocket *sock,unsigned selected) { @@ -246,11 +246,11 @@ class CSortMerge: public CSimpleInterface, implements ISocketSelectNotify if (closing) { closing = false; #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "notifySelected calling closedown"); + LOG(MCthorDetailedDebugInfo, "notifySelected calling closedown"); #endif closedown(); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "notifySelected called closedown"); + LOG(MCthorDetailedDebugInfo, "notifySelected called closedown"); #endif done = true; donesem.signal(); @@ -515,7 +515,7 @@ protected: friend class CSortMerge; respos += vMAPL(j,i)-vMAPL(j,i-1); // note we are adding up all of the lower as we want start rowcount_t totalrows = resnum; - LOG(MCthorDetailedDebugInfo, thorJob, "Output start = %" RCPF "d, num = %" RCPF "u",respos,resnum); + LOG(MCthorDetailedDebugInfo, "Output start = %" RCPF "d, num = %" RCPF "u",respos,resnum); IArrayOf readers; IException *exc = NULL; @@ -530,7 +530,7 @@ protected: friend class CSortMerge; { if (i==partno) { - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) local, pos=%" RCPF "u len=%" RCPF "u",i,sstart,snum); + LOG(MCthorDetailedDebugInfo, "SORT Merge READ: Stream(%u) local, pos=%" RCPF "u len=%" RCPF "u",i,sstart,snum); readers.append(*slave.createMergeInputStream(sstart,snum)); } else @@ -576,7 +576,7 @@ void CSortMerge::closedown() { CriticalBlock block(crit); #ifdef _FULL_TRACE - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge: closing %s",url.str()); + LOG(MCthorDetailedDebugInfo, "SORT Merge: closing %s",url.str()); #endif if (!socket) return; @@ -611,7 +611,7 @@ void CSortMerge::closedown() throw; } started = false; - LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge: finished %s, %d rows merged",url.str(),ndone); + LOG(MCthorDetailedDebugInfo, "SORT Merge: finished %s, %d rows merged",url.str(),ndone); } IMergeTransferServer *createMergeTransferServer(ISortSlaveBase *parent) diff --git a/thorlcr/slave/backup.cpp b/thorlcr/slave/backup.cpp index 43c39a32c30..d466cc8c726 100644 --- a/thorlcr/slave/backup.cpp +++ b/thorlcr/slave/backup.cpp @@ -105,14 +105,14 @@ class CThorBackupHandler : public CSimpleInterface, implements IBackup, implemen { StringBuffer errMsg; if (!currentAbort) - LOG(MCwarning, thorJob, "%s", errMsg.append("Backup inconsistency detected, backup aborted: ").append(item->dst).str()); + LOG(MCwarning, "%s", errMsg.append("Backup inconsistency detected, backup aborted: ").append(item->dst).str()); backupIFile->remove(); } } catch (IException *e) { StringBuffer errMsg("copying: "); - LOG(MCwarning, thorJob, e, errMsg.append(item->src)); + LOG(MCwarning, e, errMsg.append(item->src)); try { backupIFile->remove(); } catch (IException *e) { EXCLOG(e); e->Release(); } if (!ignoreError) throw; diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 049f61b34a7..719899dc6a9 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -71,7 +71,7 @@ void enableThorSlaveAsDaliClient() { try { - LOG(MCdebugProgress, thorJob, "calling initClientProcess"); + LOG(MCdebugProgress, "calling initClientProcess"); initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp)); break; } @@ -79,11 +79,11 @@ void enableThorSlaveAsDaliClient() { if ((e->errorCode()!=JSOCKERR_port_in_use)) throw; - FLLOG(MCexception(e), thorJob, e,"InitClientProcess"); + FLLOG(MCexception(e), e,"InitClientProcess"); if (retry++>10) throw; e->Release(); - LOG(MCdebugProgress, thorJob, "Retrying"); + LOG(MCdebugProgress, "Retrying"); Sleep(retry*2000); } } @@ -450,7 +450,7 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, public: CKMContainer(CKJService &_service, CKeyLookupContext *_ctx) - : service(_service), ctx(_ctx), contextLogger(jhtreeCacheStatistics, thorJob) + : service(_service), ctx(_ctx), contextLogger(jhtreeCacheStatistics) { keyManager.setown(ctx->createKeyManager(&contextLogger)); StringBuffer tracing; @@ -1562,11 +1562,11 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, } virtual void reset() override { - LOG(MCthorDetailedDebugInfo, thorJob, "KJService reset()"); + LOG(MCthorDetailedDebugInfo, "KJService reset()"); processorPool->stopAll(true); processorPool->joinAll(false); clearAll(); - LOG(MCthorDetailedDebugInfo, thorJob, "KJService reset() done"); + LOG(MCthorDetailedDebugInfo, "KJService reset() done"); } virtual void start() override { @@ -1577,7 +1577,7 @@ class CKJService : public CSimpleInterfaceOf, implements IThreaded, { if (aborted) return; - LOG(MCthorDetailedDebugInfo, thorJob, "KJService stop()"); + LOG(MCthorDetailedDebugInfo, "KJService stop()"); queryNodeComm().cancel(RANK_ALL, keyLookupMpTag); processorPool->stopAll(true); processorPool->joinAll(true); @@ -1774,6 +1774,7 @@ class CJobListener : public CSimpleInterface bool doReply; OwnedPtr perf; + JobNameScope activeJobName; while (!stopped && queryNodeComm().recv(msg, 0, masterSlaveMpTag)) { doReply = true; @@ -1892,10 +1893,8 @@ class CJobListener : public CSimpleInterface ILogMsgFilter *existingLogHandler = queryLogMsgManager()->queryMonitorFilter(logHandler); dbgassertex(existingLogHandler); verifyex(queryLogMsgManager()->changeMonitorFilterOwn(logHandler, getCategoryLogMsgFilter(existingLogHandler->queryAudienceMask(), existingLogHandler->queryClassMask(), maxLogDetail))); - queryLogMsgManager()->removeJobId(thorJob.queryJobID()); - LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); - setDefaultJobId(thorJobId); + + activeJobName.set(wuid); PROGLOG("Started wuid=%s, user=%s, graph=%s [log detail level=%u]\n", wuid.get(), user.str(), graphName.get(), maxLogDetail); PROGLOG("Using query: %s", soPath.str()); diff --git a/thorlcr/slave/slwatchdog.cpp b/thorlcr/slave/slwatchdog.cpp index c3dd6078efc..30536b3ea68 100644 --- a/thorlcr/slave/slwatchdog.cpp +++ b/thorlcr/slave/slwatchdog.cpp @@ -106,7 +106,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement CriticalBlock b(crit); activeGraphs.append(*LINK(&graph)); StringBuffer str("Watchdog: Start Job "); - LOG(MCthorDetailedDebugInfo, thorJob, "%s", str.append(graph.queryGraphId()).str()); + LOG(MCthorDetailedDebugInfo, "%s", str.append(graph.queryGraphId()).str()); } virtual void stopGraph(CGraphBase &graph, MemoryBuffer *mb) override { @@ -119,7 +119,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement else { StringBuffer str("Watchdog: Stop Job "); - LOG(MCthorDetailedDebugInfo, thorJob, "%s", str.append(graph.queryGraphId()).str()); + LOG(MCthorDetailedDebugInfo, "%s", str.append(graph.queryGraphId()).str()); if (mb) { DelayedSizeMarker sizeMark(*mb); @@ -138,7 +138,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement #endif stopped = true; threaded.join(); - LOG(MCdebugProgress, thorJob, "Stopped watchdog"); + LOG(MCdebugProgress, "Stopped watchdog"); } } virtual void debugRequest(MemoryBuffer &msg, const char *request) const override @@ -173,7 +173,7 @@ class CGraphProgressHandlerBase : public CInterfaceOf, implement // IThreaded virtual void threadmain() override { - LOG(MCthorDetailedDebugInfo, thorJob, "Watchdog: thread running"); + LOG(MCthorDetailedDebugInfo, "Watchdog: thread running"); gatherAndSend(); // send initial data assertex(HEARTBEAT_INTERVAL>=8); unsigned count = HEARTBEAT_INTERVAL+getRandom()%8-4; diff --git a/thorlcr/slave/thslavemain.cpp b/thorlcr/slave/thslavemain.cpp index 896a6b042a7..e93c69d96d8 100644 --- a/thorlcr/slave/thslavemain.cpp +++ b/thorlcr/slave/thslavemain.cpp @@ -119,7 +119,7 @@ static bool RegisterSelf(SocketEndpoint &masterEp) { StringBuffer slfStr; StringBuffer masterStr; - LOG(MCdebugProgress, thorJob, "registering %s - master %s",slfEp.getEndpointHostText(slfStr).str(),masterEp.getEndpointHostText(masterStr).str()); + LOG(MCdebugProgress, "registering %s - master %s",slfEp.getEndpointHostText(slfStr).str(),masterEp.getEndpointHostText(masterStr).str()); try { SocketEndpoint ep = masterEp; @@ -234,11 +234,11 @@ static bool RegisterSelf(SocketEndpoint &masterEp) OERRLOG("Failed to connect to all nodes"); else PROGLOG("verified mp connection to rest of cluster"); - LOG(MCdebugProgress, thorJob, "registered %s",slfStr.str()); + LOG(MCdebugProgress, "registered %s",slfStr.str()); } catch (IException *e) { - FLLOG(MCexception(e), thorJob, e,"slave registration error"); + FLLOG(MCexception(e), e,"slave registration error"); e->Release(); return false; } @@ -258,7 +258,7 @@ bool UnregisterSelf(IException *e) StringBuffer slfStr; slfEp.getEndpointHostText(slfStr); - LOG(MCdebugProgress, thorJob, "Unregistering slave : %s", slfStr.str()); + LOG(MCdebugProgress, "Unregistering slave : %s", slfStr.str()); try { CMessageBuffer msg; @@ -266,16 +266,16 @@ bool UnregisterSelf(IException *e) serializeException(e, msg); // NB: allows exception to be NULL if (!queryWorldCommunicator().send(msg, masterNode, MPTAG_THORREGISTRATION, 60*1000)) { - LOG(MCerror, thorJob, "Failed to unregister slave : %s", slfStr.str()); + LOG(MCerror, "Failed to unregister slave : %s", slfStr.str()); return false; } - LOG(MCdebugProgress, thorJob, "Unregistered slave : %s", slfStr.str()); + LOG(MCdebugProgress, "Unregistered slave : %s", slfStr.str()); isRegistered = false; return true; } catch (IException *e) { if (!jobListenerStopped) - FLLOG(MCexception(e), thorJob, e,"slave unregistration error"); + FLLOG(MCexception(e), e,"slave unregistration error"); e->Release(); } return false; @@ -284,9 +284,9 @@ bool UnregisterSelf(IException *e) bool ControlHandler(ahType type) { if (ahInterrupt == type) - LOG(MCdebugProgress, thorJob, "CTRL-C detected"); + LOG(MCdebugProgress, "CTRL-C detected"); else if (!jobListenerStopped) - LOG(MCdebugProgress, thorJob, "SIGTERM detected"); + LOG(MCdebugProgress, "SIGTERM detected"); bool unregOK = false; if (!jobListenerStopped) { @@ -330,7 +330,7 @@ ILogMsgHandler *startSlaveLog() queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler()); #endif - LOG(MCdebugProgress, thorJob, "Opened log file %s", lf->queryLogFileSpec()); + LOG(MCdebugProgress, "Opened log file %s", lf->queryLogFileSpec()); } else { @@ -340,13 +340,12 @@ ILogMsgHandler *startSlaveLog() if (getComponentConfigSP()->getProp("@workunit", wuid)) { LogMsgJobId thorJobId = queryLogMsgManager()->addJobId(wuid); - thorJob.setJobID(thorJobId); setDefaultJobId(thorJobId); } } //setupContainerizedStorageLocations(); - LOG(MCdebugProgress, thorJob, "Build %s", hpccBuildInfo.buildTag); + LOG(MCdebugProgress, "Build %s", hpccBuildInfo.buildTag); return logHandler; } @@ -427,7 +426,7 @@ int main( int argc, const char *argv[] ) #ifdef USE_MP_LOG startLogMsgParentReceiver(); - LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp)); + LOG(MCdebugProgress, "MPServer started on port %d", getFixedPort(TPORT_mp)); #endif SocketEndpoint masterEp(master); @@ -456,7 +455,7 @@ int main( int argc, const char *argv[] ) if (err) { IException *e = makeErrnoExceptionV(-1, "Failed to change dir to '%s'", thorPath.str()); - FLLOG(MCexception(e), thorJob, e); + FLLOG(MCexception(e), e); throw e; } @@ -507,16 +506,16 @@ int main( int argc, const char *argv[] ) useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead")); - LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR); + LOG(MCdebugProgress, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR); #ifdef _WIN32 ULARGE_INTEGER userfree; ULARGE_INTEGER total; ULARGE_INTEGER free; if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) { unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart); - LOG(MCdebugInfo, thorJob, "Total disk space = %" I64F "d k", total.QuadPart/1000); - LOG(MCdebugInfo, thorJob, "Free disk space = %" I64F "d k", free.QuadPart/1000); - LOG(MCdebugInfo, thorJob, "%d%% disk free\n",pc); + LOG(MCdebugInfo, "Total disk space = %" I64F "d k", total.QuadPart/1000); + LOG(MCdebugInfo, "Free disk space = %" I64F "d k", free.QuadPart/1000); + LOG(MCdebugInfo, "%d%% disk free\n",pc); } #endif @@ -581,12 +580,12 @@ int main( int argc, const char *argv[] ) slaveMain(jobListenerStopped, slaveLogHandler); } - LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK"); + LOG(MCdebugProgress, "ThorSlave terminated OK"); } catch (IException *e) { if (!jobListenerStopped) - FLLOG(MCexception(e), thorJob, e,"ThorSlave"); + FLLOG(MCexception(e), e,"ThorSlave"); unregisterException.setown(e); } #ifndef _CONTAINERIZED diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp index 2a3450ad6a5..d64d06da1a2 100644 --- a/thorlcr/thorutil/thbuf.cpp +++ b/thorlcr/thorutil/thbuf.cpp @@ -801,7 +801,7 @@ class CSharedWriteAheadBase : public CSimpleInterface, implements ISharedSmartBu { unsigned whichChunk = queryCOutput(output).currentChunkNum - lowestChunk; #ifdef TRACE_WRITEAHEAD - LOG(MCthorDetailedDebugInfo, thorJob, "output=%d, chunk=%d (whichChunk=%d)", output, currentChunkNum, whichChunk); + LOG(MCthorDetailedDebugInfo, "output=%d, chunk=%d (whichChunk=%d)", output, currentChunkNum, whichChunk); #endif rowSet.setown(readRows(output, whichChunk)); assertex(rowSet); @@ -1523,7 +1523,7 @@ class CSharedWriteAheadDisk : public CSharedWriteAheadBase Owned chunk = savedChunks.dequeue(); if (!chunk) break; } - LOG(MCthorDetailedDebugInfo, thorJob, "CSharedWriteAheadDisk: highOffset=%" I64F "d", highOffset); + LOG(MCthorDetailedDebugInfo, "CSharedWriteAheadDisk: highOffset=%" I64F "d", highOffset); } virtual void reset() { diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index ff98bd8c015..0c38400b5b4 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -144,7 +144,7 @@ void ActPrintLogArgs(const CGraphElementBase *container, const ActLogEnum flags, return; // suppress logging child activities unless thorlog_all flag StringBuffer res; ActPrintLogArgsPrep(res, container, flags, format, args); - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void ActPrintLogArgs(const CGraphElementBase *container, IException *e, const ActLogEnum flags, const LogMsgCategory &logCat, const char *format, va_list args) @@ -156,7 +156,7 @@ void ActPrintLogArgs(const CGraphElementBase *container, IException *e, const Ac res.append(" : "); e->errorMessage(res); } - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void ActPrintLogEx(const CGraphElementBase *container, const ActLogEnum flags, const LogMsgCategory &logCat, const char *format, ...) @@ -168,7 +168,7 @@ void ActPrintLogEx(const CGraphElementBase *container, const ActLogEnum flags, c va_start(args, format); ActPrintLogArgsPrep(res, container, flags, format, args); va_end(args); - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void ActPrintLog(const CActivityBase *activity, const char *format, ...) @@ -213,7 +213,7 @@ void GraphPrintLogArgs(CGraphBase *graph, const ActLogEnum flags, const LogMsgCa return; // suppress logging from child graph unless thorlog_all flag StringBuffer res; GraphPrintLogArgsPrep(res, graph, flags, logCat, format, args); - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void GraphPrintLogArgs(CGraphBase *graph, IException *e, const ActLogEnum flags, const LogMsgCategory &logCat, const char *format, va_list args) @@ -227,7 +227,7 @@ void GraphPrintLogArgs(CGraphBase *graph, IException *e, const ActLogEnum flags, res.append(" : "); e->errorMessage(res); } - LOG(logCat, thorJob, "%s", res.str()); + LOG(logCat, "%s", res.str()); } void GraphPrintLog(CGraphBase *graph, IException *e, const char *format, ...) @@ -659,12 +659,12 @@ class CTempNameHandler if (file.isFile()==fileBool::foundYes) { if (log) - LOG(MCdebugInfo, thorJob, "Deleting %s", file.queryFilename()); + LOG(MCdebugInfo, "Deleting %s", file.queryFilename()); try { file.remove(); } catch (IException *e) { if (log) - FLLOG(MCwarning, thorJob, e); + FLLOG(MCwarning, e); e->Release(); } } @@ -717,7 +717,7 @@ class CTempNameHandler catch (IException *e) { if (log) - FLLOG(MCwarning, thorJob, e); + FLLOG(MCwarning, e); e->Release(); } subDirPath.clear(); @@ -756,7 +756,7 @@ void GetTempFilePath(StringBuffer &name, const char *suffix) void SetTempDir(const char *rootTempDir, const char *uniqueSubDir, const char *tempPrefix, bool clearDir) { TempNameHandler.setTempDir(rootTempDir, uniqueSubDir, tempPrefix, clearDir); - LOG(MCdebugProgress, thorJob, "temporary rootTempdir: %s, uniqueSubDir: %s, prefix: %s", rootTempDir, uniqueSubDir, tempPrefix); + LOG(MCdebugProgress, "temporary rootTempdir: %s, uniqueSubDir: %s, prefix: %s", rootTempDir, uniqueSubDir, tempPrefix); } void ClearTempDir() @@ -764,7 +764,7 @@ void ClearTempDir() try { TempNameHandler.clear(true); - LOG(MCthorDetailedDebugInfo, thorJob, "temp directory cleared"); + LOG(MCthorDetailedDebugInfo, "temp directory cleared"); } catch (IException *e) { @@ -812,8 +812,6 @@ void loadCmdProp(IPropertyTree *tree, const char *cmdProp) } } -LogMsgJobInfo thorJob(UnknownJob, UnknownUser); // configured at job start (in manager and workers) - void ensureDirectoryForFile(const char *fName) { if (!recursiveCreateDirectoryForFile(fName)) @@ -823,7 +821,7 @@ void ensureDirectoryForFile(const char *fName) // Not recommended to be used from slaves as tend to be one or more trying at same time. void reportExceptionToWorkunit(IConstWorkUnit &workunit,IException *e, ErrorSeverity severity) { - LOG(MCwarning, thorJob, e, "Reporting exception to WU"); + LOG(MCwarning, e, "Reporting exception to WU"); Owned wu = &workunit.lock(); if (wu) { @@ -1160,12 +1158,12 @@ void CFifoFileCache::deleteFile(IFile &ifile) try { if (!ifile.remove()) - FLLOG(MCoperatorWarning, thorJob, "CFifoFileCache: Failed to remove file (missing) : %s", ifile.queryFilename()); + FLLOG(MCoperatorWarning, "CFifoFileCache: Failed to remove file (missing) : %s", ifile.queryFilename()); } catch (IException *e) { StringBuffer s("Failed to remove file: "); - FLLOG(MCoperatorWarning, thorJob, e, s.append(ifile.queryFilename())); + FLLOG(MCoperatorWarning, e, s.append(ifile.queryFilename())); } } diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index f378ab1e604..6ebfd8f0850 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -122,8 +122,6 @@ #define THOR_SEM_RETRY_TIMEOUT 2 // Logging -extern graph_decl LogMsgJobInfo thorJob; - enum ThorExceptionAction { tea_null, tea_warning, tea_abort, tea_shutdown }; enum RegistryCode:unsigned { rc_register, rc_deregister }; @@ -284,7 +282,7 @@ class graph_decl CTimeoutTrigger : public CInterface, implements IThreaded void stop() { running = false; todo.signal(); } void inform(IException *e) { - LOG(MCdebugProgress, thorJob, "INFORM [%s]", description.get()); + LOG(MCdebugProgress, "INFORM [%s]", description.get()); CriticalBlock block(crit); if (exception.get()) e->Release(); @@ -299,7 +297,7 @@ class graph_decl CTimeoutTrigger : public CInterface, implements IThreaded CriticalBlock block(crit); IException *e = exception.getClear(); if (e) - LOG(MCdebugProgress, thorJob, "CLEARING TIMEOUT [%s]", description.get()); + LOG(MCdebugProgress, "CLEARING TIMEOUT [%s]", description.get()); todo.signal(); return e; } diff --git a/tools/hidl/hidlcomp.cpp b/tools/hidl/hidlcomp.cpp index f9dba61e2d7..b2328b909cb 100644 --- a/tools/hidl/hidlcomp.cpp +++ b/tools/hidl/hidlcomp.cpp @@ -633,40 +633,6 @@ esp_xlate_info *esp_xlat(const char *from, bool defaultToString) return (defaultToString) ? &esp_xlate_table[0] : NULL; } -//TODO: this is not bullet proof. better idea?? -// Return: NULL if no need to be defined, otherwise the type to be defined -// Note: the caller needs to free memory -static char* getToBeDefinedType(const char* type) -{ - const char* colon = strchr(type, ':'); - const char* bareType = colon ? colon+1 : type; - - if (strnicmp(type, "xsd",colon-type)==0) - return NULL; - - /* - for (unsigned i=0; esp_xlate_table[i].meta_type!=NULL; i++) - { - if (stricmp(bareType,esp_xlate_table[i].xsd_type)==0) - return NULL; - } - */ - - if (strnicmp(type, "tns", colon-type)!=0) - { - char msg[128]; - sprintf(msg, "*** unhandled type: %s", type); - outs(msg); - yyerror(msg); - return NULL; - } - - if (strncmp(bareType, "ArrayOf", sizeof("ArrayOf")-1)==0) - return strdup(bareType+sizeof("ArrayOf")-1); - else - return strdup(bareType); -} - static const char *MetaTypeToXsdType(const char *val) { esp_xlate_info *xlation=esp_xlat(val); @@ -2484,11 +2450,6 @@ void EspMessageInfo::write_esp_ipp() outf("\tCX%s(const char* defvalue_) : SoapEnumParamNew()\n", name_, name_); outf("\t{ doInit(); setDefaultValue(defvalue_); }\n"); - - // getXsdDefinition - outs("\tstatic void getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &schema, BoolHash &added)\n"); - outs("\t{ getSharedInstance().getXsdDefinitionInternal(context,request,schema,added); }\n"); - // getMapInfo() outs("\tstatic void getMapInfo(IMapInfo& info, BoolHash& added) { getSharedInstance().getMapInfo_(info,added); }\n\n"); outf("\tstatic const char* stringOf(C%s val) { return getSharedInstance().toString(val); }\n\n",name_); @@ -2503,47 +2464,9 @@ void EspMessageInfo::write_esp_ipp() // TODO: getMapInfo_() internal implementation outs("\tvoid getMapInfo_(IMapInfo& info, BoolHash& added) { }\n"); - // handle descriptions - { - outs("\tvoid getXsdDefinitionInternal(IEspContext &context, CHttpRequest* request, StringBuffer &schema, BoolHash &added)\n"); - bool hasDesc = false; - for (pi = getParams(); pi!=NULL; pi=pi->next) - { - const char* desc = pi->getMetaString("desc",NULL); - if (desc) { hasDesc = true; break; } - } - if (hasDesc) - { - outs("\t{\n\t\tconst char* descriptions [] = {"); - for (pi = getParams(); pi!=NULL; pi=pi->next) - { - const char* desc = pi->getMetaString("desc",NULL); - outf("%s,", desc ? desc : "NULL"); - } - outs("};\n"); - outs("\t\tgetXsdDefinition_(context,request,schema,added,descriptions);\n\t}\n"); - } - else - outf("\t{ getXsdDefinition_(context,request,schema,added,NULL); }\n"); - } - outf("\tvoid doInit();\n"); outs("};\n\n"); - - // array -#if 0 - //outf("MAKEValueArray(C%s, %sArray);\n\n", name_, name_); - outf("inline C%s Array__Member2Param(C%s &src) { return src; }\n", name_, name_); - outf("inline void Array__Assign(C%s & dest, C%s const & src){ dest = src; }\n", name_, name_); - outf("inline bool Array__Equal(C%s const & m, C%s const p) { return m==p; }\n", name_, name_); - outf("inline void Array__Destroy(C%s & /*next* /) { }\n", name_); - //outf("MAKECopyArrayOf(simple, simple, array) - outf("MAKEArrayOf(C%s, C%s, %sArray)\n\n", name_,name_,name_); - //outf("class %sArray : public ArrayOf {};\n\n",name_,name_,name_); - outs("\n"); -#endif - return; } @@ -2639,22 +2562,10 @@ void EspMessageInfo::write_esp_ipp() outf("\t\treturn \"%s\";\n", name_); outs("\t}\n\n"); - //method ==> getXsdDefinition - outs("\tstatic StringBuffer &getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &schema, BoolHash&added, const char *xns=\"xsd\", const char *wsns=\"tns\", unsigned flags=1)\n"); - outs("\t{\n"); - outs("\t\treturn getXsdDefinition(context, request, queryXsdElementName(), schema, added, xns, wsns, flags);\n"); - outs("\t}\n"); - - //method ==> getXsdDefinition - outs("\tstatic StringBuffer &getXsdDefinition(IEspContext &context, CHttpRequest* request, const char *msgTypeName, StringBuffer &schema, BoolHash&added, const char *xns=\"xsd\", const char *wsns=\"tns\", unsigned flags=1);\n"); - //method ==> getMapInfo outs("\tstatic void getMapInfo(IMapInfo& info);\n"); outs("\tstatic void getMapInfo(IMapInfo& info, BoolHash& added);\n"); - //method ==> getHtmlForm - outs("\tstatic StringBuffer &getHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &form, bool includeFormTag=true, const char *prefix=NULL);\n"); - //method ==> hasCustomHttpContent outs("\tstatic bool hasCustomHttpContent()\n"); outs("\t{\n"); @@ -2664,9 +2575,6 @@ void EspMessageInfo::write_esp_ipp() outs("\t\treturn false;\n"); outs("\t}\n"); - //method ==> serializeHtml - outs("\tStringBuffer &serializeHtml(IEspContext &context, const char *serv, const char *method, StringBuffer &html);\n"); - //method ==> serialize (IRpcMessage&) outs("\n\tvoid serialize(IRpcMessage& rpc_resp);\n"); @@ -2721,8 +2629,6 @@ void EspMessageInfo::write_esp_ipp() outs("\n\tbool unserialize(IEspContext* ctx, IProperties& params, MapStrToBuf *attachments, const char *basepath=NULL);\n"); } - //outs("\n\tvoid unserialize(const char * msg);\n"); - if (espm_type_==espm_response) { outs("\n\tvirtual void setRedirectUrl(const char *url)\n"); @@ -2959,516 +2865,6 @@ void EspMessageInfo::write_esp() outs("\tunserialize(ctx,*params,attachments, NULL);\n}\n"); } - //======================================================================================= - //method ==> getXsdDefinition - // @context: the context could affect the schema, e.g., the version, the URL parameters - // @flags: 0x001 - define the struct/enum as complexType. - // 0x010 - do not include group-type enclosure (i.e., just define the fields, e.g., as part of other complex type definition) - // 0x100 - do not include fields at all (only to include definitions of fields, recursively) - bool isExtSimpleType = getMetaInt("element", 0)!=0; - outf("\nStringBuffer &C%s::getXsdDefinition(IEspContext &context, CHttpRequest* request, const char *msgTypeName, StringBuffer &schema, BoolHash &added, const char *xns, const char *wsns, unsigned flags)\n", name_); - outs("{\n"); - - indentReset(1); - indentOuts("if (!(flags & 0x100))\n\t{\n"); - - indentOuts(1,"IProperties *props = request->queryParameters();\n"); - indentOuts("if(msgTypeName) {\n"); - indentOuts(1,"if(added.getValue(msgTypeName))\n"); - indentOuts1(1,"return schema;\n"); - indentOuts("else\n"); - indentOuts1(1,"added.setValue(msgTypeName, 1);\n"); - indentOuts(-1,"}\n"); - - indentOuts("if (flags & 0x01) {\n"); - bool isEmptyComplexType = true; //a complex type with no children, no attribute and no parent - if (isExtSimpleType || getParentName() || hasNonAttributeChild() || (espm_type_==espm_response && getMetaInt("exceptions_inline", 0))) - isEmptyComplexType = false; - else - { - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (!pi->getMetaInt("attribute", 0) && !pi->getMetaInt("hidden", 0) && !pi->getMetaInt("hidden_soap", 0)) - isEmptyComplexType = false; - } - } - - if (isEmptyComplexType) - { - if (espm_type_==espm_struct) - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - else - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - } - else if (espm_type_==espm_struct) - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - else - indentOuts(1,"schema.appendf(\"\\n\", msgTypeName);\n"); - - if (isExtSimpleType) - indentOuts("schema.append(\"\\n\");\n"); - indentOuts(-1, "}\n"); - - // native arrays - typedef std::map EspNativeArrays; - EspNativeArrays nativeArrays; - // esp struct arrays - typedef std::set EspStructArrays; - EspStructArrays structArrays; - - //no element children for extended simple type - if (!isEmptyComplexType && !isExtSimpleType) - { - const char *xsdGroupType = getXsdGroupType(); - - bool hasChild = hasNonAttributeChild(); - bool exceptions = espm_type_==espm_response && getMetaInt("exceptions_inline", 0); - bool needGroupType = exceptions || hasChild || parent; - if (needGroupType) - { - indentOuts("if (!(flags & 0x10)) {\n"); - indentOutf1(1,"schema.append(\"\");\n", xsdGroupType); - if (exceptions) - indentOuts("schema.append(\"\");\n"); - indentOuts("}\n"); - } - - if (parent) - indentOutf("C%s::getXsdDefinition(context, request, NULL, schema, added, xns, wsns, 0x10);\n", parent); - - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (!pi->getMetaInt("attribute", 0) && !pi->getMetaInt("hidden", 0) && !pi->getMetaInt("hidden_soap", 0)) - { - enum {definedtype, inline_primitive_array, inline_array, complextype} complexity=definedtype; - StrBuffer buffer; - - const char *xsd_type = pi->getMetaXsdType(); - if (xsd_type) - { - buffer.append(xsd_type); - } - else if (pi->flags & PF_TEMPLATE) - { - if (!strcmp(pi->templ, "ESParray")) - { - if (pi->isPrimitiveArray()) - { - if (pi->getMetaString("item_tag", NULL)) - complexity=inline_primitive_array; - else { - const char* type = pi->getArrayImplType(); - nativeArrays[pi->getArrayItemXsdType()] = VStrBuffer("Esp%s", type).str(); - buffer.appendf("tns:Esp%s", type); - } - } - else - { - buffer.append("tns:"); - - if (pi->getMetaString("item_tag", NULL)) - complexity=inline_array; - else { - structArrays.insert(pi->typname); - buffer.append("ArrayOf"); - } - buffer.append(pi->typname); - } - } - else - buffer.append("xsd:string"); - } - else if (pi->kind==TK_ESPSTRUCT || pi->kind==TK_ESPENUM) - { - buffer.appendf("tns:%s",pi->typname); - } - else - { - buffer.append(pi->getXsdType()); - } - - - const char *xsdtype = buffer.str(); - const char *xsdns = ""; - - if (xsdtype) - { - int count= buffer.length(); - if (count>0 && buffer.charAt(count-1)=='\"') - buffer.setCharAt(count-1,0); - if (*xsdtype=='\"') - xsdtype++; - if (strchr(xsdtype, ':')==NULL) - xsdns="xsd:"; - } - - bool hasMapInfo = pi->hasMapInfo(); - if (hasMapInfo) - { - indentOutf("if (!context.suppressed(\"%s\",\"%s\")) {\n", this->name_, pi->name); - indentInc(1); - } - - const char *access=pi->getMetaString("access", NULL); - if (access) - { - indentOuts("SecAccessFlags acc;\n"); - indentOutf("if (context.authorizeFeature(%s, acc) && acc>=SecAccess_Read) {\n", access); - indentInc(1); - } - - StrBuffer xmlTag; - const char* tagName = pi->getMetaStringValue(xmlTag,"xml_tag") ? xmlTag.str() : pi->name; - - switch (complexity) - { - case inline_array: - { - indentOutf("schema.append(\"\\n\");\n", tagName); - indentOuts("schema.append(\"\\n\");\n"); - indentOutf("schema.append(\"\");\n", pi->getMetaString("item_tag", "Item"), xsdtype); - indentOuts("schema.append(\"\");\n"); - indentOutf("schema.append(\"\");\n"); - break; - } - case inline_primitive_array: - { - indentOutf("schema.append(\"\");\n", tagName); - indentOuts("schema.append(\"\");\n"); - indentOutf("schema.append(\"getMetaString("item_tag", "Item"), pi->getArrayItemXsdType()); - - int cols = pi->getMetaInt("cols",0); - int rows = pi->getMetaInt("rows",0); - if (cols>0 || rows>0) - { - outs("\");\n"); - indentOuts("if (context.queryOptions()&ESPCTX_ALL_ANNOTATION)\n"); - indentOuts(1,"schema.append(\"> 0) - outf(" formCols=\\\"%d\\\"", cols); - if (rows>0) - outf(" formRows=\\\"%d\\\"", rows); - outs("/>\\n\");\n"); - indentOuts(-1,"else\n"); - indentOuts("\tschema.append(\"/>\\n\");\n"); - } - else - outs("/>\\n\");\n"); - - indentOuts("schema.append(\"\\n\");\n"); - indentOutf("schema.append(\"\\n\");\n"); - break; - } - case definedtype: - { - indentOutf("schema.append(\"getMetaInt("required", 0)) ? 1 : 0; - if (minOccurs==0) - outs(" minOccurs=\\\"0\\\""); - - // default value - if (pi->hasMetaTag("default")) - { - if (pi->kind==TK_CHAR || pi->kind==TK_UNSIGNEDCHAR || strcmp(xsdtype,"string")==0) - { - const char* val = pi->getMetaString("default",NULL); - if (val && *val) - { - // remove quotes - if (*val=='"' || *val=='\'') - { - char* s = strdup(val); - *(s+strlen(s)-1) = 0; - outf(" default=\\\"%s\\\"", s+1); - free(s); - } - else - outf(" default=\\\"%s\\\"", val); - } - } - else if (pi->kind==TK_DOUBLE || pi->kind==TK_FLOAT || strcmp(xsdtype,"double")==0) - { - double val = pi->getMetaDouble("default"); - if (val==0) - val = pi->getMetaInt("default"); // auto conversion - if (val!=0) - outf(" default=\\\"%g\\\"", val); - } - else if (pi->kind==TK_BOOL || strcmp(xsdtype,"bool")==0) - { - outf(" default=\\\"%s\\\"", pi->getMetaInt("default") ? "true" : "false"); - } - else if (pi->kind==TK_ESPENUM) - { - const char* val = pi->getMetaString("default",NULL); - if (val && *val) - { - // remove quotes - if (*val=='"' || *val=='\'') - { - char* s = strdup(val); - *(s+strlen(s)-1) = 0; - outf(" default=\\\"%s\\\"", s+1); - free(s); - } - else - outf(" default=\\\"%s\\\"", val); - } - else - { - int value = pi->getMetaInt("default", -1); - if (value != -1) - outf(" default=\\\"%d\\\"", value); - } - } - else// assume it is integer - { - int val = pi->getMetaInt("default"); - if (val) - outf(" default=\\\"%d\\\"", val); - } - } - - // name & type - outf(" name=\\\"%s\\\" type=\\\"%s%s\\\"", tagName, xsdns, xsdtype); - - // check for annotations - StrBuffer annot; - - const char* formType = pi->getMetaInt("password") ? "password" : NULL; - if (formType) - annot.appendf(" formType=\\\"%s\\\"",formType); - - bool collapsed = pi->getMetaInt("collapsed")?true:false;; - if (collapsed) - annot.append(" collapsed=\\\"true\\\""); - - int cols = pi->getMetaInt("cols",0); - if (cols>0) - annot.appendf(" formCols=\\\"%d\\\"", cols); - - int rows = pi->getMetaInt("rows",0); - if (rows>0) - annot.appendf(" formRows=\\\"%d\\\"", rows); - - StrBuffer tmp; - pi->getMetaStringValue(tmp,"form_ui"); - if (tmp.length()) { - StrBuffer encoded; - encodeXML(tmp.str(),encoded); - annot.appendf(" ui=\\\"%s\\\"", printfEncode(encoded.str(), tmp.clear()).str()); - } - - pi->getMetaStringValue(tmp.clear(), "html_head"); - if (tmp.length()) { - StrBuffer encoded; - encodeXML(tmp.str(),encoded); - annot.appendf(" html_head=\\\"%s\\\"", printfEncode(encoded.str(), tmp.clear()).str()); - } - - if (annot.length()) - { - outs("\");\n"); - indentOuts("if (context.queryOptions()&ESPCTX_ALL_ANNOTATION)\n"); - indentOuts(1,"schema.append(\"> \\n\");\n"); - indentOuts(-1,"else\n"); - indentOuts("\tschema.append(\"/>\\n\");\n"); - } - else - outs("/>\\n\");\n"); - - break; - } - default: - break; - } - - if (access) - indentOuts(-1,"}\n"); - - //if (optional) - if (hasMapInfo) - indentOuts(-1,"}\n"); - } - } - - if (needGroupType) - { - indentOuts("if (!(flags & 0x10))\n"); - indentOutf1(1,"schema.append(\"\\n\");\n", xsdGroupType); - } - - } //!isEmptyComplexType && !isExtSimpleType - - if (!isEmptyComplexType) - { - //attributes last - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (pi->getMetaInt("attribute")!=0 && !pi->getMetaInt("hidden")) - { - StrBuffer tmp; - const char* tagName = pi->getMetaStringValue(tmp,"xml_tag") ? tmp.str() : pi->name; - indentOutf("schema.append(\"\");\n", tagName, pi->getXsdType()); - } - } - - indentOuts("if (flags & 0x01) {\n"); - if (isExtSimpleType) - indentOuts1(1,"schema.append(\"\\n\");\n"); - if (espm_type_==espm_struct) - indentOuts1(1,"schema.append(\"\\n\");\n"); - else - indentOuts1(1,"schema.append(\"\\n\");\n"); - indentOuts("}\n"); - } - - indentOuts(-1,"}\n"); // if (flags & 0x100) - //------------------------------------------------------------------------- - // try to figure out structs without ESPuses - // true if it is with map info, once a STRUCT or ENUM is there without map info, it is false - const int MIT_HasMap = 0x01; - const int MIT_IsEnum = 0x02; - const int MIT_IsStruct = 0x04; - enum MapInfoType { MIT_EnumFalse=MIT_IsEnum, MIT_EnumTrue=MIT_HasMap|MIT_IsEnum, - MIT_StructFalse=MIT_IsStruct,MIT_StructTrue=MIT_IsStruct|MIT_HasMap }; - typedef std::map TypeMap; - TypeMap typesNeeded; - - bool parenthesisOpened = false; - if (parent) - { - outs("\tif (!(flags & 0x10) || (flags & 0x100))\n"); - outs("\t{\n"); - parenthesisOpened = true; - outf("\t\tC%s::getXsdDefinition(context, request, schema, added, xns, wsns, 0x110);\n", parent); - typesNeeded[parent] = false; - } - for (pi=getParams();pi!=NULL;pi=pi->next) - { - // we still need to make it usable for the service writer - //if (pi->getMetaInt("hidden", 0) || pi->getMetaInt("hidden_soap", 0)) - // continue; - std::string thisType; - - const char *xsd_type=pi->getMetaXsdType(); - if (xsd_type) - { - char* buf = strdup(xsd_type[0]=='"' ? xsd_type+1 : xsd_type); - size_t count= strlen(buf); - if (count>0 && buf[count-1]=='\"') - buf[count-1]=0; - - char* typeName = getToBeDefinedType(buf); - if (typeName) - { - thisType = typeName; - free(typeName); - } - free(buf); - } - - else if (pi->kind==TK_ESPSTRUCT || pi->kind==TK_ESPENUM) - { - if (pi->typname) - thisType = pi->typname; - else - outs("\t\t*** pi->typname is empty!!\n"); - } - - else if (pi->flags & PF_TEMPLATE) - { - if (strcmp(pi->templ, "ESParray")==0) - { - if (pi->typname && strcmp(pi->typname, "string")!=0 && strcmp(pi->typname, "EspTextFile")!=0) - thisType = pi->typname; - } - else - outf("\t\t// *** skip field: name=%s, type=%s\n", pi->name, pi->typname); - } - - if (thisType.length() && strcmp(thisType.c_str(),name_)!=0) - { - TypeMap::const_iterator it = typesNeeded.find(thisType); - if (it==typesNeeded.end() || (*it).second & MIT_HasMap) // thisType is not in typesNeeded or it is there with version info - { - if (!parenthesisOpened) - { - outs("\tif (!(flags & 0x10) || (flags & 0x100))\n\t{\n"); - parenthesisOpened = true; - } - - - if (pi->hasMapInfo()) - { - outf("\t\tif (!context.suppressed(\"%s\",\"%s\"))\n\t", this->name_, pi->name); - typesNeeded[thisType] = (pi->kind==TK_ESPENUM) ? MIT_EnumTrue : MIT_StructTrue; - } - else - typesNeeded[thisType] = (pi->kind==TK_ESPENUM) ? MIT_EnumFalse : MIT_StructFalse; - if (pi->kind == TK_ESPENUM) - outf("\t\tCX%s::getXsdDefinition(context, request, schema, added);\n", thisType.c_str()); - else - outf("\t\tC%s::getXsdDefinition(context, request, schema, added);\n", thisType.c_str()); - } - } - } - - if (parenthesisOpened) - outs("\t}\n"); - - // native arrays - for (EspNativeArrays::const_iterator it = nativeArrays.begin(); it!=nativeArrays.end(); it++) - { - outf("\tif (added.getValue(\"%s\")==NULL) {\n", (*it).second.c_str()); - outf("\t\taddEspNativeArray(schema,\"%s\",\"%s\");\n", (*it).first.c_str(), (*it).second.c_str()); - outf("\t\tadded.setValue(\"%s\",1);\n", (*it).second.c_str()); - outf("\t}\n"); - } - - // struct arrays - for (EspStructArrays::const_iterator it2 = structArrays.begin(); it2!=structArrays.end(); it2++) - { - const char* type = (*it2).c_str(); - outf("\tif (added.getValue(\"%s\") && added.getValue(\"ArrayOf%s\")==NULL) {\n", type, type); - outf("\t\tschema.append(\"\\n\");\n",type); - outf("\t\tschema.append(\"\\n\");\n"); - outf("\t\tschema.append(\"\\n\");\n", type, type); - outf("\t\tschema.append(\"\\n\");\n"); - outf("\t\tschema.append(\"\\n\");\n"); - outf("\t\tadded.setValue(\"ArrayOf%s\",1);\n", type); - outf("\t}\n"); - } - - /* - //typesNeeded.erase(name_); // Do not include self: avoid recursive call - - if (typesNeeded.size()>0) - { - outs("\tif (!(flags & 0x10))\n"); - outs("\t{\n"); - for (TypeMap::const_iterator it = typesNeeded.begin(); it!=typesNeeded.end(); ++it) - { - const char* type = (*it).first.c_str(); - const char* fld = (*it).second.c_str(); - if (fld[0] != 0) - { - outf("\t\tif (!context.suppressed(\"%s\",\"%s\"))\n\t", this->name_, fld); - outf("\t\t\tC%s::getXsdDefinition(context, request, schema, added);\n", type); - } - else - outf("\t\tC%s::getXsdDefinition(context, request, schema, added);\n", type); - } - outs("\t}\n"); - } - */ - outs("\treturn schema;\n"); - outs("}\n"); - //======================================================================================= //method ==> getMapInfo outf("\nvoid C%s::getMapInfo(IMapInfo& info) { BoolHash added; getMapInfo(info, added); }\n",name_); @@ -3502,224 +2898,9 @@ void EspMessageInfo::write_esp() } } - if (typesNeeded.size()>0) - { - for (TypeMap::const_iterator it = typesNeeded.begin(); it!=typesNeeded.end(); ++it) - { - outf("\tif (!added.getValue(\"%s\"))\n", (*it).first.c_str()); - outf("\t{\n"); - outf("\t\tadded.setValue(\"%s\",1);\n", (*it).first.c_str()); - outf("\t\tC%s%s::getMapInfo(info,added);\n",((*it).second & MIT_IsEnum)?"X":"",(*it).first.c_str()); - outf("\t}\n"); - } - } - outs("}\n"); - //======================================================================================= - //method ==> getHtmlForm - //TODO: move includeFormTag into onGetForm() to reduce significant generated code indentReset(); - indentOutf("\nStringBuffer &C%s::getHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &form, bool includeFormTag, const char *prefix)\n", name_); - - indentOuts("{\n"); - indentOuts(1,"IProperties *props = request->queryParameters();\n"); - bool hasAttachment=false; - for (pi=getParams();pi;pi=pi->next) - { - if (pi->typname && !stricmp("EspTextFile", pi->typname)) - hasAttachment=true; - } - - indentOuts("if (includeFormTag) {\n"); - indentOutf(1,"StringBuffer params, versionTag;\n"); - indentOuts("bool hasVersion = getUrlParams(props,params);\n"); - indentOuts("if (!hasVersion) versionTag.appendf(\"%cver_=%g\",params.length()?'&':'?',context.getClientVersion());\n"); - indentOutf("form.appendf(\"\\n
\\n\", serv, method, params.str(), versionTag.str());\n", - hasAttachment ? "multipart/form-data" : "application/x-www-form-urlencoded"); - indentOutf(-1,"}\n"); - if (parent) - indentOutf("C%s::getHtmlForm(context, request, serv, method, form, false, prefix);\n", parent); - - indentOuts("StringBuffer extfix;\n"); - indentOuts("form.append(\" \\n\");\n"); - - for (pi=getParams();pi!=NULL;pi=pi->next) - { - bool hasMapInfo = pi->hasMapInfo(); - if (hasMapInfo) - { - indentOutf("if (!context.suppressed(\"%s\",\"%s\")) {\n", this->name_, pi->name); - indentInc(1); - } - - if (!pi->getMetaInt("hidden")) - { - int rows = pi->getMetaInt("rows", 5); - - StrBuffer label; - if (!pi->getMetaStringValue(label,"label")) - if (!pi->getMetaStringValue(label,"xml_tag")) - label = pi->name; - - if (pi->getMetaInt("rows") || (pi->flags & PF_TEMPLATE)) - { - int cols = pi->getMetaInt("cols", 50); - - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->name); - indentOutf("form.appendf(\"\");\n"); - } - else - { - esp_xlate_info *espinfo = esp_xlat(pi); - if (pi->kind==TK_ESPSTRUCT) - { - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->getXmlTag()); - indentOutf("form.append(\"\").append(\"\");\n"); - } - else if (espinfo->access_kind==TK_BOOL) - { - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->getXmlTag()); - - // handle default value - indentOutf("\n\tform.appendf(\" \\n\", extfix.str());\n", - label.str(), pi->getMetaInt("default",0)?"checked=\\\"1\\\"":""); - } - else if (espinfo->access_kind==TK_INT || espinfo->access_kind==TK_UNSIGNED - || espinfo->access_kind==TK_SHORT || espinfo->access_kind==TK_UNSIGNEDSHORT) - { - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix) extfix.append(prefix).append(\".\");\n"); - indentOutf("\textfix.append(\"%s\");\n", pi->getXmlTag()); - - int cols = pi->getMetaInt("cols", 20); - indentOutf("form.appendf(\" \\n\");\n"); - } - //TODO: handle default value for float, double, long, longlong etc - else - { - const char *xsdtype = pi->getXsdType(); - bool isDate = (!stricmp(xsdtype, "date")); - const char *inputType = "text"; - if (pi->typname && !strcmp(pi->typname, "EspTextFile")) - inputType="file"; - else if (pi->getMetaInt("password")) - inputType="password"; - - indentOuts("extfix.clear();\n"); - indentOuts("if (prefix && *prefix)\n"); - indentOuts1(1,"extfix.append(prefix).append(\".\");\n"); - indentOutf("extfix.append(\"%s\");\n", pi->getXmlTag()); - - // handle default value - StrBuffer tmp; - const char* def = pi->getMetaStringValue(tmp,"default") ? tmp.str() : ""; - - int cols = pi->getMetaInt("cols", isDate ? 20 : 50); - indentOutf("form.appendf(\" \\n\");\n"); - } - } - } - if (hasMapInfo) - indentOuts(-1,"}\n"); - } - - indentReset(1); - indentOuts("if (includeFormTag) {\n"); - indentOuts(1,"form.append(\"\");\n"); - indentOuts(-1,"}\n"); - - indentOuts("form.append(\"
%s: \");\n", label.str()); - if (!pi->isEspStringArray()) - { - // handle default value - StrBuffer tmp; - const char* def = pi->getMetaStringValue(tmp,"default") ? tmp.str() : ""; - - indentOutf("form.appendf(\"\", extfix.str());\n", - cols, rows, def); - - indentOutf("form.append(\"
\");\n"); - } - else - { - indentOutf("form.appendf(\"\", extfix.str());\n", cols, rows); - } - - indentOutf("form.append(\"
%s:
\");\n", label.str()); - indentOutf("C%s::getHtmlForm(context, request, serv, method, form, false, extfix.str());\n", pi->typname); - indentOuts("form.append(\"
%s?
%s: getMetaInt("default"); - if (defValue!=0) - indentOutf(" value=\\\"%d\\\"", defValue); - - outs("/>\", extfix.str());\n"); - indentOuts("form.append(\"
%s: \", extfix.str());\n", label.str(), inputType, cols, def); - if (isDate) - indentOutf("form.append(\"\\\"Pick
\");\n", pi->name); - indentOuts("form.append(\"
\");\n"); - indentOuts("form.append(\"  
\");\n"); - indentOuts("if (includeFormTag)\n"); - indentOuts(1,"form.append(\"
\");\n"); - indentOuts(-1,"return form;\n"); - indentOuts(-1,"}\n"); - - //method ==> serializeHtml - outf("\nStringBuffer &C%s::serializeHtml(IEspContext &context, const char *serv, const char *method, StringBuffer &html)\n", name_); - outs("{\n"); - if (getMetaInt("serialize_html")) - { - outs("\thtml.append(\"

\");\n"); - - if (contentVar) - { - if (!(contentVar->flags & PF_TEMPLATE)) - { - if (contentVar->typname!=NULL && strcmp(contentVar->typname, "EspResultSet")==0) - { - outf("\thtml.appendf(\"
%s:
\");\n", contentVar->name); - outf("\tEspHttpBinding::formatHtmlResultSet(context, serv, method, m_%s->str(), html);", contentVar->name); - } - else if (contentVar->typname==NULL || strcmp(contentVar->typname, "binary")!=0) - { - outf("\thtml.appendf(\"
%s:
\");\n", contentVar->name, contentVar->name, contentVar->name); - } - } - } - else - { - for (pi=getParams();pi!=NULL;pi=pi->next) - { - if (!(pi->flags & PF_TEMPLATE) && (pi->kind!=TK_ESPSTRUCT)) - { - if (pi->typname!=NULL && strcmp(pi->typname, "EspResultSet")==0) - { - outf("\thtml.appendf(\"
%s:
\");\n", pi->name); - outf("\tEspHttpBinding::formatHtmlResultSet(context, serv, method, m_%s->str(), html);", pi->name); - } - else if (pi->typname==NULL || strcmp(pi->typname, "binary")!=0) - { - outf("\thtml.appendf(\"
%s:
\");\n", pi->name, pi->name, pi->name); - } - } - } - } - - outs("\thtml.append(\"

\");\n"); - } - outs("\treturn html;\n"); - outs("}\n"); - //method ==> serialize (IRpcMessage&) outf("\nvoid C%s::serialize(IRpcMessage& rpc_resp)\n{\n", name_); @@ -3988,9 +3169,7 @@ void EspMessageInfo::write_esp() outf("\tdouble clientVer = ctx ? ctx->getClientVersion() : -1;\n"); } -#if 1 // not respecting nil_remove: backward compatible - for (pi=getParams();pi!=NULL;pi=pi->next) { if (pi->getMetaInt("attribute")) @@ -4182,232 +3361,6 @@ void EspMessageInfo::write_esp() free(uname); } -#else - // respect nil_remove: may cause backward compatibility problem - - bool nilRemove = getMetaInt("nil_remove", 0)!=0; - - indentReset(1); - for (pi=getParams();pi!=NULL;pi=pi->next) - { - char *uname=strdup(pi->name); - *uname = upperchar(*uname); - - if (pi->hasMapInfo()) - pi->write_mapinfo_check(1,"ctx"); - - if (pi->flags & PF_TEMPLATE) // array - { - indentOutf("{\n"); - indentInc(1); - if (pi->isPrimitiveArray()) - { - const char *item_tag = pi->getMetaString("item_tag", "Item"); - const char *type = pi->getArrayImplType(); - indentOutf("%s& v = src.get%s();\n",type,uname); - indentOutf("int size = v.length();\n"); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"<%s>\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOuts("for (int i=0;ikind) - { - case TK_BOOL: - case TK_SHORT: - case TK_INT: fmt = "d"; break; - case TK_UNSIGNED: fmt = "u"; break; - case TK_LONG: fmt = "ld"; break; - case TK_UNSIGNEDLONG: fmt = "lu"; break; - case TK_FLOAT: - case TK_DOUBLE: fmt = "g"; break; - case TK_null: - case TK_CHAR: fmt = "s"; break; - default: - { - char buf[128]; - sprintf(buf,"Unhandled array type: %s (%s)", getTypeKindName(pi->kind), name_); - yyerror(buf); - } - } - - indentOutf1(1,"buffer.appendf(\"<%s>%%%s\",v.item(i));\n",item_tag,fmt,item_tag); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - else if (pi->typname) - { - indentOutf("IArrayOf& v = src.get%s();\n",pi->typname,uname); - indentOuts("int size = v.length();\n"); - const char *item_tag = pi->getMetaString("item_tag", pi->typname); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"<%s>\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("for (int i=0;i\");\n",item_tag); - indentOutf("C%s::serializer(ctx,v.item(i),buffer,false);\n",pi->typname); - indentOutf("buffer.append(\"\");\n",item_tag); - indentOutf(-1,"}\n"); - if (nilRemove) - { - indentOutf("if (size>0)\n"); - indentOutf1(1,"buffer.append(\"\");\n", pi->getXmlTag()); - } - else - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - else - { - indentOutf("**** TODO: unhandled array: type=%s, name=%s, xsd-type=%s\n", pi->typname, uname, pi->getXsdType()); - } - indentOutf(-1,"}\n"); - } - else if (pi->kind == TK_ESPSTRUCT) - { - indentOutf("{\n"); - indentInc(1); - if (nilRemove) - { - indentOutf("StringBuffer tmp;\n"); - indentOutf("C%s::serializer(ctx,src.get%s(), tmp, false);\n", pi->typname, uname); - indentOutf("if (tmp.length()>0)\n"); - const char* tag = pi->getXmlTag(); - indentOutf1(1,"buffer.appendf(\"<%s>%%s\",tmp.str());\n", tag, tag); - } - else - { - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("C%s::serializer(ctx,src.get%s(), buffer, false);\n", pi->typname, uname); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - indentOutf(-1,"}\n"); - } - else - { - esp_xlate_info* info = esp_xlat(pi); - switch(info->access_kind) - { - case TK_CHAR: - indentOuts("{\n"); - indentOutf(1,"const char* s = src.get%s();\n", uname); - if (nilRemove) - { - indentOutf("if (s && *s)\n"); - indentInc(1); - } - - if (!getMetaInt("encode",1)) - { - indentOutf("buffer.appendf(\"<%s>%%s\",s);\n",pi->name,pi->name); - } - else - { - if (nilRemove) - indentOutf1(-1,"{\n"); - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("encodeUtf8XML(s,buffer);\n"); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - if (nilRemove) - indentOutf1(-1,"}\n"); - } - if (nilRemove) - indentInc(-1); - indentOuts(-1,"}\n"); - break; - - case TK_INT: - case TK_LONG: - case TK_SHORT: - indentOuts("{\n"); - indentOutf(1,"%s n = src.get%s();\n", esp_xlat(pi)->access_type, uname); - if (nilRemove) - { - indentOuts("if (n)\n"); - indentInc(1); - } - indentOutf("buffer.appendf(\"<%s>%%d\", n);\n", pi->name,pi->name); - if (nilRemove) - indentInc(-1); - indentOuts(-1,"}\n"); - break; - - case TK_BOOL: - indentOuts("{\n"); - indentOutf(1,"%s b = src.get%s();\n", esp_xlat(pi)->access_type, uname); - if (nilRemove) - { - indentOuts("if (b)\n"); - indentInc(1); - } - const char* tag = pi->getXmlTag(); - indentOutf("buffer.append(\"<%s>1\");\n", tag,tag); - if (nilRemove) - indentInc(-1); - indentOuts(-1,"}\n"); - break; - - default: - if (pi->kind == TK_STRUCT && info->eam_type == EAM_jmbin) // binary - { - //TODO: should we encode binary data? - indentOuts("{\n"); - if (nilRemove) - { - indentOutf(1,"StringBuffer tmp;\n"); - indentOutf("JBASE64_Encode(src.get%s().toByteArray(), src.get%s().length(), tmp, true);\n", uname, uname); - indentOutf("if (tmp.length()>0)\n"); - const char* tag = pi->getXmlTag(); - indentOutf1(1,"buffer.appendf(\"<%s>%%s\",tmp.str());\n", tag,tag); - } - else - { - indentOutf(1,"buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("JBASE64_Encode(src.get%s().toByteArray(), src.get%s().length(), buffer, ture);\n", uname, uname); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - indentOuts(-1,"}\n"); - } - else - { - indentOuts("{\n"); - if (nilRemove) - { - indentOutf(1,"//*** default kind: %s; type=%s, name=%s\n", getTypeKindName(pi->kind), pi->typname, uname); - indentOutf("StringBuffer tmp(src.get%s());\n",uname); - indentOutf("if (tmp.length()>0)\n"); - indentOutf1(1,"buffer.appendf(\"<%s>%%s\",tmp.str());\n", pi->name,pi->name); - } - else - { - indentOutf(1,"//*** default kind: %s; type=%s, name=%s\n", getTypeKindName(pi->kind), pi->typname, uname); - indentOutf("buffer.append(\"<%s>\");\n", pi->getXmlTag()); - indentOutf("buffer.append(src.get%s());\n", uname); - indentOutf("buffer.append(\"\");\n", pi->getXmlTag()); - } - indentOuts(-1,"}\n"); - } - break; - } - } - - free(uname); - } -#endif outf("\tif (keepRootTag)\n\t\tbuffer.append(\"\");\n", name_); outs("}\n"); @@ -4993,11 +3946,8 @@ void EspServInfo::write_esp_binding_ipp() //method ==> processRequest outs("\tvirtual int processRequest(IRpcMessage* rpc_call, IRpcMessage* rpc_response);\n"); - //method ==> getXsdDefinition - outs("\tint getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &content, const char *service, const char *method, bool mda);\n"); - - //method ==> getMethodHtmlForm - outs("\tvirtual int getMethodHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &page, bool bIncludeFormTag);\n"); + // method ===> getServiceXmlFilename + outs("\tint getServiceXmlFilename(StringBuffer &filename);\n"); //method ==> getQualifiedNames outs("\tint getQualifiedNames(IEspContext& ctx, MethodInfoArray & methods);\n"); @@ -5021,7 +3971,6 @@ void EspServInfo::write_esp_binding_ipp() outs("\tvirtual int onGetForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method);\n"); //Method ==> onGetXForm - //if (getMetaInt("use_new_form",0)) outs("\tvirtual int onGetXForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method);\n"); //Method ==> supportGeneratedForms @@ -5117,7 +4066,6 @@ void EspServInfo::write_esp_binding_ipp() void EspServInfo::write_esp_binding(const char *packagename) { EspMethodInfo *mthi=NULL; - int useMethodName = getMetaInt("use_method_name", 0); StrBuffer wsdlVer; bool hasVersion = getMetaVerInfo(tags,"version",wsdlVer); if (!hasVersion) @@ -5136,7 +4084,6 @@ void EspServInfo::write_esp_binding(const char *packagename) outf("{\n"); outf("\tinit_strings();\n"); outf("\tsetWsdlVersion(%s);\n", wsdlVer.str()); - outf("\tserviceXmlFilename.append(\"%s.xml\");\n", packagename); outf("}\n"); outf("\nC%sSoapBinding::C%sSoapBinding(IPropertyTree* cfg, const char *bindname, const char *procname, http_soap_log_level level):CHttpSoapBinding(cfg, bindname, procname, level)\n", name_, name_); @@ -5144,7 +4091,6 @@ void EspServInfo::write_esp_binding(const char *packagename) outf("\tinit_strings();\n"); outf("\tinit_metrics();\n"); outf("\tsetWsdlVersion(%s);\n", wsdlVer.str()); - outf("\tserviceXmlFilename.append(\"%s.xml\");\n", packagename); outf("}\n"); outf("\nvoid C%sSoapBinding::init_strings()\n", name_); @@ -5360,89 +4306,12 @@ void EspServInfo::write_esp_binding(const char *packagename) outs("\treturn -1;\n"); outs("}\n"); - //method ==> getXsdDefinition packagename is the base ecm filename - outf("\nint C%sSoapBinding::getXsdDefinition(IEspContext &context, CHttpRequest* request, StringBuffer &content, const char *service, const char *method, bool mda)\n", name_); - outs("{\n"); - - outs("\tBoolHash added;\n"); - - - // version - if (hasVersion) - { - outs("\tif (context.getClientVersion()<=0)\n"); - outf("\t\tcontext.setClientVersion(%s);\n\n", wsdlVer.str()); - } - outs("\tDBGLOG(\"Client version: %g\", context.getClientVersion());\n"); - - // getXSDDefinition(StringBuffer &content, const char *packageName, const char *serviceName, double version, const char *method); -// outf( "getXSDDefinition(content, \"%s\", \"%s\", context.getClientVersion(), method);\n\n", packagename, name_); - - indentReset(1); - outf(1, "bool fullservice = (!Utils::strcasecmp(service, \"%s\"));\n", name_); - indentOuts("bool allMethods = (method==NULL || *method==0);\n"); - bool needPropsDeclare = true; - bool needAccessFlagDeclare = true; - for (mthi=methods;mthi!=NULL;mthi=mthi->next) - { - const char *requestName = (useMethodName) ? mthi->getName() : mthi->getReq(); - const char *optional=mthi->getMetaString("optional", NULL); - const char *access=mthi->getMetaString("access", NULL); - - if (optional) - { - if (needPropsDeclare) - { - indentOuts("IProperties *props = request->queryParameters();\n"); - needPropsDeclare = false; - } - indentOutf("if (props && props->hasProp(%s)) {\n", optional); - indentInc(1); - } - if (access) - { - if (needAccessFlagDeclare) - { - indentOuts("SecAccessFlags acc;\n"); - needAccessFlagDeclare = false; - } - indentOutf("if (context.authorizeFeature(%s, acc) && acc>=SecAccess_Read) {\n", access); - indentInc(1); - } - - indentOutf("if ((allMethods&&(fullservice||isMethodInSubService(context, service, \"%s\"))) || Utils::strcasecmp(method, \"%s\")==0)\n", mthi->getName(), mthi->getName()); - indentOuts("{\n"); - indentInc(1); - bool hasMap = mthi->write_mapinfo_check("context"); - indentOutf("C%s::getMapInfo(context.queryMapInfo());\n", mthi->getReq()); - indentOutf("C%s::getMapInfo(context.queryMapInfo());\n", mthi->getResp()); - indentOutf("C%s::getXsdDefinition(context, request, \"%s\", content, added);\n", mthi->getReq(), requestName); - indentOutf("C%s::getXsdDefinition(context, request, content, added);\n", mthi->getResp()); - if (hasMap) - indentOuts(-1,"}\n"); - indentOuts(-1,"}\n"); - if (access) - indentOuts(-1,"}\n"); - if (optional) - indentOuts(-1,"}\n"); - } - indentOuts("return 0;\n"); - indentOuts(-1,"}\n"); - //method ==> getMethodHtmlForm - outf("\nint C%sSoapBinding::getMethodHtmlForm(IEspContext &context, CHttpRequest* request, const char *serv, const char *method, StringBuffer &page, bool bIncludeFormTag)\n", name_); + //method ==> getServiceXmlFilename for xsd and wsdl transformations + outf("\nint C%sSoapBinding::getServiceXmlFilename(StringBuffer &filename)\n", name_); outs("{\n"); - - outf("\tDBGLOG(\"Client version: %%g\", context.getClientVersion());\n"); - - for (mthi=methods;mthi!=NULL;mthi=mthi->next) - { - outf("\tif (Utils::strcasecmp(method, \"%s\")==0)\n", mthi->getName()); - outs("\t{\n"); - outf("\t\tC%s::getHtmlForm(context, request, serv, method, page);\n", mthi->getReq()); - outs("\t}\n"); - } - outs("\treturn 0;\n"); + outf("\tfilename.append(\"%s.xml\");\n", packagename); + outs("\treturn 1;\n"); outs("}\n"); //method ==> getQualifiedNames @@ -5980,7 +4849,7 @@ void EspServInfo::write_esp_service_ipp() outf("\timplements IEsp%s\n", name_); outs("{\n"); outs("private:\n"); - outs("\tIEspContainer* m_container;\n"); + outs("\tIEspContainer* m_container = nullptr;\n"); outs("public:\n"); outs("\tIMPLEMENT_IINTERFACE;\n\n"); outf("\tC%s(){}\n\tvirtual ~C%s(){}\n", name_, name_); @@ -6035,14 +4904,12 @@ void EspServInfo::write_esp_client_ipp() outs("\tStringBuffer soap_url;\n"); //dom - outs("\tStringBuffer soap_userid;\n"); outs("\tStringBuffer soap_password;\n"); outs("\tStringBuffer soap_realm;\n"); outs("\tStringBuffer soap_action;\n"); outs("\tlong soap_reqid = 0;\n"); - outs("\npublic:\n"); outs("\tIMPLEMENT_IINTERFACE;\n\n"); diff --git a/tools/swapnode/swapnodelib.cpp b/tools/swapnode/swapnodelib.cpp index 8a8df9a1108..649c6dcfd09 100644 --- a/tools/swapnode/swapnodelib.cpp +++ b/tools/swapnode/swapnodelib.cpp @@ -42,8 +42,6 @@ #define SDS_LOCK_TIMEOUT 30000 #define SWAPNODE_RETRY_TIME (1000 * 60 * 60 * 1) // 1hr -static const LogMsgJobInfo swapnodeJob(UnknownJob, UnknownUser); - static bool ensureThorIsDown(const char *cluster, bool nofail, bool wait) { bool retry = false; @@ -781,7 +779,7 @@ class CAutoSwapNode : public CSwapNode StringBuffer msg; msg.appendf("AUTOSWAPNODE: cluster %s node %d: swapped out %s, swapped in %s", groupName.get(), badrank.item(i4) + 1, from.str(), to.str()); emailSwap(msg.str()); - FLLOG(MCoperatorError, swapnodeJob, "%s", msg.str()); + FLLOG(MCoperatorError, "%s", msg.str()); } } return true; diff --git a/vcpkg.json.in b/vcpkg.json.in index 62861804aa7..23d90885709 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -103,7 +103,9 @@ }, { "name": "libmysql", - "platform": "@VCPKG_MYSQLEMBED@ & !(windows & x86)" + "platform": "@VCPKG_MYSQLEMBED@ & !(windows & x86)", + "default-features": false, + "features": [] }, { "name": "librdkafka", diff --git a/vcpkg_overlays/libmysql/cross-build.patch b/vcpkg_overlays/libmysql/cross-build.patch new file mode 100644 index 00000000000..a91118b4f85 --- /dev/null +++ b/vcpkg_overlays/libmysql/cross-build.patch @@ -0,0 +1,162 @@ +diff --git a/cmake/os/Darwin.cmake b/cmake/os/Darwin.cmake +index 725b9bd..289bb3d 100644 +--- a/cmake/os/Darwin.cmake ++++ b/cmake/os/Darwin.cmake +@@ -31,13 +31,9 @@ ENDIF() + # We require at least XCode 10.0 + IF(NOT FORCE_UNSUPPORTED_COMPILER) + IF(MY_COMPILER_IS_CLANG) +- CHECK_C_SOURCE_RUNS(" +- int main() +- { +- return (__clang_major__ < 10); +- }" HAVE_SUPPORTED_CLANG_VERSION) +- IF(NOT HAVE_SUPPORTED_CLANG_VERSION) +- MESSAGE(FATAL_ERROR "XCode 10.0 or newer is required!") ++ IF (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10) ++ MESSAGE(FATAL_ERROR ++ "XCode 10.0 or newer is required! Compiler version was ${CMAKE_CXX_COMPILER_VERSION}") + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "Unsupported compiler!") +diff --git a/libmysql/CMakeLists.txt b/libmysql/CMakeLists.txt +index 8e22453..2b63413 100644 +--- a/libmysql/CMakeLists.txt ++++ b/libmysql/CMakeLists.txt +@@ -259,6 +259,7 @@ ELSEIF(FREEBSD) + MESSAGE(STATUS "BSD built in DNS SRV APIs") + ELSE() + FIND_LIBRARY(RESOLV_LIBRARY NAMES resolv) ++ FIND_LIBRARY(RESOLV_LIBRARY NAMES resolv PATHS ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES} NO_DEFAULT_PATH) + IF (RESOLV_LIBRARY) + LIST(APPEND LIBS_TO_LINK ${RESOLV_LIBRARY}) + SET(HAVE_UNIX_DNS_SRV 1 PARENT_SCOPE) +@@ -413,6 +414,7 @@ MYSQL_ADD_EXECUTABLE(libmysql_api_test + ${CMAKE_CURRENT_BINARY_DIR}/api_test.c + LINK_LIBRARIES libmysql ${LIBRT} + SKIP_INSTALL ++ EXCLUDE_FROM_ALL + ) + # Clang/UBSAN needs this on some platforms. + SET_TARGET_PROPERTIES(libmysql_api_test PROPERTIES LINKER_LANGUAGE CXX) +@@ -439,6 +441,6 @@ ADD_CUSTOM_COMMAND( + COMMAND libmysql_api_test + > ${CMAKE_CURRENT_BINARY_DIR}/libmysql_api_test.out + ) +-MY_ADD_CUSTOM_TARGET(run_libmysql_api_test ALL ++MY_ADD_CUSTOM_TARGET(run_libmysql_api_test + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libmysql_api_test.out + ) +diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt +index 8e93387..d971acb 100644 +--- a/scripts/CMakeLists.txt ++++ b/scripts/CMakeLists.txt +@@ -69,7 +69,6 @@ ADD_CUSTOM_COMMAND( + + # Add target for the above to be built + MY_ADD_CUSTOM_TARGET(GenFixPrivs +- ALL + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/mysql_fix_privilege_tables_sql.h + ) + +diff --git a/scripts/sys_schema/CMakeLists.txt b/scripts/sys_schema/CMakeLists.txt +index aaba357..a0e5265 100644 +--- a/scripts/sys_schema/CMakeLists.txt ++++ b/scripts/sys_schema/CMakeLists.txt +@@ -221,7 +221,6 @@ MY_ADD_CUSTOM_TARGET(sql_commands + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/sql_commands.h) + + MY_ADD_CUSTOM_TARGET(GenSysSchemaC +- ALL + DEPENDS comp_sql sql_commands + COMMENT "Generating Sys Schema C code" + ) +diff --git a/share/CMakeLists.txt b/share/CMakeLists.txt +index 28cde40..be916cb 100644 +--- a/share/CMakeLists.txt ++++ b/share/CMakeLists.txt +@@ -47,8 +47,12 @@ SET(dirs + ukrainian + ) + +-FOREACH (dir ${dirs}) +- INSTALL(DIRECTORY ${CMAKE_BINARY_DIR}/${INSTALL_MYSQLSHAREDIR}/${dir} ++ set(src_dir ${CMAKE_BINARY_DIR}/share/libmysql) ++ if(CMAKE_CROSSCOMPILING) ++ set(src_dir ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql) ++ endif() ++ FOREACH (dir ${dirs}) ++ INSTALL(DIRECTORY ${src_dir}/${dir} + DESTINATION ${INSTALL_MYSQLSHAREDIR} + COMPONENT Server + ) +diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt +index 665b35d..88f0743 100644 +--- a/strings/CMakeLists.txt ++++ b/strings/CMakeLists.txt +@@ -69,6 +69,13 @@ SET(ZH_HANS_SRC_FILE ${CMAKE_SOURCE_DIR}/strings/lang_data/zh_hans.txt) + SET(ZH_HANS_DST_FILE ${CMAKE_BINARY_DIR}/strings/uca900_zh_tbls.cc) + SET(JA_HANS_SRC_FILE ${CMAKE_SOURCE_DIR}/strings/lang_data/ja_hans.txt) + SET(JA_HANS_DST_FILE ${CMAKE_BINARY_DIR}/strings/uca900_ja_tbls.cc) ++if(CMAKE_CROSSCOMPILING) ++ file(COPY ++ "${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/uca900_zh_tbls.cc" ++ "${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/uca900_ja_tbls.cc" ++ DESTINATION "${CMAKE_BINARY_DIR}/strings" ++ ) ++else() + ADD_CUSTOM_COMMAND(OUTPUT ${ZH_HANS_DST_FILE} + ${JA_HANS_DST_FILE} + COMMAND uca9dump zh +@@ -79,6 +86,8 @@ ADD_CUSTOM_COMMAND(OUTPUT ${ZH_HANS_DST_FILE} + --out_file=${JA_HANS_DST_FILE} + DEPENDS uca9dump ${ZH_HANS_SRC_FILE} ${JA_HANS_SRC_FILE} + ) ++install(FILES "${ZH_HANS_DST_FILE}" "${JA_HANS_DST_FILE}" DESTINATION "share/libmysql/${PROJECT_VERSION}") ++endif() + + SET_SOURCE_FILES_PROPERTIES( + ${JA_HANS_DST_FILE} ${ZH_HANS_DST_FILE} +diff --git a/utilities/CMakeLists.txt b/utilities/CMakeLists.txt +index da34524..dc397da 100644 +--- a/utilities/CMakeLists.txt ++++ b/utilities/CMakeLists.txt +@@ -39,6 +39,21 @@ MYSQL_ADD_EXECUTABLE(comp_client_err + SKIP_INSTALL + ) + ++if(CMAKE_CROSSCOMPILING) ++ file(COPY ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/include/mysql/mysqld_error.h ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/mysqlclient_ername.h ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/mysqld_ername.h ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/mysqld_errmsg.h ++ DESTINATION ${PROJECT_BINARY_DIR}/include ++ ) ++ file(COPY ++ ${VCPKG_INSTALLED_DIR}/${VCPKG_HOST_TRIPLET}/share/libmysql/${PROJECT_VERSION}/errmsg.sys ++ DESTINATION ${PROJECT_BINARY_DIR}/share/libmysql/english ++ ) ++ add_custom_target(GenClientError) ++ add_custom_target(GenError) ++else() + ADD_CUSTOM_COMMAND(OUTPUT ${PROJECT_BINARY_DIR}/include/mysqlclient_ername.h + COMMAND comp_client_err + --in_file=${PROJECT_SOURCE_DIR}/include/errmsg.h +@@ -97,6 +112,16 @@ MYSQL_ADD_EXECUTABLE(range_check_err + SKIP_INSTALL + ) + ++ install( ++ FILES ++ ${PROJECT_BINARY_DIR}/include/mysqlclient_ername.h ++ ${PROJECT_BINARY_DIR}/include/mysqld_ername.h ++ ${PROJECT_BINARY_DIR}/include/mysqld_errmsg.h ++ ${PROJECT_BINARY_DIR}/share/libmysql/english/errmsg.sys ++ DESTINATION share/libmysql/${PROJECT_VERSION} ++ ) ++endif() ++ + # Set InnoDB mutex type + ADD_DEFINITIONS(-DMUTEX_EVENT) + diff --git a/vcpkg_overlays/libmysql/dependencies.patch b/vcpkg_overlays/libmysql/dependencies.patch new file mode 100644 index 00000000000..0129d71e974 --- /dev/null +++ b/vcpkg_overlays/libmysql/dependencies.patch @@ -0,0 +1,140 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e17ec1d0..465e42f0 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -818,8 +818,12 @@ INCLUDE(fileutils) + INCLUDE(zlib) + INCLUDE(zstd) + INCLUDE(lz4) +-INCLUDE(icu) +-INCLUDE(libevent) ++macro(MYSQL_CHECK_ICU) ++endmacro() ++macro(MYSQL_CHECK_LIBEVENT) ++endmacro() ++macro(WARN_MISSING_SYSTEM_LIBEVENT) ++endmacro() + INCLUDE(ssl) + INCLUDE(sasl) + INCLUDE(ldap) +@@ -837,7 +841,8 @@ INCLUDE(curl) + INCLUDE(rapidjson) + INCLUDE(fprofile) + INCLUDE(gloves) +-INCLUDE(fido2) ++macro(MYSQL_CHECK_FIDO) ++endmacro() + INCLUDE(win_jemalloc) + + IF(UNIX) +@@ -1942,7 +1947,7 @@ MYSQL_CHECK_RAPIDJSON() + MYSQL_CHECK_FIDO() + MYSQL_CHECK_FIDO_DLLS() + +-IF(APPLE) ++IF(0) + GET_FILENAME_COMPONENT(HOMEBREW_BASE ${HOMEBREW_HOME} DIRECTORY) + IF(EXISTS ${HOMEBREW_BASE}/include/boost) + FOREACH(SYSTEM_LIB ICU LIBEVENT LZ4 PROTOBUF ZSTD FIDO) +diff --git a/cmake/boost.cmake b/cmake/boost.cmake +index c979055c..53e55fcf 100644 +--- a/cmake/boost.cmake ++++ b/cmake/boost.cmake +@@ -301,7 +301,7 @@ IF(NOT BOOST_MAJOR_VERSION EQUAL 10) + COULD_NOT_FIND_BOOST() + ENDIF() + +-IF(NOT BOOST_MINOR_VERSION EQUAL 77) ++IF(NOT BOOST_MINOR_VERSION EQUAL 77 AND NOT IGNORE_BOOST_VERSION) + MESSAGE(WARNING "Boost minor version found is ${BOOST_MINOR_VERSION} " + "we need 77" + ) +diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake +index 7107f4bd..bb7f3733 100644 +--- a/cmake/libutils.cmake ++++ b/cmake/libutils.cmake +@@ -534,7 +534,7 @@ MACRO(MERGE_CONVENIENCE_LIBRARIES TARGET_ARG) + + # On Windows, ssleay32.lib/libeay32.lib or libssl.lib/libcrypto.lib + # must be merged into mysqlclient.lib +- IF(WIN32 AND ${TARGET} STREQUAL "mysqlclient") ++ IF(0) + SET(LINKER_EXTRA_FLAGS "") + FOREACH(LIB ${SSL_LIBRARIES}) + STRING_APPEND(LINKER_EXTRA_FLAGS " ${LIB}") +diff --git a/cmake/lz4.cmake b/cmake/lz4.cmake +index 10e7e8c8..48772329 100644 +--- a/cmake/lz4.cmake ++++ b/cmake/lz4.cmake +@@ -46,7 +46,7 @@ FUNCTION(FIND_SYSTEM_LZ4) + FIND_PATH(LZ4_INCLUDE_DIR + NAMES lz4frame.h) + FIND_LIBRARY(LZ4_SYSTEM_LIBRARY +- NAMES lz4) ++ NAMES lz4d lz4 NAMES_PER_DIR) + IF (LZ4_INCLUDE_DIR AND LZ4_SYSTEM_LIBRARY) + SET(SYSTEM_LZ4_FOUND 1 CACHE INTERNAL "") + ADD_LIBRARY(lz4_interface INTERFACE) +diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake +index 04e3af87..8ae52c82 100644 +--- a/cmake/ssl.cmake ++++ b/cmake/ssl.cmake +@@ -273,7 +273,18 @@ MACRO (MYSQL_CHECK_SSL) + FIND_ALTERNATIVE_SYSTEM_SSL() + ENDIF() + +- IF(WITH_SSL STREQUAL "system" OR WITH_SSL_PATH OR ALTERNATIVE_SYSTEM_SSL) ++ IF(WITH_SSL STREQUAL "system") ++ find_package(OpenSSL REQUIRED) ++ set(OPENSSL_LIBRARY OpenSSL::SSL CACHE STRING "") ++ set(CRYPTO_LIBRARY OpenSSL::Crypto CACHE STRING "") ++ find_program(OPENSSL_EXECUTABLE openssl ++ DOC "path to the openssl executable") ++ set(SSL_DEFINES "-DHAVE_OPENSSL") ++ add_library(ext::openssl ALIAS OpenSSL::SSL) ++ set(SSL_LIBRARIES ext::openssl) ++ set(OPENSSL_APPLINK_C "${OPENSSL_APPLINK_SOURCE}") ++ include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) ++ ELSEIF(WITH_SSL STREQUAL "system" OR WITH_SSL_PATH OR ALTERNATIVE_SYSTEM_SSL) + IF((APPLE OR WIN32) AND WITH_SSL STREQUAL "system") + # FindOpenSSL.cmake knows about + # http://www.slproweb.com/products/Win32OpenSSL.html +diff --git a/cmake/zlib.cmake b/cmake/zlib.cmake +index 3781fe09..ac312ad0 100644 +--- a/cmake/zlib.cmake ++++ b/cmake/zlib.cmake +@@ -124,7 +124,7 @@ MACRO (MYSQL_CHECK_ZLIB) + MESSAGE(FATAL_ERROR "WITH_ZLIB must be bundled or system") + ENDIF() + +- ADD_LIBRARY(ext::zlib ALIAS zlib_interface) ++ ADD_LIBRARY(ext::zlib ALIAS ZLIB::ZLIB) + + IF(ZLIB_VERSION VERSION_LESS MIN_ZLIB_VERSION_REQUIRED) + MESSAGE(FATAL_ERROR +diff --git a/cmake/zstd.cmake b/cmake/zstd.cmake +index 425426d1..0ae1a907 100644 +--- a/cmake/zstd.cmake ++++ b/cmake/zstd.cmake +@@ -90,15 +90,16 @@ MACRO (MYSQL_CHECK_ZSTD) + IF(WITH_ZSTD STREQUAL "bundled") + MYSQL_USE_BUNDLED_ZSTD() + ELSEIF(WITH_ZSTD STREQUAL "system") +- FIND_SYSTEM_ZSTD() +- IF (NOT SYSTEM_ZSTD_FOUND) +- MESSAGE(FATAL_ERROR "Cannot find system zstd libraries.") +- ENDIF() ++ find_package(ZSTD NAMES zstd REQUIRED) + ELSE() + MESSAGE(FATAL_ERROR "WITH_ZSTD must be bundled or system") + ENDIF() + +- ADD_LIBRARY(ext::zstd ALIAS zstd_interface) ++ if(TARGET zstd::libzstd_shared) ++ add_library(ext::zstd ALIAS zstd::libzstd_shared) ++ else() ++ add_library(ext::zstd ALIAS zstd::libzstd_static) ++ endif() + + IF(ZSTD_VERSION VERSION_LESS MIN_ZSTD_VERSION_REQUIRED) + MESSAGE(FATAL_ERROR diff --git a/vcpkg_overlays/libmysql/fix_dup_symbols.patch b/vcpkg_overlays/libmysql/fix_dup_symbols.patch new file mode 100644 index 00000000000..352fd8af89f --- /dev/null +++ b/vcpkg_overlays/libmysql/fix_dup_symbols.patch @@ -0,0 +1,20 @@ +diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt +index 058967b..bcd8841 100644 +--- a/client/CMakeLists.txt ++++ b/client/CMakeLists.txt +@@ -43,7 +43,6 @@ MYSQL_ADD_EXECUTABLE(mysql + pattern_matcher.cc + readline.cc + client_query_attributes.cc +- multi_factor_passwordopt-vars.cc + ${CMAKE_CURRENT_SOURCE_DIR}/common/user_registration.cc + LINK_LIBRARIES mysqlclient client_base ${EDITLINE_LIBRARY} + ) +@@ -226,7 +226,6 @@ SET(MYSQLBINLOG_SOURCES + ${CMAKE_SOURCE_DIR}/sql/binlog_reader.cc + ${CMAKE_SOURCE_DIR}/sql/stream_cipher.cc + ${CMAKE_SOURCE_DIR}/sql/rpl_log_encryption.cc +- ${CMAKE_SOURCE_DIR}/libbinlogevents/src/trx_boundary_parser.cpp + ) + + SET(MYSQLBINLOG_LIBRARIES diff --git a/vcpkg_overlays/libmysql/install-exports.patch b/vcpkg_overlays/libmysql/install-exports.patch new file mode 100644 index 00000000000..3cf34ed3be5 --- /dev/null +++ b/vcpkg_overlays/libmysql/install-exports.patch @@ -0,0 +1,121 @@ +diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake +index baf49cd5..c45fda71 100644 +--- a/cmake/install_macros.cmake ++++ b/cmake/install_macros.cmake +@@ -113,8 +113,30 @@ FUNCTION(MYSQL_INSTALL_TARGET target_arg) + IF(ARG_NAMELINK_SKIP) + SET(LIBRARY_INSTALL_ARGS NAMELINK_SKIP) + ENDIF() ++ if(target STREQUAL "libmysql" OR target STREQUAL "mysqlclient") ++ target_include_directories(${target} INTERFACE $) ++ if(target STREQUAL "mysqlclient") ++ set_target_properties(${target} PROPERTIES EXPORT_NAME libmysql) # uniform ++ endif() ++ FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/unofficial-libmysql-config.cmake" ++"include(CMakeFindDependencyMacro) ++find_dependency(OpenSSL) ++find_dependency(Threads) ++find_dependency(ZLIB) ++find_dependency(zstd CONFIG) ++include(\"\${CMAKE_CURRENT_LIST_DIR}/unofficial-libmysql-targets.cmake\") ++if(NOT TARGET ${target}) # legacy vcpkg ++ add_library(${target} INTERFACE IMPORTED) ++ set_target_properties(${target} PROPERTIES INTERFACE_LINK_LIBRARIES unofficial::libmysql::libmysql) ++endif() ++") ++ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/unofficial-libmysql-config.cmake" DESTINATION "share/unofficial-libmysql") ++ install(EXPORT "unofficial-libmysql-targets" NAMESPACE unofficial::libmysql:: DESTINATION "share/unofficial-libmysql") ++ set(LIBRARY_EXPORT_ARGS EXPORT unofficial-libmysql-targets) ++ endif() + INSTALL(TARGETS ${target} +- RUNTIME DESTINATION ${ARG_DESTINATION} ${COMP} ++ ${LIBRARY_EXPORT_ARGS} ++ RUNTIME DESTINATION bin ${COMP} + ARCHIVE DESTINATION ${ARG_DESTINATION} ${COMP} + LIBRARY DESTINATION ${ARG_DESTINATION} ${COMP} ${LIBRARY_INSTALL_ARGS}) + SET(INSTALL_LOCATION ${ARG_DESTINATION} ) +diff --git a/cmake/libutils.cmake b/cmake/libutils.cmake +index a5333987..c954bfb1 100644 +--- a/cmake/libutils.cmake ++++ b/cmake/libutils.cmake +@@ -385,8 +385,12 @@ MACRO(MERGE_LIBRARIES_SHARED TARGET_ARG) + IF(ARG_NAMELINK_SKIP) + SET(INSTALL_ARGS NAMELINK_SKIP) + ENDIF() ++ if(INSTALL_STATIC_LIBRARIES) ++ set_target_properties(${TARGET} PROPERTIES EXCLUDE_FROM_ALL 1) ++ else() + MYSQL_INSTALL_TARGET(${TARGET} DESTINATION "${DESTINATION}" ${COMP} + ${INSTALL_ARGS}) ++ endif() + ENDIF() + + IF(WIN32) +@@ -563,6 +567,8 @@ MACRO(MERGE_CONVENIENCE_LIBRARIES TARGET_ARG) + ENDIF() + IF(INSTALL_STATIC_LIBRARIES) + MYSQL_INSTALL_TARGET(${TARGET} DESTINATION "${INSTALL_LIBDIR}" ${COMP}) ++ else() ++ set_target_properties(${TARGET} PROPERTIES EXCLUDE_FROM_ALL 1) + ENDIF() + ENDIF() + ENDMACRO(MERGE_CONVENIENCE_LIBRARIES) +diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt +index c4ef5694..8e933877 100644 +--- a/scripts/CMakeLists.txt ++++ b/scripts/CMakeLists.txt +@@ -315,10 +315,14 @@ MACRO(EXTRACT_LINK_LIBRARIES target var) + FOREACH(lib ${TARGET_LIB_DEPENDS}) + IF (lib MATCHES "^\\-l") + SET(${var} "${${var}} ${lib} ") ++ elseif(lib MATCHES "^ext::(openssl|zlib|zstd)" AND NOT WIN32) ++ list(APPEND ext_targets "${CMAKE_MATCH_1}") + ELSEIF (lib MATCHES "^ext::") + STRING(REGEX MATCH "ext::([a-z]+)" MATCH_LIB "${lib}") + SET(SYSTEM_LIB ${CMAKE_MATCH_1}) + SET(${var} "${${var}} -l${SYSTEM_LIB} ") ++ elseif(TARGET "${lib}" AND NOT WIN32) ++ # merged or shared, not external + ELSEIF (lib MATCHES "^\\-L") + # Search directory. Test on FreeBSD: -L/usr/local/lib -lunwind + SET(${var} "${${var}} ${lib} ") +@@ -354,6 +358,7 @@ IF(MSVC) + GET_TARGET_PROPERTY(LIBMYSQL_OS_SHLIB_VERSION mysqlclient VERSION) + GET_TARGET_PROPERTY(LIBMYSQL_OS_OUTPUT_NAME mysqlclient OUTPUT_NAME) + ELSE() ++ EXTRACT_LINK_LIBRARIES(libmysql CLIENT_LIBS) + GET_TARGET_PROPERTY(LIBMYSQL_OS_SHLIB_VERSION libmysql VERSION) + GET_TARGET_PROPERTY(LIBMYSQL_OS_OUTPUT_NAME libmysql OUTPUT_NAME) + ENDIF() +@@ -373,6 +378,10 @@ IF (WITH_SSL STREQUAL "system") + "${CONFIG_LIBS_PRIVATE}") + ENDIF() + ENDIF() ++list(APPEND CONFIG_REQUIRES_PRIVATE ${ext_targets}) ++list(REMOVE_DUPLICATES CONFIG_REQUIRES_PRIVATE) ++list(JOIN CONFIG_REQUIRES_PRIVATE " " CONFIG_REQUIRES_PRIVATE) ++string(REPLACE "zstd" "libzstd" CONFIG_REQUIRES_PRIVATE "${CONFIG_REQUIRES_PRIVATE}" ) + + MESSAGE(STATUS "CONFIG_CLIENT_LIBS ${CONFIG_CLIENT_LIBS}") + MESSAGE(STATUS "CONFIG_LIBS_PRIVATE ${CONFIG_LIBS_PRIVATE}") +diff --git a/scripts/mysql_config.sh b/scripts/mysql_config.sh +index 6e8520be..bf1a45af 100644 +--- a/scripts/mysql_config.sh ++++ b/scripts/mysql_config.sh +@@ -119,6 +119,16 @@ fi + # Create options + libs="-L$pkglibdir@RPATH_OPTION@" + libs="$libs -l@LIBMYSQL_OS_OUTPUT_NAME@ @CONFIG_CLIENT_LIBS@" ++if test -z "${PKG_CONFIG}" ; then ++ if pkg-config --version >/dev/null 2>&1 ; then ++ PKG_CONFIG=pkg-config ++ elif pkgconf --version >/dev/null 2>&1 ; then ++ PKG_CONFIG=pkgconf ++ else ++ PKG_CONFIG=false ++ fi ++fi ++libs="$libs $(${PKG_CONFIG} --libs @CONFIG_REQUIRES_PRIVATE@)" + + cflags="-I$pkgincludedir @CFLAGS@" + cxxflags="-I$pkgincludedir @CXXFLAGS@" diff --git a/vcpkg_overlays/libmysql/libmysql-config.cmake b/vcpkg_overlays/libmysql/libmysql-config.cmake new file mode 100644 index 00000000000..ea2bbbe0ab0 --- /dev/null +++ b/vcpkg_overlays/libmysql/libmysql-config.cmake @@ -0,0 +1,6 @@ +file(READ "${CMAKE_CURRENT_LIST_DIR}/usage" usage) +message(WARNING "find_package(libmysql) is deprecated.\n${usage}") +include(CMakeFindDependencyMacro) +find_dependency(unofficial-libmysql CONFIG REQUIRED) +set(libmysql_FOUND 1) +set(MYSQL_LIBRARIES unofficial::libmysql::libmysql) diff --git a/vcpkg_overlays/libmysql/portfile.cmake b/vcpkg_overlays/libmysql/portfile.cmake new file mode 100644 index 00000000000..34565231170 --- /dev/null +++ b/vcpkg_overlays/libmysql/portfile.cmake @@ -0,0 +1,160 @@ +if (EXISTS "${CURRENT_INSTALLED_DIR}/include/mysql/mysql.h") + message(FATAL_ERROR "FATAL ERROR: ${PORT} and libmariadb are incompatible.") +endif() + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO mysql/mysql-server + REF mysql-${VERSION} + SHA512 5df45c1ce1e2c620856b9274666cf56738d6a0308c33c9c96583b494c987fb0e862e676301109b9e4732070d54e6086596a62ad342f35adc59ca9f749e37b561 + HEAD_REF master + PATCHES + dependencies.patch + install-exports.patch + fix_dup_symbols.patch + cross-build.patch +) + +vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS + INVERTED_FEATURES + host-tools FORCE_CMAKE_CROSSCOMPILING +) + +file(GLOB third_party "${SOURCE_PATH}/extra/*" "${SOURCE_PATH}/include/boost_1_70_0") +list(REMOVE_ITEM third_party "${SOURCE_PATH}/extra/libedit") +if (third_party) + file(REMOVE_RECURSE ${third_party}) +endif() + +#Skip the version check for Visual Studio +set(FORCE_UNSUPPORTED_COMPILER "") +if(VCPKG_TARGET_IS_WINDOWS) + set(FORCE_UNSUPPORTED_COMPILER 1) +endif() + +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" BUILD_STATIC_LIBS) +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "static" STATIC_CRT_LINKAGE) + +set(cross_options "") +if(VCPKG_CROSSCOMPILING) + list(APPEND cross_options + -DCMAKE_CROSSCOMPILING=1 + -DVCPKG_HOST_TRIPLET=${HOST_TRIPLET} + # required, skip try_run + -DHAVE_RAPIDJSON_WITH_STD_REGEX=1 + ) + if(NOT VCPKG_TARGET_IS_WINDOWS) + list(APPEND cross_options + # optimistic, skip try_run + -DHAVE_CLOCK_GETTIME=1 + -DHAVE_CLOCK_REALTIME=1 + # pessimistic, skip try_run + -DHAVE_C_FLOATING_POINT_FUSED_MADD=1 + -DHAVE_CXX_FLOATING_POINT_FUSED_MADD=1 + -DHAVE_SETNS=0 + ) + endif() +endif() +if ("-DFORCE_CMAKE_CROSSCOMPILING=ON" IN_LIST FEATURE_OPTIONS) + list(APPEND cross_options + -DCMAKE_CROSSCOMPILING=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + ${cross_options} + -DINSTALL_INCLUDEDIR=include/mysql + -DINSTALL_DOCDIR=share/${PORT}/doc + -DINSTALL_MANDIR=share/${PORT}/doc + -DINSTALL_INFODIR=share/${PORT}/doc + -DINSTALL_DOCREADMEDIR=share/${PORT} + -DINSTALL_SHAREDIR=share + -DINSTALL_MYSQLSHAREDIR=share/${PORT} + -DWITHOUT_SERVER=ON + -DWITH_BUILD_ID=OFF + -DWITH_UNIT_TESTS=OFF + -DENABLED_PROFILING=OFF + -DWIX_DIR=OFF + -DIGNORE_BOOST_VERSION=ON + -DWITH_TEST_TRACE_PLUGIN=OFF + -DMYSQL_MAINTAINER_MODE=OFF + -DBUNDLE_RUNTIME_LIBRARIES=OFF + -DDOWNLOAD_BOOST=OFF + -DWITH_CURL=none + -DWITH_EDITLINE=bundled # not in vcpkg + -DWITH_LZ4=system + -DWITH_RAPIDJSON=system + -DWITH_SSL=system + -DWITH_SYSTEMD=OFF + -DWITH_ZLIB=system + -DWITH_ZSTD=system + -DFORCE_UNSUPPORTED_COMPILER=${FORCE_UNSUPPORTED_COMPILER} + -DINSTALL_STATIC_LIBRARIES=${BUILD_STATIC_LIBS} + -DLINK_STATIC_RUNTIME_LIBRARIES=${STATIC_CRT_LINKAGE} + MAYBE_UNUSED_VARIABLES + BUNDLE_RUNTIME_LIBRARIES # only on windows + LINK_STATIC_RUNTIME_LIBRARIES # only on windows + WIX_DIR # only on windows + WITH_BUILD_ID # only on windows +) + +vcpkg_cmake_install(ADD_BIN_TO_PATH) +vcpkg_cmake_config_fixup(PACKAGE_NAME unofficial-libmysql) +vcpkg_fixup_pkgconfig() + +set(MYSQL_TOOLS + my_print_defaults + mysql + mysql_config_editor + mysql_migrate_keyring + mysql_secure_installation + mysql_ssl_rsa_setup + mysqladmin + mysqlbinlog + mysqlcheck + mysqldump + mysqlimport + mysqlpump + mysqlshow + mysqlslap + mysqltest + perror + zlib_decompress +) +if (NOT VCPKG_CROSSCOMPILING) + list(APPEND MYSQL_TOOLS + comp_err + ) +endif() +if (VCPKG_TARGET_IS_WINDOWS) + list(APPEND MYSQL_TOOLS + echo + ) +else() + list(APPEND MYSQL_TOOLS + mysql_config + ) +endif() + +vcpkg_copy_tools(TOOL_NAMES ${MYSQL_TOOLS} AUTO_CLEAN) + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/lib/debug" +) + +if (NOT VCPKG_TARGET_IS_WINDOWS) + set(MYSQL_CONFIG_FILE "${CURRENT_PACKAGES_DIR}/tools/libmysql/mysql_config") + vcpkg_replace_string("${MYSQL_CONFIG_FILE}" "/bin/mysql_.*config" "/tools/libmysql/mysql_.*config") + vcpkg_replace_string("${MYSQL_CONFIG_FILE}" "'${CURRENT_PACKAGES_DIR}" "\"\$basedir\"\'") +endif() + +file(INSTALL "${CURRENT_PORT_DIR}/libmysql-config.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") + +set(libedit_copying "${SOURCE_PATH}/COPYING for libedit") +file(COPY_FILE "${SOURCE_PATH}/extra/libedit/libedit-20210910-3.1/COPYING" "${libedit_copying}") +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE" "${libedit_copying}") diff --git a/vcpkg_overlays/libmysql/usage b/vcpkg_overlays/libmysql/usage new file mode 100644 index 00000000000..7ea8c37e177 --- /dev/null +++ b/vcpkg_overlays/libmysql/usage @@ -0,0 +1,4 @@ +libmysql provides CMake targets: + + find_package(unofficial-libmysql REQUIRED) + target_link_libraries(main PRIVATE unofficial::libmysql::libmysql) diff --git a/vcpkg_overlays/libmysql/vcpkg.json b/vcpkg_overlays/libmysql/vcpkg.json new file mode 100644 index 00000000000..b143f59ef7f --- /dev/null +++ b/vcpkg_overlays/libmysql/vcpkg.json @@ -0,0 +1,47 @@ +{ + "name": "libmysql", + "version": "8.0.34", + "port-version": 1, + "description": "A MySQL client library for C development", + "homepage": "https://github.com/mysql/mysql-server", + "license": "GPL-2.0-or-later", + "supports": "!android & !mingw & !uwp & !xbox", + "dependencies": [ + "boost-algorithm", + "boost-functional", + "boost-geometry", + "boost-graph", + "boost-optional", + "lz4", + { + "name": "ncurses", + "platform": "!windows | mingw" + }, + "openssl", + "rapidjson", + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + }, + "zlib", + "zstd" + ], + "default-features": [ + "host-tools" + ], + "features": { + "host-tools": { + "description": "Build CLI tools", + "dependencies": [ + { + "name": "libmysql", + "host": true + } + ] + } + } +} \ No newline at end of file