Skip to content

Commit

Permalink
HPCC-29790 Use '>' as a prefix for stage names in stats scopes
Browse files Browse the repository at this point in the history 
Signed-off-by: Gavin Halliday <[email protected]>
  • Loading branch information
@ghalliday
ghalliday committed Jan 25, 2024
1 parent 3bca1fe commit 7a190fb
Show file tree
Hide file tree
Showing 18 changed files with 146 additions and 111 deletions.
4 changes: 2 additions & 2 deletions common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3405,7 +3405,7 @@ void WuScopeFilter::finishedFilter()
{
sourceFlags &= ~(SSFsearchWorkflow);
}
else if (matchOnly(SSTcompilestage))
else if (matchOnly(SSToperation))
{
//compile stages are not stored in the graph
sourceFlags &= ~(SSFsearchGraphStats|SSFsearchGraph|SSFsearchWorkflow);
Expand Down Expand Up @@ -12346,7 +12346,7 @@ extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username,

{
Owned<IWorkUnit> wu = &cw->lock();
addTimeStamp(wu, SSTcompilestage, "compile", StWhenQueued, 0);
addTimeStamp(wu, SSToperation, ">compile", StWhenQueued, 0);
}

cw.clear();
Expand Down
4 changes: 2 additions & 2 deletions devdoc/Workunits.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ workunit dll:
creator="eclcc"
kind="SizePeakMemory"
s="compile"
scope="compile"
scope=">compile"
ts="1428933081084000"
unit="sz"
value="27885568"/>
Expand Down Expand Up @@ -1083,7 +1083,7 @@ the same time (the output filenames are derived from the input name).
creator="eclcc"
kind="SizePeakMemory"
s="compile"
scope="compile"
scope=">compile"
ts="1428933081084000"
unit="sz"
value="27885568" />
Expand Down
34 changes: 17 additions & 17 deletions ecl/eclcc/eclcc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1282,7 +1282,7 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
systemIoStartInfo.setown(new OsDiskStats(true));

if (optCompileBatchOut.isEmpty())
addTimeStamp(instance.wu, SSTcompilestage, "compile", StWhenStarted);
addTimeStamp(instance.wu, SSToperation, ">compile", StWhenStarted);
const char * sourcePathname = queryContents ? str(queryContents->querySourcePath()) : NULL;
const char * defaultErrorPathname = sourcePathname ? sourcePathname : queryAttributePath;

Expand Down Expand Up @@ -1411,7 +1411,7 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
if (exportDependencies || optMetaLocation)
parseCtx.nestedDependTree.setown(createPTree("Dependencies", ipt_fast));

addTimeStamp(instance.wu, SSTcompilestage, "compile:parse", StWhenStarted);
addTimeStamp(instance.wu, SSToperation, ">compile:>parse", StWhenStarted);
try
{
HqlLookupContext ctx(parseCtx, &errorProcessor, instance.dataServer);
Expand Down Expand Up @@ -1464,14 +1464,14 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
unsigned __int64 parseTimeNs = cycle_to_nanosec(get_cycles_now() - startCycles);
instance.stats.parseTime = (unsigned)nanoToMilli(parseTimeNs);

updateWorkunitStat(instance.wu, SSTcompilestage, "compile:parse", StTimeElapsed, NULL, parseTimeNs);
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse", StTimeElapsed, NULL, parseTimeNs);
stat_type sourceDownloadTime = localRepositoryManager.getStatistic(StTimeElapsed);
if (sourceDownloadTime)
updateWorkunitStat(instance.wu, SSTcompilestage, "compile:parse:download", StTimeElapsed, NULL, sourceDownloadTime);
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse:>download", StTimeElapsed, NULL, sourceDownloadTime);

if (optExtraStats)
{
updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
}

if (exportDependencies)
Expand Down Expand Up @@ -1576,35 +1576,35 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
if (optGatherDiskStats)
systemIoFinishInfo.setown(new OsDiskStats(true));
instance.stats.generateTime = (unsigned)nanoToMilli(totalTimeNs) - instance.stats.parseTime;
const char *scopeName = optCompileBatchOut.isEmpty() ? "compile" : "compile:generate";
const char *scopeName = optCompileBatchOut.isEmpty() ? ">compile" : ">compile:>generate";
if (optCompileBatchOut.isEmpty())
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeElapsed, NULL, totalTimeNs);
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeElapsed, NULL, totalTimeNs);

const cost_type cost = money2cost_type(calcCost(getMachineCostRate(), nanoToMilli(totalTimeNs)));
if (cost)
instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, scopeName, StCostCompile, NULL, cost, 1, 0, StatsMergeReplace);
instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, scopeName, StCostCompile, NULL, cost, 1, 0, StatsMergeReplace);

if (systemFinishTime.getTotal())
{
SystemProcessInfo systemElapsed = systemFinishTime - systemStartTime;
SystemProcessInfo processElapsed = processFinishTime - processStartTime;
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StNumSysContextSwitches, NULL, systemElapsed.getNumContextSwitches());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsUser, NULL, systemElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsSystem, NULL, systemElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsTotal, NULL, systemElapsed.getTotalNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeUser, NULL, processElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeSystem, NULL, processElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StNumSysContextSwitches, NULL, systemElapsed.getNumContextSwitches());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeOsUser, NULL, systemElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeOsSystem, NULL, systemElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeOsTotal, NULL, systemElapsed.getTotalNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeUser, NULL, processElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeSystem, NULL, processElapsed.getSystemNs());
if (processFinishTime.getPeakResidentMemory())
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizePeakMemory, NULL, processFinishTime.getPeakResidentMemory());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizePeakMemory, NULL, processFinishTime.getPeakResidentMemory());
}

if (optGatherDiskStats)
{
const BlockIoStats summaryIo = systemIoFinishInfo->querySummaryStats() - systemIoStartInfo->querySummaryStats();
if (summaryIo.rd_sectors)
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
if (summaryIo.wr_sectors)
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
}
}

Expand Down
22 changes: 11 additions & 11 deletions ecl/eclccserver/eclccserver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,8 +408,8 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (!extension || strncmp(extension, ".o", 2) == 0)
filename.set("link");

scope.append("compile:compile c++:").append(filename);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, scope, StTimeElapsed, NULL, compileTimer.elapsedNs(), 1, 0, StatsMergeReplace);
scope.append(">compile:>compile c++:").append(filename);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, scope, StTimeElapsed, NULL, compileTimer.elapsedNs(), 1, 0, StatsMergeReplace);
}

return ret;
Expand Down Expand Up @@ -683,7 +683,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
bool compileCppSeparately = config->getPropBool("@compileCppSeparately", true);
if (compileCppSeparately)
{
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
eclccCmd.appendf(" -Sx %s.cc", wuid);
}
if (workunit->getResultLimit())
Expand Down Expand Up @@ -717,15 +717,15 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (retcode == 0 && compileCppSeparately)
{
cycle_t startCompileCpp = get_cycles_now();
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile:compile c++", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile:>compile c++", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
retcode = doCompileCpp(abortWaiter, wuid, workunit->getDebugValueInt("maxCompileThreads", defaultMaxCompileThreads));
unsigned __int64 elapsed_compilecpp = cycle_to_nanosec(get_cycles_now() - startCompileCpp);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed_compilecpp, 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile:>compile c++", StTimeElapsed, NULL, elapsed_compilecpp, 1, 0, StatsMergeReplace);
}
if (compileCppSeparately)
{
unsigned __int64 elapsed_compile = cycle_to_nanosec(get_cycles_now() - startCompile);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StTimeElapsed, NULL, elapsed_compile, 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile", StTimeElapsed, NULL, elapsed_compile, 1, 0, StatsMergeReplace);
}
bool processKilled = (retcode >= 128);
//If the process is killed it is probably because it ran out of memory - so try to compile as a K8s job
Expand Down Expand Up @@ -859,8 +859,8 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid.get());
if (noteDequeued)
addTimeStamp(wu, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(wu, SSTcompilestage, "compile", StWhenK8sLaunched, 0);
addTimeStamp(wu, SSToperation, ">compile", StWhenDequeued, 0);
addTimeStamp(wu, SSToperation, ">compile", StWhenK8sLaunched, 0);
}
k8s::runJob("compile", wuid, wuid);
}
Expand Down Expand Up @@ -934,12 +934,12 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (isContainerized())
{
if (config->getPropBool("@k8sJob"))
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenK8sStarted, 0);
addTimeStamp(workunit, SSToperation, ">compile", StWhenK8sStarted, 0);
else
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(workunit, SSToperation, ">compile", StWhenDequeued, 0);
}
else
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(workunit, SSToperation, ">compile", StWhenDequeued, 0);

CSDSServerStatus serverstatus("ECLCCserverThread");
serverstatus.queryProperties()->setProp("@cluster", config->queryProp("@name"));
Expand Down
6 changes: 3 additions & 3 deletions ecl/hql/hqlexpr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1464,9 +1464,9 @@ void HqlLookupContext::reportTiming(const char * name)
__uint64 localTime = cycle_to_nanosec(elapsedCycles - childCycles);

StringBuffer scope;
scope.append("compiler::parse::").append(name);
parseCtx.statsTarget.addStatistic(SSTcompilestage, scope, StTimeTotalExecute, nullptr, totalTime, 1, 0, StatsMergeSum);
parseCtx.statsTarget.addStatistic(SSTcompilestage, scope, StTimeLocalExecute, nullptr, localTime, 1, 0, StatsMergeSum);
scope.append(">compile:>parse:>").append(name);
parseCtx.statsTarget.addStatistic(SSToperation, scope, StTimeTotalExecute, nullptr, totalTime, 1, 0, StatsMergeSum);
parseCtx.statsTarget.addStatistic(SSToperation, scope, StTimeLocalExecute, nullptr, localTime, 1, 0, StatsMergeSum);

childCycles = 0;
}
Expand Down
16 changes: 8 additions & 8 deletions ecl/hql/hqltrans.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,18 +166,18 @@ void HqlTransformStats::add(const HqlTransformStats & other)
void HqlTransformStats::gatherTransformStats(IStatisticTarget & target, const char * scope) const
{
#ifdef TRANSFORM_STATS_TIME
target.addStatistic(SSTcompilestage, scope, StTimeTotalExecute, nullptr, cycle_to_nanosec(totalTime), 1, 0, StatsMergeSum);
target.addStatistic(SSTcompilestage, scope, StTimeLocalExecute, nullptr, cycle_to_nanosec(totalTime-(childTime-recursiveTime)), 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StTimeTotalExecute, nullptr, cycle_to_nanosec(totalTime), 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StTimeLocalExecute, nullptr, cycle_to_nanosec(totalTime-(childTime-recursiveTime)), 1, 0, StatsMergeSum);
#endif
#ifdef TRANSFORM_STATS_DETAILS
if (numAnalyseCalls)
target.addStatistic(SSTcompilestage, scope, StNumAnalyseExprs, nullptr, numAnalyseCalls, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumAnalyseExprs, nullptr, numAnalyseCalls, 1, 0, StatsMergeSum);
if (numAnalyse)
target.addStatistic(SSTcompilestage, scope, StNumUniqueAnalyseExprs, nullptr, numAnalyse, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumUniqueAnalyseExprs, nullptr, numAnalyse, 1, 0, StatsMergeSum);
if (numTransformCalls)
target.addStatistic(SSTcompilestage, scope, StNumTransformExprs, nullptr, numTransformCalls, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumTransformExprs, nullptr, numTransformCalls, 1, 0, StatsMergeSum);
if (numTransforms)
target.addStatistic(SSTcompilestage, scope, StNumUniqueTransformExprs, nullptr, numTransforms, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumUniqueTransformExprs, nullptr, numTransforms, 1, 0, StatsMergeSum);
#endif
}

Expand Down Expand Up @@ -247,8 +247,8 @@ void HqlTransformerInfo::gatherTransformStats(IStatisticTarget & target) const
if (numInstances)
{
StringBuffer scope;
scope.append("compile:transform:").append(name);
target.addStatistic(SSTcompilestage, scope, StNumStarts, nullptr, numInstances, 1, 0, StatsMergeSum);
scope.append(">compile:>transform:>").append(name);
target.addStatistic(SSToperation, scope, StNumStarts, nullptr, numInstances, 1, 0, StatsMergeSum);
stats.gatherTransformStats(target, scope);
}
#endif
Expand Down
14 changes: 7 additions & 7 deletions ecl/hqlcpp/hqlecl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
noOutput = true;
{
// ensure warnings/errors are available before we do the processing...
addTimeStamp(wu, SSTcompilestage, "compile:generate", StWhenStarted);
addTimeStamp(wu, SSToperation, ">compile:>generate", StWhenStarted);
wu->commit();

cycle_t startCycles = get_cycles_now();
Expand Down Expand Up @@ -583,13 +583,13 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)

doExpand(translator);
unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
updateWorkunitStat(wu, SSTcompilestage, "compile:generate", StTimeElapsed, NULL, elapsed);
updateWorkunitStat(wu, SSToperation, ">compile:>generate", StTimeElapsed, NULL, elapsed);

if (wu->getDebugValueBool("addMemoryToWorkunit", true))
{
ProcessInfo info(ReadMemoryInfo);
if (info.getPeakResidentMemory())
wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StSizePeakMemory, NULL, info.getPeakResidentMemory(), 1, 0, StatsMergeReplace);
wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile", StSizePeakMemory, NULL, info.getPeakResidentMemory(), 1, 0, StatsMergeReplace);
}


Expand Down Expand Up @@ -633,7 +633,7 @@ void HqlDllGenerator::insertStandAloneCode()
void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
{
CCycleTimer elapsedTimer;
addTimeStamp(wu, SSTcompilestage, "compile:generate:write c++", StWhenStarted);
addTimeStamp(wu, SSToperation, ">compile:>generate:>write c++", StWhenStarted);

bool isMultiFile = translator.spanMultipleCppFiles();
CompilerType targetCompiler = translator.queryOptions().targetCompiler;
Expand All @@ -655,7 +655,7 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
}
}

updateWorkunitStat(wu, SSTcompilestage, "compile:generate:write c++", StTimeElapsed, NULL, elapsedTimer.elapsedNs());
updateWorkunitStat(wu, SSToperation, ">compile:>generate:>write c++", StTimeElapsed, NULL, elapsedTimer.elapsedNs());
}

bool HqlDllGenerator::abortRequested()
Expand All @@ -666,7 +666,7 @@ bool HqlDllGenerator::abortRequested()
bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
{
cycle_t startCycles = get_cycles_now();
addTimeStamp(wu, SSTcompilestage, "compile:compile c++", StWhenStarted);
addTimeStamp(wu, SSToperation, ">compile:>compile c++", StWhenStarted);
ForEachItemIn(i, sourceFiles)
compiler->addSourceFile(sourceFiles.item(i), sourceFlags.item(i));

Expand Down Expand Up @@ -735,7 +735,7 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
}

unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
updateWorkunitStat(wu, SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed);
updateWorkunitStat(wu, SSToperation, ">compile:>compile c++", StTimeElapsed, NULL, elapsed);
}
//Keep the files if there was a compile error.
if (ok && deleteGenerated)
Expand Down
Loading

0 comments on commit 7a190fb

Please sign in to comment.