Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC-29790 Switch to using > as a stage name prefix #17483

Merged
merged 1 commit into from
Jan 25, 2024
+146 −111
Merged

HPCC-29790 Switch to using > as a stage name prefix #17483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions common/workunit/workunit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3405,7 +3405,7 @@ void WuScopeFilter::finishedFilter()
{
sourceFlags &= ~(SSFsearchWorkflow);
}
else if (matchOnly(SSTcompilestage))
else if (matchOnly(SSToperation))
{
//compile stages are not stored in the graph
sourceFlags &= ~(SSFsearchGraphStats|SSFsearchGraph|SSFsearchWorkflow);
Expand Down Expand Up @@ -12346,7 +12346,7 @@ extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username,

{
Owned<IWorkUnit> wu = &cw->lock();
addTimeStamp(wu, SSTcompilestage, "compile", StWhenQueued, 0);
addTimeStamp(wu, SSToperation, ">compile", StWhenQueued, 0);
}

cw.clear();
Expand Down
4 changes: 2 additions & 2 deletions devdoc/Workunits.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ workunit dll:
creator="eclcc"
kind="SizePeakMemory"
s="compile"
scope="compile"
scope=">compile"
ts="1428933081084000"
unit="sz"
value="27885568"/>
Expand Down Expand Up @@ -1083,7 +1083,7 @@ the same time (the output filenames are derived from the input name).
creator="eclcc"
kind="SizePeakMemory"
s="compile"
scope="compile"
scope=">compile"
ts="1428933081084000"
unit="sz"
value="27885568" />
Expand Down
34 changes: 17 additions & 17 deletions ecl/eclcc/eclcc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1282,7 +1282,7 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
systemIoStartInfo.setown(new OsDiskStats(true));

if (optCompileBatchOut.isEmpty())
addTimeStamp(instance.wu, SSTcompilestage, "compile", StWhenStarted);
addTimeStamp(instance.wu, SSToperation, ">compile", StWhenStarted);
const char * sourcePathname = queryContents ? str(queryContents->querySourcePath()) : NULL;
const char * defaultErrorPathname = sourcePathname ? sourcePathname : queryAttributePath;

Expand Down Expand Up @@ -1411,7 +1411,7 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
if (exportDependencies || optMetaLocation)
parseCtx.nestedDependTree.setown(createPTree("Dependencies", ipt_fast));

addTimeStamp(instance.wu, SSTcompilestage, "compile:parse", StWhenStarted);
addTimeStamp(instance.wu, SSToperation, ">compile:>parse", StWhenStarted);
try
{
HqlLookupContext ctx(parseCtx, &errorProcessor, instance.dataServer);
Expand Down Expand Up @@ -1464,14 +1464,14 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
unsigned __int64 parseTimeNs = cycle_to_nanosec(get_cycles_now() - startCycles);
instance.stats.parseTime = (unsigned)nanoToMilli(parseTimeNs);

updateWorkunitStat(instance.wu, SSTcompilestage, "compile:parse", StTimeElapsed, NULL, parseTimeNs);
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse", StTimeElapsed, NULL, parseTimeNs);
stat_type sourceDownloadTime = localRepositoryManager.getStatistic(StTimeElapsed);
if (sourceDownloadTime)
updateWorkunitStat(instance.wu, SSTcompilestage, "compile:parse:download", StTimeElapsed, NULL, sourceDownloadTime);
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse:>download", StTimeElapsed, NULL, sourceDownloadTime);

if (optExtraStats)
{
updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
updateWorkunitStat(instance.wu, SSToperation, ">compile:>parse", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
}

if (exportDependencies)
Expand Down Expand Up @@ -1576,35 +1576,35 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
if (optGatherDiskStats)
systemIoFinishInfo.setown(new OsDiskStats(true));
instance.stats.generateTime = (unsigned)nanoToMilli(totalTimeNs) - instance.stats.parseTime;
const char *scopeName = optCompileBatchOut.isEmpty() ? "compile" : "compile:generate";
const char *scopeName = optCompileBatchOut.isEmpty() ? ">compile" : ">compile:>generate";
if (optCompileBatchOut.isEmpty())
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeElapsed, NULL, totalTimeNs);
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeElapsed, NULL, totalTimeNs);

const cost_type cost = money2cost_type(calcCost(getMachineCostRate(), nanoToMilli(totalTimeNs)));
if (cost)
instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, scopeName, StCostCompile, NULL, cost, 1, 0, StatsMergeReplace);
instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, scopeName, StCostCompile, NULL, cost, 1, 0, StatsMergeReplace);

if (systemFinishTime.getTotal())
{
SystemProcessInfo systemElapsed = systemFinishTime - systemStartTime;
SystemProcessInfo processElapsed = processFinishTime - processStartTime;
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StNumSysContextSwitches, NULL, systemElapsed.getNumContextSwitches());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsUser, NULL, systemElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsSystem, NULL, systemElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsTotal, NULL, systemElapsed.getTotalNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeUser, NULL, processElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeSystem, NULL, processElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StNumSysContextSwitches, NULL, systemElapsed.getNumContextSwitches());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeOsUser, NULL, systemElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeOsSystem, NULL, systemElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeOsTotal, NULL, systemElapsed.getTotalNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeUser, NULL, processElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StTimeSystem, NULL, processElapsed.getSystemNs());
if (processFinishTime.getPeakResidentMemory())
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizePeakMemory, NULL, processFinishTime.getPeakResidentMemory());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizePeakMemory, NULL, processFinishTime.getPeakResidentMemory());
}

if (optGatherDiskStats)
{
const BlockIoStats summaryIo = systemIoFinishInfo->querySummaryStats() - systemIoStartInfo->querySummaryStats();
if (summaryIo.rd_sectors)
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
if (summaryIo.wr_sectors)
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
updateWorkunitStat(instance.wu, SSToperation, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
}
}

Expand Down
22 changes: 11 additions & 11 deletions ecl/eclccserver/eclccserver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,8 +408,8 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (!extension || strncmp(extension, ".o", 2) == 0)
filename.set("link");

scope.append("compile:compile c++:").append(filename);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, scope, StTimeElapsed, NULL, compileTimer.elapsedNs(), 1, 0, StatsMergeReplace);
scope.append(">compile:>compile c++:").append(filename);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, scope, StTimeElapsed, NULL, compileTimer.elapsedNs(), 1, 0, StatsMergeReplace);
}

return ret;
Expand Down Expand Up @@ -683,7 +683,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
bool compileCppSeparately = config->getPropBool("@compileCppSeparately", true);
if (compileCppSeparately)
{
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
eclccCmd.appendf(" -Sx %s.cc", wuid);
}
if (workunit->getResultLimit())
Expand Down Expand Up @@ -717,15 +717,15 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (retcode == 0 && compileCppSeparately)
{
cycle_t startCompileCpp = get_cycles_now();
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile:compile c++", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile:>compile c++", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
retcode = doCompileCpp(abortWaiter, wuid, workunit->getDebugValueInt("maxCompileThreads", defaultMaxCompileThreads));
unsigned __int64 elapsed_compilecpp = cycle_to_nanosec(get_cycles_now() - startCompileCpp);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed_compilecpp, 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile:>compile c++", StTimeElapsed, NULL, elapsed_compilecpp, 1, 0, StatsMergeReplace);
}
if (compileCppSeparately)
{
unsigned __int64 elapsed_compile = cycle_to_nanosec(get_cycles_now() - startCompile);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StTimeElapsed, NULL, elapsed_compile, 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile", StTimeElapsed, NULL, elapsed_compile, 1, 0, StatsMergeReplace);
}
bool processKilled = (retcode >= 128);
//If the process is killed it is probably because it ran out of memory - so try to compile as a K8s job
Expand Down Expand Up @@ -859,8 +859,8 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid.get());
if (noteDequeued)
addTimeStamp(wu, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(wu, SSTcompilestage, "compile", StWhenK8sLaunched, 0);
addTimeStamp(wu, SSToperation, ">compile", StWhenDequeued, 0);
addTimeStamp(wu, SSToperation, ">compile", StWhenK8sLaunched, 0);
}
k8s::runJob("compile", wuid, wuid);
}
Expand Down Expand Up @@ -934,12 +934,12 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (isContainerized())
{
if (config->getPropBool("@k8sJob"))
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenK8sStarted, 0);
addTimeStamp(workunit, SSToperation, ">compile", StWhenK8sStarted, 0);
else
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(workunit, SSToperation, ">compile", StWhenDequeued, 0);
}
else
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(workunit, SSToperation, ">compile", StWhenDequeued, 0);

CSDSServerStatus serverstatus("ECLCCserverThread");
serverstatus.queryProperties()->setProp("@cluster", config->queryProp("@name"));
Expand Down
6 changes: 3 additions & 3 deletions ecl/hql/hqlexpr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1464,9 +1464,9 @@ void HqlLookupContext::reportTiming(const char * name)
__uint64 localTime = cycle_to_nanosec(elapsedCycles - childCycles);

StringBuffer scope;
scope.append("compiler::parse::").append(name);
parseCtx.statsTarget.addStatistic(SSTcompilestage, scope, StTimeTotalExecute, nullptr, totalTime, 1, 0, StatsMergeSum);
parseCtx.statsTarget.addStatistic(SSTcompilestage, scope, StTimeLocalExecute, nullptr, localTime, 1, 0, StatsMergeSum);
scope.append(">compile:>parse:>").append(name);
parseCtx.statsTarget.addStatistic(SSToperation, scope, StTimeTotalExecute, nullptr, totalTime, 1, 0, StatsMergeSum);
parseCtx.statsTarget.addStatistic(SSToperation, scope, StTimeLocalExecute, nullptr, localTime, 1, 0, StatsMergeSum);

childCycles = 0;
}
Expand Down
16 changes: 8 additions & 8 deletions ecl/hql/hqltrans.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,18 +166,18 @@ void HqlTransformStats::add(const HqlTransformStats & other)
void HqlTransformStats::gatherTransformStats(IStatisticTarget & target, const char * scope) const
{
#ifdef TRANSFORM_STATS_TIME
target.addStatistic(SSTcompilestage, scope, StTimeTotalExecute, nullptr, cycle_to_nanosec(totalTime), 1, 0, StatsMergeSum);
target.addStatistic(SSTcompilestage, scope, StTimeLocalExecute, nullptr, cycle_to_nanosec(totalTime-(childTime-recursiveTime)), 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StTimeTotalExecute, nullptr, cycle_to_nanosec(totalTime), 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StTimeLocalExecute, nullptr, cycle_to_nanosec(totalTime-(childTime-recursiveTime)), 1, 0, StatsMergeSum);
#endif
#ifdef TRANSFORM_STATS_DETAILS
if (numAnalyseCalls)
target.addStatistic(SSTcompilestage, scope, StNumAnalyseExprs, nullptr, numAnalyseCalls, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumAnalyseExprs, nullptr, numAnalyseCalls, 1, 0, StatsMergeSum);
if (numAnalyse)
target.addStatistic(SSTcompilestage, scope, StNumUniqueAnalyseExprs, nullptr, numAnalyse, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumUniqueAnalyseExprs, nullptr, numAnalyse, 1, 0, StatsMergeSum);
if (numTransformCalls)
target.addStatistic(SSTcompilestage, scope, StNumTransformExprs, nullptr, numTransformCalls, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumTransformExprs, nullptr, numTransformCalls, 1, 0, StatsMergeSum);
if (numTransforms)
target.addStatistic(SSTcompilestage, scope, StNumUniqueTransformExprs, nullptr, numTransforms, 1, 0, StatsMergeSum);
target.addStatistic(SSToperation, scope, StNumUniqueTransformExprs, nullptr, numTransforms, 1, 0, StatsMergeSum);
#endif
}

Expand Down Expand Up @@ -247,8 +247,8 @@ void HqlTransformerInfo::gatherTransformStats(IStatisticTarget & target) const
if (numInstances)
{
StringBuffer scope;
scope.append("compile:transform:").append(name);
target.addStatistic(SSTcompilestage, scope, StNumStarts, nullptr, numInstances, 1, 0, StatsMergeSum);
scope.append(">compile:>transform:>").append(name);
target.addStatistic(SSToperation, scope, StNumStarts, nullptr, numInstances, 1, 0, StatsMergeSum);
stats.gatherTransformStats(target, scope);
}
#endif
Expand Down
14 changes: 7 additions & 7 deletions ecl/hqlcpp/hqlecl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
noOutput = true;
{
// ensure warnings/errors are available before we do the processing...
addTimeStamp(wu, SSTcompilestage, "compile:generate", StWhenStarted);
addTimeStamp(wu, SSToperation, ">compile:>generate", StWhenStarted);
wu->commit();

cycle_t startCycles = get_cycles_now();
Expand Down Expand Up @@ -583,13 +583,13 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)

doExpand(translator);
unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
updateWorkunitStat(wu, SSTcompilestage, "compile:generate", StTimeElapsed, NULL, elapsed);
updateWorkunitStat(wu, SSToperation, ">compile:>generate", StTimeElapsed, NULL, elapsed);

if (wu->getDebugValueBool("addMemoryToWorkunit", true))
{
ProcessInfo info(ReadMemoryInfo);
if (info.getPeakResidentMemory())
wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StSizePeakMemory, NULL, info.getPeakResidentMemory(), 1, 0, StatsMergeReplace);
wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSToperation, ">compile", StSizePeakMemory, NULL, info.getPeakResidentMemory(), 1, 0, StatsMergeReplace);
}


Expand Down Expand Up @@ -633,7 +633,7 @@ void HqlDllGenerator::insertStandAloneCode()
void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
{
CCycleTimer elapsedTimer;
addTimeStamp(wu, SSTcompilestage, "compile:generate:write c++", StWhenStarted);
addTimeStamp(wu, SSToperation, ">compile:>generate:>write c++", StWhenStarted);

bool isMultiFile = translator.spanMultipleCppFiles();
CompilerType targetCompiler = translator.queryOptions().targetCompiler;
Expand All @@ -655,7 +655,7 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
}
}

updateWorkunitStat(wu, SSTcompilestage, "compile:generate:write c++", StTimeElapsed, NULL, elapsedTimer.elapsedNs());
updateWorkunitStat(wu, SSToperation, ">compile:>generate:>write c++", StTimeElapsed, NULL, elapsedTimer.elapsedNs());
}

bool HqlDllGenerator::abortRequested()
Expand All @@ -666,7 +666,7 @@ bool HqlDllGenerator::abortRequested()
bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
{
cycle_t startCycles = get_cycles_now();
addTimeStamp(wu, SSTcompilestage, "compile:compile c++", StWhenStarted);
addTimeStamp(wu, SSToperation, ">compile:>compile c++", StWhenStarted);
ForEachItemIn(i, sourceFiles)
compiler->addSourceFile(sourceFiles.item(i), sourceFlags.item(i));

Expand Down Expand Up @@ -735,7 +735,7 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
}

unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
updateWorkunitStat(wu, SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed);
updateWorkunitStat(wu, SSToperation, ">compile:>compile c++", StTimeElapsed, NULL, elapsed);
}
//Keep the files if there was a compile error.
if (ok && deleteGenerated)
Expand Down
Loading
Loading