Skip to content

Commit

Permalink
HPCC-29790 Switch to using > as a stage name prefix
Browse files Browse the repository at this point in the history 
Signed-off-by: Gavin Halliday <[email protected]>
  • Loading branch information
@ghalliday
ghalliday committed Jun 22, 2023
1 parent 833be7a commit 88f19cc
Show file tree
Hide file tree
Showing 14 changed files with 112 additions and 51 deletions.
4 changes: 2 additions & 2 deletions common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3381,7 +3381,7 @@ void WuScopeFilter::finishedFilter()
{
sourceFlags &= ~(SSFsearchWorkflow);
}
else if (matchOnly(SSTcompilestage))
else if (matchOnly(SSTstage))
{
//compile stages are not stored in the graph
sourceFlags &= ~(SSFsearchGraphStats|SSFsearchGraph|SSFsearchWorkflow);
Expand Down Expand Up @@ -12262,7 +12262,7 @@ extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username,

{
Owned<IWorkUnit> wu = &cw->lock();
addTimeStamp(wu, SSTcompilestage, "compile", StWhenQueued, 0);
addTimeStamp(wu, SSTstage, "compile", StWhenQueued, 0);
}

cw.clear();
Expand Down
33 changes: 18 additions & 15 deletions ecl/eclcc/eclcc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1281,7 +1281,7 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
systemIoStartInfo.setown(new OsDiskStats(true));

if (optCompileBatchOut.isEmpty())
addTimeStamp(instance.wu, SSTcompilestage, "compile", StWhenStarted);
addTimeStamp(instance.wu, SSTstage, "compile", StWhenStarted);
const char * sourcePathname = queryContents ? str(queryContents->querySourcePath()) : NULL;
const char * defaultErrorPathname = sourcePathname ? sourcePathname : queryAttributePath;

Expand Down Expand Up @@ -1410,7 +1410,7 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
if (exportDependencies || optMetaLocation)
parseCtx.nestedDependTree.setown(createPTree("Dependencies", ipt_fast));

addTimeStamp(instance.wu, SSTcompilestage, "compile:parse", StWhenStarted);
addTimeStamp(instance.wu, SSTstage, "compile:>parse", StWhenStarted);
try
{
HqlLookupContext ctx(parseCtx, &errorProcessor, instance.dataServer);
Expand Down Expand Up @@ -1463,11 +1463,14 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
unsigned __int64 parseTimeNs = cycle_to_nanosec(get_cycles_now() - startCycles);
instance.stats.parseTime = (unsigned)nanoToMilli(parseTimeNs);

updateWorkunitStat(instance.wu, SSTcompilestage, "compile:parse", StTimeElapsed, NULL, parseTimeNs);
updateWorkunitStat(instance.wu, SSTstage, "compile:>parse", StTimeElapsed, NULL, parseTimeNs);
stat_type sourceDownloadTime = localRepositoryManager.getStatistic(StTimeElapsed);
if (sourceDownloadTime)
updateWorkunitStat(instance.wu, SSTstage, "compile:>parse:>download", StTimeElapsed, NULL, sourceDownloadTime);

if (optExtraStats)
{
updateWorkunitStat(instance.wu, SSTcompilestage, "compile:cache", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
updateWorkunitStat(instance.wu, SSTstage, "compile:cache", StNumAttribsProcessed, NULL, parseCtx.numAttribsProcessed);
}

if (exportDependencies)
Expand Down Expand Up @@ -1574,33 +1577,33 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
instance.stats.generateTime = (unsigned)nanoToMilli(totalTimeNs) - instance.stats.parseTime;
const char *scopeName = optCompileBatchOut.isEmpty() ? "compile" : "compile:generate";
if (optCompileBatchOut.isEmpty())
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeElapsed, NULL, totalTimeNs);
updateWorkunitStat(instance.wu, SSTstage, scopeName, StTimeElapsed, NULL, totalTimeNs);

const cost_type cost = money2cost_type(calcCost(getMachineCostRate(), nanoToMilli(totalTimeNs)));
if (cost)
instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, scopeName, StCostCompile, NULL, cost, 1, 0, StatsMergeReplace);
instance.wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, scopeName, StCostCompile, NULL, cost, 1, 0, StatsMergeReplace);

if (systemFinishTime.getTotal())
{
SystemProcessInfo systemElapsed = systemFinishTime - systemStartTime;
SystemProcessInfo processElapsed = processFinishTime - processStartTime;
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StNumSysContextSwitches, NULL, systemElapsed.getNumContextSwitches());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsUser, NULL, systemElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsSystem, NULL, systemElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeOsTotal, NULL, systemElapsed.getTotalNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeUser, NULL, processElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StTimeSystem, NULL, processElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StNumSysContextSwitches, NULL, systemElapsed.getNumContextSwitches());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StTimeOsUser, NULL, systemElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StTimeOsSystem, NULL, systemElapsed.getSystemNs());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StTimeOsTotal, NULL, systemElapsed.getTotalNs());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StTimeUser, NULL, processElapsed.getUserNs());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StTimeSystem, NULL, processElapsed.getSystemNs());
if (processFinishTime.getPeakResidentMemory())
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizePeakMemory, NULL, processFinishTime.getPeakResidentMemory());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StSizePeakMemory, NULL, processFinishTime.getPeakResidentMemory());
}

if (optGatherDiskStats)
{
const BlockIoStats summaryIo = systemIoFinishInfo->querySummaryStats() - systemIoStartInfo->querySummaryStats();
if (summaryIo.rd_sectors)
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StSizeOsDiskRead, NULL, summaryIo.rd_sectors * summaryIo.getSectorSize());
if (summaryIo.wr_sectors)
updateWorkunitStat(instance.wu, SSTcompilestage, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
updateWorkunitStat(instance.wu, SSTstage, scopeName, StSizeOsDiskWrite, NULL, summaryIo.wr_sectors * summaryIo.getSectorSize());
}
}

Expand Down
20 changes: 10 additions & 10 deletions ecl/eclccserver/eclccserver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
filename.set("link");

scope.append("compile:compile c++:").append(filename);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, scope, StTimeElapsed, NULL, compileTimer.elapsedNs(), 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, scope, StTimeElapsed, NULL, compileTimer.elapsedNs(), 1, 0, StatsMergeReplace);
}

return ret;
Expand Down Expand Up @@ -682,7 +682,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
bool compileCppSeparately = config->getPropBool("@compileCppSeparately", true);
if (compileCppSeparately)
{
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, "compile", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
eclccCmd.appendf(" -Sx %s.cc", wuid);
}
if (workunit->getResultLimit())
Expand Down Expand Up @@ -716,15 +716,15 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (retcode == 0 && compileCppSeparately)
{
cycle_t startCompileCpp = get_cycles_now();
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile:compile c++", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, "compile:compile c++", StWhenStarted, NULL, getTimeStampNowValue(), 1, 0, StatsMergeAppend);
retcode = doCompileCpp(abortWaiter, wuid, workunit->getDebugValueInt("maxCompileThreads", defaultMaxCompileThreads));
unsigned __int64 elapsed_compilecpp = cycle_to_nanosec(get_cycles_now() - startCompileCpp);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed_compilecpp, 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, "compile:compile c++", StTimeElapsed, NULL, elapsed_compilecpp, 1, 0, StatsMergeReplace);
}
if (compileCppSeparately)
{
unsigned __int64 elapsed_compile = cycle_to_nanosec(get_cycles_now() - startCompile);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StTimeElapsed, NULL, elapsed_compile, 1, 0, StatsMergeReplace);
workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, "compile", StTimeElapsed, NULL, elapsed_compile, 1, 0, StatsMergeReplace);
}
bool processKilled = (retcode >= 128);
//If the process is killed it is probably because it ran out of memory - so try to compile as a K8s job
Expand Down Expand Up @@ -851,8 +851,8 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid.get());
if (noteDequeued)
addTimeStamp(wu, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(wu, SSTcompilestage, "compile", StWhenK8sLaunched, 0);
addTimeStamp(wu, SSTstage, "compile", StWhenDequeued, 0);
addTimeStamp(wu, SSTstage, "compile", StWhenK8sLaunched, 0);
}
runK8sJob("compile", wuid, wuid);
}
Expand Down Expand Up @@ -917,12 +917,12 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
if (isContainerized())
{
if (config->getPropBool("@k8sJob"))
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenK8sStarted, 0);
addTimeStamp(workunit, SSTstage, "compile", StWhenK8sStarted, 0);
else
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(workunit, SSTstage, "compile", StWhenDequeued, 0);
}
else
addTimeStamp(workunit, SSTcompilestage, "compile", StWhenDequeued, 0);
addTimeStamp(workunit, SSTstage, "compile", StWhenDequeued, 0);

CSDSServerStatus serverstatus("ECLCCserverThread");
serverstatus.queryProperties()->setProp("@cluster", config->queryProp("@name"));
Expand Down
6 changes: 3 additions & 3 deletions ecl/hql/hqlexpr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1464,9 +1464,9 @@ void HqlLookupContext::reportTiming(const char * name)
__uint64 localTime = cycle_to_nanosec(elapsedCycles - childCycles);

StringBuffer scope;
scope.append("compiler::parse::").append(name);
parseCtx.statsTarget.addStatistic(SSTcompilestage, scope, StTimeTotalExecute, nullptr, totalTime, 1, 0, StatsMergeSum);
parseCtx.statsTarget.addStatistic(SSTcompilestage, scope, StTimeLocalExecute, nullptr, localTime, 1, 0, StatsMergeSum);
scope.append("compile:>parse:>").append(name);
parseCtx.statsTarget.addStatistic(SSTstage, scope, StTimeTotalExecute, nullptr, totalTime, 1, 0, StatsMergeSum);
parseCtx.statsTarget.addStatistic(SSTstage, scope, StTimeLocalExecute, nullptr, localTime, 1, 0, StatsMergeSum);

childCycles = 0;
}
Expand Down
12 changes: 12 additions & 0 deletions ecl/hql/hqlrepository.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,16 @@ void EclRepositoryManager::gatherPackagesUsed(StringArray & used) const
}
}

unsigned __int64 EclRepositoryManager::getStatistic(StatisticKind kind) const
{
switch (kind)
{
case StTimeElapsed:
return cycle_to_nanosec(gitDownloadCycles);
}
return 0;
}

void EclRepositoryManager::processArchive(IPropertyTree * archiveTree)
{
IArrayOf<IEclRepository> savedSources; // also includes -D options
Expand Down Expand Up @@ -793,6 +803,7 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
}

bool ok = false;
CCycleTimer gitDownloadTimer;
if (alreadyExists)
{
if (options.updateRepos)
Expand All @@ -819,6 +830,7 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
ok = true;
}
}
gitDownloadCycles += gitDownloadTimer.elapsedCycles();

if (!ok)
throw makeStringExceptionV(99, "Cannot locate the source code for dependency '%s'. --fetchrepos not enabled", defaultUrl);
Expand Down
2 changes: 2 additions & 0 deletions ecl/hql/hqlrepository.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ class HQL_API EclRepositoryManager
void gatherPackagesUsed(StringArray & used) const;
void inherit(const EclRepositoryManager & other);
void kill();
unsigned __int64 getStatistic(StatisticKind kind) const;

void processArchive(IPropertyTree * archiveTree);
IEclPackage * queryDependentRepository(IIdAtom * name, const char * defaultUrl, IEclSourceCollection * overrideSources);
Expand Down Expand Up @@ -86,6 +87,7 @@ class HQL_API EclRepositoryManager
std::vector<DependencyInfo> dependencies;
IArrayOf<IEclRepository> sharedSources; // plugins, std library, bundles
IArrayOf<IEclRepository> allSources; // also includes -D options
cycle_t gitDownloadCycles = 0;

//Include all options in a nested struct to make it easy to ensure they are cloned
struct {
Expand Down
14 changes: 7 additions & 7 deletions ecl/hql/hqltrans.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,18 +166,18 @@ void HqlTransformStats::add(const HqlTransformStats & other)
void HqlTransformStats::gatherTransformStats(IStatisticTarget & target, const char * scope) const
{
#ifdef TRANSFORM_STATS_TIME
target.addStatistic(SSTcompilestage, scope, StTimeTotalExecute, nullptr, cycle_to_nanosec(totalTime), 1, 0, StatsMergeSum);
target.addStatistic(SSTcompilestage, scope, StTimeLocalExecute, nullptr, cycle_to_nanosec(totalTime-(childTime-recursiveTime)), 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StTimeTotalExecute, nullptr, cycle_to_nanosec(totalTime), 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StTimeLocalExecute, nullptr, cycle_to_nanosec(totalTime-(childTime-recursiveTime)), 1, 0, StatsMergeSum);
#endif
#ifdef TRANSFORM_STATS_DETAILS
if (numAnalyseCalls)
target.addStatistic(SSTcompilestage, scope, StNumAnalyseExprs, nullptr, numAnalyseCalls, 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StNumAnalyseExprs, nullptr, numAnalyseCalls, 1, 0, StatsMergeSum);
if (numAnalyse)
target.addStatistic(SSTcompilestage, scope, StNumUniqueAnalyseExprs, nullptr, numAnalyse, 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StNumUniqueAnalyseExprs, nullptr, numAnalyse, 1, 0, StatsMergeSum);
if (numTransformCalls)
target.addStatistic(SSTcompilestage, scope, StNumTransformExprs, nullptr, numTransformCalls, 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StNumTransformExprs, nullptr, numTransformCalls, 1, 0, StatsMergeSum);
if (numTransforms)
target.addStatistic(SSTcompilestage, scope, StNumUniqueTransformExprs, nullptr, numTransforms, 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StNumUniqueTransformExprs, nullptr, numTransforms, 1, 0, StatsMergeSum);
#endif
}

Expand Down Expand Up @@ -248,7 +248,7 @@ void HqlTransformerInfo::gatherTransformStats(IStatisticTarget & target) const
{
StringBuffer scope;
scope.append("compile:transform:").append(name);
target.addStatistic(SSTcompilestage, scope, StNumStarts, nullptr, numInstances, 1, 0, StatsMergeSum);
target.addStatistic(SSTstage, scope, StNumStarts, nullptr, numInstances, 1, 0, StatsMergeSum);
stats.gatherTransformStats(target, scope);
}
#endif
Expand Down
14 changes: 7 additions & 7 deletions ecl/hqlcpp/hqlecl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
noOutput = true;
{
// ensure warnings/errors are available before we do the processing...
addTimeStamp(wu, SSTcompilestage, "compile:generate", StWhenStarted);
addTimeStamp(wu, SSTstage, "compile:generate", StWhenStarted);
wu->commit();

cycle_t startCycles = get_cycles_now();
Expand Down Expand Up @@ -583,13 +583,13 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)

doExpand(translator);
unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
updateWorkunitStat(wu, SSTcompilestage, "compile:generate", StTimeElapsed, NULL, elapsed);
updateWorkunitStat(wu, SSTstage, "compile:generate", StTimeElapsed, NULL, elapsed);

if (wu->getDebugValueBool("addMemoryToWorkunit", true))
{
ProcessInfo info(ReadMemoryInfo);
if (info.getPeakResidentMemory())
wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTcompilestage, "compile", StSizePeakMemory, NULL, info.getPeakResidentMemory(), 1, 0, StatsMergeReplace);
wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTstage, "compile", StSizePeakMemory, NULL, info.getPeakResidentMemory(), 1, 0, StatsMergeReplace);
}


Expand Down Expand Up @@ -633,7 +633,7 @@ void HqlDllGenerator::insertStandAloneCode()
void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
{
CCycleTimer elapsedTimer;
addTimeStamp(wu, SSTcompilestage, "compile:generate:write c++", StWhenStarted);
addTimeStamp(wu, SSTstage, "compile:generate:write c++", StWhenStarted);

bool isMultiFile = translator.spanMultipleCppFiles();
CompilerType targetCompiler = translator.queryOptions().targetCompiler;
Expand All @@ -655,7 +655,7 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
}
}

updateWorkunitStat(wu, SSTcompilestage, "compile:generate:write c++", StTimeElapsed, NULL, elapsedTimer.elapsedNs());
updateWorkunitStat(wu, SSTstage, "compile:generate:write c++", StTimeElapsed, NULL, elapsedTimer.elapsedNs());
}

bool HqlDllGenerator::abortRequested()
Expand All @@ -666,7 +666,7 @@ bool HqlDllGenerator::abortRequested()
bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
{
cycle_t startCycles = get_cycles_now();
addTimeStamp(wu, SSTcompilestage, "compile:compile c++", StWhenStarted);
addTimeStamp(wu, SSTstage, "compile:compile c++", StWhenStarted);
ForEachItemIn(i, sourceFiles)
compiler->addSourceFile(sourceFiles.item(i), sourceFlags.item(i));

Expand Down Expand Up @@ -735,7 +735,7 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
}

unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
updateWorkunitStat(wu, SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed);
updateWorkunitStat(wu, SSTstage, "compile:compile c++", StTimeElapsed, NULL, elapsed);
}
//Keep the files if there was a compile error.
if (ok && deleteGenerated)
Expand Down
2 changes: 1 addition & 1 deletion ecl/hqlcpp/hqlhtcpp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6237,7 +6237,7 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
void HqlCppTranslator::ensureWorkUnitUpdated()
{
if (timeReporter)
updateWorkunitTimings(wu(), SSTcompilestage, StTimeTotalExecute, timeReporter);
updateWorkunitTimings(wu(), SSTstage, StTimeTotalExecute, timeReporter);
}

double HqlCppTranslator::getComplexity(IHqlExpression * expr, ClusterType cluster)
Expand Down
Loading

0 comments on commit 88f19cc

Please sign in to comment.