Skip to content

Commit

Permalink
Support for isOpt
Browse files Browse the repository at this point in the history 
Signed-off-by: Richard Chapman <[email protected]>
  • Loading branch information
@richardkchapman
richardkchapman committed Jun 19, 2024
1 parent 40dc79d commit c6b6a42
Show file tree
Hide file tree
Showing 7 changed files with 103 additions and 51 deletions.
85 changes: 61 additions & 24 deletions common/pkgfiles/referencedfilelist.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -965,32 +965,13 @@ void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm)

bool ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
{
Owned<IConstWUGraphIterator> graphs = &cw->getGraphs(GraphTypeActivities);
ForEach(*graphs)
SummaryMap files;
if (cw->getSummary(SummaryType::ReadFile, files) &&
cw->getSummary(SummaryType::ReadIndex, files))
{
Owned <IPropertyTree> xgmml = graphs->query().getXGMMLTree(false, false);
Owned<IPropertyTreeIterator> iter = xgmml->getElements("//node[att/@name='_*ileName']");
ForEach(*iter)
for (const auto& [lName, isOpt] : files)
{
IPropertyTree &node = iter->query();
bool isOpt = false;
const char *logicalName = node.queryProp("att[@name='_fileName']/@value");
if (!logicalName)
logicalName = node.queryProp("att[@name='_indexFileName']/@value");
if (!logicalName)
continue;

isOpt = node.getPropBool("att[@name='_isIndexOpt']/@value");
if (!isOpt)
isOpt = node.getPropBool("att[@name='_isOpt']/@value");

ThorActivityKind kind = (ThorActivityKind) node.getPropInt("att[@name='_kind']/@value", TAKnone);
//not likely to be part of roxie queries, but for forward compatibility:
if(kind==TAKdiskwrite || kind==TAKspillwrite || kind==TAKindexwrite || kind==TAKcsvwrite || kind==TAKxmlwrite || kind==TAKjsonwrite)
continue;
if (node.getPropBool("att[@name='_isSpill']/@value") ||
node.getPropBool("att[@name='_isTransformSpill']/@value"))
continue;
const char *logicalName = lName.c_str();
StringArray subfileNames;
unsigned flags = isOpt ? RefFileOptional : RefFileNotOptional;
if (pkg)
Expand Down Expand Up @@ -1018,6 +999,62 @@ bool ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackag
ensureFile(logicalName, flags, NULL, false, &subfileNames);
}
}
else
{
Owned<IConstWUGraphIterator> graphs = &cw->getGraphs(GraphTypeActivities);
ForEach(*graphs)
{
Owned <IPropertyTree> xgmml = graphs->query().getXGMMLTree(false, false);
Owned<IPropertyTreeIterator> iter = xgmml->getElements("//node[att/@name='_*ileName']");
ForEach(*iter)
{
IPropertyTree &node = iter->query();
bool isOpt = false;
const char *logicalName = node.queryProp("att[@name='_fileName']/@value");
if (!logicalName)
logicalName = node.queryProp("att[@name='_indexFileName']/@value");
if (!logicalName)
continue;

isOpt = node.getPropBool("att[@name='_isIndexOpt']/@value");
if (!isOpt)
isOpt = node.getPropBool("att[@name='_isOpt']/@value");

ThorActivityKind kind = (ThorActivityKind) node.getPropInt("att[@name='_kind']/@value", TAKnone);
//not likely to be part of roxie queries, but for forward compatibility:
if(kind==TAKdiskwrite || kind==TAKspillwrite || kind==TAKindexwrite || kind==TAKcsvwrite || kind==TAKxmlwrite || kind==TAKjsonwrite)
continue;
if (node.getPropBool("att[@name='_isSpill']/@value") ||
node.getPropBool("att[@name='_isTransformSpill']/@value"))
continue;
StringArray subfileNames;
unsigned flags = isOpt ? RefFileOptional : RefFileNotOptional;
if (pkg)
{
const char *pkgid = pkg->locateSuperFile(logicalName);
if (pkgid)
{
flags |= (RefFileSuper | RefFileInPackage);
Owned<ISimpleSuperFileEnquiry> ssfe = pkg->resolveSuperFile(logicalName);
if (ssfe && ssfe->numSubFiles()>0)
{
unsigned count = ssfe->numSubFiles();
while (count--)
{
StringBuffer subfile;
ssfe->getSubFileName(count, subfile);
ensureFile(subfile, RefSubFile | RefFileInPackage, pkgid, false, nullptr);
subfileNames.append(subfile);
}
}
}
ensureFile(logicalName, flags, pkgid, pkg->isCompulsory(), &subfileNames);
}
else
ensureFile(logicalName, flags, NULL, false, &subfileNames);
}
}
}
return pkg ? pkg->isCompulsory() : false;
}

Expand Down
17 changes: 12 additions & 5 deletions common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8747,11 +8747,18 @@ bool CLocalWorkUnit::getSummary(SummaryType type, SummaryMap &map) const
IPropertyTree *list = p->queryPropTree(xpath);
if (!list)
return false;
Owned<IPropertyTreeIterator> r = list->getElements("v");
ForEach(*r)
Owned<IPropertyTreeIterator> v = list->getElements("v");
ForEach(*v)
{
const char *val = r->query().queryProp(".");
map[val] = true;
const char *name = v->query().queryProp(".");
map[name] = false;
}
Owned<IPropertyTreeIterator> o = list->getElements("o");
ForEach(*o)
{
const char *name = o->query().queryProp(".");
if (map.find(name) != map.end())
map[name] = true;
}
return true;
}
Expand All @@ -8762,7 +8769,7 @@ void CLocalWorkUnit::setSummary(SummaryType type, const SummaryMap &map)
IPropertyTree *summaries = ensurePTree(p, "Summaries");
IPropertyTree *list = ensurePTree(summaries, summaryTypeName(type));
for (const auto& [key, value] : map)
list->addProp("v", key.c_str());
list->addProp(value ? "o" : "v", key.c_str());
}

void CLocalWorkUnit::setDebugValueInt(const char *propname, int value, bool overwrite)
Expand Down
1 change: 1 addition & 0 deletions common/workunit/workunit.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1181,6 +1181,7 @@ interface IConstWUScopeIterator : extends IScmIterator
//! IWorkUnit
//! Provides high level access to WorkUnit "header" data.

// Be sure to update summaryTypeName in workunit.cpp if adding anything here
enum class SummaryType
{
First,
Expand Down
12 changes: 6 additions & 6 deletions ecl/hqlcpp/hqlckey.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1192,7 +1192,7 @@ void HqlCppTranslator::buildKeyedJoinExtra(ActivityInstance & instance, IHqlExpr

//virtual const char * getFileName() = 0; // Returns filename of raw file fpos'es refer into
if (info->isFullJoin())
buildFilenameFunction(instance, instance.createctx, WaFilename, "getFileName", info->queryFileFilename(), hasDynamicFilename(info->queryFile()), SummaryType::ReadFile);
buildFilenameFunction(instance, instance.createctx, WaFilename, "getFileName", info->queryFileFilename(), hasDynamicFilename(info->queryFile()), SummaryType::ReadFile, info->isKeyOpt());

//virtual bool diskAccessRequired() = 0;
if (info->isFullJoin())
Expand Down Expand Up @@ -1229,7 +1229,7 @@ void HqlCppTranslator::buildKeyJoinIndexReadHelper(ActivityInstance & instance,
info->buildExtractIndexReadFields(instance.startctx);

//virtual const char * getIndexFileName() = 0;
buildFilenameFunction(instance, instance.startctx, WaIndexname, "getIndexFileName", info->queryKeyFilename(), hasDynamicFilename(info->queryKey()), SummaryType::ReadIndex);
buildFilenameFunction(instance, instance.startctx, WaIndexname, "getIndexFileName", info->queryKeyFilename(), hasDynamicFilename(info->queryKey()), SummaryType::ReadIndex, info->isKeyOpt());

//virtual IOutputMetaData * queryIndexRecordSize() = 0;
LinkedHqlExpr indexExpr = info->queryOriginalKey();
Expand Down Expand Up @@ -1489,7 +1489,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyedDistribute(BuildCtx & ctx
doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);

//virtual const char * getIndexFileName() = 0;
buildFilenameFunction(*instance, instance->startctx, WaIndexname, "getIndexFileName", keyFilename, dynamic, SummaryType::ReadIndex);
buildFilenameFunction(*instance, instance->startctx, WaIndexname, "getIndexFileName", keyFilename, dynamic, SummaryType::ReadIndex, info.isKeyOpt());

//virtual IOutputMetaData * queryIndexRecordSize() = 0;
LinkedHqlExpr indexExpr = info.queryRawKey();
Expand Down Expand Up @@ -1583,7 +1583,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyDiff(BuildCtx & ctx, IHqlEx
noteAllFieldsUsed(updated);

//virtual const char * getOutputName() = 0;
buildFilenameFunction(*instance, instance->startctx, WaOutputFilename, "getOutputName", output, hasDynamicFilename(expr), SummaryType::WriteFile);
buildFilenameFunction(*instance, instance->startctx, WaOutputFilename, "getOutputName", output, hasDynamicFilename(expr), SummaryType::WriteFile, false);

//virtual int getSequence() = 0;
doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
Expand Down Expand Up @@ -1626,10 +1626,10 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyPatch(BuildCtx & ctx, IHqlE
noteAllFieldsUsed(original);

//virtual const char * getPatchName() = 0;
buildFilenameFunction(*instance, instance->startctx, WaPatchFilename, "getPatchName", patch, true, SummaryType::ReadFile);
buildFilenameFunction(*instance, instance->startctx, WaPatchFilename, "getPatchName", patch, true, SummaryType::ReadFile, false);

//virtual const char * getOutputName() = 0;
buildFilenameFunction(*instance, instance->startctx, WaOutputFilename, "getOutputName", output, hasDynamicFilename(expr), SummaryType::WriteIndex);
buildFilenameFunction(*instance, instance->startctx, WaOutputFilename, "getOutputName", output, hasDynamicFilename(expr), SummaryType::WriteIndex, false);

//virtual int getSequence() = 0;
doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
Expand Down
15 changes: 11 additions & 4 deletions ecl/hqlcpp/hqlcpp.ipp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1887,7 +1887,7 @@ public:
void doBuildUserFunctionReturn(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * value);

void addFilenameConstructorParameter(ActivityInstance & instance, WuAttr attr, IHqlExpression * expr, SummaryType summaryType);
void buildFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, WuAttr attr, const char * name, IHqlExpression * expr, bool isDynamic, SummaryType summaryType);
void buildFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, WuAttr attr, const char * name, IHqlExpression * expr, bool isDynamic, SummaryType summaryType, bool isOpt);
void buildRefFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, WuAttr attr, const char * name, IHqlExpression * dataset);
void createAccessFunctions(StringBuffer & helperFunc, BuildCtx & declarectx, unsigned prio, const char * interfaceName, const char * object);

Expand All @@ -1911,7 +1911,7 @@ protected:
void buildIteratorNext(BuildCtx & ctx, IHqlExpression * iter, IHqlExpression * row);
bool shouldEvaluateSelectAsAlias(BuildCtx & ctx, IHqlExpression * expr);
IWUResult * createWorkunitResult(int sequence, IHqlExpression * nameExpr);
void noteFilename(ActivityInstance & instance, WuAttr attr, IHqlExpression * expr, bool isDynamic, SummaryType summaryType);
void noteFilename(ActivityInstance & instance, WuAttr attr, IHqlExpression * expr, bool isDynamic, SummaryType summaryType, bool isOpt);
bool checkGetResultContext(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
void buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * boundTarget, const CHqlBoundTarget * targetAssign);
void buildGetResultSetInfo(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * boundTarget, const CHqlBoundTarget * targetAssign);
Expand Down Expand Up @@ -2144,9 +2144,16 @@ protected:

// These are used to generate workunit summary info, to avoid having to walk the xgmml to get it
SummaryMap summaries[(int) SummaryType::NumItems];
void noteSummaryInfo(const char *name, SummaryType type)
void noteSummaryInfo(const char *name, SummaryType type, bool isOpt)
{
summaries[(int) type][name] = true;
SummaryMap &map = summaries[(int) type];
if (isOpt)
{
if (map.find(name) == map.end())
map[name] = true;
}
else
map[name] = false;
}
};

Expand Down
20 changes: 10 additions & 10 deletions ecl/hqlcpp/hqlhtcpp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3386,17 +3386,17 @@ void HqlCppTranslator::addFilenameConstructorParameter(ActivityInstance & instan
{
OwnedHqlExpr folded = foldHqlExpression(expr);
instance.addConstructorParameter(folded);
noteFilename(instance, attr, folded, false, summaryType);
noteFilename(instance, attr, folded, false, summaryType, false);
}

void HqlCppTranslator::buildFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, WuAttr attr, const char * name, IHqlExpression * expr, bool isDynamic, SummaryType summaryType)
void HqlCppTranslator::buildFilenameFunction(ActivityInstance & instance, BuildCtx & classctx, WuAttr attr, const char * name, IHqlExpression * expr, bool isDynamic, SummaryType summaryType, bool isOpt)
{
OwnedHqlExpr folded = foldHqlExpression(expr);
doBuildVarStringFunction(classctx, name, folded);
noteFilename(instance, attr, folded, isDynamic, summaryType);
noteFilename(instance, attr, folded, isDynamic, summaryType, isOpt);
}

void HqlCppTranslator::noteFilename(ActivityInstance & instance, WuAttr attr, IHqlExpression * expr, bool isDynamic, SummaryType summaryType)
void HqlCppTranslator::noteFilename(ActivityInstance & instance, WuAttr attr, IHqlExpression * expr, bool isDynamic, SummaryType summaryType, bool isOpt)
{
if (options.addFilesnamesToGraph)
{
Expand All @@ -3417,7 +3417,7 @@ void HqlCppTranslator::noteFilename(ActivityInstance & instance, WuAttr attr, IH
StringBuffer propValue;
folded->queryValue()->getStringValue(propValue);
instance.addAttribute(attr, propValue);
noteSummaryInfo(propValue, summaryType);
noteSummaryInfo(propValue, summaryType, isOpt);
}
}
if (isDynamic)
Expand Down Expand Up @@ -3477,7 +3477,7 @@ void HqlCppTranslator::buildRefFilenameFunction(ActivityInstance & instance, Bui
break;
}

buildFilenameFunction(instance, classctx, attr, name, filename, hasDynamicFilename(table), summaryType);
buildFilenameFunction(instance, classctx, attr, name, filename, hasDynamicFilename(table), summaryType, table->hasAttribute(optAtom));
}

void HqlCppTranslator::buildConnectInputOutput(BuildCtx & ctx, ActivityInstance * instance, ABoundActivity * table, unsigned outputIndex, unsigned inputIndex, const char * label, bool nWay)
Expand Down Expand Up @@ -10669,7 +10669,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
buildInstancePrefix(instance);

//virtual const char * getFileName() { return "x.d00"; }
buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", filename, hasDynamicFilename(expr), SummaryType::WriteIndex);
buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", filename, hasDynamicFilename(expr), SummaryType::WriteIndex, false);

//virtual unsigned getFlags() = 0;
IHqlExpression * updateAttr = expr->queryAttribute(updateAtom);
Expand Down Expand Up @@ -10720,7 +10720,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH

IHqlExpression * indexNameAttr = expr->queryAttribute(indexAtom);
if (indexNameAttr)
buildFilenameFunction(*instance, instance->startctx, WaDistributeIndexname, "getDistributeIndexName", indexNameAttr->queryChild(0), hasDynamicFilename(expr), SummaryType::ReadIndex);
buildFilenameFunction(*instance, instance->startctx, WaDistributeIndexname, "getDistributeIndexName", indexNameAttr->queryChild(0), hasDynamicFilename(expr), SummaryType::ReadIndex, false);

buildExpiryHelper(instance->createctx, expr->queryAttribute(expireAtom));
buildUpdateHelper(instance->createctx, *instance, dataset, updateAttr);
Expand Down Expand Up @@ -11071,7 +11071,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
if (filename && filename->getOperator() != no_pipe)
{
bool isDynamic = expr->hasAttribute(resultAtom) || hasDynamicFilename(expr);
buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", filename, isDynamic, SummaryType::WriteFile);
buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", filename, isDynamic, SummaryType::WriteFile, false);
if (!filename->isConstant())
constFilename = false;
}
Expand Down Expand Up @@ -18060,7 +18060,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
StringBuffer serviceName;
getUTF8Value(serviceName, service);
instance->addAttribute(WaServiceName, serviceName);
noteSummaryInfo(serviceName, SummaryType::Service);
noteSummaryInfo(serviceName, SummaryType::Service, false);
}

enum class ReqFormat { NONE, XML, JSON, FORM_ENCODED };
Expand Down
4 changes: 2 additions & 2 deletions ecl/hqlcpp/hqlsource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1185,7 +1185,7 @@ void SourceBuilder::buildFilenameMember()
summaryType = SummaryType::ReadIndex;
break;
}
translator.buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", nameExpr, translator.hasDynamicFilename(tableExpr), summaryType);
translator.buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", nameExpr, translator.hasDynamicFilename(tableExpr), summaryType, tableExpr->hasAttribute(optAtom));
}

void SourceBuilder::buildReadMembers(IHqlExpression * expr)
Expand Down Expand Up @@ -4854,7 +4854,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityXmlRead(BuildCtx & ctx, IHqlEx
fieldUsage->noteAll();

//---- virtual const char * getFileName() { return "x.d00"; } ----
buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", filename, hasDynamicFilename(tableExpr), SummaryType::ReadIndex);
buildFilenameFunction(*instance, instance->startctx, WaFilename, "getFileName", filename, hasDynamicFilename(tableExpr), SummaryType::ReadIndex, tableExpr->hasAttribute(optAtom));
buildEncryptHelper(instance->startctx, tableExpr->queryAttribute(encryptAtom));

bool usesContents = false;
Expand Down

0 comments on commit c6b6a42

Please sign in to comment.