Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…
Browse files Browse the repository at this point in the history 
…9.6.x
  • Loading branch information
@GordonSmith
GordonSmith committed Apr 29, 2024
2 parents d5a8ba7 + 55b6c59 commit 05793e0
Show file tree
Hide file tree
Showing 37 changed files with 1,611 additions and 228 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/build-assets.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,8 +259,8 @@ jobs:
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build -DVCPKG_FILES_DIR=/hpcc-dev -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-$(echo '${{ needs.preamble.outputs.community_tag }}' | sed 's/community_//' ).zip *.pdf"
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-$(echo '${{ needs.preamble.outputs.community_tag }}' | sed 's/community_//' ).zip *.pdf"
- name: Upload Artifacts for ECLIDE build
if: ${{ !matrix.ln && !matrix.container && matrix.documentation }}
Expand Down
2 changes: 1 addition & 1 deletion common/workunit/workunit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13911,7 +13911,7 @@ extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, c
OwnedIFile target = createIFile(destPathName);
if (!target->exists())
{
source->copyTo(target, 0, NULL, true);
source->copyTo(target, 0, NULL, false);
}
query->addAssociatedFile(type, destPathName, "localhost", description, crc, minActivity, maxActivity);
// Should we delete the local files? No - they may not be finished with
Expand Down
38 changes: 24 additions & 14 deletions dali/base/dadfs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3566,23 +3566,33 @@ class CDistributedFile: public CDistributedFileBase<IDistributedFile>

offset_t maxPartSz = 0, minPartSz = (offset_t)-1, totalPartSz = 0;

maxSkewPart = 0;
minSkewPart = 0;
for (unsigned p=0; p<np; p++)
try
{
IDistributedFilePart &part = queryPart(p);
offset_t size = part.getFileSize(true, false);
if (size > maxPartSz)
{
maxPartSz = size;
maxSkewPart = p;
}
if (size < minPartSz)
maxSkewPart = 0;
minSkewPart = 0;
for (unsigned p=0; p<np; p++)
{
minPartSz = size;
minSkewPart = p;
IDistributedFilePart &part = queryPart(p);
offset_t size = part.getFileSize(true, false);
if (size > maxPartSz)
{
maxPartSz = size;
maxSkewPart = p;
}
if (size < minPartSz)
{
minPartSz = size;
minSkewPart = p;
}
totalPartSz += size;
}
totalPartSz += size;
}
catch (IException *e)
{
// guard against getFileSize throwing an exception (if parts missing)
EXCLOG(e);
e->Release();
return false;
}
offset_t avgPartSz = totalPartSz / np;
if (0 == avgPartSz)
Expand Down
95 changes: 95 additions & 0 deletions dali/daliadmin/daadmin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2264,6 +2264,101 @@ void getxref(const char *dst)
conn->close();
}

void checkFileSizeOne(IUserDescriptor *user, const char *lfn, bool fix)
{
try
{
CDfsLogicalFileName dlfn;
dlfn.set(lfn);
Owned<IDistributedFile> dFile = queryDistributedFileDirectory().lookup(dlfn, user, AccessMode::tbdRead, false, false, nullptr, defaultPrivilegedUser, 30000); // 30 sec timeout
if (dFile)
{
if (dFile->querySuperFile())
WARNLOG("Skipping: file '%s' is a superfile", lfn);
else
{
bool fileLocked = false;
COnScopeExit ensureFileUnlock([&]() { if (fileLocked) dFile->unlockProperties(); });
unsigned numParts = dFile->numParts();
for (unsigned p=0; p<numParts; p++)
{
IDistributedFilePart &part = dFile->queryPart(p);
IPropertyTree &attrs = part.queryAttributes();
if (!attrs.hasProp("@size"))
{
if (fix)
{
offset_t partSize;
try
{
partSize = part.getFileSize(true, true);
if (!fileLocked)
{
// we lock the file once, so that the individual part lock/unlocks are effectively a NOP
dFile->lockProperties(30000);
fileLocked = true;
PROGLOG("File '%s' has missing @size attributes", lfn);
}
part.lockProperties(30000);
}
catch (IException *e)
{
EXCLOG(e);
e->Release();
continue;
}
COnScopeExit ensurePartUnlock([&]() { part.unlockProperties(); });
PROGLOG("Part %u: Setting @size to %" I64F "u", p+1, partSize);
attrs.setPropInt64("@size", partSize);
}
else
PROGLOG("File '%s' missing @size on part %u", lfn, p+1);
}
}
}
}
else
WARNLOG("File '%s' not found", lfn);
}
catch (IException *e)
{
EXCLOG(e);
e->Release();
}
}

void checkFileSize(IUserDescriptor *user, const char *lfnPattern, bool fix)
{
if (containsWildcard(lfnPattern))
{
unsigned count = 0;
Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator(lfnPattern, user, true, false); // no supers
CCycleTimer timer;
if (iter->first())
{
while (true)
{
IPropertyTree &attr = iter->query();
const char *lfn = attr.queryProp("@name");
checkFileSizeOne(user, lfn, fix);
++count;

if (!iter->next())
break;
else if (timer.elapsedCycles() >= queryOneSecCycles()*10) // log every 10 secs
{
PROGLOG("Processed %u files", count);
timer.reset();
}
}
}
PROGLOG("Total files processed %u files", count);
}
else
checkFileSizeOne(user, lfnPattern, fix);
}


struct CTreeItem : public CInterface
{
String *tail;
Expand Down
1 change: 1 addition & 0 deletions dali/daliadmin/daadmin.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ extern DALIADMIN_API void listmatches(const char *path, const char *match, const
extern DALIADMIN_API void dfsreplication(const char *clusterMask, const char *lfnMask, unsigned redundancy, bool dryRun);
extern DALIADMIN_API void migrateFiles(const char *srcGroup, const char *tgtGroup, const char *filemask, const char *_options);
extern DALIADMIN_API void getxref(const char *dst);
extern DALIADMIN_API void checkFileSize(IUserDescriptor *user, const char *lfnPattern, bool fix);

extern DALIADMIN_API void listworkunits(const char *test, const char *min, const char *max);
extern DALIADMIN_API void workunittimings(const char *wuid);
Expand Down
56 changes: 32 additions & 24 deletions dali/daliadmin/daliadmin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,49 +55,53 @@ void usage(const char *exe)
printf(" count <xpath> -- counts xpath matches\n");
printf("\n");
printf("Logical File meta information commands:\n");
printf(" dfsfile <logicalname> -- get meta information for file\n");
printf(" dfsmeta <logicalname> <storage> -- get new meta information for file\n");
printf(" setdfspartattr <logicalname> <part> <attribute> [<value>] -- set attribute of a file part to value, or delete the attribute if not provided\n");
printf(" dfspart <logicalname> <part> -- get meta information for part num\n");
printf(" checksuperfile <superfilename> [fix=true|false] -- check superfile links consistent and optionally fix\n");
printf(" checksubfile <subfilename> -- check subfile links to parent consistent\n");
printf(" checkfilesize <logicalfilemask> [fix=true|false] -- check file size attributes and optionally fix");
printf(" cleanscopes -- remove empty scopes\n");
printf(" clusternodes <clustername> [filename] -- get IPs for cluster group. Written to optional filename if provided\n");
printf(" dfscheck -- verify dfs file information is valid\n");
printf(" dfscompratio <logicalname> -- returns compression ratio of file\n");
printf(" dfscsv <logicalnamemask> -- get csv info. for files matching mask\n");
printf(" dfsexists <logicalname> -- sets return value to 0 if file exists\n");
printf(" dfsfile <logicalname> -- get meta information for file\n");
printf(" dfsgroup <logicalgroupname> [filename] -- get IPs for logical group (aka cluster). Written to optional filename if provided\n");
printf(" clusternodes <clustername> [filename] -- get IPs for cluster group. Written to optional filename if provided\n");
printf(" dfsls [<logicalname>] [options]-- get list of files within a scope (options=lrs)\n");
printf(" dfsmap <logicalname> -- get part files (primary and replicates)\n");
printf(" dfsexists <logicalname> -- sets return value to 0 if file exists\n");
printf(" dfsmeta <logicalname> <storage> -- get new meta information for file\n");
printf(" dfsparents <logicalname> -- list superfiles containing file\n");
printf(" dfspart <logicalname> <part> -- get meta information for part num\n");
printf(" dfsperm <logicalname> -- returns LDAP permission for file\n");
printf(" dfsreplication <clustermask> <logicalnamemask> <redundancy-count> [dryrun] -- set redundancy for files matching mask, on specified clusters only\n");
printf(" dfsscopes <mask> -- lists logical scopes (mask = * for all)\n");
printf(" dfsunlink <logicalname> -- unlinks file from all super parents\n");
printf(" dfsverify <logicalname> -- verifies parts exist, returns 0 if ok\n");
printf(" setprotect <logicalname> <id> -- overwrite protects logical file\n");
printf(" unprotect <logicalname> <id> -- unprotect (if id=* then clear all)\n");
printf(" listprotect <logicalnamemask> <id-mask> -- list protected files\n");
printf(" checksuperfile <superfilename> [fix=true|false] -- check superfile links consistent and optionally fix\n");
printf(" checksubfile <subfilename> -- check subfile links to parent consistent\n");
printf(" holdlock <logicalfile> <read|write> -- hold a lock to the logical-file until a key is pressed");
printf(" listexpires <logicalnamemask> -- lists logical files with expiry value\n");
printf(" listprotect <logicalnamemask> <id-mask> -- list protected files\n");
printf(" listrelationships <primary> <secondary>\n");
printf(" dfsperm <logicalname> -- returns LDAP permission for file\n");
printf(" dfscompratio <logicalname> -- returns compression ratio of file\n");
printf(" dfsscopes <mask> -- lists logical scopes (mask = * for all)\n");
printf(" cleanscopes -- remove empty scopes\n");
printf(" normalizefilenames [<logicalnamemask>] -- normalize existing logical filenames that match, e.g. .::.::scope::.::name -> scope::name\n");
printf(" dfsreplication <clustermask> <logicalnamemask> <redundancy-count> [dryrun] -- set redundancy for files matching mask, on specified clusters only\n");
printf(" holdlock <logicalfile> <read|write> -- hold a lock to the logical-file until a key is pressed");
printf(" setdfspartattr <logicalname> <part> <attribute> [<value>] -- set attribute of a file part to value, or delete the attribute if not provided\n");
printf(" setprotect <logicalname> <id> -- overwrite protects logical file\n");
printf(" unprotect <logicalname> <id> -- unprotect (if id=* then clear all)\n");
printf("\n");
printf("Workunit commands:\n");
printf(" listworkunits [<prop>=<val> [<lower> [<upper>]]] -- list workunits that match prop=val in workunit name range lower to upper\n");
printf(" listmatches <connection xpath> [<match xpath>=<val> [<property xpaths>]] -- <property xpaths> is comma separated list of xpaths\n");
printf(" workunittimings <WUID>\n");
printf("\n");
printf("Other dali server and misc commands:\n");
printf(" serverlist <mask> -- list server IPs (mask optional)\n");
printf(" clusterlist <mask> -- list clusters (mask optional)\n");
printf(" auditlog <fromdate> <todate> <match>\n");
printf(" cleanglobalwuid [dryrun] [noreconstruct]\n");
printf(" clusterlist <mask> -- list clusters (mask optional)\n");
printf(" coalesce -- force transaction coalesce\n");
printf(" mpping <server-ip> -- time MP connect\n");
printf(" dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
printf(" daliping [ <num> ] -- time dali server connect\n");
printf(" getxref <destxmlfile> -- get all XREF information\n");
printf(" dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
printf(" migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
printf(" mpping <server-ip> -- time MP connect\n");
printf(" serverlist <mask> -- list server IPs (mask optional)\n");
printf(" translatetoxpath logicalfile [File|SuperFile|Scope]\n");
printf(" unlock <xpath or logicalfile> <[path|file]> -- unlocks either matching xpath(s) or matching logical file(s), can contain wildcards\n");
printf(" validatestore [fix=<true|false>]\n"
" [verbose=<true|false>]\n"
Expand All @@ -106,9 +110,6 @@ void usage(const char *exe)
printf(" wuidcompress <wildcard> <type> -- scan workunits that match <wildcard> and compress resources of <type>\n");
printf(" wuiddecompress <wildcard> <type> -- scan workunits that match <wildcard> and decompress resources of <type>\n");
printf(" xmlsize <filename> [<percentage>] -- analyse size usage in xml file, display individual items above 'percentage' \n");
printf(" migratefiles <src-group> <target-group> [<filemask>] [dryrun] [createmaps] [listonly] [verbose]\n");
printf(" translatetoxpath logicalfile [File|SuperFile|Scope]\n");
printf(" cleanglobalwuid [dryrun] [noreconstruct]\n");
printf("\n");
printf("Common options\n");
printf(" server=<dali-server-ip> -- server ip\n");
Expand Down Expand Up @@ -148,6 +149,8 @@ int main(int argc, const char* argv[])
StringBuffer tmps;
for (int i=1;i<argc;i++) {
const char *param = argv[i];
if (startsWith(param, "--"))
continue; // handled by loadConfiguration
if ((memcmp(param,"server=",7)==0)||
(memcmp(param,"logfile=",8)==0)||
(memcmp(param,"rawlog=",7)==0)||
Expand Down Expand Up @@ -474,6 +477,11 @@ int main(int argc, const char* argv[])
CHECKPARAMS(1,1);
getxref(params.item(1));
}
else if (strieq(cmd,"checkfilesize")) {
CHECKPARAMS(1,1);
bool fix = props->getPropBool("fix");
checkFileSize(userDesc, params.item(1), fix);
}
else if (strieq(cmd,"dalilocks")) {
CHECKPARAMS(0,2);
bool filesonly = false;
Expand Down
Loading

0 comments on commit 05793e0

Please sign in to comment.