Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/candidate-9.6.x'
Browse files Browse the repository at this point in the history 
Signed-off-by: Gordon Smith <[email protected]>
  • Loading branch information
@GordonSmith
GordonSmith committed Apr 5, 2024
2 parents 2844583 + 2c6805b commit 8998b32
Show file tree
Hide file tree
Showing 17 changed files with 189 additions and 101 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-assets.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ jobs:
sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\
cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES_PASSPHRASE=${{ secrets.SIGN_MODULES_PASSPHRASE }} -DSIGN_MODULES_KEYID=${{ secrets.SIGN_MODULES_KEYID }} -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DSIGN_MODULES=OFF -DMAKE_DOCS_ONLY=ON -DUSE_NATIVE_LIBRARIES=ON -DDOCS_AUTO=ON -DDOC_LANGS=ALL && \
cmake --build /hpcc-dev/build --parallel $(nproc) --target all"
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/EN_US && zip ALL_HPCC_DOCS_EN_US-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cd /hpcc-dev/build/Release/docs/PT_BR && zip ALL_HPCC_DOCS_PT_BR-${{ needs.preamble.outputs.community_tag }}.zip *.pdf"
Expand Down
6 changes: 3 additions & 3 deletions dali/base/dadfs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1256,7 +1256,7 @@ protected: friend class CDistributedFile;
return ret;
}
virtual bool removePhysicalPartFiles(const char *logicalName, IFileDescriptor *fileDesc, IMultiException *mexcept, unsigned numParallelDeletes=0) override;
virtual void setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout);
virtual void setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout);
};


Expand Down Expand Up @@ -11635,11 +11635,11 @@ void CDistributedFileDirectory::setFileAccessed(CDfsLogicalFileName &dlfn,IUserD
checkDfsReplyException(mb);
}

void CDistributedFileDirectory::setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout)
void CDistributedFileDirectory::setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali, unsigned foreigndalitimeout)
{
CDfsLogicalFileName dlfn;
dlfn.set(logicalName);
setFileAccessed(dlfn, nullptr, dt, foreigndali, foreigndalitimeout);
setFileAccessed(dlfn, udesc, dt, foreigndali, foreigndalitimeout);
}

void CDistributedFileDirectory::setFileProtect(CDfsLogicalFileName &dlfn,IUserDescriptor *user, const char *owner, bool set, const INode *foreigndali,unsigned foreigndalitimeout)
Expand Down
2 changes: 1 addition & 1 deletion dali/base/dadfs.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,7 @@ interface IDistributedFileDirectory: extends IInterface
// useful to clearup after temporary unpublished file.
virtual bool removePhysicalPartFiles(const char *logicalName, IFileDescriptor *fileDesc, IMultiException *mexcept, unsigned numParallelDeletes=0) = 0;

virtual void setFileAccessed(const char *logicalName, const CDateTime &dt, const INode *foreigndali=nullptr, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0;
virtual void setFileAccessed(IUserDescriptor* udesc, const char *logicalName, const CDateTime &dt, const INode *foreigndali=nullptr, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) = 0;
};


Expand Down
43 changes: 13 additions & 30 deletions docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
similarity to actual persons, living or dead, is purely
coincidental.</para>

<para></para>
<para />
</legalnotice>

<xi:include href="common/Version.xml"
Expand All @@ -64,10 +64,10 @@
<chapter id="ContainerizedHPCCOverview">
<title>Containerized HPCC Overview</title>

<para>Starting with version 8.0, the HPCC
Systems<superscript>®</superscript> Platform is focusing on containerized
deployments. This is useful for cloud-based deployments (large or small)
or local testing/development deployments.</para>
<para>Since version 8.0, the HPCC Systems<superscript>®</superscript>
Platform started focusing significantly on containerized deployments. This
is useful for cloud-based deployments (large or small) or local
testing/development deployments.</para>

<para>Docker containers managed by Kubernetes (K8s) is a new target
operating environment, alongside continued support for traditional
Expand All @@ -81,19 +81,18 @@
to run, but a significant change in how components are configured, how and
when they start up, and where they store their data.</para>

<para>This book focuses on containerized deployments. The first section is
about using Docker containers and Helm charts locally. Docker and Helm do
a lot of the work for you. The second part uses the same techniques in the
cloud.</para>
<para>This book focuses on these containerized deployments. The first
section is about using Docker containers and Helm charts locally. Docker
and Helm do a lot of the work for you. The second part uses the same
techniques in the cloud.</para>

<para>For local small deployments (for development and testing), we
suggest using Docker Desktop and Helm. This is useful for learning,
development, and testing.</para>

<para>For Cloud deployments, you can use any flavor of Cloud services, if
it supports Docker, Kubernetes, and Helm. This book, however, will focus
on Microsoft Azure for Cloud Services. Future versions may include
specifics for other Cloud providers.</para>
on Microsoft Azure for Cloud Services. </para>

<para>If you want to manually manage your local or Cloud deployment, you
can still use the traditional installers and Configuration Manager, but
Expand All @@ -106,28 +105,12 @@
be easy for someone familiar with Kubernetes and Helm to install and
manage the HPCC Systems platform.</para>

<variablelist>
<varlistentry>
<term>Note:</term>

<listitem>
<para>The traditional bare-metal version of the HPCC Systems
platform is mature and has been heavily used in commercial
applications for almost two decades and is fully intended for
production use. The containerized version is new and is not yet 100%
ready for production. In addition, aspects of that version could
change without notice. We encourage you to use it and provide
feedback so we can make this version as robust as a bare-metal
installation.</para>
</listitem>
</varlistentry>
</variablelist>

<sect1 id="barevscontainer">
<title>Bare-metal vs Containers</title>

<para>If you are familiar with the HPCC Systems platform, there are a
few fundamental changes to note.</para>
<para>If you are familiar with the traditional bare-metal HPCC Systems
platform deployments, there are a few fundamental changes to
note.</para>

<sect2 id="processesandpods">
<title>Processes and Pods, not Machines</title>
Expand Down
23 changes: 18 additions & 5 deletions docs/EN_US/ECLStandardLibraryReference/SLR-Mods/CreateExternalDirectory.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
<primary>File.CreateExternalDirectory</primary>
</indexterm><indexterm>
<primary>CreateExternalDirectory</primary>
</indexterm>(</emphasis> <emphasis> location, path </emphasis> <emphasis
</indexterm>(</emphasis> <emphasis> location, path <emphasis
role="bold">[</emphasis> <emphasis>, planename</emphasis> <emphasis
role="bold">]</emphasis> </emphasis> <emphasis
role="bold">)</emphasis></para>

<informaltable colsep="1" frame="all" rowsep="1">
Expand All @@ -24,7 +26,8 @@
<entry><emphasis>location</emphasis></entry>

<entry>A null-terminated string containing the IP address of the
remote machine.</entry>
remote machine. Optional if <emphasis>planename</emphasis> is
provided. </entry>
</row>

<row>
Expand All @@ -33,6 +36,14 @@
<entry>A null-terminated string containing the directory path to
create.</entry>
</row>

<row>
<entry><emphasis>planename</emphasis></entry>

<entry>A null-terminated string containing name of the data plane
containing the file. Optional if <emphasis>location</emphasis> is
provided, but <emphasis>planename</emphasis> is preferred.</entry>
</row>
</tbody>
</tgroup>
</informaltable>
Expand All @@ -44,8 +55,10 @@

<para>Example:</para>

<programlisting format="linespecific">IP := '10.150.254.6';
path := '/c$/training/import/NewDir';
STD.File.CreateExternalDirectory(IP,path);
<programlisting format="linespecific">IMPORT STD;
IP := '';
path := '/var/lib/HPCCSystems/dropzone/advancedtraining/';
planename := 'mydropzone';
STD.File.CreateExternalDirectory(IP,path,planename);
</programlisting>
</sect1>
23 changes: 18 additions & 5 deletions docs/EN_US/ECLStandardLibraryReference/SLR-Mods/DeleteExternalFile.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
<primary>File.DeleteExternalFile</primary>
</indexterm><indexterm>
<primary>DeleteExternalFile</primary>
</indexterm>(</emphasis> <emphasis> location, path </emphasis> <emphasis
</indexterm>(</emphasis> <emphasis> location, path <emphasis
role="bold">[</emphasis> <emphasis>, planename</emphasis> <emphasis
role="bold">]</emphasis> </emphasis> <emphasis
role="bold">)</emphasis></para>

<informaltable colsep="1" frame="all" rowsep="1">
Expand All @@ -24,7 +26,8 @@
<entry><emphasis>location</emphasis></entry>

<entry>A null-terminated string containing the IP address of the
remote machine.</entry>
remote machine. Optional if <emphasis>planename</emphasis> is
provided. </entry>
</row>

<row>
Expand All @@ -33,6 +36,14 @@
<entry>A null-terminated string containing the path/name of the file
to remove.</entry>
</row>

<row>
<entry><emphasis>planename</emphasis></entry>

<entry>A null-terminated string containing name of the data plane
containing the file. Optional if <emphasis>location</emphasis> is
provided, but <emphasis>planename</emphasis> is preferred.</entry>
</row>
</tbody>
</tgroup>
</informaltable>
Expand All @@ -44,8 +55,10 @@

<para>Example:</para>

<programlisting format="linespecific">IP := '10.150.254.6';
infile := '/c$/training/import/AdvancedECL/people';
STD.File.DeleteExternalFile(IP,infile);
<programlisting format="linespecific">IMPORT STD;
IP := '';
infile := '/var/lib/HPCCSystems/dropzone/originalperson';
planename := 'mydropzone';
STD.File.DeleteExternalFile(IP,infile,planename);
</programlisting>
</sect1>
27 changes: 20 additions & 7 deletions docs/EN_US/ECLStandardLibraryReference/SLR-Mods/MoveExternalFile.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@
<primary>File.MoveExternalFile</primary>
</indexterm><indexterm>
<primary>MoveExternalFile</primary>
</indexterm>(</emphasis> <emphasis> location, frompath, topath </emphasis>
<emphasis role="bold">)</emphasis></para>
</indexterm>(</emphasis> <emphasis> location, frompath, topath <emphasis
role="bold">[</emphasis> <emphasis>, planename</emphasis> <emphasis
role="bold">]</emphasis> </emphasis> <emphasis
role="bold">)</emphasis></para>

<informaltable colsep="1" frame="all" rowsep="1">
<tgroup cols="2">
Expand All @@ -24,7 +26,8 @@
<entry><emphasis>location</emphasis></entry>

<entry>A null-terminated string containing the IP address of the
remote machine.</entry>
remote machine. Optional if <emphasis>planename</emphasis> is
provided. </entry>
</row>

<row>
Expand All @@ -40,6 +43,14 @@
<entry>A null-terminated string containing the path/name of the
target file.</entry>
</row>

<row>
<entry><emphasis>planeName</emphasis></entry>

<entry>A null-terminated string containing name of the data plane
containing the file. Optional if <emphasis>location</emphasis> is
provided, but <emphasis>planename</emphasis> is preferred.</entry>
</row>
</tbody>
</tgroup>
</informaltable>
Expand All @@ -53,8 +64,10 @@

<para>Example:</para>

<programlisting format="linespecific">IP := '10.150.254.6';
infile := '/c$/training/import/AdvancedECL/people';
outfile := '/c$/training/import/DFUtest/people';
STD.File.MoveExternalFile(IP,infile,outfile);</programlisting>
<programlisting format="linespecific">IMPORT STD;
IP := '';
infile := '/var/lib/HPCCSystems/dropzone/originalperson';
outfile := '/var/lib/HPCCSystems/dropzone/originalperson_bak';
planename := 'mydropzone';
STD.File.MoveExternalFile(IP,infile,outfile,planename);</programlisting>
</sect1>
34 changes: 24 additions & 10 deletions docs/EN_US/ECLStandardLibraryReference/SLR-Mods/RemoteDirectory.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@
<sect1 id="RemoteDirectory">
<title>RemoteDirectory</title>

<para><emphasis role="bold">STD.File.RemoteDirectory<indexterm>
<para><emphasis role="bold">STD.File.RemoteDirectory(</emphasis> <emphasis>
machineIP, dir </emphasis> <emphasis role="bold">[</emphasis> <emphasis>,
mask </emphasis> <emphasis role="bold">][</emphasis> <emphasis>, recurse
</emphasis> <emphasis role="bold">]</emphasis><emphasis
role="bold">[</emphasis> <emphasis>, planeName</emphasis> <emphasis
role="bold">]</emphasis> <emphasis> </emphasis> <emphasis role="bold">)
<indexterm>
<primary>STD.File.RemoteDirectory</primary>
</indexterm><indexterm>
<primary>File.RemoteDirectory</primary>
</indexterm><indexterm>
<primary>RemoteDirectory</primary>
</indexterm>(</emphasis> <emphasis> machineIP, dir </emphasis> <emphasis
role="bold">[</emphasis> <emphasis>, mask </emphasis> <emphasis
role="bold">][</emphasis> <emphasis>, recurse </emphasis> <emphasis
role="bold">]</emphasis> <emphasis> </emphasis> <emphasis
role="bold">)</emphasis></para>
</indexterm></emphasis></para>

<informaltable colsep="1" frame="all" rowsep="1">
<tgroup cols="2">
Expand All @@ -27,7 +29,8 @@
<entry><emphasis>machineIP</emphasis></entry>

<entry>A null-terminated string containing the IP address of the
remote machine.</entry>
remote machine. Optional if <emphasis>planeName</emphasis> is
provided. </entry>
</row>

<row>
Expand All @@ -54,6 +57,14 @@
omitted, the default is FALSE.</entry>
</row>

<row>
<entry><emphasis>planeName</emphasis></entry>

<entry>A null-terminated string containing name of the data plane
containing the file. Optional if <emphasis>machineIP</emphasis> is
provided, but <emphasis>planeName</emphasis> is preferred.</entry>
</row>

<row>
<entry>Return:<emphasis> </emphasis></entry>

Expand Down Expand Up @@ -84,7 +95,10 @@ END;</programlisting>

<para>Example:</para>

<programlisting format="linespecific">OUTPUT(STD.File.RemoteDirectory('edata12','\in','*.d00'));
OUTPUT(STD.File.RemoteDirectory('10.150.254.6',
'/var/lib/HPCCSystems/hpcc-data/thor/','acc*',TRUE));</programlisting>
<programlisting format="linespecific">IMPORT STD;
machineIP := '';
dir := '/var/lib/HPCCSystems/dropzone/training';
recurse:= FALSE;
planeName := 'mydropzone';
OUTPUT(STD.File.RemoteDirectory(machineIP,dir,'*.csv',recurse,planeName));</programlisting>
</sect1>
10 changes: 5 additions & 5 deletions esp/services/ws_dfsservice/ws_dfsservice.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ enum LfnMetaOpts : byte
LfnMOptTls = 0x02,
};
BITMASK_ENUM(LfnMetaOpts);
static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, LfnMetaOpts opts, IPropertyTree *metaRoot, IPropertyTree *meta)
static void populateLFNMeta(IUserDescriptor *userDesc, const char *logicalName, unsigned __int64 leaseId, LfnMetaOpts opts, IPropertyTree *metaRoot, IPropertyTree *meta)
{
CDfsLogicalFileName lfn;
lfn.set(logicalName);
Expand All @@ -59,7 +59,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, L

assertex(!lfn.isMulti()); // not supported, don't think needs to be/will be.

Owned<IPropertyTree> tree = queryDistributedFileDirectory().getFileTree(logicalName, nullptr);
Owned<IPropertyTree> tree = queryDistributedFileDirectory().getFileTree(logicalName, userDesc);
if (!tree)
return;
if (hasMask(opts, LfnMOptRemap))
Expand Down Expand Up @@ -111,7 +111,7 @@ static void populateLFNMeta(const char *logicalName, unsigned __int64 leaseId, L
{
IPropertyTree &sub = *(orderedSubFiles[f]);
sub.getProp("@name", subname.clear());
populateLFNMeta(subname, leaseId, opts, metaRoot, fileMeta);
populateLFNMeta(userDesc, subname, leaseId, opts, metaRoot, fileMeta);
}
}
}
Expand Down Expand Up @@ -182,7 +182,7 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r
opts |= LfnMOptTls;

Owned<IPropertyTree> responseTree = createPTree();
populateLFNMeta(logicalName, leaseId, opts, responseTree, responseTree);
populateLFNMeta(userDesc, logicalName, leaseId, opts, responseTree, responseTree);

// serialize response
MemoryBuffer respMb, compressedRespMb;
Expand All @@ -198,7 +198,7 @@ bool CWsDfsEx::onDFSFileLookup(IEspContext &context, IEspDFSFileLookupRequest &r
// Really this should be done at end (or at end as well), but this is same as existing DFS lookup.
CDateTime dt;
dt.setNow();
queryDistributedFileDirectory().setFileAccessed(logicalName, dt);
queryDistributedFileDirectory().setFileAccessed(userDesc, logicalName, dt);

LOG(MCauditInfo,",FileAccess,EspProcess,READ,%s,%u,%s", logicalName, timeoutSecs, userID.str());
}
Expand Down
2 changes: 1 addition & 1 deletion fs/dafsserver/dafsserver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3367,7 +3367,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface
*/
unsigned ms = timer.elapsedMs();
totalThrottleDelay += ms;
PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %u milliseconds, proceeding as cpu(%u)<throttleCPULimit(%u)", title.get(), getRFCText(cmd), cpu, ms, cpuThreshold);
PROGLOG("Throttler(%s): transaction delayed [cmd=%s] for : %u milliseconds, proceeding as cpu(%u)<throttleCPULimit(%u)", title.get(), getRFCText(cmd), ms, cpu, cpuThreshold);
hadSem = false;
}
else
Expand Down
Loading

0 comments on commit 8998b32

Please sign in to comment.