Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Jake Smith <[email protected]>

common/remote/hooks/git/gitfile.cpp

@@ -174,19 +174,19 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
         return buf.length() > LFSsiglen && memcmp(buf.toByteArray(), LFSsig, LFSsiglen)==0;
     }
 public:
-    GitRepositoryFileIO(GitCommitTree * commitTree, const char *gitDirectory, const git_oid * oid, const char * gitUser)
+    GitRepositoryFileIO(const char * filename, GitCommitTree * commitTree, const char *gitDirectory, const git_oid * oid, const char * gitUser)
     {
         git_blob *blob = nullptr;
         int error = git_blob_lookup(&blob, git_tree_owner(commitTree->queryTree()), oid);
         if (error)
-            throw MakeStringException(0, "git git_blob_lookup returned exit status %d", error);
+            throw MakeStringException(0, "git git_blob_lookup for '%s' returned exit status %d", filename, error);
 
         git_object_size_t blobsize = git_blob_rawsize(blob);
         const void * data = git_blob_rawcontent(blob);
         buf.append(blobsize, data);
         git_blob_free(blob);
         if (isLFSfile())
-            readLfsContents(gitDirectory, gitUser);
+            readLfsContents(filename, gitDirectory, gitUser);
     }
     virtual size32_t read(offset_t pos, size32_t len, void * data)
     {
@@ -229,16 +229,21 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
     }
 
 protected:
-    void readLfsContents(const char *gitDirectory, const char * gitUser)
+    void readLfsContents(const char * filename, const char *gitDirectory, const char * gitUser)
     {
         EnvironmentVector env;
         Owned<IFile> extractedKey;
 
         //If fetching from git and the username is specified then use the script file to provide the username/password
         //Only support retrieving the password as a secret - not as a filename
+        //NB: This code should be kept in sync with runGitCommand() in hqlrepository.cpp (and ideally combined)
         if (!isEmptyString(gitUser))
         {
+            env.emplace_back("HPCC_GIT_USERNAME", gitUser);
+
+            // If gituser is specified never prompt for credentials, otherwise the server can hang.
             env.emplace_back("GIT_TERMINAL_PROMPT", "0");
+
             StringBuffer scriptPath;
             getPackageFolder(scriptPath);
             addPathSepChar(scriptPath).append("bin/hpccaskpass.sh");
@@ -254,8 +259,10 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
                     env.emplace_back("HPCC_GIT_PASSPATH", extractedKey->queryFilename());
                 }
                 else
-                    DBGLOG("Secret doesn't contain password for git user %s", gitUser);
+                    OWARNLOG("Secret doesn't contain password for git user %s", gitUser);
             }
+            else
+                OWARNLOG("No secret found for git user %s", gitUser);
         }
         Owned<IPipeProcess> pipe = createPipeProcess();
         for (const auto & cur : env)
@@ -275,7 +282,7 @@ class GitRepositoryFileIO : implements CSimpleInterfaceOf<IFileIO>
         if (retcode)
         {
             buf.clear();  // Can't rely on destructor to clean this for me
-            throw MakeStringException(0, "git-lfs returned exit status %d", retcode);
+            throw MakeStringException(0, "git-lfs for '%s' (user %s) returned exit status %d", filename, gitUser ? gitUser : "", retcode);
         }
     }
 
@@ -388,12 +395,12 @@ class GitRepositoryFile : implements IFile, public CInterface
     virtual IFileIO * open(IFOmode mode, IFEflags extraFlags) override
     {
         assertex(mode==IFOread && isExisting && !isDir);
-        return new GitRepositoryFileIO(commitTree, gitDirectory, &oid, gitUser);
+        return new GitRepositoryFileIO(fullName, commitTree, gitDirectory, &oid, gitUser);
     }
     virtual IFileIO * openShared(IFOmode mode, IFSHmode shmode, IFEflags extraFlags) override
     {
         assertex(mode==IFOread && isExisting && !isDir);
-        return new GitRepositoryFileIO(commitTree, gitDirectory, &oid, gitUser);
+        return new GitRepositoryFileIO(fullName, commitTree, gitDirectory, &oid, gitUser);
     }
 
 

dali/dfu/dfurun.cpp

@@ -1429,15 +1429,19 @@ class CDFUengine: public CInterface, implements IDFUengine
                     // keys default wrap for copy
                     if (destination->getWrap()||(iskey&&(cmd==DFUcmd_copy)))
                     {
-                        if (destination->getNumPartsOverride())
-                            throw makeStringExceptionV(-1, "DestinationNumPartOverride is provided but %s", destination->getWrap()?"getWrap is true":"is copying a key");
-                        dst->setNumPartsOverride(srcFile->numParts());
+                        unsigned numOverrideParts = destination->getNumPartsOverride();
+                        if (numOverrideParts)
+                        {
+                            if (srcFile->numParts() != numOverrideParts)
+                                throw makeStringExceptionV(-1, "Destination NumPartsOverride is provided but %s", (iskey&&(cmd==DFUcmd_copy))?"not supported when copying a key":"getWrap is true");
+                        }
+                        dst->setNumParts(srcFile->numParts());
                     }
                     else if (plane)
                     {
                         // use destination defaultSprayParts if requestor doesn't provide num parts
                         if (plane->hasProp("@defaultSprayParts") && destination->getNumPartsOverride()==0)
-                            dst->setNumPartsOverride(plane->getPropInt("@defaultSprayParts"));
+                            dst->setNumParts(plane->getPropInt("@defaultSprayParts"));
                     }
                 }
                 break;

dali/dfu/dfuwu.cpp

@@ -1378,7 +1378,7 @@ class CDFUfileSpec: public CLinkedDFUWUchild, implements IDFUfileSpec
     {
         queryRoot()->setProp("@partmask",val);
     }
-    void setNumParts(unsigned val)
+    virtual void setNumParts(unsigned val) override
     {
         queryRoot()->setPropInt("@numparts",val);
 

dali/dfu/dfuwu.hpp

@@ -281,6 +281,7 @@ interface IDFUfileSpec: extends IConstDFUfileSpec
     virtual void setFromXML(const char *xml) = 0;
     virtual void setCompressed(bool set) = 0;
     virtual void setWrap(bool val) = 0;
+    virtual void setNumParts(unsigned val) = 0;
     virtual void setNumPartsOverride(unsigned num) = 0;
     virtual void setReplicateOffset(int val) = 0;           // sets for all clusters
     virtual void setDiffKey(const char *keyname) = 0;

dali/ft/filecopy.cpp

@@ -3826,7 +3826,7 @@ void FileSprayer::updateTargetProperties()
         curProps.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), prevNumReads + totalNumReads);
         curProps.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), legacyReadCost + prevReadCost + totalReadCost);
     }
-    progressReport->setFileAccessCost(cost_type2money(totalReadCost+totalWriteCost));
+    progressReport->setFileAccessCost(totalReadCost+totalWriteCost);
     if (error)
         throw error.getClear();
 }

docs/EN_US/ContainerizedHPCC/ContainerizedMods/LocalDeployment.xml

@@ -453,7 +453,7 @@ OUTPUT(allPeople,,'MyData::allPeople',THOR,OVERWRITE);
     <emphasis role="strong">C:\hpccdata\hpcc-data\tutorial</emphasis>, you can
     then reference the file using this syntax:</para>
 
-    <para><programlisting>'~plane::data::tutorial::originalperson'</programlisting><variablelist>
+    <para><programlisting>'~plane::hpcc-data::tutorial::originalperson'</programlisting><variablelist>
         <varlistentry>
           <term>Note:</term>
 

docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml

@@ -287,6 +287,25 @@
       url="http://httpd.apache.org/docs/2.2/programs/htpasswd.html">http://httpd.apache.org/docs/2.2/programs/htpasswd.html</ulink>.</para>
     </sect3>
   </sect2>
+<sect2 id="SingleUserSecMgr">
+    <title>Single User Security Manager</title>
+
+    <para>The Single User security manager is a specialized security manager
+    that allows a username/password combination to be specified on the ESP
+    startup command line. At runtime, when you attempt to access any
+    authenticating ESP feature, such as ECL Watch, you must specify a
+    username/password combination.</para>
+
+    <para>A single user security manager could be useful for a custom
+    deployment where you do not want to configure an entire LDAP server or
+    create a Linux HTPASSWD file, such as a classroom environment or a custom
+    HPCC Systems Virtual Machine.</para>
+
+    <para>See the <ulink
+    url="https://hpccsystems.com/training/documentation/all/"><emphasis>Security
+    Manager Plugin Framework</emphasis></ulink> document for more information
+    on configuring and deploying Security Manager plugins.</para>
+  </sect2>
 
   <sect2 id="Using_LDAP_Authentication" role="brk">
     <title>Using LDAP Authentication</title>

ecl/eclcc/eclcc.cpp

@@ -1252,6 +1252,12 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
     }
 
     IErrorReceiver & errorProcessor = *severityMapper;
+    //Associate the error handler - so that failures to fetch from git can be reported as errors, but also mapped
+    //to warnings to ensure that automated tasks do not fail because the could not connect to git.
+    localRepositoryManager.setErrorReceiver(&errorProcessor);
+    //Ensure the error processor is cleared up when we exit this function
+    COnScopeExit scoped([&]() { localRepositoryManager.setErrorReceiver(NULL); });
+
     //All dlls/exes are essentially cloneable because you may be running multiple instances at once
     //The only exception would be a dll created for a one-time query.  (Currently handled by eclserver.)
     instance.wu->setCloneable(true);
@@ -1507,7 +1513,10 @@ void EclCC::processSingleQuery(const EclRepositoryManager & localRepositoryManag
         {
             StringBuffer s;
             e->errorMessage(s);
-            errorProcessor.reportError(3, s.str(), defaultErrorPathname, 1, 0, 0);
+            unsigned errorCode = e->errorCode();
+            if ((errorCode < HQL_ERROR_START) || (errorCode > HQL_ERROR_END))
+                errorCode = ERR_UNKNOWN_EXCEPTION;
+            errorProcessor.reportError(errorCode, s.str(), defaultErrorPathname, 1, 0, 0);
             e->Release();
         }
 

ecl/hql/hqlerrors.hpp

@@ -442,6 +442,10 @@
 #define ERR_INVALID_PROBABILITY     2403
 #define ERR_DEFAULT_VIRTUAL_CLASH   2404
 #define ERR_EMBEDERROR              2405
+#define ERR_FAIL_CLONE_REPO         2406
+#define ERR_FAIL_UPDATE_REPO        2407
+#define ERR_CANNOT_RESOLVE_BRANCH   2408
+#define ERR_UNKNOWN_EXCEPTION       2409
 
 #define ERR_CPP_COMPILE_ERROR       2999
 

ecl/hql/hqlgram.y

@@ -3837,7 +3837,7 @@ soapFlag
                             if ($3.queryExprType()->isBoolean())
                                 parser->normalizeExpression($3, type_boolean, true);
                             else
-                                parser->normalizeExpression($3, type_int, true);
+                                parser->normalizeExpression($3, type_int, false);
                             $$.setExpr(createExprAttribute(persistAtom, $3.getExpr()), $1);
                         }
     ;

ecl/hql/hqlrepository.cpp

@@ -821,14 +821,18 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
     }
 
     bool ok = false;
+    Owned<IError> error;
     CCycleTimer gitDownloadTimer;
     if (alreadyExists)
     {
         if (options.updateRepos)
         {
             unsigned retCode = runGitCommand(nullptr, "fetch origin", repoPath, true);
             if (retCode != 0)
-                DBGLOG("Failed to download the latest version of %s", defaultUrl);
+            {
+                VStringBuffer msg("Failed to download the latest version of '%s' error code (%u)", defaultUrl, retCode);
+                error.setown(createError(CategoryError, SeverityError, ERR_FAIL_UPDATE_REPO, msg.str(), nullptr, 0, 0, 0));
+            }
         }
 
         ok = true;
@@ -844,17 +848,30 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
             VStringBuffer params("clone %s \"%s\" --no-checkout", repoUrn.str(), repo.str());
             unsigned retCode = runGitCommand(nullptr, params, options.eclRepoPath, true);
             if (retCode != 0)
-                throw makeStringExceptionV(99, "Failed to clone dependency '%s'", defaultUrl);
+            {
+                VStringBuffer msg("Failed to clone dependency '%s' error code (%u)", defaultUrl, retCode);
+                error.setown(createError(CategoryError, SeverityError, ERR_FAIL_CLONE_REPO, msg.str(), nullptr, 0, 0, 0));
+            }
             ok = true;
         }
     }
     gitDownloadCycles += gitDownloadTimer.elapsedCycles();
+    if (error)
+    {
+        if (errorReceiver)
+        {
+            error.setown(errorReceiver->mapError(error)); // MORE: This mapping should really be done within reportError()
+            errorReceiver->report(error);
+        }
+        else
+            throw error.getClear();
+    }
 
     if (!ok)
         throw makeStringExceptionV(99, "Cannot locate the source code for dependency '%s'.  --fetchrepos not enabled", defaultUrl);
 
     if (startsWith(version, "semver:"))
-        throw makeStringExceptionV(99, "Semantic versioning not yet supported for dependency '%s'.", defaultUrl);
+        throw makeStringExceptionV(ERR_CANNOT_RESOLVE_BRANCH, "Semantic versioning not yet supported for dependency '%s'.", defaultUrl);
 
     // Really the version should be a SHA, but for flexibility version could be a sha, a tag or a branch (on origin).
     // Check for a remote branch first - because it appears that when git clones a repo, it creates a local branch for
@@ -879,7 +896,7 @@ IEclSourceCollection * EclRepositoryManager::resolveGitCollection(const char * r
         DBGLOG("Version '%s' resolved to sha '%s'", version.str(), sha.str());
 
     if (sha.isEmpty())
-        throw makeStringExceptionV(99, "Branch/tag '%s' could not be found for dependency '%s'.", version.str(), defaultUrl);
+        throw makeStringExceptionV(ERR_CANNOT_RESOLVE_BRANCH, "Branch/tag '%s' could not be found for dependency '%s'.", version.str(), defaultUrl);
 
     path.append(repoPath).appendf("/.git/{%s", sha.str());
     if (options.gitUser)
@@ -937,6 +954,7 @@ unsigned EclRepositoryManager::runGitCommand(StringBuffer * output, const char *
     Owned<IFile> extractedKey;
     EnvironmentVector env;
     //If fetching from git and the username is specified then use the script file to provide the username/password
+    //NB: This code should be kept in sync with readLfsContents() in gitfile.cpp (and ideally combined)
     if (needCredentials)
     {
         //If the username is supplied, then get the secret and write it to a temporary location.

ecl/hql/hqlrepository.hpp

@@ -74,7 +74,10 @@ class HQL_API EclRepositoryManager
     }
 
     IEclSourceCollection * resolveGitCollection(const char * repoPath, const char * defaultUrl);
-
+    void setErrorReceiver(IErrorReceiver * _errorReceiver) const
+    {
+        errorReceiver = _errorReceiver;
+    }
 protected:
     IEclRepository * createNewSourceFileEclRepository(IErrorReceiver *errs, const char * path, unsigned flags, unsigned trace, bool includeInArchive);
     IEclRepository * createGitRepository(IErrorReceiver *errs, const char * path, const char * urn, unsigned flags, unsigned trace, bool includeInArchive);
@@ -85,6 +88,7 @@ class HQL_API EclRepositoryManager
     IEclPackage * queryRepository(IIdAtom * name, const char * defaultUrl, IEclSourceCollection * overrideSource, bool includeDefinitions);
 
 private:
+    mutable IErrorReceiver * errorReceiver = nullptr; // mutable to allow const methods to set it, it logically doesn't change the object
     using DependencyInfo = std::pair<std::string, Shared<IEclPackage>>;
     CIArrayOf<EclRepositoryMapping> repos;
     std::vector<DependencyInfo> dependencies;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -18242,7 +18242,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
         else if (matchesBoolean(persistArg, true))
             persistArg = nullptr;
         else if (!matchesConstantValue(persistArg, 0)) // Avoid generating 0 since that is the default implementation
-            doBuildUnsignedFunction(instance->createctx, "getPersistMaxRequests", persistArg);
+            doBuildUnsignedFunction(instance->startctx, "getPersistMaxRequests", persistArg);
     }
 
     //virtual unsigned getFlags()