diff --git a/cmake_modules/parse_cmake.sh b/cmake_modules/parse_cmake.sh index ab85a24a866..badee4cf861 100755 --- a/cmake_modules/parse_cmake.sh +++ b/cmake_modules/parse_cmake.sh @@ -138,6 +138,8 @@ function update_version_file() local _new_sequence=$3 local _new_minor=$4 local _new_major=$5 + local _new_timestamp=$(date -u "+%Y-%m-%dT%H:%M:%SZ") + if [ -z "$_new_minor" ] ; then _new_minor=$HPCC_MINOR fi @@ -152,6 +154,7 @@ function update_version_file() -e "\"s/HPCC_POINT +$HPCC_POINT *\)/HPCC_POINT $_new_point )/\"" \ -e "\"s/HPCC_SEQUENCE +$HPCC_SEQUENCE *\)/HPCC_SEQUENCE $_new_sequence )/\"" \ -e "\"s/HPCC_MATURITY +\"$HPCC_MATURITY\" *\)/HPCC_MATURITY \"$_new_maturity\" )/\"" \ + -e "\"s/HPCC_TAG_TIMESTAMP.*\)/HPCC_TAG_TIMESTAMP \"$_new_timestamp\" )/\"" \ -i.bak $VERSIONFILE fi if [ -z "$DRYRUN" ] ; then @@ -161,6 +164,7 @@ function update_version_file() -e "s/HPCC_POINT +$HPCC_POINT *\)/HPCC_POINT $_new_point )/" \ -e "s/HPCC_SEQUENCE +$HPCC_SEQUENCE *\)/HPCC_SEQUENCE $_new_sequence )/" \ -e "s/HPCC_MATURITY +\"$HPCC_MATURITY\" *\)/HPCC_MATURITY \"$_new_maturity\" )/" \ + -e "s/HPCC_TAG_TIMESTAMP.*\)/HPCC_TAG_TIMESTAMP \"$_new_timestamp\" )/" \ -i.bak $VERSIONFILE cat $VERSIONFILE else @@ -170,6 +174,7 @@ function update_version_file() -e "s/HPCC_POINT +$HPCC_POINT *\)/HPCC_POINT $_new_point )/" \ -e "s/HPCC_SEQUENCE +$HPCC_SEQUENCE *\)/HPCC_SEQUENCE $_new_sequence )/" \ -e "s/HPCC_MATURITY +\"$HPCC_MATURITY\" *\)/HPCC_MATURITY \"$_new_maturity\" )/" \ + -e "s/HPCC_TAG_TIMESTAMP.*\)/HPCC_TAG_TIMESTAMP \"$_new_timestamp\" )/" \ $VERSIONFILE fi } diff --git a/esp/bindings/http/platform/httpbinding.cpp b/esp/bindings/http/platform/httpbinding.cpp index 933c549af33..0cbcaffcc09 100644 --- a/esp/bindings/http/platform/httpbinding.cpp +++ b/esp/bindings/http/platform/httpbinding.cpp @@ -159,7 +159,10 @@ class CEspCorsAllowedOrigin : public CInterfaceOf { StringBuffer scheme; StringBuffer unused; - splitUrlIsolateScheme(_origin, unused, unused, scheme, hostPort, unused); + StringBuffer port; + splitUrlIsolateScheme(_origin, unused, unused, scheme, hostPort, port, unused); + if (port.length()) + hostPort.append(':').append(port); //Allow-Max-Age of 7200 (2 hours) matches the limit in chrome const char *allowedMaxAge = allowed->queryProp("@maxAge"); diff --git a/esp/scm/ws_logaccess.ecm b/esp/scm/ws_logaccess.ecm index 8fa42f07e96..71717e9ada4 100644 --- a/esp/scm/ws_logaccess.ecm +++ b/esp/scm/ws_logaccess.ecm @@ -78,7 +78,7 @@ ESPenum LogColumnValueType : string enum("enum") }; -ESPStruct LogColumn +ESPStruct [nil_remove] LogColumn { string Name; ESPenum LogColumnType LogType; diff --git a/initfiles/examples/embed/python-catch.ecl b/initfiles/examples/embed/python-catch.ecl index 11b06dbfc7e..341be6550ef 100644 --- a/initfiles/examples/embed/python-catch.ecl +++ b/initfiles/examples/embed/python-catch.ecl @@ -1,4 +1,4 @@ -IMPORT Python; +IMPORT Python3 AS Python; /* This example illustrates and tests the use of embedded Python @@ -6,17 +6,17 @@ IMPORT Python; // Mapping of exceptions from Python to ECL -integer testThrow(integer val) := EMBED(Python) -raise Exception('Error from Python') +INTEGER testThrow(integer val) := EMBED(Python) + raise Exception('Error from Python') ENDEMBED; // Can't catch an expression(only a dataset) -d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold'); +d := DATASET([{ 1, '' }], { INTEGER a, STRING m} ) : STORED('nofold'); -d t := transform +d t := TRANSFORM self.a := FAILCODE; self.m := FAILMESSAGE; self := []; end; -catch(d(testThrow(a) = a), onfail(t)); +CATCH(d(testThrow(a) = a), onfail(t)); diff --git a/initfiles/examples/embed/python-import.ecl b/initfiles/examples/embed/python-import.ecl index 6015476fcfd..19a4f61d145 100644 --- a/initfiles/examples/embed/python-import.ecl +++ b/initfiles/examples/embed/python-import.ecl @@ -1,4 +1,4 @@ -IMPORT python; +IMPORT Python3 AS Python; /* This example illustrates a call to a Python functions defined in the Python module python_cat.py diff --git a/initfiles/examples/embed/python-simple.ecl b/initfiles/examples/embed/python-simple.ecl index 7dfd3cbfd9e..53d88daa47e 100644 --- a/initfiles/examples/embed/python-simple.ecl +++ b/initfiles/examples/embed/python-simple.ecl @@ -1,4 +1,4 @@ -IMPORT Python; +IMPORT Python3 AS Python; /* This example illustrates and tests the use of embedded Python @@ -6,75 +6,75 @@ IMPORT Python; // Scalar parameters and resuls -integer add1(integer val) := EMBED(Python) -val+1 +INTEGER add1(INTEGER val) := EMBED(Python) + val+1 ENDEMBED; -string add2(string val) := EMBED(Python) -val+'1' +STRING add2(STRING val) := EMBED(Python) + val+'1' ENDEMBED; -string add3(varstring val) := EMBED(Python) -val+'1' +STRING add3(VARSTRING val) := EMBED(Python) + val+'1' ENDEMBED; -utf8 add4(utf8 val) := EMBED(Python) -val+'1' +UTF8 add4(UTF8 val) := EMBED(Python) + val+'1' ENDEMBED; -unicode add5(unicode val) := EMBED(Python) -val+'1' +UNICODE add5(UNICODE val) := EMBED(Python) + val+'1' ENDEMBED; -utf8 add6(utf8 val) := EMBED(Python) -return val+'1' +UTF8 add6(UTF8 val) := EMBED(Python) + return val+'1' ENDEMBED; -unicode add7(unicode val) := EMBED(Python) -return val+'1' +UNICODE add7(UNICODE val) := EMBED(Python) + return val+'1' ENDEMBED; -data testData(data val) := EMBED(Python) -val[0] = val[0] + 1 -return val +DATA testData(DATA val) := EMBED(Python) + val[0] = val[0] + 1 + return val ENDEMBED; // Sets in ECL map to Python lists -set of integer testSet(set of integer val) := EMBED(Python) -return sorted(val) +SET OF INTEGER testSet(SET OF INTEGER val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of string testSet2(set of string val) := EMBED(Python) -return sorted(val) +SET OF STRING testSet2(SET OF STRING val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of string testSet3(set of string8 val) := EMBED(Python) -return sorted(val) +SET OF STRING testSet3(SET OF STRING8 val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of utf8 testSet4(set of utf8 val) := EMBED(Python) -return sorted(val) +SET OF UTF8 testSet4(SET OF UTF8 val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of varstring testSet5(set of varstring val) := EMBED(Python) -return sorted(val) +SET OF VARSTRING testSet5(SET OF VARSTRING val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of varstring8 testSet6(set of varstring8 val) := EMBED(Python) -return sorted(val) +SET OF VARSTRING8 testSet6(SET OF VARSTRING8 val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of unicode testSet7(set of unicode val) := EMBED(Python) -return sorted(val) +SET OF UNICODE testSet7(SET OF UNICODE val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of unicode8 testSet8(set of unicode8 val) := EMBED(Python) -return sorted(val) +SET OF UNICODE8 testSet8(SET OF UNICODE8 val) := EMBED(Python) + return sorted(val) ENDEMBED; -set of data testSet9(set of data val) := EMBED(Python) -return val +SET OF DATA testSet9(SET OF DATA val) := EMBED(Python) + return val ENDEMBED; // Now run the tests diff --git a/initfiles/examples/embed/python-simple2.ecl b/initfiles/examples/embed/python-simple2.ecl index 13f612e0189..34f2a4c3758 100644 --- a/initfiles/examples/embed/python-simple2.ecl +++ b/initfiles/examples/embed/python-simple2.ecl @@ -1,11 +1,11 @@ -import python; +IMPORT Python3 AS Python; /* This example illustrates and tests the use of embedded Python. In this example the python that is embedded is more complex, including a definition of a function */ -string anagram(string word) := EMBED(Python) +STRING anagram(string word) := EMBED(Python) def anagram(w): if word == 'cat': return 'act' diff --git a/initfiles/examples/embed/python-stream.ecl b/initfiles/examples/embed/python-stream.ecl index ecadf05c647..4080d0366e7 100644 --- a/initfiles/examples/embed/python-stream.ecl +++ b/initfiles/examples/embed/python-stream.ecl @@ -1,4 +1,4 @@ -IMPORT Python; +IMPORT Python3 AS Python; /* This example illustrates and tests the use of embedded Python. @@ -9,17 +9,17 @@ IMPORT Python; // These are the record structures we will be returning - note the child records, datasets and dictionaries in it childrec := RECORD - string name => unsigned value; + STRING name => UNSIGNED value; END; eclRecord := RECORD STRING name1; STRING10 name2; LINKCOUNTED DATASET(childrec) childnames; - LINKCOUNTED DICTIONARY(childrec) childdict{linkcounted}; + LINKCOUNTED DICTIONARY(childrec) childdict{LINKCOUNTED}; childrec r; - unsigned1 val1; - integer1 val2; + UNSIGNED1 val1; + INTEGER1 val2; UTF8 u1; UNICODE u2; UNICODE8 u3; @@ -30,43 +30,43 @@ eclRecord := RECORD END; namerec := RECORD - string name; + STRING name; END; namerec2 := RECORD - string name; - string name2; + STRING name; + STRING name2; END; // To return a dataset, we can return a list of tuples, each one correponding to a field in the resulting ECL record // In this example, the fields are mapped by position // Just to spice things up we proved a couple of parameters too -dataset(eclRecord) streamedNames(data d, utf8 u) := EMBED(Python) +DATASET(eclRecord) streamedNames(DATA d, utf8 u) := EMBED(Python) return [ \ ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1, U'là', U'là', U'là', 0x01000000, d, False, {"1","2"}), \ ("John", "Smith", [], [], ("c", 3), 250, -1, U'là', U'là', u, 0x02000000, d, True, []) \ ] ENDEMBED; -output(streamedNames(d'AA', u'là')); +OUTPUT(streamedNames(d'AA', u'là')); // We can also return a dataset by using a Python generator, which will be lazy-evaluated as the records are required by ECL code... -dataset(childrec) testGenerator(unsigned lim) := EMBED(Python) +DATASET(childrec) testGenerator(unsigned lim) := EMBED(Python) num = 0 while num < lim: yield ("Generated", num) num += 1 ENDEMBED; -output (testGenerator(10)); +OUTPUT (testGenerator(10)); // If the returned tuples are namedtuples, we map fields by name rather than by position // Test use of Python named tuple... -dataset(childrec) testNamedTuples() := EMBED(Python) +DATASET(childrec) testNamedTuples() := EMBED(Python) import collections ChildRec = collections.namedtuple("childrec", "value, name") # Note - order is reverse of childrec - but works as we get fields by name c1 = ChildRec(1, "name1") @@ -74,7 +74,7 @@ dataset(childrec) testNamedTuples() := EMBED(Python) return [ c1, c2 ] ENDEMBED; -output(testNamedTuples()); +OUTPUT(testNamedTuples()); // To return a record, just return a tuple (or namedtuple) @@ -82,51 +82,51 @@ childrec testRecord(integer value, string s) := EMBED(Python) return (s, value) ENDEMBED; -output(testRecord(1,'Hello').value); -output(testRecord(1,'Hello').name); +OUTPUT(testRecord(1,'Hello').value); +OUTPUT(testRecord(1,'Hello').name); // If the record has a single field, you don't need to put the field into a tuple... -dataset(namerec) testMissingTuple1(unsigned lim) := EMBED(Python) +DATASET(namerec) testMissingTuple1(unsigned lim) := EMBED(Python) return [ '1', '2', '3' ] ENDEMBED; -output (testMissingTuple1(10)); +OUTPUT (testMissingTuple1(10)); // ... but you can if you want -dataset(namerec) testMissingTuple2(unsigned lim) := EMBED(Python) +DATASET(namerec) testMissingTuple2(unsigned lim) := EMBED(Python) return [ ('1'), ('2'), ('3') ] ENDEMBED; -output (testMissingTuple2(10)); +OUTPUT (testMissingTuple2(10)); // You can define a transform in Python, using a function that returns a record (i.e. a Python tuple) // Note that the tuple we pass to Python is a namedtuple -transform(childrec) testTransform(namerec inrec, unsigned c) := EMBED(Python) +TRANSFORM(childrec) testTransform(namerec inrec, unsigned c) := EMBED(Python) return (inrec.name, c) ENDEMBED; -d := dataset([{'Richard'},{'Gavin'}], namerec); +d := DATASET([{'Richard'},{'Gavin'}], namerec); -output(project(d, testTransform(LEFT, COUNTER))); +OUTPUT(PROJECT(d, testTransform(LEFT, COUNTER))); // Most transforms take a record as the input, but it's not a requirement -transform(childrec) testTransformNoRow(unsigned lim) := EMBED(Python) +TRANSFORM(childrec) testTransformNoRow(unsigned lim) := EMBED(Python) return ("Hello", lim) ENDEMBED; -output(row(testTransformNoRow(10))); +OUTPUT(ROW(testTransformNoRow(10))); // When passing datasets to Python, we get an iterator of named tuples // They are actually implemented as generators, meaning they are lazy-evaluated names := DATASET([{'Richard'}, {'James'}, {'Andrew'}], namerec); -string datasetAsIterator(dataset(namerec) input) := EMBED(Python) +STRING datasetAsIterator(DATASET(namerec) input) := EMBED(Python) s = '' for n in input: s = s + ' ' + n.name return s; ENDEMBED; -output(datasetAsIterator(names)); +OUTPUT(datasetAsIterator(names)); diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index 39723c6741e..185f7d128a9 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -836,6 +836,7 @@ constexpr LogMsgCategory MCdebugInfo(MSGAUD_programmer, MSGCLS_information, Debu constexpr LogMsgCategory MCauditInfo(MSGAUD_audit, MSGCLS_information, AudMsgThreshold); constexpr LogMsgCategory MCoperatorInfo(MSGAUD_operator, MSGCLS_information, InfoMsgThreshold); constexpr LogMsgCategory MCoperatorMetric(MSGAUD_operator, MSGCLS_metric, ErrMsgThreshold); +constexpr LogMsgCategory MCmonitorEvent(MSGAUD_monitor, MSGCLS_event, ProgressMsgThreshold); /* * Function to determine log level (detail) for exceptions, based on log message class diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index b99e6265c7c..7fa24b4b794 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -143,7 +143,7 @@ static void validateKeyName(const char *key) throw makeStringExceptionV(-1, "Invalid secret key name %s", key); } -static void splitUrlAddress(const char *address, size_t len, StringBuffer &host, StringBuffer *port) +static void splitUrlAddress(const char *address, size_t len, StringBuffer &host, StringBuffer &port) { if (!address || len==0) return; @@ -154,14 +154,11 @@ static void splitUrlAddress(const char *address, size_t len, StringBuffer &host, { host.append(sep - address, address); len = len - (sep - address) - 1; - if (port) - port->append(len, sep+1); - else - host.append(':').append(len, sep+1); + port.append(len, sep+1); } } -static void splitUrlAuthority(const char *authority, size_t authorityLen, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer *port) +static void splitUrlAuthority(const char *authority, size_t authorityLen, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port) { if (!authority || authorityLen==0) return; @@ -184,6 +181,14 @@ static void splitUrlAuthority(const char *authority, size_t authorityLen, String } } +static void splitUrlAuthorityHostPort(const char *authority, size_t authorityLen, StringBuffer &user, StringBuffer &password, StringBuffer &hostPort) +{ + StringBuffer port; + splitUrlAuthority(authority, authorityLen, user, password, hostPort, port); + if (port.length()) + hostPort.append(':').append(port); +} + static inline void extractUrlProtocol(const char *&url, StringBuffer *scheme) { if (!url) @@ -224,7 +229,7 @@ extern jlib_decl void splitFullUrl(const char *url, StringBuffer &user, StringBu const char *authority = nullptr; size_t authorityLen = 0; splitUrlSections(url, authority, authorityLen, path, nullptr); - splitUrlAuthority(authority, authorityLen, user, password, host, &port); + splitUrlAuthority(authority, authorityLen, user, password, host, port); } extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path) @@ -232,15 +237,15 @@ extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user const char *authority = nullptr; size_t authorityLen = 0; splitUrlSections(url, authority, authorityLen, path, &schemeHostPort); - splitUrlAuthority(authority, authorityLen, user, password, schemeHostPort, nullptr); + splitUrlAuthorityHostPort(authority, authorityLen, user, password, schemeHostPort); } -extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &scheme, StringBuffer &hostPort, StringBuffer &path) +extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &scheme, StringBuffer &host, StringBuffer &port, StringBuffer &path) { const char *authority = nullptr; size_t authorityLen = 0; splitUrlSections(url, authority, authorityLen, path, &scheme); - splitUrlAuthority(authority, authorityLen, user, password, hostPort, nullptr); + splitUrlAuthority(authority, authorityLen, user, password, host, port); } @@ -260,8 +265,23 @@ extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secret { secretName.set("http-connect-"); //Having the host and port visible will help with manageability wherever the secret is stored - if (scheme && !strnicmp("https", scheme, 5)) - secretName.append("ssl-"); + if (scheme) + { + if (!strnicmp("http", scheme, 4)) + { + if ('s' == scheme[4]) + { + if (443 == port) + port = 0; // suppress default port, such that with or without, the generated secret name will be the same + secretName.append("ssl-"); + } + else if (':' == scheme[4]) + { + if (80 == port) + port = 0; // suppress default port, such that with or without, the generated secret name will be the same + } + } + } secretName.append(host); //port is optionally already part of host replaceExtraHostAndPortChars(secretName); @@ -290,13 +310,14 @@ extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secret StringBuffer username; StringBuffer urlPassword; StringBuffer scheme; - StringBuffer hostPort; + StringBuffer host; + StringBuffer port; StringBuffer path; - splitUrlIsolateScheme(url, username, urlPassword, scheme, hostPort, path); + splitUrlIsolateScheme(url, username, urlPassword, scheme, host, port, path); if (!isEmptyString(inputUsername)) username.set(inputUsername); - - return generateDynamicUrlSecretName(secretName, scheme, username, hostPort, 0, path); + unsigned portNum = port.length() ? atoi(port) : 0; + return generateDynamicUrlSecretName(secretName, scheme, username, host, portNum, path); } //--------------------------------------------------------------------------------------------------------------------- diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index 81bada07763..39dddeeb3d2 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -37,7 +37,7 @@ extern jlib_decl void setSecretMount(const char * path); extern jlib_decl void setSecretTimeout(unsigned timeoutMs); extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId = nullptr, const char * optVersion = nullptr); -extern jlib_decl ISecret * resolveSecret(const char *category, const char * name, const char * optRequiredVault); +extern jlib_decl ISecret * resolveSecret(const char *category, const char * name, const char * optRequiredVault, const char* optVersion); extern jlib_decl bool getSecretKeyValue(MemoryBuffer & result, const IPropertyTree *secret, const char * key); extern jlib_decl bool getSecretKeyValue(StringBuffer & result, const IPropertyTree *secret, const char * key); @@ -56,7 +56,7 @@ extern jlib_decl IPropertyTree *createIssuerTlsClientConfig(const char *issuer, extern jlib_decl void splitFullUrl(const char *url, bool &https, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port, StringBuffer &fullpath); extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path); -extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &scheme, StringBuffer &hostPort, StringBuffer &path); +extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &scheme, StringBuffer &host, StringBuffer &port, StringBuffer &path); extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *scheme, const char *userPasswordPair, const char *host, unsigned port, const char *path); extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *url, const char *username); diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 88ccae0ec6b..2eaa65d7c7f 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -163,7 +163,7 @@ class CSpan : public CInterfaceOf { StringBuffer out; toLog(out); - DBGLOG("Span end: {%s}", out.str()); + LOG(MCmonitorEvent, "Span end: {%s}", out.str()); } const char * getSpanID() const @@ -449,7 +449,7 @@ class CSpan : public CInterfaceOf StringBuffer out; toLog(out); - DBGLOG("Span start: {%s}", out.str()); + LOG(MCmonitorEvent, "Span start: {%s}", out.str()); } } diff --git a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp index 0ed01dd2c78..fad1077b23d 100644 --- a/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp +++ b/system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp @@ -490,9 +490,9 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns if (!isEmptyString(m_podSearchColName)) columns.appendf("%s, ", m_podSearchColName.str()); - columns.appendf("%s, %s, %s, %s, %s, %s, %s", + columns.appendf("%s, %s, %s, %s, %s, %s, %s, %s", m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(), - m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol); + m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol, defaultHPCCLogProcIDCol); } bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName, bool targetsV2) diff --git a/testing/regress/ecl/embedforward.ecl b/testing/regress/ecl/embedforward.ecl index d1f1d299d0e..01abb5ab973 100644 --- a/testing/regress/ecl/embedforward.ecl +++ b/testing/regress/ecl/embedforward.ecl @@ -16,11 +16,11 @@ ############################################################################## */ //class=embedded -//class=python2 +//class=python3 //nothor -IMPORT Python; +IMPORT Python3 As Python; // Test whether you can use EMBED inside a FORWARD diff --git a/testing/regress/ecl/embedpy2-catch.ecl b/testing/regress/ecl/embedpy2-catch.ecl deleted file mode 100644 index ec0f91ae240..00000000000 --- a/testing/regress/ecl/embedpy2-catch.ecl +++ /dev/null @@ -1,41 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -//nothor - -//Thor doesn't handle CATCH properly, see HPCC-9059 -//skip type==thorlcr TBD - -IMPORT Python; - -integer testThrow(integer val) := EMBED(Python) -raise Exception('Error from Python') -ENDEMBED; - -// Can't catch an expression(only a dataset) -d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold'); - -d t := transform - self.a := FAILCODE; - self.m := FAILMESSAGE; - self := []; -end; - -catch(d(testThrow(a) = a), onfail(t)); diff --git a/testing/regress/ecl/embedpy2-fold.ecl b/testing/regress/ecl/embedpy2-fold.ecl deleted file mode 100644 index ffb71351cb3..00000000000 --- a/testing/regress/ecl/embedpy2-fold.ecl +++ /dev/null @@ -1,127 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -//nothor - -IMPORT Python; - -integer add1(integer val) := EMBED(Python:FOLD) -val+1 -ENDEMBED; - -string add2(string val) := EMBED(Python:FOLD) -val+'1' -ENDEMBED; - -string add3(varstring val) := EMBED(Python:FOLD) -val+'1' -ENDEMBED; - -utf8 add4(utf8 val) := EMBED(Python:FOLD) -val+'1' -ENDEMBED; - -unicode add5(unicode val) := EMBED(Python:FOLD) -val+'1' -ENDEMBED; - -utf8 add6(utf8 val) := EMBED(Python:FOLD) -return val+'1' -ENDEMBED; - -unicode add7(unicode val) := EMBED(Python:FOLD) -return val+'1' -ENDEMBED; - -data testData(data val) := EMBED(Python:FOLD) -val[0] = val[0] + 1 -return val -ENDEMBED; - -set of integer testSet(set of integer val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of string testSet2(set of string val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of string testSet3(set of string8 val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of utf8 testSet4(set of utf8 val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of varstring testSet5(set of varstring val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of varstring8 testSet6(set of varstring8 val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of unicode testSet7(set of unicode val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of unicode8 testSet8(set of unicode8 val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -set of data testSet9(set of data val) := EMBED(Python:FOLD) -return val[0:2] -ENDEMBED; - -// And a test that is NOT foldable: - -string myvalue := 'test' : STORED('myvalue'); - -integer add1a(integer val) := EMBED(Python:FOLD) -val+1 -ENDEMBED; - - -ASSERT(add1(10)=11, CONST); -ASSERT(add1a(10)=11); -ASSERT(add2('Hello')='Hello1', CONST); -ASSERT(add3('World')='World1', CONST); -ASSERT(add4(U'Oh là là Straße')=U'Oh là là Straße1', CONST); -ASSERT(add5(U'Стоял')=U'Стоял1', CONST); -ASSERT(add6(U'Oh là là Straße')=U'Oh là là Straße1', CONST); -ASSERT(add7(U'Стоял')=U'Стоял1', CONST); - -ASSERT(add2('Oh là là Straße')='Oh là là Straße1', CONST); // Passing latin chars - should be untranslated - -ASSERT(testData(D'ax')=D'bx', CONST); -ASSERT(testSet([1,3,2])=[1,3], CONST); -ASSERT(testSet2(['red','green','yellow'])=['red','green'],CONST); -ASSERT(testSet3(['one','two','three'])=['one','two'],CONST); - -ASSERT(testSet4([U'Oh', U'là', U'Straße'])=[U'Oh', U'là'], CONST); -ASSERT(testSet5(['Un','Deux','Trois'])=['Un','Deux'], CONST); -ASSERT(testSet6(['Uno','Dos','Tre'])=['Uno','Dos'], CONST); - -ASSERT(testSet7([U'On', U'der', U'Straße'])=[U'On', U'der'], CONST); -ASSERT(testSet8([U'Aus', U'zum', U'Straße'])=[U'Aus', U'zum'], CONST); -ASSERT(testSet9([D'Aus', D'zum', D'Strade'])=[D'Aus', D'zum'], CONST); - -OUTPUT('ok'); diff --git a/testing/regress/ecl/embedpy2.ecl b/testing/regress/ecl/embedpy2.ecl deleted file mode 100644 index 80bfd8dc51b..00000000000 --- a/testing/regress/ecl/embedpy2.ecl +++ /dev/null @@ -1,176 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -//nothor - -IMPORT Python; - -integer add1(integer val) := EMBED(Python) -val+1 -ENDEMBED; - -string add2(string val) := EMBED(Python) -val+'1' -ENDEMBED; - -string add3(varstring val) := EMBED(Python) -val+'1' -ENDEMBED; - -utf8 add4(utf8 val) := EMBED(Python) -val+'1' -ENDEMBED; - -unicode add5(unicode val) := EMBED(Python) -val+'1' -ENDEMBED; - -utf8 add6(utf8 val) := EMBED(Python) -return val+'1' -ENDEMBED; - -unicode add7(unicode val) := EMBED(Python) -return val+'1' -ENDEMBED; - -data testData(data val) := EMBED(Python) -val[0] = val[0] + 1 -return val -ENDEMBED; - -set of integer testSet(set of integer val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of string testSet2(set of string val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of string testSet3(set of string8 val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of utf8 testSet4(set of utf8 val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of varstring testSet5(set of varstring val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of varstring8 testSet6(set of varstring8 val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of unicode testSet7(set of unicode val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of unicode8 testSet8(set of unicode8 val) := EMBED(Python) -return sorted(val) -ENDEMBED; - -set of data testSet9(set of data val) := EMBED(Python) -return val -ENDEMBED; - -real8 realdivide(integer v1, integer v2) := EMBED(Python) -from __future__ import division -return v1/v2 -ENDEMBED; - -unsigned8 truncdivide(integer v1, integer v2) := EMBED(Python) -return v1/v2 -ENDEMBED; - - -add1(10); -add2('Hello'); -add3('World'); -add4(U'Oh là là Straße'); -add5(U'Стоял'); -add6(U'Oh là là Straße'); -add7(U'Стоял'); - -add2('Oh là là Straße'); // Passing latin chars - should be untranslated -realdivide(3,2); -truncdivide(3,2); - -testData(D'aa'); -testSet([1,3,2]); -testSet2(['red','green','yellow']); -testSet3(['one','two','three']); -testSet4([U'Oh', U'là', U'Straße']); -testSet5(['Un','Deux','Trois']); -testSet6(['Uno','Dos','Tre']); -testSet7([U'On', U'der', U'Straße']); -testSet8([U'Aus', U'zum', U'Straße']); -testSet9([D'Aus', D'zum', D'Strade']); - -s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER))); -s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2))); - SUM(NOFOLD(s1 + s2), a); - -s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER))); -s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2)))); - SUM(NOFOLD(s1a + s2a), a); - -s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1)); -s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1)); - SUM(NOFOLD(s1b + s2b), a); - -s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1'))); -s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1'))); - SUM(NOFOLD(s1c + s2c), a); - -unsigned persistscope1(unsigned a) := EMBED(Python: globalscope('yo'),persist('workunit')) - global b - b = a + 1 - return a -ENDEMBED; - -unsigned usepersistscope1(unsigned a) := EMBED(Python: globalscope('yo'),persist('workunit')) - global b - return a + b -ENDEMBED; - -unsigned persistscope2(unsigned a) := EMBED(Python: globalscope('yi'),persist('workunit')) - global b - b = a + 11 - return a -ENDEMBED; - -unsigned usepersistscope2(unsigned a) := EMBED(Python: globalscope('yi'),persist('workunit')) - global b - return a + b -ENDEMBED; - -unsigned usepersistscope2b := EMBED(Python: globalscope('yi'),persist('workunit')) - global b - return b -ENDEMBED; - -sequential( - persistscope1(1), - persistscope2(1), - usepersistscope1(1), - usepersistscope2(1), - usepersistscope2b -); diff --git a/testing/regress/ecl/embedpy2b.ecl b/testing/regress/ecl/embedpy2b.ecl deleted file mode 100644 index 4679aee341a..00000000000 --- a/testing/regress/ecl/embedpy2b.ecl +++ /dev/null @@ -1,37 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -import python; - -string anagram(string word) := EMBED(Python) - def anagram(w): - if word == 'cat': - return 'act' - else: - return w - - return anagram(word) -ENDEMBED; - -anagram('dog'); -anagram('cat'); - - - diff --git a/testing/regress/ecl/issue16712.ecl b/testing/regress/ecl/issue16712.ecl deleted file mode 100644 index af41adf08ea..00000000000 --- a/testing/regress/ecl/issue16712.ecl +++ /dev/null @@ -1,57 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -IMPORT Python; - -nested := RECORD - SET OF integer value; -END; - -parent := RECORD - DATASET(nested) nest; -END; - -DATASET(parent) getP() := EMBED(C++) - __result = rtlMalloc(46); - __lenResult = 46; - memcpy(__result, "\x2a\x00\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00" -"\x00\x00\x10\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00", 46); -ENDEMBED; - -unsigned pcode(DATASET(parent) p) := EMBED(Python) - count = 0 - for child in p: - for c2 in child.nest: - for c3 in c2: - for c4 in c3: - count += c4 - - return count -ENDEMBED; - -ds := getp(); -ds2 := DATASET([{[{[1,2]},{[3,4]}]}], parent); -sequential( - output(ds), - output(ds2), - pcode(ds); - pcode(ds2); - '' -); diff --git a/testing/regress/ecl/py2embedactivity.ecl b/testing/regress/ecl/py2embedactivity.ecl deleted file mode 100644 index 64efa7872d8..00000000000 --- a/testing/regress/ecl/py2embedactivity.ecl +++ /dev/null @@ -1,69 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -IMPORT Python; - -r := RECORD - UNSIGNED id; - STRING name; -END; - -m(unsigned numRows, boolean isLocal = false, unsigned numParallel = 0) := MODULE - EXPORT streamed dataset(r) myDataset(unsigned numRows = numRows) := EMBED(Python : activity, local(isLocal), parallel(numParallel)) - numSlaves = __activity__.numSlaves - numParallel = numSlaves * __activity__.numStrands - rowsPerPart = (numRows + numParallel - 1) / numParallel - thisSlave = __activity__.slave - thisIndex = thisSlave * __activity__.numStrands + __activity__.strand - first = thisIndex * rowsPerPart - last = first + rowsPerPart - if first > numRows: - first = numRows - if last > numRows: - last = numRows - - names = [ "Gavin", "Richard", "John", "Bart" ] - while first < last: - yield (first, names[first % 4 ]) - first += 1 - ENDEMBED; -END; - -r2 := RECORD - UNSIGNED id; - DATASET(r) child; -END; - -sequential( - //Global activity - fixed number of rows - output(m(10).myDataset()); - - //Local version of the activity - output(count(m(10, isLocal := true).myDataset()) = CLUSTERSIZE * 10); - - //Check that stranding (if implemented) still generates unique records - output(COUNT(DEDUP(m(1000, numParallel := 5).myDataset(), id, ALL))); - - //Check that the activity can also be executed in a child query - output(DATASET(10, TRANSFORM(r2, SELF.id := COUNTER; SELF.child := m(COUNTER).myDataset()))); - - //Test stranding inside a child query - output(DATASET(10, TRANSFORM(r2, SELF.id := COUNTER; SELF.child := m(COUNTER, NumParallel := 3).myDataset()))); -); \ No newline at end of file diff --git a/testing/regress/ecl/py2import.ecl b/testing/regress/ecl/py2import.ecl deleted file mode 100644 index 006a4a4cdd5..00000000000 --- a/testing/regress/ecl/py2import.ecl +++ /dev/null @@ -1,31 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -import ^ as root; -import python; -HPCCBaseDir := #IFDEFINED(root.HPCCBaseDir, '/opt/HPCCSystems/'); - -string pcat(string a, string b) := IMPORT(Python, HPCCBaseDir + 'examples/embed/python_cat.cat':time); -pcat('Hello ', 'world!'); - -integer padd(integer a, integer b) := EMBED(Python :time) - return a + b -ENDEMBED; -padd(1, 2)*5; diff --git a/testing/regress/ecl/py2streame.ecl b/testing/regress/ecl/py2streame.ecl deleted file mode 100644 index bd76198b8e7..00000000000 --- a/testing/regress/ecl/py2streame.ecl +++ /dev/null @@ -1,93 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -IMPORT Python; - -childrec := RECORD - string name => unsigned value; -END; - -namesRecord := RECORD - STRING name1; - STRING10 name2; - LINKCOUNTED DATASET(childrec) childnames; - LINKCOUNTED DICTIONARY(childrec) childdict{linkcounted}; - childrec r; - unsigned1 val1; - integer1 val2; - UTF8 u1; - UNICODE u2; - UNICODE8 u3; - BIG_ENDIAN unsigned4 val3; - DATA d; - BOOLEAN b; - SET OF STRING ss1; -END; - -dataset(namesRecord) blockedNames(string prefix) := EMBED(Python) - return ["Gavin","John","Bart"] -ENDEMBED; - -_linkcounted_ dataset(namesRecord) linkedNames(string prefix) := EMBED(Python) - return ["Gavin","John","Bart"] -ENDEMBED; - -dataset(namesRecord) streamedNames(data d, utf8 u) := EMBED(Python) - return [ \ - ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1, U'là', U'là', U'là', 1, d, False, ["1","2"]), \ - ("John", "Smith", [], [], ("c", 3), 250, -1, U'là', U'là', u, 2, d, True, set(["3"])) \ - ] -ENDEMBED; - -// Test use of Python generator object for lazy evaluation... - -dataset(childrec) testGenerator(unsigned lim) := EMBED(Python:time) - num = 0 - while num < lim: - yield ("Generate:", num) - num += 1 -ENDEMBED; - -output(streamedNames(d'AA', u'là')); -output (testGenerator(10)); - -// Test what happens when two threads pull from a generator -c := testGenerator(1000); -count(c(value < 500)); -count(c(value > 500)); - -// Test Python code returning named tuples -childrec tnamed(string s) := EMBED(Python) - import collections; - childrec = collections.namedtuple("childrec", "value,name") - return childrec(1,s) -ENDEMBED; - -output(tnamed('Yo').name); - -// Test passing records into Python - -dataset(namesRecord) streamInOut(dataset(namesRecord) recs) := EMBED(Python) - for rec in recs: - if rec.name1 == 'Gavin': - yield rec -ENDEMBED; - -output(streamInOut(streamedNames(d'AA', u'là'))); diff --git a/testing/regress/ecl/py2streame2.ecl b/testing/regress/ecl/py2streame2.ecl deleted file mode 100644 index 81ed71268e1..00000000000 --- a/testing/regress/ecl/py2streame2.ecl +++ /dev/null @@ -1,88 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -IMPORT Python; - -childrec := RECORD - string name => unsigned value; -END; - -namerec := RECORD - string name; -END; - -namerec2 := RECORD - string name; - string name2; -END; - -// Test use of Python generator object for lazy evaluation... - -dataset(childrec) testGenerator(unsigned lim) := EMBED(Python) - num = 0 - while num < lim: - yield ("Generate:", num) - num += 1 -ENDEMBED; - -// Test use of Python named tuple... - -dataset(childrec) testNamedTuple(unsigned lim) := EMBED(Python) - import collections - ChildRec = collections.namedtuple("childrec", "value, name") # Note - order is reverse of childrec - but works as we get fields by name - c1 = ChildRec(1, "name1") - c2 = ChildRec(name="name2", value=2) - return [ c1, c2 ] -ENDEMBED; - -// Test 'missing tuple' case... - -dataset(namerec) testMissingTuple1(unsigned lim) := EMBED(Python) - return [ '1', '2', '3' ] -ENDEMBED; - -dataset(namerec) testMissingTuple2(unsigned lim) := EMBED(Python) - return [ ('1'), ('2'), ('3') ] -ENDEMBED; - -// Test returning a row -childrec testRowReturn(unsigned lim) := EMBED(Python) - return ("Hello", lim) -ENDEMBED; - -// Test defining a transform -transform(childrec) testTransform(unsigned lim) := EMBED(Python) - return ("Hello", lim) -ENDEMBED; - -// Test a transform with input and output rows -transform(childrec) testTransform2(namerec inrec, unsigned p) := EMBED(Python) - return (inrec.name, p) -ENDEMBED; - -output (testGenerator(10)); -output (testNamedTuple(10)); -output (testMissingTuple1(10)); -output (testMissingTuple2(10)); - -output(testRowReturn(10)); -output(row(testTransform(10))); -d := dataset([{'Richard'},{'dsfg'}], namerec); -output(project(d, testTransform2(LEFT, 10))); diff --git a/testing/regress/ecl/py2streame3.ecl b/testing/regress/ecl/py2streame3.ecl deleted file mode 100644 index 4f09b86d7d7..00000000000 --- a/testing/regress/ecl/py2streame3.ecl +++ /dev/null @@ -1,44 +0,0 @@ -/*############################################################################## - - HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -############################################################################## */ - -//class=embedded -//class=python2 - -IMPORT Python; - -childrec := RECORD - string name => unsigned value; -END; - -titleRec := { string title }; -titles := dataset(['', 'Mr. ', 'Rev. '], titleRec); - -// Test defining a transform -transform(childrec) testTransformTitle(titleRec inrec, unsigned lim) := EMBED(Python) - return (inrec.title, lim) -ENDEMBED; - -// Test defining a transform -//MORE: The embed function shpo -transform(childrec) testTransformTitle2(_linkcounted_ row(titleRec) inrec, unsigned lim) := EMBED(Python) - return (inrec.title, lim) -ENDEMBED; - -sequential( -output(project(titles, testTransformTitle(LEFT, 10))); -output(project(titles, testTransformTitle2(LEFT, 10))); -);