From 7caa41bd18e674df6c75b23dd434d424530b77b4 Mon Sep 17 00:00:00 2001 From: Terrence Asselin Date: Thu, 12 Oct 2023 19:44:27 -0500 Subject: [PATCH] HPCC-30310 esdl ecl output to stdout Update esdl ecl command to allow output to stdout by making the outputDir parameter optional. When using stdout, restrict the use of options to those combinations that result in a single file output. Disabled verbose output in the call to EsdlCmdHelper::convertECMtoESXDL to avoid printing extra text to stdout. Update esdl regression test with cases for esdl ecl, though the suite needs independent updates to bring tests of the other commands in line with recent platform changes. This will be handled in the ticket HPCC-30699. Signed-off-by: Terrence Asselin --- testing/esp/esdlcmd/esdlcmd-test.py | 137 +++++++++++++++++- .../esdlcmd/key/ecl-incl/allversionreport.ecl | 37 +++++ testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl | 40 +++++ .../ecl-stdout-incl-rollup/from-stdout.ecl | 63 ++++++++ .../key/ecl-stdout-single/from-stdout.ecl | 40 +++++ tools/esdlcmd/esdl2ecl.cpp | 134 ++++++++++------- 6 files changed, 397 insertions(+), 54 deletions(-) create mode 100644 testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl create mode 100644 testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl create mode 100644 testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl create mode 100644 testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl diff --git a/testing/esp/esdlcmd/esdlcmd-test.py b/testing/esp/esdlcmd/esdlcmd-test.py index 99c258d42df..627495982fc 100755 --- a/testing/esp/esdlcmd/esdlcmd-test.py +++ b/testing/esp/esdlcmd/esdlcmd-test.py @@ -52,6 +52,7 @@ def __init__(self, stats, exe_path, output_base, test_path): self.test_path = test_path self.stats = stats +# This class is base for commands related to services: wsdl, xsd, cpp and java. class TestCaseBase: """Settings for a specific test case.""" @@ -103,6 +104,108 @@ def validate_results(self): logging.debug('TestCaseBase implementation called, no comparison run') return False +# This class is the base for the 'transform' commands: ecl and xml. +# In a future update, investigate refactoring the base classes so there is a single +# parent with any shared capabilities. +# +# When writing to stdout, the key directory should contain a file named 'from-stdout.ecl' +# that contains the expected output. +class TestCaseTransformBase: + def __init__(self, run_settings, name, command, esdl_file, xsl_path, use_stdout, expected_err=None, options=None): + self.run_settings = run_settings + self.name = name + self.command = command + self.esdl_path = (self.run_settings.test_path / 'inputs' / esdl_file) + #self.service = service + self.xsl_path = xsl_path + self.options = options + self.output_path = Path(self.run_settings.output_base) / name + self.stdout = use_stdout + self.expected_err = expected_err + if self.stdout: + self.args = [ + str(run_settings.exe_path), + self.command, + self.esdl_path, + '-cde', + self.xsl_path, + ] + else: + self.args = [ + str(run_settings.exe_path), + self.command, + self.esdl_path, + self.output_path, + '-cde', + self.xsl_path, + ] + + if options: + self.args.extend(options) + + self.result = None + + def run_test(self): + safe_mkdir(self.output_path) + logging.debug("Test %s args: %s", self.name, str(self.args)) + self.result = subprocess.run(self.args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + if self.expected_err != None and self.expected_err == self.result.stderr: + success = True + elif self.result.returncode != 0: + logging.error('Error running "esdl %s" for test "%s": %s', self.command, self.name, self.result.stderr) + success = False + else: + success = self.validate_results() + + self.run_settings.stats.add_count(success) + + def is_same(self, dir1, dir2): + """ + Compare two directory trees content. + Return False if they differ, True is they are the same. + """ + compared = DirectoryCompare(dir1, dir2) + if (compared.left_only or compared.right_only or compared.diff_files + or compared.funny_files): + return False + for subdir in compared.common_dirs: + if not self.is_same(os.path.join(dir1, subdir), os.path.join(dir2, subdir)): + return False + return True + + def validate_results(self): + """Compare test case results to the known key. + + Return True if the two are identical or False otherwise. + """ + outName = self.output_path + key = (self.run_settings.test_path / 'key' / self.name) + + # When output was stdout, write the captured stdout text to a file named 'from-stdout.ecl' + # in the output directory. This allows us to use the same method of comparing the key + # and result. + if self.stdout: + if self.result.stdout != None and len(self.result.stdout) > 0: + with open((outName / 'from-stdout.ecl'), 'w', encoding='utf-8') as f: + f.write(self.result.stdout) + + if (not key.exists()): + logging.error('Missing key file %s', str(key)) + return False + + if (not outName.exists()): + logging.error('Missing output for test %s', self.name) + return False + + if (not self.is_same(str(key), str(outName))): + logging.debug('Comparing key %s to output %s', str(key), str(outName)) + logging.error('Test failed: %s', self.name) + return False + else: + logging.debug('Passed: %s', self.name) + return True + class TestCaseXSD(TestCaseBase): """Test case for the wsdl or xsd commands. @@ -212,6 +315,8 @@ def parse_options(): the parsed options and arguments. """ + command_values = ['all', 'cpp', 'ecl', 'java', 'wsdl', 'xsd'] + parser = argparse.ArgumentParser(description=DESC) parser.add_argument('testroot', help='Path of the root folder of the esdlcmd testing project') @@ -231,7 +336,15 @@ def parse_options(): help='Enable debug logging of test cases', action='store_true', default=False) + parser.add_argument('-c', '--commands', + help='esdl commands to run tests for, use once for each command or pass "all" to test all commands. Defaults to "all".', + action="append", choices=command_values) + args = parser.parse_args() + + if args.commands == None: + args.commands = ['all'] + return args @@ -278,6 +391,11 @@ def main(): stats = Statistics() run_settings = TestRun(stats, exe_path, args.outdir, test_path) + esdl_includes_path = str(test_path / 'inputs') + + expected_err_multi_file_incl = '\nOutput to stdout is not supported for multiple files. Either add the Rollup\noption or specify an output directory.\n' + expected_err_multi_file_expanded = '\nOutput to stdout is not supported for multiple files. Remove the Output expanded\n XML option or specify an output directory.\n' + test_cases = [ # wsdl TestCaseXSD(run_settings, 'wstest-wsdl-default', 'wsdl', 'ws_test.ecm', 'WsTest', @@ -390,10 +508,27 @@ def main(): # A single element is created for each request structure defined. This is default behavior. TestCaseXSD(run_settings, 'use-request-name', 'wsdl', 'ws_userequestname.ecm', 'WsUseRequestName', xsl_base_path), + + # ecl + TestCaseTransformBase(run_settings, 'ecl-stdout-single', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True), + + TestCaseTransformBase(run_settings, 'ecl-stdout-incl-err', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True, + expected_err=expected_err_multi_file_incl, options=['-I', esdl_includes_path, '--includes']), + + TestCaseTransformBase(run_settings, 'ecl-stdout-expanded-err', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True, + expected_err=expected_err_multi_file_expanded, options=['-x']), + + TestCaseTransformBase(run_settings, 'ecl-stdout-incl-rollup', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=True, + options=['-I', esdl_includes_path, '--includes', '--rollup']), + + TestCaseTransformBase(run_settings, 'ecl-incl', 'ecl', 'ws_test.ecm', xsl_base_path, use_stdout=False, + options=['-I', esdl_includes_path, '--includes']) + ] for case in test_cases: - case.run_test() + if 'all' in args.commands or case.command in args.commands: + case.run_test() logging.info('Success count: %d', stats.successCount) logging.info('Failure count: %d', stats.failureCount) diff --git a/testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl b/testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl new file mode 100644 index 00000000000..009048a0674 --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-incl/allversionreport.ecl @@ -0,0 +1,37 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from allversionreport.xml. ***/ +/*===================================================*/ + + +EXPORT allversionreport := MODULE + +EXPORT t_FooBar := RECORD + UTF8 Foo {XPATH('Foo')}; + UTF8 Bar {XPATH('Bar')}; +END; + +EXPORT t_AllVersionArrays := RECORD + SET OF UTF8 StringArray {XPATH('StringArray/Item'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) FooBarArray {XPATH('FooBarArray/FooBar'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) NamedItemFooBarArray {XPATH('NamedItemFooBarArray/NamedItem'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! +END; + +EXPORT t_AllVersionReportRequest := RECORD + UTF8 OptionalDeveloperStringVal {XPATH('OptionalDeveloperStringVal')};//hidden[developer] + INTEGER Annotate20ColsIntVal {XPATH('Annotate20ColsIntVal')}; + t_AllVersionArrays Arrays {XPATH('Arrays')}; + UTF8 UnrelentingForce {XPATH('UnrelentingForce')}; //values['1','2','3',''] +END; + +EXPORT t_AllVersionReportResponse := RECORD + UTF8 ResultVal {XPATH('ResultVal')}; + t_AllVersionArrays ResultArrays {XPATH('ResultArrays')}; +END; + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from allversionreport.xml. ***/ +/*===================================================*/ + diff --git a/testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl b/testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl new file mode 100644 index 00000000000..89894f075b8 --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-incl/ws_test.ecl @@ -0,0 +1,40 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + + +EXPORT ws_test := MODULE + +EXPORT t_MinVersionReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlRequest +EXPORT t_WsTestPingRequest := RECORD +END; +*/ + +EXPORT t_MinVersionReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlResponse +EXPORT t_WsTestPingResponse := RECORD +END; +*/ + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + diff --git a/testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl b/testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl new file mode 100644 index 00000000000..f558558fa0d --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-stdout-incl-rollup/from-stdout.ecl @@ -0,0 +1,63 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + + +EXPORT ws_test := MODULE + +EXPORT t_FooBar := RECORD + UTF8 Foo {XPATH('Foo')}; + UTF8 Bar {XPATH('Bar')}; +END; + +EXPORT t_AllVersionArrays := RECORD + SET OF UTF8 StringArray {XPATH('StringArray/Item'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) FooBarArray {XPATH('FooBarArray/FooBar'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! + DATASET(t_FooBar) NamedItemFooBarArray {XPATH('NamedItemFooBarArray/NamedItem'), MAXCOUNT(1)}; // max_count must be specified in ESDL defintion! +END; + +EXPORT t_AllVersionReportRequest := RECORD + UTF8 OptionalDeveloperStringVal {XPATH('OptionalDeveloperStringVal')};//hidden[developer] + INTEGER Annotate20ColsIntVal {XPATH('Annotate20ColsIntVal')}; + t_AllVersionArrays Arrays {XPATH('Arrays')}; + UTF8 UnrelentingForce {XPATH('UnrelentingForce')}; //values['1','2','3',''] +END; + +EXPORT t_MinVersionReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlRequest +EXPORT t_WsTestPingRequest := RECORD +END; +*/ + +EXPORT t_AllVersionReportResponse := RECORD + UTF8 ResultVal {XPATH('ResultVal')}; + t_AllVersionArrays ResultArrays {XPATH('ResultArrays')}; +END; + +EXPORT t_MinVersionReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlResponse +EXPORT t_WsTestPingResponse := RECORD +END; +*/ + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + diff --git a/testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl b/testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl new file mode 100644 index 00000000000..89894f075b8 --- /dev/null +++ b/testing/esp/esdlcmd/key/ecl-stdout-single/from-stdout.ecl @@ -0,0 +1,40 @@ +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + + +EXPORT ws_test := MODULE + +EXPORT t_MinVersionReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportRequest := RECORD + UTF8 RequestString {XPATH('RequestString')}; +END; + +EXPORT t_VersionRangeReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlRequest +EXPORT t_WsTestPingRequest := RECORD +END; +*/ + +EXPORT t_MinVersionReportResponse := RECORD + UTF8 ResponseString {XPATH('ResponseString')}; +END; + +/*Empty record generated from empty EsdlResponse +EXPORT t_WsTestPingResponse := RECORD +END; +*/ + + +END; + +/*** Not to be hand edited (changes will be lost on re-generation) ***/ +/*** ECL Interface generated by esdl2ecl version 1.0 from ws_test.xml. ***/ +/*===================================================*/ + diff --git a/tools/esdlcmd/esdl2ecl.cpp b/tools/esdlcmd/esdl2ecl.cpp index fa5ff0b7e80..ef2c5f84ffd 100644 --- a/tools/esdlcmd/esdl2ecl.cpp +++ b/tools/esdlcmd/esdl2ecl.cpp @@ -103,7 +103,7 @@ class EsdlIndexedPropertyTrees { fileName.append(srcext); StringBuffer esxml; - EsdlCmdHelper::convertECMtoESXDL(fileName.str(), srcfile, esxml, loadincludes && rollUp, true, true, isIncludedESDL, includePath); + EsdlCmdHelper::convertECMtoESXDL(fileName.str(), srcfile, esxml, loadincludes && rollUp, false, true, isIncludedESDL, includePath); src = createPTreeFromXMLString(esxml, 0); } else if (!srcext || !*srcext || stricmp(srcext, XML_FILE_EXTENSION)==0) @@ -202,31 +202,26 @@ class Esdl2EclCmd : public EsdlConvertCmd return false; } - //First two parameters' order is fixed. - for (int par = 0; par < 2 && !iter.done(); par++) + //First parameter's order is fixed. + const char *arg = iter.query(); + if (*arg != '-') + optSource.set(arg); + else + { + usage(); + return false; + } + + + if (iter.next()) { - const char *arg = iter.query(); + arg = iter.query(); if (*arg != '-') { - if (optSource.isEmpty()) - optSource.set(arg); - else if (optOutDirPath.isEmpty()) - optOutDirPath.set(arg); - else - { - fprintf(stderr, "\nunrecognized argument detected before required parameters: %s\n", arg); - usage(); - return false; - } + optOutDirPath.set(arg); + optStdout = false; + iter.next(); } - else - { - fprintf(stderr, "\noption detected before required parameters: %s\n", arg); - usage(); - return false; - } - - iter.next(); } for (; !iter.done(); iter.next()) @@ -281,7 +276,24 @@ class Esdl2EclCmd : public EsdlConvertCmd virtual bool finalizeOptions(IProperties *globals) { - return EsdlConvertCmd::finalizeOptions(globals); + // We can't call EsdlConvertCmd::finalizeOptions because it requires outOutDirPath + if (optStdout) + { + if (optOutputExpandedXML) + { + fprintf(stderr, "\nOutput to stdout is not supported for multiple files. Remove the Output expanded\n XML option or specify an output directory.\n"); + usage(); + return false; + } + else if (optProcessIncludes && !optRollUpEclToSingleFile) + { + fprintf(stderr, "\nOutput to stdout is not supported for multiple files. Either add the Rollup\noption or specify an output directory.\n"); + usage(); + return false; + } + } + + return true; } virtual int processCMD() @@ -378,10 +390,16 @@ class Esdl2EclCmd : public EsdlConvertCmd virtual void usage() { fputs("\nUsage:\n\n" - "esdl ecl sourcePath outputPath [options]\n" - "\nsourcePath must be absolute path to the ESDL Definition file containing the" + "esdl ecl sourcePath [outputPath] [options]\n" + "\n" + "sourcePath must be absolute path to the ESDL Definition file containing the\n" "EsdlService definition for the service you want to work with.\n" - "outputPath must be the absolute path where the ECL output with be created.\n" + "\n" + "outputPath, if supplied, must be the absolute path where the ECL output will be\n" + "created. When outputPath is omitted options must generate a single file which is\n" + "written to stdout. This means to write to stdout, you must not use -x/--expandedxml,\n" + "nor can you use --includes without also using --rollup.\n" + "\n" " Options:\n" " -x, --expandedxml Output expanded XML files\n" " --includes Process all included files\n" @@ -390,7 +408,7 @@ class Esdl2EclCmd : public EsdlConvertCmd " --ecl-imports Comma-delimited import list to be attached to output ECL\n" " each entry generates a corresponding import *.\n" " --ecl-header Text included in target header (must be valid ECL) \n" - " --utf8- Don't use UTF8 strings" + " --utf8- Don't use UTF8 strings\n" " " ESDLOPT_INCLUDE_PATH_USAGE ,stdout); } @@ -398,10 +416,11 @@ class Esdl2EclCmd : public EsdlConvertCmd void outputEcl(const char *srcpath, const char *file, const char *path, const char *types, const char * xml, const char * eclimports, const char * eclheader) { DBGLOG("Generating ECL file for %s", file); - + StringBuffer outfile; StringBuffer filePath; StringBuffer fileName; StringBuffer fileExt; + StringBuffer fullSrcFile; splitFilename(file, NULL, &filePath, &fileName, &fileExt); @@ -409,30 +428,30 @@ class Esdl2EclCmd : public EsdlConvertCmd if (!strnicmp(finger, "wsm_", 4)) finger+=4; - StringBuffer outfile; - if (path && *path) - { - outfile.append(path); - if (outfile.length() && !strchr("/\\", outfile.charAt(outfile.length()-1))) - outfile.append('/'); - } - outfile.append(finger).append(".ecl"); + fullSrcFile.append(srcpath); + addPathSepChar(fullSrcFile, PATHSEPCHAR).append(file); + if (!optStdout) { - //If the target output file cannot be accessed, this operation will - //throw, and will be caught and reported at the shell level. - Owned ofile = createIFile(outfile.str()); - if (ofile) + if (path && *path) { - Owned fileIO = ofile->open(IFOcreate); - fileIO.clear(); + outfile.append(path); + if (outfile.length() && !strchr("/\\", outfile.charAt(outfile.length()-1))) + outfile.append('/'); } - } + outfile.append(finger).append(".ecl"); - StringBuffer fullname(srcpath); - if (fullname.length() && !strchr("/\\", fullname.charAt(fullname.length()-1))) - fullname.append('/'); - fullname.append(fileName.str()).append(".xml"); + { + //If the target output file cannot be accessed, this operation will + //throw, and will be caught and reported at the shell level. + Owned ofile = createIFile(outfile.str()); + if (ofile) + { + Owned fileIO = ofile->open(IFOcreate); + fileIO.clear(); + } + } + } StringBuffer expstr; expstr.append(""); @@ -468,10 +487,10 @@ class Esdl2EclCmd : public EsdlConvertCmd params->setProp("utf8strings", optUseUtf8Strings ? "yes" : "no"); StringBuffer esdl2eclxslt (optHPCCCompFilesDir.get()); esdl2eclxslt.append("/xslt/esdl2ecl.xslt"); - esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str()); + esdl2eclxsltTransform(expstr.str(), esdl2eclxslt.str(), params, outfile.str(), fullSrcFile.str()); } - void esdl2eclxsltTransform(const char* xml, const char* sheet, IProperties *params, const char *filename) + void esdl2eclxsltTransform(const char* xml, const char* sheet, IProperties *params, const char *outputFilename, const char *srcFilename) { StringBuffer xsl; xsl.loadFile(sheet); @@ -494,15 +513,23 @@ class Esdl2EclCmd : public EsdlConvertCmd } } - trans->setResultTarget(filename); - try { - trans->transform(); + if (optStdout) + { + StringBuffer output; + trans->transform(output); + fputs(output.str(), stdout); + } + else + { + trans->setResultTarget(outputFilename); + trans->transform(); + } } catch(...) { - fprintf(stderr, "Error transforming Esdl to ECL file %s", filename); + fprintf(stderr, "Error transforming Esdl file %s to ECL file", srcFilename); } } @@ -511,6 +538,7 @@ class Esdl2EclCmd : public EsdlConvertCmd bool optProcessIncludes; bool optOutputExpandedXML; bool optUseUtf8Strings = true; + bool optStdout = true; StringAttr optHPCCCompFilesDir; StringAttr optECLIncludesList; StringAttr optECLHeaderBlock;