diff --git a/.github/workflows/capgen_unit_tests.yaml b/.github/workflows/capgen_unit_tests.yaml index 441250c9..363d19e3 100644 --- a/.github/workflows/capgen_unit_tests.yaml +++ b/.github/workflows/capgen_unit_tests.yaml @@ -3,7 +3,7 @@ name: Capgen Unit Tests on: workflow_dispatch: pull_request: - branches: [feature/capgen, main] + branches: [develop, main] jobs: unit_tests: diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index c57461cb..791c5084 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -3,7 +3,7 @@ name: Python package on: workflow_dispatch: pull_request: - branches: [feature/capgen, main] + branches: [develop, main] jobs: build: diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index b67de540..3cb4ee19 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -55,7 +55,7 @@ def parse_arguments(): verbose = args.verbose debug = args.debug if args.suites: - sdfs = ['suite_{0}.xml'.format(x) for x in args.suites.split(',')] + sdfs = ['{0}.xml'.format(x) for x in args.suites.split(',')] else: sdfs = None builddir = args.builddir @@ -181,8 +181,22 @@ def parse_suites(suites_dir, sdfs): logging.info('Parsing suite definition files ...') suites = [] for sdf in sdfs: - logging.info('Parsing suite definition file {0} ...'.format(os.path.join(suites_dir, sdf))) - suite = Suite(sdf_name=os.path.join(suites_dir, sdf)) + sdf_file=os.path.join(suites_dir, sdf) + if not os.path.exists(sdf_file): + # If suite file not found, check old filename convention (suite_[suitename].xml) + sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") + if os.path.exists(sdf_file_legacy): + logging.warning("Parsing suite definition file using legacy naming convention") + logging.warning(f"Filename {os.path.basename(sdf_file_legacy)}") + logging.warning(f"Suite name {sdf}") + sdf_file=sdf_file_legacy + else: + logging.critical(f"Suite definition file {sdf_file} not found.") + success = False + return (success, suites) + + logging.info(f'Parsing suite definition file {sdf_file} ...') + suite = Suite(sdf_name=sdf_file) success = suite.parse() if not success: logging.error('Parsing suite definition file {0} failed.'.format(sdf)) diff --git a/scripts/ccpp_track_variables.py b/scripts/ccpp_track_variables.py index 60d31c1e..64dfc518 100755 --- a/scripts/ccpp_track_variables.py +++ b/scripts/ccpp_track_variables.py @@ -69,7 +69,7 @@ def create_metadata_filename_dict(metapath): with that scheme""" metadata_dict = {} - scheme_filenames = glob.glob(os.path.join(metapath, "*.meta")) + scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"), recursive=True) if not scheme_filenames: raise Exception(f'No files found in {metapath} with ".meta" extension') diff --git a/scripts/common.py b/scripts/common.py index c29ccda2..f5ba7f1d 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -73,7 +73,7 @@ CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}' # Filename pattern for suite definition files -SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$') +SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^(.*)\.xml$') # Maximum number of concurrent CCPP instances per MPI task CCPP_NUM_INSTANCES = 200 diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index d8b68000..c05fa5b4 100755 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -348,6 +348,14 @@ def __init__(self, run_env, table_name_in=None, table_type_in=None, # end if self.__start_context = ParseContext(context=self.__pobj) self.__init_from_file(known_ddts, self.__run_env, skip_ddt_check=skip_ddt_check) + # Set absolute path for all dependencies + path = os.path.dirname(self.__pobj.filename) + if self.relative_path: + path = os.path.join(path, self.relative_path) + # end if + for ind, dep in enumerate(self.__dependencies): + self.__dependencies[ind] = os.path.abspath(os.path.join(path, dep)) + # end for # end if def __init_from_file(self, known_ddts, run_env, skip_ddt_check=False): diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 4f00b9b9..889b2c1d 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -691,11 +691,16 @@ def parse(self, make_call_tree=False): suite_xml = tree.getroot() self._name = suite_xml.get('name') # Validate name of suite in XML tag against filename; could be moved to common.py - if not (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)): - logging.critical("Invalid suite name {0} in suite definition file {1}.".format( - self._name, self._sdf_name)) - success = False - return success + if not (os.path.basename(self._sdf_name) == '{}.xml'.format(self._name)): + if (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)): + logging.debug("Parsing suite using legacy naming convention") + logging.debug(f"Filename {os.path.basename(self._sdf_name)}") + logging.debug(f"Suite name {format(self._name)}") + else: + logging.critical("Invalid suite name {0} in suite definition file {1}.".format( + self._name, self._sdf_name)) + success = False + return success # Check if suite name is too long if len(self._name) > SUITE_NAME_MAX_CHARS: diff --git a/scripts/parse_tools/xml_tools.py b/scripts/parse_tools/xml_tools.py index 0ff56b3a..164b169b 100644 --- a/scripts/parse_tools/xml_tools.py +++ b/scripts/parse_tools/xml_tools.py @@ -51,6 +51,21 @@ def call_command(commands, logger, silent=False): False >>> call_command(['ls'], _LOGGER) True + >>> try: + ... call_command(['ls','--invalid-option'], _LOGGER) + ... except CCPPError as e: + ... print(str(e)) + Execution of 'ls --invalid-option' failed with code: 2 + Error output: ls: unrecognized option '--invalid-option' + Try 'ls --help' for more information. + >>> try: + ... os.chdir(os.path.dirname(__file__)) + ... call_command(['ls', os.path.basename(__file__), 'foo.bar.baz'], _LOGGER) + ... except CCPPError as e: + ... print(str(e)) + Execution of 'ls xml_tools.py foo.bar.baz' failed with code: 2 + xml_tools.py + Error output: ls: cannot access 'foo.bar.baz': No such file or directory """ result = False outstr = '' @@ -66,9 +81,17 @@ def call_command(commands, logger, silent=False): result = False else: cmd = ' '.join(commands) - emsg = "Execution of '{}' failed with code:\n" - outstr = emsg.format(cmd, err.returncode) - outstr += "{}".format(err.output) + outstr = f"Execution of '{cmd}' failed with code: {err.returncode}\n" + outstr += f"{err.output.decode('utf-8', errors='replace').strip()}" + if hasattr(err, 'stderr') and err.stderr: + stderr_str = err.stderr.decode('utf-8', errors='replace').strip() + if stderr_str: + if err.output: + outstr += os.linesep + # end if + outstr += f"Error output: {stderr_str}" + # end if + # end if raise CCPPError(outstr) from err # end if # end of try diff --git a/scripts/var_props.py b/scripts/var_props.py index d73ed241..21477b4a 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -940,6 +940,13 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, # end if # end if if self.__compat: + # Only "none" units are case-insensitive + if var1_units.lower() == 'none': + var1_units = 'none' + # end if + if var2_units.lower() == 'none': + var2_units = 'none' + # end if # Check units argument if var1_units != var2_units: # Try to find a set of unit conversions diff --git a/test/capgen_test/run_test b/test/capgen_test/run_test index e5ac6e12..3d21e2c2 100755 --- a/test/capgen_test/run_test +++ b/test/capgen_test/run_test @@ -130,7 +130,7 @@ ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_ddt_suite_cap.F90" ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_temp_suite_cap.F90" process_list="adjusting=temp_calc_adjust,setter=temp_set" module_list="environ_conditions,make_ddt,setup_coeffs,temp_adjust,temp_calc_adjust,temp_set" -dependencies="bar.F90,foo.F90,qux.F90" +dependencies="${scriptdir}/adjust/qux.F90,${scriptdir}/bar.F90,${scriptdir}/foo.F90" suite_list="ddt_suite;temp_suite" required_vars_ddt="ccpp_error_code,ccpp_error_message,horizontal_dimension" required_vars_ddt="${required_vars_ddt},horizontal_loop_begin" diff --git a/test/capgen_test/temp_adjust.meta b/test/capgen_test/temp_adjust.meta index 02b5fa73..420e9112 100644 --- a/test/capgen_test/temp_adjust.meta +++ b/test/capgen_test/temp_adjust.meta @@ -1,6 +1,8 @@ [ccpp-table-properties] name = temp_adjust type = scheme + dependencies = qux.F90 + relative_path = adjust [ccpp-arg-table] name = temp_adjust_run type = scheme diff --git a/test/capgen_test/temp_calc_adjust.meta b/test/capgen_test/temp_calc_adjust.meta index 60a86865..437de934 100644 --- a/test/capgen_test/temp_calc_adjust.meta +++ b/test/capgen_test/temp_calc_adjust.meta @@ -2,7 +2,6 @@ name = temp_calc_adjust type = scheme dependencies = foo.F90, bar.F90 - dependencies = qux.F90 [ccpp-arg-table] name = temp_calc_adjust_run type = scheme diff --git a/test/unit_tests/test_metadata_table.py b/test/unit_tests/test_metadata_table.py index fe6154ad..08248150 100755 --- a/test/unit_tests/test_metadata_table.py +++ b/test/unit_tests/test_metadata_table.py @@ -15,9 +15,10 @@ import os import unittest -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(TEST_DIR, "sample_files") +UNIT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DIR = os.path.abspath(os.path.join(UNIT_TEST_DIR, os.pardir)) +SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, "scripts")) +SAMPLE_FILES_DIR = os.path.join(UNIT_TEST_DIR, "sample_files") if not os.path.exists(SCRIPTS_DIR): raise ImportError("Cannot find scripts directory") @@ -374,11 +375,15 @@ def test_dependencies_rel_path(self): titles = [elem.table_name for elem in result] self.assertEqual(len(dependencies), 4) - self.assertIn('machine.F', dependencies, msg="Dependency 'machine.F' is expected but not found") - self.assertIn('physcons.F90', dependencies, msg="Dependency 'physcons.F90' is expected but not found") - self.assertIn('GFDL_parse_tracers.F90', dependencies, msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found") - self.assertIn('rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90', dependencies, \ - msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found") + phys_dir = os.path.join(TEST_DIR, "ccpp", "physics", "physics") + self.assertIn(os.path.join(phys_dir, 'machine.F'), dependencies, \ + msg="Dependency 'machine.F' is expected but not found") + self.assertIn(os.path.join(phys_dir, 'physcons.F90'), dependencies, \ + msg="Dependency 'physcons.F90' is expected but not found") + self.assertIn(os.path.join(phys_dir, 'GFDL_parse_tracers.F90'), dependencies, \ + msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found") + self.assertIn(os.path.join(phys_dir, 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90'), dependencies, \ + msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found") self.assertIn(rel_path, "../../ccpp/physics/physics") self.assertEqual(len(result), 1) @@ -399,4 +404,3 @@ def test_invalid_table_properties_type(self): if __name__ == "__main__": unittest.main() - diff --git a/test/var_compatibility_test/test_host.meta b/test/var_compatibility_test/test_host.meta index 5d861764..da71b182 100644 --- a/test/var_compatibility_test/test_host.meta +++ b/test/var_compatibility_test/test_host.meta @@ -26,7 +26,7 @@ [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP - units = none + units = None dimensions = () type = character kind = len=512 diff --git a/test_prebuild/test_blocked_data/suite_blocked_data_suite.xml b/test_prebuild/test_blocked_data/blocked_data_suite.xml similarity index 100% rename from test_prebuild/test_blocked_data/suite_blocked_data_suite.xml rename to test_prebuild/test_blocked_data/blocked_data_suite.xml diff --git a/test_prebuild/unit_tests/test_metadata_parser.py b/test_prebuild/unit_tests/test_metadata_parser.py index febc3d93..1d4b4819 100644 --- a/test_prebuild/unit_tests/test_metadata_parser.py +++ b/test_prebuild/unit_tests/test_metadata_parser.py @@ -45,7 +45,7 @@ def test_MetadataTable_parse_table(tmpdir): assert metadata_header.table_name == "" assert metadata_header.table_type == "scheme" assert metadata_header.relative_path == "path" - assert metadata_header.dependencies == ["a.f", "b.f"] + assert metadata_header.dependencies == [os.path.join(tmpdir, metadata_header.relative_path,"a.f"), os.path.join(tmpdir, metadata_header.relative_path,"b.f")] # check metadata section assert len(metadata_header.sections()) == 1