Merge branch 'develop' of https://github.com/NCAR/ccpp-framework into…

… feature/bugfix_for_autoconv

.github/workflows/capgen_unit_tests.yaml

@@ -3,7 +3,7 @@ name: Capgen Unit Tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [feature/capgen, main]
+    branches: [develop, main]
 
 jobs:
   unit_tests:

.github/workflows/python.yaml

@@ -3,7 +3,7 @@ name: Python package
 on:
   workflow_dispatch:
   pull_request:
-    branches: [feature/capgen, main]
+    branches: [develop, main]
 
 jobs:
   build:

scripts/ccpp_prebuild.py

@@ -55,7 +55,7 @@ def parse_arguments():
     verbose = args.verbose
     debug = args.debug
     if args.suites:
-        sdfs = ['suite_{0}.xml'.format(x) for x in args.suites.split(',')]
+        sdfs = ['{0}.xml'.format(x) for x in args.suites.split(',')]
     else:
         sdfs = None
     builddir = args.builddir
@@ -181,8 +181,22 @@ def parse_suites(suites_dir, sdfs):
     logging.info('Parsing suite definition files ...')
     suites = []
     for sdf in sdfs:
-        logging.info('Parsing suite definition file {0} ...'.format(os.path.join(suites_dir, sdf)))
-        suite = Suite(sdf_name=os.path.join(suites_dir, sdf))
+        sdf_file=os.path.join(suites_dir, sdf)
+        if not os.path.exists(sdf_file):
+            # If suite file not found, check old filename convention (suite_[suitename].xml)
+            sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}")
+            if os.path.exists(sdf_file_legacy):
+                logging.warning("Parsing suite definition file using legacy naming convention")
+                logging.warning(f"Filename {os.path.basename(sdf_file_legacy)}")
+                logging.warning(f"Suite name {sdf}")
+                sdf_file=sdf_file_legacy
+            else:
+                logging.critical(f"Suite definition file {sdf_file} not found.")
+                success = False
+                return (success, suites)
+
+        logging.info(f'Parsing suite definition file {sdf_file} ...')
+        suite = Suite(sdf_name=sdf_file)
         success = suite.parse()
         if not success:
             logging.error('Parsing suite definition file {0} failed.'.format(sdf))

scripts/ccpp_track_variables.py

@@ -69,7 +69,7 @@ def create_metadata_filename_dict(metapath):
        with that scheme"""
 
     metadata_dict = {}
-    scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"))
+    scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"), recursive=True)
     if not scheme_filenames:
         raise Exception(f'No files found in {metapath} with ".meta" extension')
 

scripts/common.py

@@ -73,7 +73,7 @@
 CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}'
 
 # Filename pattern for suite definition files
-SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$')
+SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^(.*)\.xml$')
 
 # Maximum number of concurrent CCPP instances per MPI task
 CCPP_NUM_INSTANCES = 200

scripts/metadata_table.py

@@ -348,6 +348,14 @@ def __init__(self, run_env, table_name_in=None, table_type_in=None,
             # end if
             self.__start_context = ParseContext(context=self.__pobj)
             self.__init_from_file(known_ddts, self.__run_env, skip_ddt_check=skip_ddt_check)
+            # Set absolute path for all dependencies
+            path = os.path.dirname(self.__pobj.filename)
+            if self.relative_path:
+                path = os.path.join(path, self.relative_path)
+            # end if
+            for ind, dep in enumerate(self.__dependencies):
+                self.__dependencies[ind] = os.path.abspath(os.path.join(path, dep))
+            # end for
         # end if
 
     def __init_from_file(self, known_ddts, run_env, skip_ddt_check=False):

scripts/mkstatic.py

@@ -691,11 +691,16 @@ def parse(self, make_call_tree=False):
         suite_xml = tree.getroot()
         self._name = suite_xml.get('name')
         # Validate name of suite in XML tag against filename; could be moved to common.py
-        if not (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)):
-            logging.critical("Invalid suite name {0} in suite definition file {1}.".format(
-                                                               self._name, self._sdf_name))
-            success = False
-            return success
+        if not (os.path.basename(self._sdf_name) == '{}.xml'.format(self._name)):
+            if (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)):
+                logging.debug("Parsing suite using legacy naming convention")
+                logging.debug(f"Filename {os.path.basename(self._sdf_name)}")
+                logging.debug(f"Suite name {format(self._name)}")
+            else:
+                logging.critical("Invalid suite name {0} in suite definition file {1}.".format(
+                                                                   self._name, self._sdf_name))
+                success = False
+                return success
 
         # Check if suite name is too long
         if len(self._name) > SUITE_NAME_MAX_CHARS:

scripts/parse_tools/xml_tools.py

@@ -51,6 +51,21 @@ def call_command(commands, logger, silent=False):
     False
     >>> call_command(['ls'], _LOGGER)
     True
+    >>> try:
+    ...    call_command(['ls','--invalid-option'], _LOGGER)
+    ... except CCPPError as e:
+    ...    print(str(e))
+    Execution of 'ls --invalid-option' failed with code: 2
+    Error output: ls: unrecognized option '--invalid-option'
+    Try 'ls --help' for more information.
+    >>> try:
+    ...    os.chdir(os.path.dirname(__file__))
+    ...    call_command(['ls', os.path.basename(__file__), 'foo.bar.baz'], _LOGGER)
+    ... except CCPPError as e:
+    ...    print(str(e))
+    Execution of 'ls xml_tools.py foo.bar.baz' failed with code: 2
+    xml_tools.py
+    Error output: ls: cannot access 'foo.bar.baz': No such file or directory
     """
     result = False
     outstr = ''
@@ -66,9 +81,17 @@ def call_command(commands, logger, silent=False):
             result = False
         else:
             cmd = ' '.join(commands)
-            emsg = "Execution of '{}' failed with code:\n"
-            outstr = emsg.format(cmd, err.returncode)
-            outstr += "{}".format(err.output)
+            outstr = f"Execution of '{cmd}' failed with code: {err.returncode}\n"
+            outstr += f"{err.output.decode('utf-8', errors='replace').strip()}"
+            if hasattr(err, 'stderr') and err.stderr:
+                stderr_str = err.stderr.decode('utf-8', errors='replace').strip()
+                if stderr_str:
+                    if err.output:
+                        outstr += os.linesep
+                    # end if
+                    outstr += f"Error output: {stderr_str}"
+                # end if
+            # end if
             raise CCPPError(outstr) from err
         # end if
     # end of try

scripts/var_props.py

@@ -940,6 +940,13 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units,
             # end if
         # end if
         if self.__compat:
+            # Only "none" units are case-insensitive
+            if var1_units.lower() == 'none':
+                var1_units = 'none'
+            # end if
+            if var2_units.lower() == 'none':
+                var2_units = 'none'
+            # end if
             # Check units argument
             if var1_units != var2_units:
                 # Try to find a set of unit conversions

test/capgen_test/run_test

@@ -130,7 +130,7 @@ ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_ddt_suite_cap.F90"
 ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_temp_suite_cap.F90"
 process_list="adjusting=temp_calc_adjust,setter=temp_set"
 module_list="environ_conditions,make_ddt,setup_coeffs,temp_adjust,temp_calc_adjust,temp_set"
-dependencies="bar.F90,foo.F90,qux.F90"
+dependencies="${scriptdir}/adjust/qux.F90,${scriptdir}/bar.F90,${scriptdir}/foo.F90"
 suite_list="ddt_suite;temp_suite"
 required_vars_ddt="ccpp_error_code,ccpp_error_message,horizontal_dimension"
 required_vars_ddt="${required_vars_ddt},horizontal_loop_begin"

test/capgen_test/temp_adjust.meta

@@ -1,6 +1,8 @@
 [ccpp-table-properties]
   name = temp_adjust
   type = scheme
+  dependencies = qux.F90
+  relative_path = adjust
 [ccpp-arg-table]
   name = temp_adjust_run
   type = scheme

test/capgen_test/temp_calc_adjust.meta

@@ -2,7 +2,6 @@
   name = temp_calc_adjust
   type = scheme
   dependencies = foo.F90, bar.F90
-  dependencies = qux.F90
 [ccpp-arg-table]
   name = temp_calc_adjust_run
   type = scheme

test/unit_tests/test_metadata_table.py

@@ -15,9 +15,10 @@
 import os
 import unittest
 
-TEST_DIR = os.path.dirname(os.path.abspath(__file__))
-SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, os.pardir, "scripts"))
-SAMPLE_FILES_DIR = os.path.join(TEST_DIR, "sample_files")
+UNIT_TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DIR = os.path.abspath(os.path.join(UNIT_TEST_DIR, os.pardir))
+SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, "scripts"))
+SAMPLE_FILES_DIR = os.path.join(UNIT_TEST_DIR, "sample_files")
 
 if not os.path.exists(SCRIPTS_DIR):
     raise ImportError("Cannot find scripts directory")
@@ -374,11 +375,15 @@ def test_dependencies_rel_path(self):
         titles = [elem.table_name for elem in result]
 
         self.assertEqual(len(dependencies), 4)
-        self.assertIn('machine.F', dependencies, msg="Dependency 'machine.F' is expected but not found")
-        self.assertIn('physcons.F90', dependencies, msg="Dependency 'physcons.F90' is expected but not found")
-        self.assertIn('GFDL_parse_tracers.F90', dependencies, msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found")
-        self.assertIn('rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90', dependencies, \
-                       msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found")
+        phys_dir = os.path.join(TEST_DIR, "ccpp", "physics", "physics")
+        self.assertIn(os.path.join(phys_dir, 'machine.F'), dependencies, \
+                      msg="Dependency 'machine.F' is expected but not found")
+        self.assertIn(os.path.join(phys_dir, 'physcons.F90'), dependencies, \
+                      msg="Dependency 'physcons.F90' is expected but not found")
+        self.assertIn(os.path.join(phys_dir, 'GFDL_parse_tracers.F90'), dependencies, \
+                      msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found")
+        self.assertIn(os.path.join(phys_dir, 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90'), dependencies, \
+                      msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found")
 
         self.assertIn(rel_path, "../../ccpp/physics/physics")
         self.assertEqual(len(result), 1)
@@ -399,4 +404,3 @@ def test_invalid_table_properties_type(self):
 
 if __name__ == "__main__":
     unittest.main()
-

test/var_compatibility_test/test_host.meta

@@ -26,7 +26,7 @@
 [ errmsg ]
   standard_name = ccpp_error_message
   long_name = Error message for error handling in CCPP
-  units = none
+  units = None
   dimensions = ()
   type = character
   kind = len=512

test_prebuild/unit_tests/test_metadata_parser.py

@@ -45,7 +45,7 @@ def test_MetadataTable_parse_table(tmpdir):
     assert metadata_header.table_name == "<name>"
     assert metadata_header.table_type == "scheme"
     assert metadata_header.relative_path == "path"
-    assert metadata_header.dependencies == ["a.f", "b.f"]
+    assert metadata_header.dependencies == [os.path.join(tmpdir, metadata_header.relative_path,"a.f"), os.path.join(tmpdir, metadata_header.relative_path,"b.f")]
 
     # check metadata section
     assert len(metadata_header.sections()) == 1