From c272ac7f69ec38dc7ce69f4e597070867cd8e9ef Mon Sep 17 00:00:00 2001 From: Steve Wardle Date: Tue, 15 Mar 2022 09:48:12 +0000 Subject: [PATCH] Import @64490 2019.01.1 --- mule/LICENCE.txt | 2 +- mule/docs/source/conf.py | 6 +- mule/lib/mule/__init__.py | 43 ++++++++++---- mule/lib/mule/pp.py | 54 ++++++++++++----- mule/lib/mule/tests/__init__.py | 34 +++++++++++ mule/lib/mule/tests/test_stashmaster | 44 ++++++++++++++ mule/lib/mule/tests/unit/test_Field.py | 16 +++-- .../tests/unit/test_stashmaster_fromumfile.py | 15 ++--- mule/setup.py | 31 +++++----- um_packing/LICENCE.txt | 2 +- um_packing/lib/um_packing/__init__.py | 2 +- um_packing/setup.py | 2 +- um_spiral_search/LICENCE.txt | 2 +- .../lib/um_spiral_search/__init__.py | 2 +- um_spiral_search/setup.py | 2 +- um_sstpert/LICENCE.txt | 2 +- um_sstpert/lib/um_sstpert/__init__.py | 2 +- um_sstpert/setup.py | 2 +- um_utils/LICENCE.txt | 2 +- um_utils/docs/source/conf.py | 6 +- um_utils/lib/um_utils/__init__.py | 2 +- um_utils/lib/um_utils/cumf.py | 58 +++++++++++++------ .../um_utils/tests/cumf/output/default.txt | 2 +- .../tests/cumf/output/ignore_missing.txt | 2 +- .../tests/cumf/output/ignore_template.txt | 2 +- .../tests/cumf/output/report_successes.txt | 2 +- .../tests/cumf/output/show_missing.txt | 2 +- .../tests/cumf/output/show_missing_full.txt | 2 +- .../tests/cumf/output/show_missing_maxone.txt | 2 +- um_utils/setup.py | 2 +- um_wafccb/LICENCE.txt | 2 +- um_wafccb/lib/um_wafccb/__init__.py | 2 +- um_wafccb/setup.py | 2 +- 33 files changed, 244 insertions(+), 109 deletions(-) create mode 100644 mule/lib/mule/tests/test_stashmaster diff --git a/mule/LICENCE.txt b/mule/LICENCE.txt index b967e6b..d1aaf75 100644 --- a/mule/LICENCE.txt +++ b/mule/LICENCE.txt @@ -1,5 +1,5 @@ !-------------------------------------------------------------------------------! -! (C) Crown Copyright 2018, Met Office. All rights reserved. ! +! (C) Crown Copyright 2019, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/mule/docs/source/conf.py b/mule/docs/source/conf.py index ff6173e..788c7fb 100644 --- a/mule/docs/source/conf.py +++ b/mule/docs/source/conf.py @@ -48,16 +48,16 @@ # General information about the project. project = u'Mule' -copyright = u'2018, UM Systems Team' +copyright = u'2019, UM Systems Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2018.07.1' +version = '2019.01.1' # The full version, including alpha/beta/rc tags. -release = '2018.07.1' +release = '2019.01.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/mule/lib/mule/__init__.py b/mule/lib/mule/__init__.py index 2bf65ab..a046c59 100644 --- a/mule/lib/mule/__init__.py +++ b/mule/lib/mule/__init__.py @@ -57,7 +57,7 @@ from contextlib import contextmanager from mule.stashmaster import STASHmaster -__version__ = "2018.07.1" +__version__ = "2019.01.1" # UM fixed length header names and positions _UM_FIXED_LENGTH_HEADER = [ @@ -744,23 +744,31 @@ def _get_raw_payload_bytes(self): data = self._data_provider._read_bytes() return data - def _can_copy_deferred_data(self, required_lbpack, required_bacc): + def _can_copy_deferred_data(self, required_lbpack, required_bacc, + required_word): """ Return whether or not it is possible to simply re-use the bytes making up the field; for this to be possible the data must be - unmodified, and the requested output packing must be the same - as the input packing. + unmodified, and the requested output packing and disk word size must + be the same as the input. """ # Whether or not this is possible depends on if the Field's # data provider has been wrapped in any operations compatible = hasattr(self._data_provider, "_read_bytes") if compatible: + # Is the packing code the same src_lbpack = self._data_provider.source.lbpack - src_bacc = self._data_provider.source.bacc - # The packing words are compatible if nothing else is different. - compatible = (required_lbpack == src_lbpack and - required_bacc == src_bacc) + compatible = required_lbpack == src_lbpack + + # If it's WGDOS packing, the accuracy matters too + if src_lbpack == 1: + src_bacc = self._data_provider.source.bacc + compatible = compatible and required_bacc == src_bacc + else: + # Otherwise the disk size matters + src_word = self._data_provider.DISK_RECORD_SIZE + compatible = compatible and required_word == src_word return compatible @@ -1514,7 +1522,7 @@ def default_from_raw(values): # Check number format is valid if num_format not in (0, 2, 3): msg = 'Unsupported number format (lbpack N4): {0}' - raise ValueError(msg.format(format)) + raise ValueError(msg.format(num_format)) # With that check out of the way remove the N4 digit and # proceed with the N1 - N3 digits @@ -1688,14 +1696,27 @@ def _write_to_file(self, output_file): if field.lbpack % 10 == 1 and int(field.bacc) == -99: field.lbpack = 10*(field.lbpack//10) - if field._can_copy_deferred_data(field.lbpack, field.bacc): + if field._can_copy_deferred_data( + field.lbpack, field.bacc, self.WORD_SIZE): # The original, unread file data is encoded as wanted, # so extract the raw bytes and write them back out # again unchanged; however first trim off any existing # padding to allow the code below to re-pad the output data_bytes = field._get_raw_payload_bytes() - data_bytes = data_bytes[:field.lblrec*self.WORD_SIZE] + data_bytes = data_bytes[ + :field.lblrec * + field._data_provider.DISK_RECORD_SIZE] output_file.write(data_bytes) + + # Calculate lblrec and lbnrec based on what will be + # written (just in case they are wrong or have come + # from a pp file) + field.lblrec = ( + field._data_provider.DISK_RECORD_SIZE * + field.lblrec // self.WORD_SIZE) + field.lbnrec = ( + field.lblrec - + (field.lblrec % -self._WORDS_PER_SECTOR)) else: # Strip just the n1-n3 digits from the lbpack value diff --git a/mule/lib/mule/pp.py b/mule/lib/mule/pp.py index 6945626..959357a 100644 --- a/mule/lib/mule/pp.py +++ b/mule/lib/mule/pp.py @@ -50,12 +50,24 @@ class PPField3(PPField, mule.Field3): # Mapping to go from release number to field object FIELD_SELECT = {2: PPField2, 3: PPField3} + +# Similarly, adjust the record size for the read providers required here +PP_WORD_SIZE = 4 + + +class _ReadPPProviderUnpacked(mule.ff._ReadFFProviderCray32Packed): + DISK_RECORD_SIZE = PP_WORD_SIZE + + +class _ReadPPProviderWGDOSPacked(mule.ff._ReadFFProviderWGDOSPacked): + DISK_RECORD_SIZE = PP_WORD_SIZE + # Create mappings for the lbpack n3-n1 digits (similar to how the mule file # classes contain mappings like these). The only real difference is that the # "Unpacked" provider uses the 32-bit class (since PP files are 32-bit) _READ_PROVIDERS = { - "000": mule.ff._ReadFFProviderCray32Packed, - "001": mule.ff._ReadFFProviderWGDOSPacked, + "000": _ReadPPProviderUnpacked, + "001": _ReadPPProviderWGDOSPacked, } _WRITE_OPERATORS = { @@ -150,7 +162,7 @@ def fields_from_pp_file(pp_file_obj_or_path): # This should be equivalent to lbnrec, but can sometimes be set to # zero... so to allow the existing provider to work add this value # to the reference field's headers - field_ref.lbnrec = reclen//4 + field_ref.lbnrec = reclen//PP_WORD_SIZE # Associate the provider offset = pp_file.tell() @@ -167,10 +179,16 @@ def fields_from_pp_file(pp_file_obj_or_path): provider = _READ_PROVIDERS[lbpack321](field_ref, pp_file, offset) field = type(field_ref)(ints, reals, provider) + # Change the DTYPE variables back to 64-bit - this is slightly hacky + # but *only* the UM File logic in the main part of Mule utilises this, + # and it will go wrong if it gets a PPField with it set to 32-bit + field.DTYPE_REAL = ">f8" + field.DTYPE_INT = ">i8" + # Now check if the field contains extra data if field.lbext > 0: # Skip past the field data only (relative seek to avoid overflows) - pp_file.seek((field.lblrec - field.lbext)*4, 1) + pp_file.seek((field.lblrec - field.lbext)*PP_WORD_SIZE, 1) # Save the current file position start = pp_file.tell() @@ -179,7 +197,7 @@ def fields_from_pp_file(pp_file_obj_or_path): # end of the record is reached vectors = {} ext_consumed = 0 - while pp_file.tell() - start < field.lbext*4: + while pp_file.tell() - start < field.lbext*PP_WORD_SIZE: # First read the code vector_code = np.fromfile(pp_file, ">i4", 1)[0] @@ -247,13 +265,11 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None): # Similar to the mule file classes, the unpacking of data can be # skipped if the packing and accuracy are unchanged and the fields - # were already PP fields - if (field._can_copy_deferred_data(field.lbpack, field.bacc) and - isinstance(field, PPField)): - + # have the appropriate word size on disk + if (field._can_copy_deferred_data( + field.lbpack, field.bacc, PP_WORD_SIZE)): # Get the raw bytes containing the data data_bytes = field._get_raw_payload_bytes() - else: # If the field has been modified follow a similar set of steps to # the mule file classes @@ -266,8 +282,13 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None): data_bytes, _ = _WRITE_OPERATORS[lbpack321].to_bytes(field) - field.lblrec = len(data_bytes)//4 - field.lbnrec = len(data_bytes)//4 + # Either way, make sure the header addresses are correct for a pp file + # both lbegin and lblnrec are zeroed, as pp files don't require a + # direct access seek point (lbegin) and being sequential they also + # don't have field padding (lbnrec) + field.lbegin = 0 + field.lbnrec = 0 + field.lblrec = len(data_bytes)//PP_WORD_SIZE # If the field appears to be variable resolution, attach the # relevant extra data (requires that a UM file object was given) @@ -302,7 +323,7 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None): cdc = umfile.column_dependent_constants # Calculate U vectors - if grid_type == 18: # U points + if grid_type in (11, 18): # U points vector[1] = cdc.lambda_u if stagger == "new_dynamics": vector[12] = cdc.lambda_p @@ -328,7 +349,7 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None): cdc.lambda_u[-1]]) # Calculate V vectors - if grid_type == 19: # V points + if grid_type in (11, 19): # V points vector[2] = rdc.phi_v if stagger == "new_dynamics": vector[14] = rdc.phi_p @@ -366,7 +387,8 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None): # Calculate the record length (pp files are not direct-access, so each # record begins and ends with its own length) - lookup_reclen = np.array((len(ints) + len(reals))*4).astype(">i4") + lookup_reclen = np.array( + (len(ints) + len(reals))*PP_WORD_SIZE).astype(">i4") # Write the first record (the field header) pp_file.write(lookup_reclen) @@ -378,7 +400,7 @@ def fields_to_pp_file(pp_file_obj_or_path, field_or_fields, umfile=None): reclen = len(data_bytes) if vector: - reclen += extra_len*4 + reclen += extra_len*PP_WORD_SIZE keys = [1, 2, 12, 13, 14, 15] sizes = ([field.lbnpt, field.lbrow] + [field.lbnpt]*2 + [field.lbrow]*2) diff --git a/mule/lib/mule/tests/__init__.py b/mule/lib/mule/tests/__init__.py index db4abfc..140014d 100644 --- a/mule/lib/mule/tests/__init__.py +++ b/mule/lib/mule/tests/__init__.py @@ -31,6 +31,40 @@ import unittest as tests from mule import Field +import mule.stashmaster + +# Override the STASHmaster to pickup the one local to these tests +mule.stashmaster.STASHMASTER_PATH_PATTERN = os.path.join( + os.path.dirname(__file__), "test_stashmaster") + +# Test for availability of mock (which is *not* a standard library at Python 2 +# (it was added to the standard library unittest at Python 3) +try: + if six.PY2: + import mock + elif six.PY3: + import unittest.mock as mock +except ImportError: + MOCK_AVAILABLE = False +else: + MOCK_AVAILABLE = True + + +def skip_mock(fn): + """ + Decorator which can be used to skip a test if the mock library isn't + available. This will completely disable the definition of a method or + class if it is decorated like this: + + @skip_mock + def test_which_uses_mock(*args): + etc + + """ + skip = tests.skipIf( + condition=not MOCK_AVAILABLE, + reason="Test required 'mock'") + return skip(fn) def _testdata_path(): diff --git a/mule/lib/mule/tests/test_stashmaster b/mule/lib/mule/tests/test_stashmaster new file mode 100644 index 0000000..1e510a2 --- /dev/null +++ b/mule/lib/mule/tests/test_stashmaster @@ -0,0 +1,44 @@ +H1| SUBMODEL_NUMBER=1 +H2| SUBMODEL_NAME=ATMOS +H3| UM_VERSION=X.X +# +#|Model |Sectn | Item |Name | +#|Space |Point | Time | Grid |LevelT|LevelF|LevelL|PseudT|PseudF|PseudL|LevCom| +#| Option Codes | Version Mask | Halo | +#|DataT |DumpP | PC1 PC2 PC3 PC4 PC5 PC6 PC7 PC8 PC9 PCA | +#|Rotate| PPF | USER | LBVC | BLEV | TLEV |RBLEVV| CFLL | CFFF | +# +#=============================================================================== +# +1| 1 | 3 | 236 |TEMPERATURE AT 1.5M | +2| 0 | 0 | 1 | 1 | 5 | -1 | -1 | 0 | 0 | 0 | 0 | +3| 000000000000000000000000000000 | 00000000000100000001 | 3 | +4| 1 | 2 | -6 -3 -3 -3 -12 21 -12 -99 -99 -99 | +5| 0 | 16 | 0 | 1 | 0 | 0 | 0 | 9999 | 58 | +# +1| 1 | 8 | 225 |DEEP SOIL TEMP. AFTER HYDROLOGY DEGK| +2| 0 | 0 | 1 | 2 | 6 | 8 | 9 | 0 | 0 | 0 | 1 | +3| 000000000000000000000000000000 | 00000000000010000000 | 3 | +4| 1 | 2 | -3 -3 -3 -3 -6 -99 -6 -99 -99 -99 | +5| 0 | 23 | 0 | 6 | 0 | 0 | 0 | 9999 | 344 | +# +1| 1 | 0 | 33 |OROGRAPHY (/STRAT LOWER BC) | +2| 2 | 0 | 1 | 1 | 5 | -1 | -1 | 0 | 0 | 0 | 0 | +3| 000000000000010000000000000000 | 00000000000000000001 | 3 | +4| 1 | 2 | -99 -3 -3 -3 -6 30 -3 -99 -99 -99 | +5| 0 | 1 | 0 | 129 | 0 | 0 | 0 | 9999 | 73 | +# +1| 1 | 16 | 4 |TEMPERATURE ON THETA LEVELS | +2| 0 | 0 | 1 | 1 | 2 | 1 | 2 | 0 | 0 | 0 | 1 | +3| 000000000000000000000000000000 | 00000000000000000001 | 3 | +4| 1 | 2 | -3 -10 -3 -3 -14 21 -3 -99 -99 -99 | +5| 0 | 16 | 0 | 65 | 0 | 0 | 0 | 0 | 3 | +# +#=============================================================================== +# +1| -1 | -1 | -1 |END OF FILE MARK | +2| 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +3| 000000000000000000000000000000 | 00000000000000000000 | 0 | +4| 0 | 0 | -99 -99 -99 -99 -30 -99 -99 -99 -99 -99 | +5| 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +# diff --git a/mule/lib/mule/tests/unit/test_Field.py b/mule/lib/mule/tests/unit/test_Field.py index 8276a3a..ef6424a 100644 --- a/mule/lib/mule/tests/unit/test_Field.py +++ b/mule/lib/mule/tests/unit/test_Field.py @@ -29,12 +29,6 @@ import mule.tests as tests from mule import Field, _NullReadProvider -import six -if six.PY2: - import mock -elif six.PY3: - import unittest.mock as mock - class Test_int_headers(tests.MuleTest): def test(self): @@ -70,12 +64,16 @@ def _check_formats(self, old_bacc=-6, new_bacc=-6, absent_provider=False): - lookup_entry = mock.Mock(lbpack=old_lbpack, bacc=old_bacc) + class dummy_lookup(object): + lbpack = old_lbpack + bacc = old_bacc + lookup_entry = dummy_lookup() + provider = _NullReadProvider(lookup_entry, None, None) if absent_provider: provider = None field = Field(list(range(45)), list(range(19)), provider) - return field._can_copy_deferred_data(new_lbpack, new_bacc) + return field._can_copy_deferred_data(new_lbpack, new_bacc, 8) def test_okay_simple(self): self.assertTrue(self._check_formats(1234, 1234)) @@ -87,7 +85,7 @@ def test_fail_nodata(self): self.assertFalse(self._check_formats(1234, 1234, absent_provider=True)) def test_fail_different_bacc(self): - self.assertFalse(self._check_formats(1234, 1234, new_bacc=-8)) + self.assertFalse(self._check_formats(1, 1, new_bacc=-8)) if __name__ == '__main__': diff --git a/mule/lib/mule/tests/unit/test_stashmaster_fromumfile.py b/mule/lib/mule/tests/unit/test_stashmaster_fromumfile.py index fadd787..d53aa39 100644 --- a/mule/lib/mule/tests/unit/test_stashmaster_fromumfile.py +++ b/mule/lib/mule/tests/unit/test_stashmaster_fromumfile.py @@ -29,12 +29,6 @@ import mule.tests as tests from mule.stashmaster import STASHmaster -import six -if six.PY2: - from mock import patch -elif six.PY3: - from unittest.mock import patch - class FakeFixedLength(object): def __init__(self, dataset_type, model_version, mdi): @@ -43,11 +37,12 @@ def __init__(self, dataset_type, model_version, mdi): self.MDI = mdi +# Skip the definition of these tests if the "mock" module isn't available +@tests.skip_mock class TestStashmasterFromUmfile(tests.MuleTest): - def test_fromumfile_mdi_version(self): self.fixed_length_header = FakeFixedLength(1, -99, -99) - with patch('warnings.warn') as warn: + with tests.mock.patch('warnings.warn') as warn: STASHmaster.from_umfile(self) expected_msg = ('Fixed length header does not define the UM model ' 'version number, unable to load STASHmaster file.') @@ -55,7 +50,7 @@ def test_fromumfile_mdi_version(self): def test_fromfile_ancil(self): self.fixed_length_header = FakeFixedLength(4, 1003, -99) - with patch('warnings.warn') as warn: + with tests.mock.patch('warnings.warn') as warn: STASHmaster.from_umfile(self) expected_msg = ('Ancillary files do not define the UM version ' 'number in the Fixed Length Header. ' @@ -66,7 +61,7 @@ def test_fromfile_ancil(self): def test_fromumfile_non_ancil(self): self.fixed_length_header = FakeFixedLength(1, 1003, -99) to_patch = 'mule.stashmaster.STASHmaster.from_version' - with patch(to_patch) as from_version: + with tests.mock.patch(to_patch) as from_version: STASHmaster.from_umfile(self) from_version.assert_called_once_with('10.3') diff --git a/mule/setup.py b/mule/setup.py index 85366aa..66e29ad 100644 --- a/mule/setup.py +++ b/mule/setup.py @@ -46,18 +46,19 @@ def run(self): setuptools.setup( - name='mule', - version='2018.07.1', - description='Unified Model Fields File interface', - author='UM Systems Team', - url='https://code.metoffice.gov.uk/trac/um', - cmdclass={'clean': CleanCommand}, - package_dir={'': 'lib'}, - packages=['mule', - 'mule.tests', - 'mule.tests.unit', - 'mule.tests.integration', - 'mule.example_code'], - package_data={'mule': - [os.path.relpath(path, "lib/mule") - for path in glob('lib/mule/tests/test_datafiles/*')]}) + name='mule', + version='2019.01.1', + description='Unified Model Fields File interface', + author='UM Systems Team', + url='https://code.metoffice.gov.uk/trac/um', + cmdclass={'clean': CleanCommand}, + package_dir={'': 'lib'}, + packages=['mule', + 'mule.tests', + 'mule.tests.unit', + 'mule.tests.integration', + 'mule.example_code'], + package_data={'mule': + [os.path.relpath(path, "lib/mule") + for path in (glob('lib/mule/tests/test_datafiles/*') + + ['lib/mule/tests/test_stashmaster'])]}) diff --git a/um_packing/LICENCE.txt b/um_packing/LICENCE.txt index 68d56ec..a1a05af 100644 --- a/um_packing/LICENCE.txt +++ b/um_packing/LICENCE.txt @@ -1,6 +1,6 @@ !-------------------------------------------------------------------------------! ! ! -! (C) Crown Copyright 2018, Met Office. All rights reserved. ! +! (C) Crown Copyright 2019, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/um_packing/lib/um_packing/__init__.py b/um_packing/lib/um_packing/__init__.py index d6736e6..7aa0725 100644 --- a/um_packing/lib/um_packing/__init__.py +++ b/um_packing/lib/um_packing/__init__.py @@ -20,4 +20,4 @@ from .um_packing import wgdos_pack, wgdos_unpack, get_shumlib_version -__version__ = "2018.07.1" +__version__ = "2019.01.1" diff --git a/um_packing/setup.py b/um_packing/setup.py index ef01d2b..6bcfcc1 100644 --- a/um_packing/setup.py +++ b/um_packing/setup.py @@ -51,7 +51,7 @@ def run(self): setuptools.setup( name='um_packing', - version='2018.07.1', + version='2019.01.1', description='Unified Model packing library extension', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', diff --git a/um_spiral_search/LICENCE.txt b/um_spiral_search/LICENCE.txt index 68d56ec..a1a05af 100644 --- a/um_spiral_search/LICENCE.txt +++ b/um_spiral_search/LICENCE.txt @@ -1,6 +1,6 @@ !-------------------------------------------------------------------------------! ! ! -! (C) Crown Copyright 2018, Met Office. All rights reserved. ! +! (C) Crown Copyright 2019, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/um_spiral_search/lib/um_spiral_search/__init__.py b/um_spiral_search/lib/um_spiral_search/__init__.py index 78c953b..010d3b8 100644 --- a/um_spiral_search/lib/um_spiral_search/__init__.py +++ b/um_spiral_search/lib/um_spiral_search/__init__.py @@ -21,4 +21,4 @@ from .um_spiral_search import spiral_search -__version__ = "2018.07.1" +__version__ = "2019.01.1" diff --git a/um_spiral_search/setup.py b/um_spiral_search/setup.py index ae211e0..9d222a7 100644 --- a/um_spiral_search/setup.py +++ b/um_spiral_search/setup.py @@ -51,7 +51,7 @@ def run(self): setuptools.setup( name='um_spiral_search', - version='2018.07.1', + version='2019.01.1', description='Unified Model Spiral Search extension', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', diff --git a/um_sstpert/LICENCE.txt b/um_sstpert/LICENCE.txt index 68d56ec..a1a05af 100644 --- a/um_sstpert/LICENCE.txt +++ b/um_sstpert/LICENCE.txt @@ -1,6 +1,6 @@ !-------------------------------------------------------------------------------! ! ! -! (C) Crown Copyright 2018, Met Office. All rights reserved. ! +! (C) Crown Copyright 2019, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/um_sstpert/lib/um_sstpert/__init__.py b/um_sstpert/lib/um_sstpert/__init__.py index 09654e0..378db6b 100644 --- a/um_sstpert/lib/um_sstpert/__init__.py +++ b/um_sstpert/lib/um_sstpert/__init__.py @@ -48,7 +48,7 @@ from um_utils.version import report_modules from um_utils.pumf import _banner -__version__ = "2018.07.1" +__version__ = "2019.01.1" def gen_pert_field(clim_fields, alpha, ens_member, date): diff --git a/um_sstpert/setup.py b/um_sstpert/setup.py index 9ef44d2..c9b4201 100644 --- a/um_sstpert/setup.py +++ b/um_sstpert/setup.py @@ -51,7 +51,7 @@ def run(self): setuptools.setup( name='um_sstpert', - version='2018.07.1', + version='2019.01.1', description='Unified Model SST-perturbation extension and utility', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', diff --git a/um_utils/LICENCE.txt b/um_utils/LICENCE.txt index 68d56ec..a1a05af 100644 --- a/um_utils/LICENCE.txt +++ b/um_utils/LICENCE.txt @@ -1,6 +1,6 @@ !-------------------------------------------------------------------------------! ! ! -! (C) Crown Copyright 2018, Met Office. All rights reserved. ! +! (C) Crown Copyright 2019, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/um_utils/docs/source/conf.py b/um_utils/docs/source/conf.py index 177ec45..61a1100 100644 --- a/um_utils/docs/source/conf.py +++ b/um_utils/docs/source/conf.py @@ -48,16 +48,16 @@ # General information about the project. project = u'UM Utilities' -copyright = u'2018, UM Systems Team' +copyright = u'2019, UM Systems Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2018.07.1' +version = '2019.01.1' # The full version, including alpha/beta/rc tags. -release = '2018.07.1' +release = '2019.01.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/um_utils/lib/um_utils/__init__.py b/um_utils/lib/um_utils/__init__.py index a21c391..eeac864 100644 --- a/um_utils/lib/um_utils/__init__.py +++ b/um_utils/lib/um_utils/__init__.py @@ -19,4 +19,4 @@ # along with these utilities. # If not, see . -__version__ = "2018.07.1" +__version__ = "2019.01.1" diff --git a/um_utils/lib/um_utils/cumf.py b/um_utils/lib/um_utils/cumf.py index 8e356cf..025f2a4 100644 --- a/um_utils/lib/um_utils/cumf.py +++ b/um_utils/lib/um_utils/cumf.py @@ -91,6 +91,7 @@ class as keyword arguments. The available settings are: import argparse import textwrap import numpy as np +import warnings from six import StringIO from collections import defaultdict from um_utils.stashmaster import STASHmaster @@ -978,11 +979,13 @@ def summary_report(comparison, stdout=None): # Report on the maximum RMS diff percentage if comparison.max_rms_diff_1[0] > 0.0: stdout.write( - "Maximum RMS diff as % of data in file 1: {0!r} (field {1})\n" + "Maximum RMS diff as % of data in file 1: " + "{0:<18.17g} (field {1})\n" .format(*comparison.max_rms_diff_1)) if comparison.max_rms_diff_2[0] > 0.0: stdout.write( - "Maximum RMS diff as % of data in file 2: {0!r} (field {1})\n" + "Maximum RMS diff as % of data in file 2: " + "{0:<18.17g} (field {1})\n" .format(*comparison.max_rms_diff_2)) if (comparison.max_rms_diff_1[0] > 0.0 or @@ -1186,21 +1189,21 @@ def report_index_errors(diffmap, stdout, name_mapping): stdout.write("Data differences:\n") stdout.write(" Number of point differences : {0}/{1}\n" .format(*comp_field.compared)) - stdout.write(" Maximum absolute difference : {0!r}\n" + stdout.write(" Maximum absolute difference : {0:<18.17g}\n" .format(comp_field.max_diff)) - stdout.write(" RMS difference : {0!r}\n" + stdout.write(" RMS difference : {0:<18.17g}\n" .format(comp_field.rms_diff)) if comp_field.rms_norm_diff_1 is None: stdout.write(" RMS diff as % of file_1 data : " "NaN (File 1 data all zero) \n") else: - stdout.write(" RMS diff as % of file_1 data : {0!r}\n" + stdout.write(" RMS diff as % of file_1 data : {0:<18.17g}\n" .format(comp_field.rms_norm_diff_1)) if comp_field.rms_norm_diff_2 is None: stdout.write(" RMS diff as % of file_2 data : " "NaN (File 2 data all zero) \n") else: - stdout.write(" RMS diff as % of file_2 data : {0!r}\n" + stdout.write(" RMS diff as % of file_2 data : {0:<18.17g}\n" .format(comp_field.rms_norm_diff_2)) stdout.write("\n") @@ -1405,20 +1408,37 @@ def __call__(self, parser, namespace, values, option_string=None): new_ff.fields = [field for field in comparison.field_comparisons if not field.data_match and field.data_shape_match] - # If any of these fields require the land-sea-mask to be written out - # add it to the start of the file list here - mask_required = [field.lbpack % 100 != 0 for field in new_ff.fields] - if any(mask_required): - lsm = None - for field in um_files[0].fields: - if field.lbrel in (2, 3) and field.lbuser4 == 30: - lsm = field + # Check if a land sea mask exists in the first file + lsm = None + for field in um_files[0].fields: + if field.lbrel in (2, 3) and field.lbuser4 == 30: + lsm = field + break + + # Now double check that there weren't any differences between the + # mask in the 2 files (if there were, the output for a land/sea + # compressed field in this diff file will be very misleading) + for field in new_ff.fields: + if field.lbuser4 == 30: + # If there is a land/sea mask difference, disable it + lsm = None + + # Now check for land/sea packed fields within the diff file + for ifield, field in enumerate(new_ff.fields): + if (field.lbpack // 10) % 10 != 0: + if lsm is not None: + # If we have the LSM, add it to the diff file + new_ff.fields.insert(0, lsm) break - if lsm is None: - msg = "Unable to write diff file, land-sea mask not present" - raise ValueError(msg) - new_ff.fields.insert(0, lsm) - + else: + # Otherwise Mule won't let us output the field, so warn + # about this here and then remove it from the diff file + msg = ("Unable to output Field {0} as it is land/sea " + "packed but no suitable land-sea mask was found") + warnings.warn(msg.format(ifield + 1)) + new_ff.fields.remove(field) + + # Assuming there are still writable fields in the diff file, write it if len(new_ff.fields) > 0: new_ff.to_file(diff_file) diff --git a/um_utils/lib/um_utils/tests/cumf/output/default.txt b/um_utils/lib/um_utils/tests/cumf/output/default.txt index 9d38f5c..eada71a 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/default.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/default.txt @@ -59,7 +59,7 @@ File_1 lookup info: t1(2015/-99/-99 -99:-99:-99) lblev(-99)/blev(0.0) lbproc(-99) Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/lib/um_utils/tests/cumf/output/ignore_missing.txt b/um_utils/lib/um_utils/tests/cumf/output/ignore_missing.txt index 4c148cf..cc98694 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/ignore_missing.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/ignore_missing.txt @@ -68,7 +68,7 @@ File_1 lookup info: t1(2015/-99/-99 -99:-99:-99) lblev(-99)/blev(0.0) lbproc(-99) Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/lib/um_utils/tests/cumf/output/ignore_template.txt b/um_utils/lib/um_utils/tests/cumf/output/ignore_template.txt index 2537cae..ecf161a 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/ignore_template.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/ignore_template.txt @@ -40,7 +40,7 @@ File_1 lookup info: t1(2015/-99/-99 -99:-99:-99) lblev(-99)/blev(0.0) lbproc(-99) Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/lib/um_utils/tests/cumf/output/report_successes.txt b/um_utils/lib/um_utils/tests/cumf/output/report_successes.txt index 90c598c..b61d82e 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/report_successes.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/report_successes.txt @@ -240,7 +240,7 @@ File_1 lookup info: Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/lib/um_utils/tests/cumf/output/show_missing.txt b/um_utils/lib/um_utils/tests/cumf/output/show_missing.txt index 7a48e83..956fb17 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/show_missing.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/show_missing.txt @@ -71,7 +71,7 @@ File_1 lookup info: t1(2015/-99/-99 -99:-99:-99) lblev(-99)/blev(0.0) lbproc(-99) Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/lib/um_utils/tests/cumf/output/show_missing_full.txt b/um_utils/lib/um_utils/tests/cumf/output/show_missing_full.txt index d6b9b76..e4373df 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/show_missing_full.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/show_missing_full.txt @@ -252,7 +252,7 @@ File_1 lookup info: Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/lib/um_utils/tests/cumf/output/show_missing_maxone.txt b/um_utils/lib/um_utils/tests/cumf/output/show_missing_maxone.txt index b93ff96..cd7d251 100644 --- a/um_utils/lib/um_utils/tests/cumf/output/show_missing_maxone.txt +++ b/um_utils/lib/um_utils/tests/cumf/output/show_missing_maxone.txt @@ -69,7 +69,7 @@ File_1 lookup info: t1(2015/-99/-99 -99:-99:-99) lblev(-99)/blev(0.0) lbproc(-99) Data differences: Number of point differences : 11/12 - Maximum absolute difference : 44.0 + Maximum absolute difference : 44 RMS difference : 25.974346318370873 RMS diff as % of file_1 data : 382.97084310253518 RMS diff as % of file_2 data : 76.594168620507048 diff --git a/um_utils/setup.py b/um_utils/setup.py index fd6918e..f693d66 100644 --- a/um_utils/setup.py +++ b/um_utils/setup.py @@ -46,7 +46,7 @@ def run(self): setuptools.setup( name='um_utils', - version='2018.07.1', + version='2019.01.1', description='Unified Model Fields File utilities', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um', diff --git a/um_wafccb/LICENCE.txt b/um_wafccb/LICENCE.txt index 68d56ec..a1a05af 100644 --- a/um_wafccb/LICENCE.txt +++ b/um_wafccb/LICENCE.txt @@ -1,6 +1,6 @@ !-------------------------------------------------------------------------------! ! ! -! (C) Crown Copyright 2018, Met Office. All rights reserved. ! +! (C) Crown Copyright 2019, Met Office. All rights reserved. ! ! ! ! Redistribution and use in source and binary forms, with or without ! ! modification, are permitted provided that the following conditions are met: ! diff --git a/um_wafccb/lib/um_wafccb/__init__.py b/um_wafccb/lib/um_wafccb/__init__.py index b2a0b55..8750157 100644 --- a/um_wafccb/lib/um_wafccb/__init__.py +++ b/um_wafccb/lib/um_wafccb/__init__.py @@ -20,4 +20,4 @@ from .um_wafccb import wafccb -__version__ = "2018.07.1" +__version__ = "2019.01.1" diff --git a/um_wafccb/setup.py b/um_wafccb/setup.py index 8d84db2..959b2e5 100644 --- a/um_wafccb/setup.py +++ b/um_wafccb/setup.py @@ -51,7 +51,7 @@ def run(self): setuptools.setup( name='um_wafccb', - version='2018.07.1', + version='2019.01.1', description='Unified Model WAFC CB extension', author='UM Systems Team', url='https://code.metoffice.gov.uk/trac/um',