From b6f5d996b439f5d1b18c1d405b8f2ab3d2fd323a Mon Sep 17 00:00:00 2001 From: Stefan Codrescu Date: Tue, 25 Feb 2020 22:00:51 +0000 Subject: [PATCH] Black reformatting. --- ncflag/cli.py | 49 +++++++++++--- ncflag/flag_wrapper.py | 52 +++++++++----- setup.py | 18 ++--- test/test_real_netcdf.py | 30 +++++--- test/test_theoretical.py | 143 ++++++++++++++++++++++++++------------- 5 files changed, 198 insertions(+), 94 deletions(-) diff --git a/ncflag/cli.py b/ncflag/cli.py index c715ed5..ec29c2d 100644 --- a/ncflag/cli.py +++ b/ncflag/cli.py @@ -9,13 +9,18 @@ except pkg_resources.DistributionNotFound: version = "unknown" + def show_flags(ctx, param, ncfile): if not ncfile or ctx.resilient_parsing: return valid_flags = [] with nc.Dataset(ncfile) as nc_in: # type: nc.Dataset for k, v in nc_in.variables.items(): - if hasattr(v, "flag_values") and hasattr(v, "flag_meanings") and len(v.dimensions) == 1: + if ( + hasattr(v, "flag_values") + and hasattr(v, "flag_meanings") + and len(v.dimensions) == 1 + ): valid_flags.append(k) click.echo("Inspectable flags: %s" % map(str, valid_flags)) ctx.exit() @@ -23,28 +28,50 @@ def show_flags(ctx, param, ncfile): @click.command() @click.version_option(version, "-v", "--version") -@click.option("--show_flags", callback=show_flags, expose_value=False, is_eager=True, - type=click.Path(exists=True, dir_okay=False), help="Print the flags this tool can inspect.") +@click.option( + "--show_flags", + callback=show_flags, + expose_value=False, + is_eager=True, + type=click.Path(exists=True, dir_okay=False), + help="Print the flags this tool can inspect.", +) @click.argument("ncfile", type=click.Path(exists=True, dir_okay=False)) @click.argument("flag", type=click.STRING) @click.option("--use_time_var", type=click.STRING, default=None) -@click.option("-l", help="log level", type=click.Choice(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']), - default="WARNING") +@click.option( + "-l", + help="log level", + type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), + default="WARNING", +) def cli(ncfile, flag, use_time_var, l): logging.getLogger().setLevel(l) with nc.Dataset(ncfile) as nc_in: # type: nc.Dataset # initial checks v = nc_in.variables[flag] # type: nc.Variable - assert hasattr(v, "flag_values"), "%s is not CF compliant flag, missing flag_values" % flag - assert hasattr(v, "flag_meanings"), "%s is not CF compliant flag, missing flag_meanings" % flag - assert len(v.dimensions) == 1, "multidimensional flags are not supported, see docs and use ipython instead" + assert hasattr(v, "flag_values"), ( + "%s is not CF compliant flag, missing flag_values" % flag + ) + assert hasattr(v, "flag_meanings"), ( + "%s is not CF compliant flag, missing flag_meanings" % flag + ) + assert ( + len(v.dimensions) == 1 + ), "multidimensional flags are not supported, see docs and use ipython instead" w = FlagWrap.init_from_netcdf(v) if use_time_var is not None: t = nc_in.variables[use_time_var] # type: nc.Variable - assert t.dimensions == v.dimensions, "To print flags by time, time flag must share dimensions" - assert hasattr(t, "units"), "did not find units on time flag %s" % use_time_var + assert ( + t.dimensions == v.dimensions + ), "To print flags by time, time flag must share dimensions" + assert hasattr(t, "units"), ( + "did not find units on time flag %s" % use_time_var + ) for i, dt in enumerate(nc.num2date(t[:], t.units)): - out_time = dt.isoformat() if dt is not None else "__________________________" + out_time = ( + dt.isoformat() if dt is not None else "__________________________" + ) click.echo("%s: %s" % (out_time, w.get_flags_set_at_index(i))) else: for i in range(v.size): diff --git a/ncflag/flag_wrapper.py b/ncflag/flag_wrapper.py index ac16a69..bec2116 100644 --- a/ncflag/flag_wrapper.py +++ b/ncflag/flag_wrapper.py @@ -35,22 +35,27 @@ def __init__(self, flags, flag_meanings, flag_values, flag_masks=None): if isinstance(flag_meanings, string_types): self._flag_meanings = flag_meanings.split() # split on spaces else: - assert isinstance(flag_meanings, list), \ - "expected flag_meanings as either list of flag_meanings, or space separated string of flag_meanings" + assert isinstance( + flag_meanings, list + ), "expected flag_meanings as either list of flag_meanings, or space separated string of flag_meanings" self._flag_meanings = flag_meanings self._flag_values = np.array(flag_values).astype(self.flags.dtype) - assert len(self._flag_values) == len(self._flag_meanings), \ - "flag_meanings vs flag_values length mismatch: found {} and {}".format(len(self._flag_meanings), - len(self._flag_values)) + assert len(self._flag_values) == len( + self._flag_meanings + ), "flag_meanings vs flag_values length mismatch: found {} and {}".format( + len(self._flag_meanings), len(self._flag_values) + ) if flag_masks is None: self._flag_masks = np.full_like(self._flag_values, -1) else: self._flag_masks = np.array(flag_masks).astype(self.flags.dtype) - assert len(self._flag_masks) == len(self._flag_meanings), \ - "flag_meanings vs flag_masks length mismatch: found {} and {}".format(len(self._flag_meanings), - len(self._flag_masks)) + assert len(self._flag_masks) == len( + self._flag_meanings + ), "flag_meanings vs flag_masks length mismatch: found {} and {}".format( + len(self._flag_meanings), len(self._flag_masks) + ) # This is only for use with init_from_netcdf to hold the reference to nc_var so that # the caller doesn't have to associate the write_to_netcdf call with an nc_var if it's the same as @@ -76,9 +81,19 @@ def init_from_netcdf(cls, nc_var, shape=None, fill=None): """ if shape is not None and fill is not None: flags = np.full(shape, fill, dtype=nc_var.dtype) - instance = cls(flags, nc_var.flag_meanings, nc_var.flag_values, getattr(nc_var, "flag_masks", None)) + instance = cls( + flags, + nc_var.flag_meanings, + nc_var.flag_values, + getattr(nc_var, "flag_masks", None), + ) else: - instance = cls(nc_var[:], nc_var.flag_meanings, nc_var.flag_values, getattr(nc_var, "flag_masks", None)) + instance = cls( + nc_var[:], + nc_var.flag_meanings, + nc_var.flag_values, + getattr(nc_var, "flag_masks", None), + ) instance._nc_var = nc_var @@ -95,13 +110,17 @@ def write_to_netcdf(self, nc_var=None): if nc_var is None and self._nc_var is not None: nc_var = self._nc_var elif nc_var is None: - raise RuntimeError("write_to_netcdf called w/o target nc_var and appears not to be init from an nc_var.") + raise RuntimeError( + "write_to_netcdf called w/o target nc_var and appears not to be init from an nc_var." + ) nc_var[:] = self.flags nc_var.flag_meanings = " ".join(self._flag_meanings) nc_var.flag_values = self._flag_values - if (not np.all(self._flag_masks == np.full_like(self._flag_values, -1)) or - getattr(nc_var, "flag_masks", None) is not None): + if ( + not np.all(self._flag_masks == np.full_like(self._flag_values, -1)) + or getattr(nc_var, "flag_masks", None) is not None + ): # only write masks if they aren't the default all bits 1 or somethign existed before nc_var.flag_masks = self._flag_masks @@ -129,7 +148,9 @@ def get(meaning): mask = ~np.ma.getmask(self.flags) # only the booleans to True potentially only where flags are not masked in the first place. - default[mask] = (self.flags[mask] & self._flag_masks[index]) == self._flag_values[index] + default[mask] = ( + self.flags[mask] & self._flag_masks[index] + ) == self._flag_values[index] return default if isinstance(flag_meaning, (list, tuple)): @@ -161,7 +182,7 @@ def reduce(self, exclude_mask=0, axis=-1): np.ma.bitwise_or.reduce(self.flags, axis=axis) & ~exclude_mask, self._flag_meanings, self._flag_values, - self._flag_masks + self._flag_masks, ) def get_flag_at_index(self, flag_meaning, i): @@ -325,4 +346,3 @@ def get_mask_for_meaning(self, flag_meaning): """ index = self._flag_meanings.index(flag_meaning) return self._flag_masks[index] - diff --git a/setup.py b/setup.py index 1026d00..5409aea 100644 --- a/setup.py +++ b/setup.py @@ -2,23 +2,19 @@ from pypandoc import convert_file setup( - name='ncflag', - version='0.2.5', + name="ncflag", + version="0.2.5", description="Utility and library to interface with CF-Compliant NetCDF flag variables.", author="Stefan Codrescu", author_email="stefan.codrescu@noaa.gov", url="https://github.com/5tefan/ncflag", packages=["ncflag"], - long_description=convert_file('README.md', 'rst'), - install_requires=[ - 'Click', - 'numpy', - 'netCDF4' - ], - entry_points=''' + long_description=convert_file("README.md", "rst"), + install_requires=["Click", "numpy", "netCDF4"], + entry_points=""" [console_scripts] ncflag=ncflag.cli:cli - ''', + """, include_package_data=False, classifiers=[ "Development Status :: 4 - Beta", @@ -27,5 +23,5 @@ "Programming Language :: Python :: 3.6", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", - ] + ], ) diff --git a/test/test_real_netcdf.py b/test/test_real_netcdf.py index dc2f335..8ff3d95 100644 --- a/test/test_real_netcdf.py +++ b/test/test_real_netcdf.py @@ -4,8 +4,11 @@ import netCDF4 as nc import os + def get_dataset(): - datafile = os.path.join(os.path.dirname(__file__), "data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc") + datafile = os.path.join( + os.path.dirname(__file__), "data/ops_exis-l1b-sfxr_g16_d20180402_v0-0-0.nc" + ) return nc.Dataset(datafile, "r") @@ -38,20 +41,29 @@ def test_misc_consistency(self): self.assertEqual(f.get_value_for_meaning("good_quality_qf"), 0) # similarly, the masking being first in the list is easy to see.. self.assertEqual(f.get_mask_for_meaning("good_quality_qf"), 524287) - + # this particular file has degraded_due_to_XRS-A_solar_maximum_channel_signal_near_zero_qf set # at index 0: original_flags = f.get_flags_set_at_index(0) - self.assertIn("degraded_due_to_XRS-A_solar_maximum_channel_signal_near_zero_qf", original_flags) + self.assertIn( + "degraded_due_to_XRS-A_solar_maximum_channel_signal_near_zero_qf", + original_flags, + ) # set a new flag... make sure it wasn't there orignally, and is there now. - f.set_flag_at_index("degraded_due_to_insufficient_number_of_integrations_after_XRS_reset_qf", 0) - self.assertNotIn("degraded_due_to_insufficient_number_of_integrations_after_XRS_reset_qf", original_flags) - self.assertIn("degraded_due_to_insufficient_number_of_integrations_after_XRS_reset_qf", f.get_flags_set_at_index(0)) - + f.set_flag_at_index( + "degraded_due_to_insufficient_number_of_integrations_after_XRS_reset_qf", 0 + ) + self.assertNotIn( + "degraded_due_to_insufficient_number_of_integrations_after_XRS_reset_qf", + original_flags, + ) + self.assertIn( + "degraded_due_to_insufficient_number_of_integrations_after_XRS_reset_qf", + f.get_flags_set_at_index(0), + ) + # try to set good_quality_qf at index 0... make sure it is now set. Setting good_quality_qf should # also implicitly unset all the other flags based on the way it's defined.... the assertEqual covers this. f.set_flag_at_index("good_quality_qf", 0) self.assertEqual(["good_quality_qf"], f.get_flags_set_at_index(0)) - - diff --git a/test/test_theoretical.py b/test/test_theoretical.py index b7aa416..6ebaa35 100644 --- a/test/test_theoretical.py +++ b/test/test_theoretical.py @@ -20,23 +20,39 @@ def test_exclusive_flag_type(self): original_flags = np.array([0, 0, 1, 2, 3, -1], dtype=np.ubyte) f = FlagWrap( - original_flags.copy(), - "good medium bad extra_bad", - np.array([0, 1, 2, 3]) + original_flags.copy(), "good medium bad extra_bad", np.array([0, 1, 2, 3]) ) - np.testing.assert_array_equal(f.get_flag("good"), np.array([1, 1, 0, 0, 0, 0]), - err_msg="only the first two are good") - np.testing.assert_array_equal(f.get_flag("medium"), np.array([0, 0, 1, 0, 0, 0]), - err_msg="only 3d element is medium") - np.testing.assert_array_equal(f.get_flag("bad"), np.array([0, 0, 0, 1, 0, 0]), - err_msg="only the 4th element is bad") - np.testing.assert_array_equal(f.get_flag("extra_bad"), np.array([0, 0, 0, 0, 1, 0]), - err_msg="only second to last is extra_bad") - np.testing.assert_array_equal(f.get_flag(["good", "bad"]), np.array([1, 1, 0, 1, 0, 0]), - err_msg="good and bad weren't merged together.") + np.testing.assert_array_equal( + f.get_flag("good"), + np.array([1, 1, 0, 0, 0, 0]), + err_msg="only the first two are good", + ) + np.testing.assert_array_equal( + f.get_flag("medium"), + np.array([0, 0, 1, 0, 0, 0]), + err_msg="only 3d element is medium", + ) + np.testing.assert_array_equal( + f.get_flag("bad"), + np.array([0, 0, 0, 1, 0, 0]), + err_msg="only the 4th element is bad", + ) + np.testing.assert_array_equal( + f.get_flag("extra_bad"), + np.array([0, 0, 0, 0, 1, 0]), + err_msg="only second to last is extra_bad", + ) + np.testing.assert_array_equal( + f.get_flag(["good", "bad"]), + np.array([1, 1, 0, 1, 0, 0]), + err_msg="good and bad weren't merged together.", + ) - self.assertTrue(len(f.get_flags_set_at_index(-1)) == 0, msg="No flags set at the end. Fill value.") + self.assertTrue( + len(f.get_flags_set_at_index(-1)) == 0, + msg="No flags set at the end. Fill value.", + ) # identity test: for flag_meaning in f._flag_meanings: @@ -47,7 +63,8 @@ def test_exclusive_flag_type(self): # exit_on_good option applies here since flags are mutually exclusive # result should be the same regardless of exit_on_good self.assertTrue( - f.get_flags_set_at_index(i, exit_on_good=True) == f.get_flags_set_at_index(i, exit_on_good=False) + f.get_flags_set_at_index(i, exit_on_good=True) + == f.get_flags_set_at_index(i, exit_on_good=False) ) self.assertTrue(f.get_flag_at_index("good", 0)) @@ -62,26 +79,36 @@ def test_exclusive_flag_type(self): f.set_flag_at_index("extra_bad", 0) # now that we've changed the first flag, make sure it's no longer good and is now extra_bad, but # importantly, nothing else changed. - np.testing.assert_array_equal(f.get_flag("extra_bad"), np.array([1, 0, 0, 0, 1, 0])) + np.testing.assert_array_equal( + f.get_flag("extra_bad"), np.array([1, 0, 0, 0, 1, 0]) + ) np.testing.assert_array_equal(f.get_flag("good"), np.array([0, 1, 0, 0, 0, 0])) # set it back to good... retest that things were back to normal f.set_flag_at_index("good", 0) # assert still in original form - np.testing.assert_array_equal(f.flags, original_flags, err_msg="Flags not back to original!") + np.testing.assert_array_equal( + f.flags, original_flags, err_msg="Flags not back to original!" + ) # continuing now to test the full vector set_flag f.set_flag("medium", [1, 0, 0, 0, 1, 0], zero_if_unset=False) - np.testing.assert_array_equal(f.get_flag("medium"), np.array([1, 0, 1, 0, 1, 0])) + np.testing.assert_array_equal( + f.get_flag("medium"), np.array([1, 0, 1, 0, 1, 0]) + ) # chance those back, they were "good" and "extra_bad" f.set_flag_at_index("good", 0) f.set_flag_at_index("extra_bad", -2) # sure intermediate state is where expected, back to original - np.testing.assert_array_equal(f.flags, original_flags, err_msg="Flags not back to original!") + np.testing.assert_array_equal( + f.flags, original_flags, err_msg="Flags not back to original!" + ) # now, going to use zero_if_unset=True which will destroy everything f.set_flag("extra_bad", [1, 1, 0, 0, 0, 0], zero_if_unset=True) - np.testing.assert_array_equal(f.get_flag("extra_bad"), np.array([1, 1, 0, 0, 0, 0])) + np.testing.assert_array_equal( + f.get_flag("extra_bad"), np.array([1, 1, 0, 0, 0, 0]) + ) np.testing.assert_array_equal(f.get_flag("good"), np.array([0, 0, 1, 1, 1, 1])) # actually, one more thing while we have this FlagWrap instance, check that find_flag works... @@ -89,7 +116,9 @@ def test_exclusive_flag_type(self): with self.assertRaises(ValueError): f.find_flag(these_dont_exist) for flag_meaning in f._flag_meanings: - np.testing.assert_array_equal(f.find_flag(these_dont_exist + [flag_meaning]), f.get_flag(flag_meaning)) + np.testing.assert_array_equal( + f.find_flag(these_dont_exist + [flag_meaning]), f.get_flag(flag_meaning) + ) # ok, we'll call it good there. def test_maskedarray_initial_flags(self): @@ -103,22 +132,28 @@ def test_maskedarray_initial_flags(self): f = FlagWrap( original_flags.copy(), ["good", "medium", "bad", "extra_bad"], - np.array([0, 1, 2, 3]) + np.array([0, 1, 2, 3]), ) # make sure that none of the flags are indicated if flags are completely masked for flag_meaning in f._flag_meanings: np.testing.assert_array_equal(f.get_flag(flag_meaning), np.zeros(5)) - f.set_flag(flag_meaning, np.zeros_like(original_flags), zero_if_unset=False) # identity test + f.set_flag( + flag_meaning, np.zeros_like(original_flags), zero_if_unset=False + ) # identity test np.testing.assert_array_equal(f.get_flag(flag_meaning), np.zeros(5)) f.set_flag("medium", [0, 1, 0, 1, 0], zero_if_unset=False) # first, check that underlying masked array looks correct. medium_value = f.get_value_for_meaning("medium") self.assertTrue(np.ma.is_masked(f.flags)) - np.testing.assert_array_equal(f.flags.mask, [1, 0, 1, 0, 1]) # make sure it's still masked - np.testing.assert_array_equal(np.ma.filled(f.flags, fill_value=9), - np.array([9, medium_value, 9, medium_value, 9])) + np.testing.assert_array_equal( + f.flags.mask, [1, 0, 1, 0, 1] + ) # make sure it's still masked + np.testing.assert_array_equal( + np.ma.filled(f.flags, fill_value=9), + np.array([9, medium_value, 9, medium_value, 9]), + ) # make sure that get_flag doesn't return masked array np.testing.assert_array_equal(f.get_flag("medium"), np.array([0, 1, 0, 1, 0])) @@ -136,14 +171,16 @@ def test_inclusive_flag(self): # as opposed to an exclusive flag where every flag_meaning is exclusive, in other words, there can only # every be one flag meaning, an inclusive flag can have multiple flag_meanings set at once. - original_flags = np.array([ - 0 | 6 | 8, # good, middle, red - 0, # good - 1, # degraded - 0 | 16, # good, blue - 1 | 4, # degraded, right - 0 | 4 | 8 | 16 # good, right, red, blue - ]) + original_flags = np.array( + [ + 0 | 6 | 8, # good, middle, red + 0, # good + 1, # degraded + 0 | 16, # good, blue + 1 | 4, # degraded, right + 0 | 4 | 8 | 16, # good, right, red, blue + ] + ) # note the way these are defined: # the lsb (least significant bit) is isolated using mask == 1, and if it's 0 -> good, 1 -> degraded @@ -159,12 +196,16 @@ def test_inclusive_flag(self): original_flags.copy(), "good degraded middle left right red blue", [0, 1, 6, 2, 4, 8, 16], - [1, 1, 6, 6, 6, 8, 16] + [1, 1, 6, 6, 6, 8, 16], ) np.testing.assert_array_equal(f.get_flag("good"), np.array([1, 1, 0, 1, 0, 1])) - np.testing.assert_array_equal(f.get_flag("degraded"), np.array([0, 0, 1, 0, 1, 0])) - np.testing.assert_array_equal(f.get_flag("middle"), np.array([1, 0, 0, 0, 0, 0])) + np.testing.assert_array_equal( + f.get_flag("degraded"), np.array([0, 0, 1, 0, 1, 0]) + ) + np.testing.assert_array_equal( + f.get_flag("middle"), np.array([1, 0, 0, 0, 0, 0]) + ) np.testing.assert_array_equal(f.get_flag("left"), np.zeros_like(original_flags)) np.testing.assert_array_equal(f.get_flag("right"), np.array([0, 0, 0, 0, 1, 1])) np.testing.assert_array_equal(f.get_flag("red"), np.array([1, 0, 0, 0, 0, 1])) @@ -179,14 +220,18 @@ def test_inclusive_flag(self): f.set_flag_at_index("degraded", 0) self.assertFalse(f.get_flag_at_index("good", 0)) self.assertTrue(f.get_flag_at_index("degraded", 0)) - np.testing.assert_array_equal(f.flags, original_flags | np.array([1, 0, 0, 0, 0, 0])) + np.testing.assert_array_equal( + f.flags, original_flags | np.array([1, 0, 0, 0, 0, 0]) + ) # set it back to good and make sure we're back to original flags f.set_flag_at_index("good", 0) np.testing.assert_array_equal(f.flags, original_flags) # test some vector set_flag operations... # !!!! NOTE: EXCELLENT SECTION TO STUDY FOR UNDERSTANDING zero_if_unset !!!!! - np.testing.assert_array_equal(f.get_flag("blue"), np.array([0, 0, 0, 1, 0, 1])) # <- covered by original_flags + np.testing.assert_array_equal( + f.get_flag("blue"), np.array([0, 0, 0, 1, 0, 1]) + ) # <- covered by original_flags # assertion, but including for clarity so reader can see what "blue" flags start out as... f.set_flag("blue", [0, 1, 0, 0, 1, 0], zero_if_unset=False) # since zero_if_unset is False and elements 4 and 6 are already 1, we expect them to still be 1... @@ -196,27 +241,31 @@ def test_inclusive_flag(self): np.testing.assert_array_equal(f.get_flag("blue"), np.array([0, 1, 0, 0, 1, 0])) # set "blue" back to original, and make sure everything else also like original f.set_flag("blue", [0, 0, 0, 1, 0, 1], zero_if_unset=True) - np.testing.assert_array_equal(f.flags, original_flags) # and now we're back to the original "blue" flags, + np.testing.assert_array_equal( + f.flags, original_flags + ) # and now we're back to the original "blue" flags, # and everything else stayed the same. # ok, since "left", "right" and "middle" are defined to be exclusive, setting all to one should # eliminate all others, but first, save those bits so it's easy to get back to original_flags without # just reassigning. - exclusive_set_original_setting = f.flags & (2 | 4) # -- this is the mask that defines this exclusive set + exclusive_set_original_setting = f.flags & ( + 2 | 4 + ) # -- this is the mask that defines this exclusive set exclusive_set = {"left", "right", "middle"} for flag_meaning in exclusive_set: # note: zero_if_unset doesn't matter here since they're all getting set - f.set_flag(flag_meaning, np.ones_like(original_flags), zero_if_unset=False) # set them all... + f.set_flag( + flag_meaning, np.ones_like(original_flags), zero_if_unset=False + ) # set them all... # test that none of the others appear for should_not_be_set in exclusive_set - {flag_meaning}: - np.testing.assert_array_equal(f.get_flag(should_not_be_set), np.zeros_like(original_flags)) + np.testing.assert_array_equal( + f.get_flag(should_not_be_set), np.zeros_like(original_flags) + ) # this following section isn't so much testing behavior of FlagWrap directly, as it is making sure # our previous operations were contained to the expected area of the flag bits. # completely zero the target area of the exlusive set (ie, 3d and 4th bits), by slightly unconventional means f.set_flag("middle", np.zeros_like(original_flags), zero_if_unset=True) f.flags |= exclusive_set_original_setting # set those back to original np.testing.assert_array_equal(f.flags, original_flags) - - - -