diff --git a/docs/source/conf.py b/docs/source/conf.py index 5d4dbd5..9be13c2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -2,34 +2,34 @@ # -- Project information -project = 'MetGenC' -copyright = '2024, NSIDC' -author = 'National Snow and Ice Data Center' +project = "MetGenC" +copyright = "2024, NSIDC" +author = "National Snow and Ice Data Center" -release = '0.6' -version = '0.6.0' +release = "0.6" +version = "0.6.0" # -- General configuration extensions = [ - 'sphinx.ext.duration', - 'sphinx.ext.doctest', - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.intersphinx', + "sphinx.ext.duration", + "sphinx.ext.doctest", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", ] intersphinx_mapping = { - 'python': ('https://docs.python.org/3/', None), - 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), + "python": ("https://docs.python.org/3/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master/", None), } -intersphinx_disabled_domains = ['std'] +intersphinx_disabled_domains = ["std"] -templates_path = ['_templates'] +templates_path = ["_templates"] # -- Options for HTML output -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # -- Options for EPUB output -epub_show_urls = 'footnote' +epub_show_urls = "footnote" diff --git a/src/nsidc/metgen/cli.py b/src/nsidc/metgen/cli.py index fd3cc97..1ce562f 100644 --- a/src/nsidc/metgen/cli.py +++ b/src/nsidc/metgen/cli.py @@ -69,6 +69,7 @@ def validate(config_filename, content_type): metgen.init_logging(configuration) metgen.validate(configuration, content_type) + @cli.command() @click.option( "-f", @@ -81,6 +82,7 @@ def assess(): """Examine a sample data file for metadata completeness""" return True + @cli.command() @click.option( "-c", diff --git a/src/nsidc/metgen/config.py b/src/nsidc/metgen/config.py index ea68a1d..cbf7ccb 100644 --- a/src/nsidc/metgen/config.py +++ b/src/nsidc/metgen/config.py @@ -4,7 +4,7 @@ import os.path from pathlib import Path -from nsidc.metgen import aws, constants +from nsidc.metgen import aws, constants, netcdf_reader class ValidationError(Exception): @@ -32,7 +32,8 @@ class Config: dry_run: bool def __post_init__(self): - self.collection = None + # data_reader: Callable[[str], dict] + self.data_reader = netcdf_reader def show(self): # TODO: add section headings in the right spot diff --git a/src/nsidc/metgen/metgen.py b/src/nsidc/metgen/metgen.py index 614e14b..67d11a7 100644 --- a/src/nsidc/metgen/metgen.py +++ b/src/nsidc/metgen/metgen.py @@ -8,6 +8,7 @@ import sys import uuid from collections.abc import Callable +from functools import cache from importlib.resources import open_text from pathlib import Path from string import Template @@ -18,7 +19,7 @@ from returns.maybe import Maybe from rich.prompt import Confirm, Prompt -from nsidc.metgen import aws, config, constants, netcdf_reader +from nsidc.metgen import aws, config, constants # ------------------------------------------------------------------- CONSOLE_FORMAT = "%(message)s" @@ -186,7 +187,6 @@ class Collection: auth_id: str version: int - data_reader: Callable[[str], dict] @dataclasses.dataclass @@ -231,15 +231,7 @@ def process(configuration: config.Config) -> None: """ Process all Granules and record the results and summary. """ - # TODO: - # Do any prep actions, like mkdir, etc - # Get real collection information from CMR - # Determine data reader based on actual data file characteristics (e.g. suffix) - configuration.collection = Collection( - configuration.auth_id, - configuration.version, - netcdf_reader.extract_metadata - ) + # TODO: Do any prep actions, like mkdir, etc # Ordered list of operations to perform on each granule operations = [ @@ -343,12 +335,21 @@ def null_operation(configuration: config.Config, granule: Granule) -> Granule: return granule +@cache +def retrieve_collection(auth_id: str, version: int): + # ummc_from_cmr = talk_to_cmr(configuration.auth_id, configuration.version) + # pull out fields from UMM-C response and use to create collection object + # with more than just auth_id and version number. + return Collection(auth_id, version) + + def granule_collection(configuration: config.Config, granule: Granule) -> Granule: """ - Associate the Collection with the Granule. + Associate collection information with the Granule. """ return dataclasses.replace( - granule, collection=configuration.collection + granule, + collection=retrieve_collection(configuration.auth_id, configuration.version), ) @@ -399,7 +400,7 @@ def create_ummg(configuration: config.Config, granule: Granule) -> Granule: # } metadata_details = {} for data_file in granule.data_filenames: - metadata_details[data_file] = granule.collection.data_reader(data_file) + metadata_details[data_file] = configuration.data_reader(data_file) # Collapse information about (possibly) multiple files into a granule summary. summary = metadata_summary(metadata_details) diff --git a/src/nsidc/metgen/netcdf_reader.py b/src/nsidc/metgen/netcdf_reader.py index 013df5a..db0b6cb 100644 --- a/src/nsidc/metgen/netcdf_reader.py +++ b/src/nsidc/metgen/netcdf_reader.py @@ -69,11 +69,13 @@ def spatial_values(netcdf): for (lon, lat) in perimeter ] + def pixel_padding(netcdf): # Adding padding should give us values that match up to the # netcdf.attrs.geospatial_bounds return abs(float(netcdf.crs.GeoTransform.split()[1])) / 2 + def thinned_perimeter(xdata, ydata): """ Extract the thinned perimeter of a grid. diff --git a/tests/test_config.py b/tests/test_config.py index 6abe408..14f3716 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -32,7 +32,7 @@ def expected_keys(): "checksum_type", "number", "dry_run", - "collection" + "data_reader", ] ) diff --git a/tests/test_metgen.py b/tests/test_metgen.py index cb77a30..01e9af5 100644 --- a/tests/test_metgen.py +++ b/tests/test_metgen.py @@ -102,22 +102,14 @@ def test_returns_datetime_range(): def test_s3_object_path_has_no_leading_slash(): - granule = metgen.Granule( - "foo", - metgen.Collection("ABCD", 2, "my_reader"), - uuid="abcd-1234" - ) + granule = metgen.Granule("foo", metgen.Collection("ABCD", 2), uuid="abcd-1234") expected = "external/ABCD/2/abcd-1234/xyzzy.bin" assert metgen.s3_object_path(granule, "xyzzy.bin") == expected def test_s3_url_simple_case(): staging_bucket_name = "xyzzy-bucket" - granule = metgen.Granule( - "foo", - metgen.Collection("ABCD", 2, "my_reader"), - uuid="abcd-1234" - ) + granule = metgen.Granule("foo", metgen.Collection("ABCD", 2), uuid="abcd-1234") expected = "s3://xyzzy-bucket/external/ABCD/2/abcd-1234/xyzzy.bin" assert metgen.s3_url(staging_bucket_name, granule, "xyzzy.bin") == expected