diff --git a/eosdis_store/dmrpp.py b/eosdis_store/dmrpp.py index 491bc2c..c2bb298 100644 --- a/eosdis_store/dmrpp.py +++ b/eosdis_store/dmrpp.py @@ -200,6 +200,10 @@ def array_to_zarr(node, dims, prefix=''): compression = child.attrib.get('compressionType') if compression == 'deflate': zarray['compressor'] = { "id": "zlib", "level": UNKNOWN_COMPRESSION_LEVEL } + elif compression == 'deflate shuffle': + zarray['compressor'] = {"id": "zlib", "level": UNKNOWN_COMPRESSION_LEVEL} + size = int(dtype[2:]) + zarray['filters'] = [{"id": "shuffle", "elementsize": size}] elif compression is None: zarray['compressor'] = None else: @@ -210,8 +214,15 @@ def array_to_zarr(node, dims, prefix=''): # NOTE - this is null in test file zarray['fill_value'] = zattrs.get('_FillValue') - #if node.attrib['name'] == 'time': - # set_trace() + # HARMONY-896: Automatic scale factor and offset filter. Not yet working with all data types + # if zattrs.get('scale_factor') or zattrs.get('add_offset'): + # zarray['filters'].append({ + # 'id': 'fixedscaleoffset', + # 'offset': zattrs.get('add_offset', 0.0), + # 'scale': zattrs.get('scale_factor', 1.0), + # 'dtype': '" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from matplotlib import pyplot as plt\n", + "\n", + "plt.rcParams[\"figure.figsize\"] = [16, 8]\n", + "plt.imshow(analysed_sst[::-1, :])\n", + "None" + ] + }, + { + "cell_type": "markdown", + "id": "1f386a03-9e9c-4457-ad2c-60f7b9012b48", + "metadata": {}, + "source": [ + "In a dozen lines of code and a few seconds, we have managed to fetch and visualize the 3.2 megabyte we needed from a 732 megabyte file using the original archive URL and no processing services" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/fixtures/20210715090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.dmrpp b/tests/fixtures/20210715090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.dmrpp new file mode 100644 index 0000000..cd6ae28 --- /dev/null +++ b/tests/fixtures/20210715090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.dmrpp @@ -0,0 +1,1979 @@ + + + + + + + + + + + sea/land field composite mask + + + -128 + + + 1 + + + 31 + + + 1 + 2 + 4 + 8 + 16 + + + open_sea land open_lake open_sea_with_ice_in_the_grid open_lake_with_ice_in_the_grid + + + mask can be used to further filter the data. + + + lon lat + + + GMT \"grdlandmask\", ice flag from sea_ice_fraction data + + + mask + + + /mask + + + 1 1447 2895 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + analysed sea surface temperature + + + sea_surface_foundation_temperature + + + kelvin + + + -32768 + + + 298.14999999999998 + + + 0.0010000000000000000 + + + -32767 + + + 32767 + + + Interim near-real-time (nrt) version using Multi-Resolution Variational Analysis (MRVA) method for interpolation; to be replaced by Final version + + + lon lat + + + MODIS_T-JPL, MODIS_A-JPL, AMSR2-REMSS, AVHRRMTA_G-NAVO, AVHRRMTB_G-NAVO, iQUAM-NOAA/NESDIS, Ice_Conc-OSISAF + + + analysed_sst + + + /analysed_sst + + + 1 1023 2047 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + sea ice area fraction + + + sea_ice_area_fraction + + + -128 + + + 0.0000000000000000 + + + 0.010000000000000000 + + + 0 + + + 100 + + + EUMETSAT OSI-SAF, copyright EUMETSAT + + + ice fraction is a dimensionless quantity between 0 and 1; it has been interpolated by a nearest neighbor approach. + + + lon lat + + + sea_ice_fraction + + + /sea_ice_fraction + + + 1 1447 2895 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + time to most recent 1km data + + + hours + + + -128 + + + -127 + + + 127 + + + MODIS and VIIRS pixels ingested by MUR + + + The grid value is hours between the analysis time and the most recent MODIS or VIIRS 1km L2P datum within 0.01 degrees from the grid point. \"Fill value\" indicates absence of such 1km data at the grid point. + + + lon lat + + + dt_1km_data + + + /dt_1km_data + + + 1 1447 2895 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + estimated error standard deviation of analysed_sst + + + kelvin + + + -32768 + + + 0.0000000000000000 + + + 0.010000000000000000 + + + 0 + + + 32767 + + + uncertainty in \"analysed_sst\" + + + lon lat + + + analysis_error + + + /analysis_error + + + 1 1023 2047 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SST anomaly from a seasonal SST climatology based on the MUR data over 2003-2014 period + + + kelvin + + + -32768 + + + 0.0000000000000000 + + + 0.0010000000000000000 + + + -32767 + + + 32767 + + + anomaly reference to the day-of-year average between 2003 and 2014 + + + lon lat + + + sst_anomaly + + + /sst_anomaly + + + 1 1023 2047 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + latitude + + + latitude + + + Y + + + degrees_north + + + -90.00000000 + + + 90.00000000 + + + geolocations inherited from the input data without correction + + + lat + + + /lat + + + 17999 + + + + + + + longitude + + + longitude + + + X + + + degrees_east + + + -180.0000000 + + + 180.0000000 + + + geolocations inherited from the input data without correction + + + lon + + + /lon + + + 36000 + + + + + + + reference time of sst field + + + time + + + T + + + seconds since 1981-01-01 00:00:00 UTC + + + Nominal time of analyzed fields + + + time + + + /time + + + + + + + + CF-1.7 + + + Daily MUR SST, Interim near-real-time (nrt) product + + + A merged, multi-sensor L4 Foundation SST analysis product from JPL. + + + http://podaac.jpl.nasa.gov/Multi-scale_Ultra-high_Resolution_MUR-SST + + + Jet Propulsion Laboratory + + + near real time (nrt) version created at nominal 1-day latency. + + + Interim-MUR(nrt) will be replaced by MUR-Final in about 3 days; MUR = \"Multi-scale Ultra-high Resolution\" + + + These data are available free of charge under data policy of JPL PO.DAAC. + + + MUR-JPL-L4-GLOB-v04.1 + + + org.ghrsst + + + 04.1nrt + + + 27665bc0-d5fc-11e1-9b23-0800200c9a66 + + + 2.0 + + + 4.1 + + + 20210716T092053Z + + + 20210715T090000Z + + + 20210715T090000Z + + + 20210714T210000Z + + + 20210715T210000Z + + + 3 + + + MODIS_T-JPL, MODIS_A-JPL, AMSR2-REMSS, AVHRRMTA_G-NAVO, AVHRRMTB_G-NAVO, iQUAM-NOAA/NESDIS, Ice_Conc-OSISAF + + + Terra, Aqua, GCOM-W, MetOp-A, MetOp-B, Buoys/Ships + + + MODIS, AMSR2, AVHRR, in-situ + + + Unidata Observation Dataset v1.0 + + + http://podaac.jpl.nasa.gov/ws/metadata/dataset/?format=iso&shortName=MUR-JPL-L4-GLOB-v04.1 + + + Oceans > Ocean Temperature > Sea Surface Temperature + + + NASA Global Change Master Directory (GCMD) Science Keywords + + + NetCDF Climate and Forecast (CF) Metadata Convention + + + -90.00000000 + + + 90.00000000 + + + -180.0000000 + + + 180.0000000 + + + 0.01 degrees + + + degrees north + + + 0.009999999776 + + + degrees east + + + 0.009999999776 + + + Please acknowledge the use of these data with the following statement: These data were provided by JPL under support by NASA MEaSUREs program. + + + JPL MUR SST project + + + ghrsst@podaac.jpl.nasa.gov + + + http://mur.jpl.nasa.gov + + + NASA Making Earth Science Data Records for Use in Research Environments (MEaSUREs) Program + + + GHRSST Project Office + + + http://www.ghrsst.org + + + ghrsst-po@nceo.ac.uk + + + L4 + + + grid + + + diff --git a/tests/test_dmrpp.py b/tests/test_dmrpp.py index 00e63dd..bb945ca 100644 --- a/tests/test_dmrpp.py +++ b/tests/test_dmrpp.py @@ -17,7 +17,7 @@ class Test(unittest.TestCase): 'https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/demo/zarr-store/f16_ssmis_20051022v7.nc.dmrpp', 'https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/demo/zarr-store/3B-HHR.MS.MRG.3IMERG.20051022-S000000-E002959.0000.V06B.HDF5.dmrpp', # MODIS data - 'https://archive.podaac.uat.earthdata.nasa.gov/podaac-uat-cumulus-protected/MODIS_A-JPL-L2P-v2019.0/20200911000001-JPL-L2P_GHRSST-SSTskin-MODIS_A-N-v02.0-fv01.0.nc.dmrpp' + 'https://archive.podaac.uat.earthdata.nasa.gov/podaac-uat-cumulus-protected/MODIS_A-JPL-L2P-v2019.0/20200911000001-JPL-L2P_GHRSST-SSTskin-MODIS_A-N-v02.0-fv01.0.nc.dmrpp', # 'https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/demo/zarr-store/3B-HHR.MS.MRG.3IMERG.20051022-S233000-E235959.1410.V06B.HDF5.dmrpp', ] @@ -114,4 +114,23 @@ def test_to_zarr_more_examples(self): fixture = json.loads(f.read()) json1 = json.dumps(fixture, sort_keys=True) json2 = json.dumps(zarr, sort_keys=True) - assert(json1 == json2) \ No newline at end of file + assert(json1 == json2) + + def test_deflate_shuffle(self): + filename = '20210715090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.dmrpp' + with open(os.path.join(testpath, 'fixtures', filename)) as f: + dmrpp = f.read() + tree = ElementTree.fromstring(dmrpp) + zarr = dmr.to_zarr(tree) + attributes = zarr['analysed_sst/.zarray'] + expected = { + 'zarr_format': 2, + 'filters': [{'id': 'shuffle', 'elementsize': 2}], + 'order': 'C', + 'dtype': '