no max chunks

NOAA-PMEL · Jul 10, 2024 · ac833a1 · ac833a1
1 parent 4f6dd65
commit ac833a1
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/mk_kechunk_goa_iron.ipynb b/mk_kechunk_goa_iron.ipynb
@@ -85,7 +85,7 @@
     "fs2 = fsspec.filesystem(\"file\")\n",
     "so = dict(default_fill_cache=False, default_cache_type='first')\n",
     "for u in flist:\n",
-    "    dachunks = NetCDF3ToZarr(u, max_chunk_size=10000)\n",
+    "    dachunks = NetCDF3ToZarr(u)\n",
     "    parts = u.split('/') # seperate file path to create a unique name for each json \n",
     "    fstem = Path(u).stem \n",
     "    outf = f'{dir}/{fstem}.json'\n",
@@ -127,8 +127,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 12.1 ms, sys: 2.88 ms, total: 15 ms\n",
-      "Wall time: 13.8 ms\n"
+      "CPU times: user 7.6 ms, sys: 2.96 ms, total: 10.6 ms\n",
+      "Wall time: 9.05 ms\n"
      ]
     }
    ],

diff --git a/read_xpublish.ipynb b/read_xpublish.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 5,
    "id": "d7572bb2-26ab-4f66-af85-586f66c137a6",
    "metadata": {},
    "outputs": [],
@@ -14,7 +14,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 6,
    "id": "019ec89c-7505-40dc-a143-56316d1435c6",
    "metadata": {},
    "outputs": [
@@ -25,7 +25,7 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mClientResponseError\u001b[0m                       Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[3], line 5\u001b[0m\n\u001b[1;32m      2\u001b[0m http_map \u001b[38;5;241m=\u001b[39m fs\u001b[38;5;241m.\u001b[39mget_mapper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://sour.pmel.noaa.gov:8970/datasets/ne_pacific_10km_data/zarr/\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# open as a zarr group\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m zg \u001b[38;5;241m=\u001b[39m \u001b[43mzarr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_consolidated\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp_map\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# or open as another Xarray Dataset\u001b[39;00m\n\u001b[1;32m      8\u001b[0m ds \u001b[38;5;241m=\u001b[39m xr\u001b[38;5;241m.\u001b[39mopen_zarr(http_map, consolidated\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
+      "Cell \u001b[0;32mIn[6], line 5\u001b[0m\n\u001b[1;32m      2\u001b[0m http_map \u001b[38;5;241m=\u001b[39m fs\u001b[38;5;241m.\u001b[39mget_mapper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://sour.pmel.noaa.gov:8970/datasets/ne_pacific_10km_data/zarr/\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# open as a zarr group\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m zg \u001b[38;5;241m=\u001b[39m \u001b[43mzarr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_consolidated\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp_map\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# or open as another Xarray Dataset\u001b[39;00m\n\u001b[1;32m      8\u001b[0m ds \u001b[38;5;241m=\u001b[39m xr\u001b[38;5;241m.\u001b[39mopen_zarr(http_map, consolidated\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
       "File \u001b[0;32m~/miniforge-pypy3/envs/xp/lib/python3.12/site-packages/zarr/convenience.py:1360\u001b[0m, in \u001b[0;36mopen_consolidated\u001b[0;34m(store, metadata_key, mode, **kwargs)\u001b[0m\n\u001b[1;32m   1357\u001b[0m         metadata_key \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmeta/root/consolidated/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m metadata_key\n\u001b[1;32m   1359\u001b[0m \u001b[38;5;66;03m# setup metadata store\u001b[39;00m\n\u001b[0;32m-> 1360\u001b[0m meta_store \u001b[38;5;241m=\u001b[39m \u001b[43mConsolidatedStoreClass\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1362\u001b[0m \u001b[38;5;66;03m# pass through\u001b[39;00m\n\u001b[1;32m   1363\u001b[0m chunk_store \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchunk_store\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;129;01mor\u001b[39;00m store\n",
       "File \u001b[0;32m~/miniforge-pypy3/envs/xp/lib/python3.12/site-packages/zarr/storage.py:3046\u001b[0m, in \u001b[0;36mConsolidatedMetadataStore.__init__\u001b[0;34m(self, store, metadata_key)\u001b[0m\n\u001b[1;32m   3043\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore \u001b[38;5;241m=\u001b[39m Store\u001b[38;5;241m.\u001b[39m_ensure_store(store)\n\u001b[1;32m   3045\u001b[0m \u001b[38;5;66;03m# retrieve consolidated metadata\u001b[39;00m\n\u001b[0;32m-> 3046\u001b[0m meta \u001b[38;5;241m=\u001b[39m json_loads(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m[\u001b[49m\u001b[43mmetadata_key\u001b[49m\u001b[43m]\u001b[49m)\n\u001b[1;32m   3048\u001b[0m \u001b[38;5;66;03m# check format of consolidated metadata\u001b[39;00m\n\u001b[1;32m   3049\u001b[0m consolidated_format \u001b[38;5;241m=\u001b[39m meta\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_consolidated_format\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n",
       "File \u001b[0;32m~/miniforge-pypy3/envs/xp/lib/python3.12/site-packages/zarr/storage.py:1446\u001b[0m, in \u001b[0;36mFSStore.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1444\u001b[0m key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_normalize_key(key)\n\u001b[1;32m   1445\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1446\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m   1447\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m   1448\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n",