From ac833a11e76e90d9b5214e7e0874d55819f274da Mon Sep 17 00:00:00 2001 From: Roland Schweitzer Date: Wed, 10 Jul 2024 09:03:55 -0700 Subject: [PATCH] no max chunks --- mk_kechunk_goa_iron.ipynb | 6 +++--- read_xpublish.ipynb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mk_kechunk_goa_iron.ipynb b/mk_kechunk_goa_iron.ipynb index b653589..394df68 100644 --- a/mk_kechunk_goa_iron.ipynb +++ b/mk_kechunk_goa_iron.ipynb @@ -85,7 +85,7 @@ "fs2 = fsspec.filesystem(\"file\")\n", "so = dict(default_fill_cache=False, default_cache_type='first')\n", "for u in flist:\n", - " dachunks = NetCDF3ToZarr(u, max_chunk_size=10000)\n", + " dachunks = NetCDF3ToZarr(u)\n", " parts = u.split('/') # seperate file path to create a unique name for each json \n", " fstem = Path(u).stem \n", " outf = f'{dir}/{fstem}.json'\n", @@ -127,8 +127,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 12.1 ms, sys: 2.88 ms, total: 15 ms\n", - "Wall time: 13.8 ms\n" + "CPU times: user 7.6 ms, sys: 2.96 ms, total: 10.6 ms\n", + "Wall time: 9.05 ms\n" ] } ], diff --git a/read_xpublish.ipynb b/read_xpublish.ipynb index e4bb288..ef091b5 100644 --- a/read_xpublish.ipynb +++ b/read_xpublish.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "d7572bb2-26ab-4f66-af85-586f66c137a6", "metadata": {}, "outputs": [], @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "id": "019ec89c-7505-40dc-a143-56316d1435c6", "metadata": {}, "outputs": [ @@ -25,7 +25,7 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mClientResponseError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[3], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m http_map \u001b[38;5;241m=\u001b[39m fs\u001b[38;5;241m.\u001b[39mget_mapper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://sour.pmel.noaa.gov:8970/datasets/ne_pacific_10km_data/zarr/\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# open as a zarr group\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m zg \u001b[38;5;241m=\u001b[39m \u001b[43mzarr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_consolidated\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp_map\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# or open as another Xarray Dataset\u001b[39;00m\n\u001b[1;32m 8\u001b[0m ds \u001b[38;5;241m=\u001b[39m xr\u001b[38;5;241m.\u001b[39mopen_zarr(http_map, consolidated\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "Cell \u001b[0;32mIn[6], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m http_map \u001b[38;5;241m=\u001b[39m fs\u001b[38;5;241m.\u001b[39mget_mapper(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://sour.pmel.noaa.gov:8970/datasets/ne_pacific_10km_data/zarr/\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# open as a zarr group\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m zg \u001b[38;5;241m=\u001b[39m \u001b[43mzarr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_consolidated\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp_map\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# or open as another Xarray Dataset\u001b[39;00m\n\u001b[1;32m 8\u001b[0m ds \u001b[38;5;241m=\u001b[39m xr\u001b[38;5;241m.\u001b[39mopen_zarr(http_map, consolidated\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", "File \u001b[0;32m~/miniforge-pypy3/envs/xp/lib/python3.12/site-packages/zarr/convenience.py:1360\u001b[0m, in \u001b[0;36mopen_consolidated\u001b[0;34m(store, metadata_key, mode, **kwargs)\u001b[0m\n\u001b[1;32m 1357\u001b[0m metadata_key \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmeta/root/consolidated/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m metadata_key\n\u001b[1;32m 1359\u001b[0m \u001b[38;5;66;03m# setup metadata store\u001b[39;00m\n\u001b[0;32m-> 1360\u001b[0m meta_store \u001b[38;5;241m=\u001b[39m \u001b[43mConsolidatedStoreClass\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1362\u001b[0m \u001b[38;5;66;03m# pass through\u001b[39;00m\n\u001b[1;32m 1363\u001b[0m chunk_store \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchunk_store\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;129;01mor\u001b[39;00m store\n", "File \u001b[0;32m~/miniforge-pypy3/envs/xp/lib/python3.12/site-packages/zarr/storage.py:3046\u001b[0m, in \u001b[0;36mConsolidatedMetadataStore.__init__\u001b[0;34m(self, store, metadata_key)\u001b[0m\n\u001b[1;32m 3043\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstore \u001b[38;5;241m=\u001b[39m Store\u001b[38;5;241m.\u001b[39m_ensure_store(store)\n\u001b[1;32m 3045\u001b[0m \u001b[38;5;66;03m# retrieve consolidated metadata\u001b[39;00m\n\u001b[0;32m-> 3046\u001b[0m meta \u001b[38;5;241m=\u001b[39m json_loads(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m[\u001b[49m\u001b[43mmetadata_key\u001b[49m\u001b[43m]\u001b[49m)\n\u001b[1;32m 3048\u001b[0m \u001b[38;5;66;03m# check format of consolidated metadata\u001b[39;00m\n\u001b[1;32m 3049\u001b[0m consolidated_format \u001b[38;5;241m=\u001b[39m meta\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzarr_consolidated_format\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n", "File \u001b[0;32m~/miniforge-pypy3/envs/xp/lib/python3.12/site-packages/zarr/storage.py:1446\u001b[0m, in \u001b[0;36mFSStore.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 1444\u001b[0m key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_normalize_key(key)\n\u001b[1;32m 1445\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1446\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap\u001b[49m\u001b[43m[\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 1447\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 1448\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n",