From 5b8d83ea74f9e4155633045cadc8cedc5317fe69 Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Thu, 14 Dec 2023 14:00:59 -0700 Subject: [PATCH 01/10] initial --- .../index_creation_scripts/README.md | 17 +++++++++++++++-- .../create_indexes-0-replicas.n1ql | 14 -------------- .../create_indexes-2-replicas.n1ql | 14 -------------- .../get_index_definitions.sh | 19 +++++++++++++++++++ 4 files changed, 34 insertions(+), 30 deletions(-) delete mode 100755 mats_metadata_and_indexes/index_creation_scripts/create_indexes-0-replicas.n1ql delete mode 100755 mats_metadata_and_indexes/index_creation_scripts/create_indexes-2-replicas.n1ql create mode 100755 mats_metadata_and_indexes/index_creation_scripts/get_index_definitions.sh diff --git a/mats_metadata_and_indexes/index_creation_scripts/README.md b/mats_metadata_and_indexes/index_creation_scripts/README.md index b08e96b3..e81af0a3 100644 --- a/mats_metadata_and_indexes/index_creation_scripts/README.md +++ b/mats_metadata_and_indexes/index_creation_scripts/README.md @@ -1,11 +1,21 @@ -# How to build the indexes +# Indexes + +## Overview + +### Cluster roles + +#### ingest / development +The standalone cluster on adb-cb1 is used for ingest data creation and import. The ingest processes query this cluster and the import processes import to this cluster. +#### production +The production cluster is what the apps read. This cluster is populated by xdcr from the ingest cluster ## Index Creation Scripts +We are trying to keep an up to date N1QL script for the standalone cluster and a different one for the production cluster (they are not the same index set) that contain the index creation statements necessary to recreate a proper set of indexes for either of those environments, or for a new environment that has one of thse roles. There are three index creation scripts - mats_metadata_and_indexes/index_creation_scripts/create_indexes-0-replicas.n1ql - This is for the standalone server since it cannot support index replication (only one node) + This is for the standalone server since it cannot support index replication (it only has one node) Copy the contents of this N1QL script and either execute it in the query window of the UI of the standalone server or execute it with the command line interface cbq. After that you must also execute the N1QL in the build_indexes.n1ql script. @@ -18,3 +28,6 @@ There are three index creation scripts This is for creating the full text search index. You can execute it as a bash script from any machine that has connectivity to the couchbase server. This script takes one parameter - the full hostname of the couchbase server. It will prompt for the avid user password. + +## Index Backups +It is difficult to keep these scripts up to date because it is essentially a manual operation. For that reason we keep a set of backups. These backups are small files that are derived from the curl based admin interface for Couchbase. \ No newline at end of file diff --git a/mats_metadata_and_indexes/index_creation_scripts/create_indexes-0-replicas.n1ql b/mats_metadata_and_indexes/index_creation_scripts/create_indexes-0-replicas.n1ql deleted file mode 100755 index 6bca1640..00000000 --- a/mats_metadata_and_indexes/index_creation_scripts/create_indexes-0-replicas.n1ql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE INDEX `adv_fcstValidEpoch_docType_subset_version_type` ON `mdata`(`fcstValidEpoch`) WHERE ((((`type` = "DD") and (`docType` = "obs")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true}; -CREATE INDEX `adv_fcstLen_region_model_fcstValidEpoch_subset_version_type_docType` ON `mdata`(`fcstLen`,`region`,`model`,`fcstValidEpoch`) WHERE ((((`subset` = "METAR") and (`version` = "V01")) and (`type` = "DD")) and (`docType` = "CTC")) WITH { "defer_build":true}; -CREATE INDEX `adv_region_fcstValidEpoch_docType_subset_version_type` ON `mdata`(`region`,`fcstValidEpoch`) WHERE ((((`type` = "DD") and (`docType` = "CTC")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true}; -CREATE INDEX `adv_type_docType_subset_fcstValidEpoch_fcstLen_model_version` ON `mdata`(`model`,`fcstValidEpoch`,`fcstLen`) WHERE ((((`type` = "DD") and (`docType` = "model")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true}; -CREATE INDEX `adv_type_subset_docType_version` ON `mdata`(`type`,`subset`,`docType`,`version`) WITH { "defer_build":true}; -CREATE INDEX `adv_model_region_fcstValidEpoch_type_docType_subset_version` ON `mdata`(`model`,`region`,`fcstValidEpoch`) WHERE ((((`type` = "DD") and (`docType` = "CTC")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true}; -CREATE INDEX `idx_type_docType_version_common` ON `mdata`(`type`,`docType`,`version`) WHERE (`subset` = "COMMON") WITH { "defer_build":true}; -CREATE INDEX `idx_type_docType_version_test` ON `mdata`(`type`,`docType`,`version`) WHERE (`subset` = "TEST") WITH { "defer_build":true}; -CREATE INDEX `ix_subset_version_model_fcstLen_fcstValidEpoc` ON `mdata`(`subset`,`version`,`model`,`fcstLen`,`fcstValidEpoch`,ceil((3600 * floor(((`fcstValidEpoch` + (3600 / 2)) / 3600))))) WHERE ((`type` = "DD") and (`docType` = "model")) WITH { "defer_build":true}; -CREATE INDEX `adv_subDocType_type_docType_version` ON `mdata`(`subDocType`) WHERE (((`type` = "DD") and (`docType` = "CTC")) and (`version` = "V01")) WITH { "defer_build":true }; -CREATE INDEX `adv_subDocType_docType_model_region_subset_fcstValidEpoch_type_version` ON `mdata`(`subDocType`,`docType`,`model`,`region`,`subset`,`fcstValidEpoch`) WHERE `version` = 'V01' AND `type` = 'DD' WITH { "defer_build":true }; -CREATE INDEX adv_status_type_version_offset_minutes_run_priority_split_schedu1027334426 ON `mdata`(`status`,`offset_minutes`,`run_priority`,split(`schedule`, ' '),lower(`subType`),`input_data_path`) WHERE `type` = 'JOB' AND `version` = 'V01' WITH { "defer_build":true}; -CREATE INDEX adv_fileType_originType_subset_type_url_mtime ON `mdata`(`fileType`,`originType`,`url`,`mtime`) WHERE ((`subset` = 'METAR') and (`type` = 'DF')) WITH { "defer_build":true}; -CREATE INDEX adv_app_meta_id_type_docType_subset_version ON `mdata`(`app`,meta().`id`) WHERE `version` = 'V01' AND `type` = 'MD' AND `docType` = 'matsGui' AND `subset` = 'COMMON' WITH { "defer_build":true}; diff --git a/mats_metadata_and_indexes/index_creation_scripts/create_indexes-2-replicas.n1ql b/mats_metadata_and_indexes/index_creation_scripts/create_indexes-2-replicas.n1ql deleted file mode 100755 index 0de2bc99..00000000 --- a/mats_metadata_and_indexes/index_creation_scripts/create_indexes-2-replicas.n1ql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE INDEX `adv_fcstValidEpoch_docType_subset_version_type` ON `mdata`(`fcstValidEpoch`) WHERE ((((`type` = "DD") and (`docType` = "obs")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_fcstLen_region_model_fcstValidEpoch_subset_version_type_docType` ON `mdata`(`fcstLen`,`region`,`model`,`fcstValidEpoch`) WHERE ((((`subset` = "METAR") and (`version` = "V01")) and (`type` = "DD")) and (`docType` = "CTC")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_region_fcstValidEpoch_docType_subset_version_type` ON `mdata`(`region`,`fcstValidEpoch`) WHERE ((((`type` = "DD") and (`docType` = "CTC")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_type_docType_subset_fcstValidEpoch_fcstLen_model_version` ON `mdata`(`model`,`fcstValidEpoch`,`fcstLen`) WHERE ((((`type` = "DD") and (`docType` = "model")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_type_subset_docType_version` ON `mdata`(`type`,`subset`,`docType`,`version`) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_model_region_fcstValidEpoch_type_docType_subset_version` ON `mdata`(`model`,`region`,`fcstValidEpoch`) WHERE ((((`type` = "DD") and (`docType` = "CTC")) and (`subset` = "METAR")) and (`version` = "V01")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `idx_type_docType_version_common` ON `mdata`(`type`,`docType`,`version`) WHERE (`subset` = "COMMON") WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `idx_type_docType_version_test` ON `mdata`(`type`,`docType`,`version`) WHERE (`subset` = "TEST") WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `ix_subset_version_model_fcstLen_fcstValidEpoc` ON `mdata`(`subset`,`version`,`model`,`fcstLen`,`fcstValidEpoch`,ceil((3600 * floor(((`fcstValidEpoch` + (3600 / 2)) / 3600))))) WHERE ((`type` = "DD") and (`docType` = "model")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_subDocType_type_docType_version` ON `mdata`(`subDocType`) WHERE (((`type` = "DD") and (`docType` = "CTC")) and (`version` = "V01")) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX `adv_subDocType_docType_model_region_subset_fcstValidEpoch_type_version` ON `mdata`(`subDocType`,`docType`,`model`,`region`,`subset`,`fcstValidEpoch`) WHERE `version` = 'V01' AND `type` = 'DD' WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX adv_status_type_version_offset_minutes_run_priority_split_schedu1027334426 ON `mdata`(`status`,`offset_minutes`,`run_priority`,split(`schedule`, ' '),lower(`subType`),`input_data_path`) WHERE `type` = 'JOB' AND `version` = 'V01' WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX adv_fileType_originType_subset_type_url_mtime ON `mdata`(`fileType`,`originType`,`url`,`mtime`) WHERE ((`subset` = 'METAR') and (`type` = 'DF')) WITH { "defer_build":true, "num_replica":2 }; -CREATE INDEX adv_app_meta_id_type_docType_subset_version ON `mdata`(`app`,meta().`id`) WHERE `version` = 'V01' AND `type` = 'MD' AND `docType` = 'matsGui' AND `subset` = 'COMMON' WITH { "defer_build":true, "num_replica":2 }; diff --git a/mats_metadata_and_indexes/index_creation_scripts/get_index_definitions.sh b/mats_metadata_and_indexes/index_creation_scripts/get_index_definitions.sh new file mode 100755 index 00000000..0ff98723 --- /dev/null +++ b/mats_metadata_and_indexes/index_creation_scripts/get_index_definitions.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "Usage $0 credentials-file" + exit 1 +fi +if [[ ! -f "$1" ]]; then + echo "$1 is not a file - exiting" + exit 1 +fi +credentials_file=$1 + +host=`grep cb_host ${credentials_file} | awk '{print $2}'` +user=`grep cb_user ${credentials_file} | awk '{print $2}'` +pwd=`grep cb_password ${credentials_file} | awk '{print $2}'` + +cred="${user}:${pwd}" + +curl --user ${cred} "http://${host}:8091/indexStatus" | jq -r '.indexes | .[] .definition' From 76f73b6b8f61a543d5031e15c1923008553d7bb5 Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Fri, 22 Dec 2023 11:39:51 -0700 Subject: [PATCH 02/10] add backfill_obs_with_RH and modify netcdf builder to add RH, windU, and windV and formatting by RUFF --- poetry.lock | 553 +++++++++++++++++- pyproject.toml | 1 + scripts/admin/cluster/rbac_export_import.py | 10 +- scripts/admin/index/index-analysis.py | 1 - .../stats/StoreAndGetStats/getStats/bucket.py | 3 +- .../StoreAndGetStats/getStats/cbcluster.py | 6 +- .../StoreAndGetStats/getStats/fileOutput.py | 3 +- .../stats/StoreAndGetStats/getStats/node.py | 2 +- .../StoreAndGetStats/getStats/nodeOutput.py | 2 +- .../StoreAndGetStats/getStats/timeSlice.py | 2 +- .../StoreAndGetStats/storeStats/bucket.py | 3 +- .../StoreAndGetStats/storeStats/cbcluster.py | 7 +- scripts/backup/automate_backup.py | 20 +- scripts/backup/config.py | 2 +- .../troubleshooting/sdk/orphans_threshold.py | 18 +- src/vxingest/netcdf_to_cb/netcdf_builder.py | 78 ++- src/vxingest/utilities/README.MD | 3 + .../utilities/backfill_obs_with_rh.py | 80 +++ .../grib2_to_cb/test_int_metar_model_grib.py | 2 + .../netcdf_to_cb/test_int_metar_obs_netcdf.py | 50 +- .../test_int_metar_partial_sums.py | 4 +- .../utilities/test_backfill_obs_with_rh.py | 82 +++ 22 files changed, 866 insertions(+), 66 deletions(-) create mode 100644 src/vxingest/utilities/README.MD create mode 100644 src/vxingest/utilities/backfill_obs_with_rh.py create mode 100644 tests/vxingest/utilities/test_backfill_obs_with_rh.py diff --git a/poetry.lock b/poetry.lock index 50b5c098..ea621794 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,34 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +[[package]] +name = "appnope" +version = "0.1.3" +description = "Disable App Nap on macOS >= 10.9" +optional = false +python-versions = "*" +files = [ + {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, + {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, +] + +[[package]] +name = "asttokens" +version = "2.4.1" +description = "Annotate AST trees with source code positions" +optional = false +python-versions = "*" +files = [ + {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, + {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, +] + +[package.dependencies] +six = ">=1.12.0" + +[package.extras] +astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] +test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] + [[package]] name = "attrs" version = "23.1.0" @@ -274,6 +303,23 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "comm" +version = "0.2.0" +description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +optional = false +python-versions = ">=3.8" +files = [ + {file = "comm-0.2.0-py3-none-any.whl", hash = "sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001"}, + {file = "comm-0.2.0.tar.gz", hash = "sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be"}, +] + +[package.dependencies] +traitlets = ">=4" + +[package.extras] +test = ["pytest"] + [[package]] name = "contourpy" version = "1.2.0" @@ -386,6 +432,44 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "debugpy" +version = "1.8.0" +description = "An implementation of the Debug Adapter Protocol for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "debugpy-1.8.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:7fb95ca78f7ac43393cd0e0f2b6deda438ec7c5e47fa5d38553340897d2fbdfb"}, + {file = "debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada"}, + {file = "debugpy-1.8.0-cp310-cp310-win32.whl", hash = "sha256:a8b7a2fd27cd9f3553ac112f356ad4ca93338feadd8910277aff71ab24d8775f"}, + {file = "debugpy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:5d9de202f5d42e62f932507ee8b21e30d49aae7e46d5b1dd5c908db1d7068637"}, + {file = "debugpy-1.8.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e"}, + {file = "debugpy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60009b132c91951354f54363f8ebdf7457aeb150e84abba5ae251b8e9f29a8a6"}, + {file = "debugpy-1.8.0-cp311-cp311-win32.whl", hash = "sha256:8cd0197141eb9e8a4566794550cfdcdb8b3db0818bdf8c49a8e8f8053e56e38b"}, + {file = "debugpy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a64093656c4c64dc6a438e11d59369875d200bd5abb8f9b26c1f5f723622e153"}, + {file = "debugpy-1.8.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:b05a6b503ed520ad58c8dc682749113d2fd9f41ffd45daec16e558ca884008cd"}, + {file = "debugpy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c6fb41c98ec51dd010d7ed650accfd07a87fe5e93eca9d5f584d0578f28f35f"}, + {file = "debugpy-1.8.0-cp38-cp38-win32.whl", hash = "sha256:46ab6780159eeabb43c1495d9c84cf85d62975e48b6ec21ee10c95767c0590aa"}, + {file = "debugpy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:bdc5ef99d14b9c0fcb35351b4fbfc06ac0ee576aeab6b2511702e5a648a2e595"}, + {file = "debugpy-1.8.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:61eab4a4c8b6125d41a34bad4e5fe3d2cc145caecd63c3fe953be4cc53e65bf8"}, + {file = "debugpy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:125b9a637e013f9faac0a3d6a82bd17c8b5d2c875fb6b7e2772c5aba6d082332"}, + {file = "debugpy-1.8.0-cp39-cp39-win32.whl", hash = "sha256:57161629133113c97b387382045649a2b985a348f0c9366e22217c87b68b73c6"}, + {file = "debugpy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:e3412f9faa9ade82aa64a50b602544efcba848c91384e9f93497a458767e6926"}, + {file = "debugpy-1.8.0-py2.py3-none-any.whl", hash = "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4"}, + {file = "debugpy-1.8.0.zip", hash = "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0"}, +] + +[[package]] +name = "decorator" +version = "5.1.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.5" +files = [ + {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, + {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, +] + [[package]] name = "eccodes" version = "1.6.1" @@ -402,6 +486,20 @@ cffi = "*" findlibs = "*" numpy = "*" +[[package]] +name = "executing" +version = "2.0.1" +description = "Get the currently executing AST node of a frame, and other information" +optional = false +python-versions = ">=3.5" +files = [ + {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, + {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, +] + +[package.extras] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] + [[package]] name = "findlibs" version = "0.0.5" @@ -499,6 +597,135 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "ipykernel" +version = "6.27.1" +description = "IPython Kernel for Jupyter" +optional = false +python-versions = ">=3.8" +files = [ + {file = "ipykernel-6.27.1-py3-none-any.whl", hash = "sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686"}, + {file = "ipykernel-6.27.1.tar.gz", hash = "sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6"}, +] + +[package.dependencies] +appnope = {version = "*", markers = "platform_system == \"Darwin\""} +comm = ">=0.1.1" +debugpy = ">=1.6.5" +ipython = ">=7.23.1" +jupyter-client = ">=6.1.12" +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +matplotlib-inline = ">=0.1" +nest-asyncio = "*" +packaging = "*" +psutil = "*" +pyzmq = ">=20" +tornado = ">=6.1" +traitlets = ">=5.4.0" + +[package.extras] +cov = ["coverage[toml]", "curio", "matplotlib", "pytest-cov", "trio"] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "trio"] +pyqt5 = ["pyqt5"] +pyside6 = ["pyside6"] +test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "ipython" +version = "8.19.0" +description = "IPython: Productive Interactive Computing" +optional = false +python-versions = ">=3.10" +files = [ + {file = "ipython-8.19.0-py3-none-any.whl", hash = "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5"}, + {file = "ipython-8.19.0.tar.gz", hash = "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +decorator = "*" +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +prompt-toolkit = ">=3.0.41,<3.1.0" +pygments = ">=2.4.0" +stack-data = "*" +traitlets = ">=5" + +[package.extras] +all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.23)", "pandas", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +black = ["black"] +doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +kernel = ["ipykernel"] +nbconvert = ["nbconvert"] +nbformat = ["nbformat"] +notebook = ["ipywidgets", "notebook"] +parallel = ["ipyparallel"] +qtconsole = ["qtconsole"] +test = ["pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath"] +test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath", "trio"] + +[[package]] +name = "jedi" +version = "0.19.1" +description = "An autocompletion tool for Python that can be used for text editors." +optional = false +python-versions = ">=3.6" +files = [ + {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, + {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, +] + +[package.dependencies] +parso = ">=0.8.3,<0.9.0" + +[package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] +qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] +testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] + +[[package]] +name = "jupyter-client" +version = "8.6.0" +description = "Jupyter protocol implementation and client libraries" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_client-8.6.0-py3-none-any.whl", hash = "sha256:909c474dbe62582ae62b758bca86d6518c85234bdee2d908c778db6d72f39d99"}, + {file = "jupyter_client-8.6.0.tar.gz", hash = "sha256:0642244bb83b4764ae60d07e010e15f0e2d275ec4e918a8f7b80fbbef3ca60c7"}, +] + +[package.dependencies] +jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +python-dateutil = ">=2.8.2" +pyzmq = ">=23.0" +tornado = ">=6.2" +traitlets = ">=5.3" + +[package.extras] +docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] + +[[package]] +name = "jupyter-core" +version = "5.5.1" +description = "Jupyter core package. A base package on which Jupyter projects rely." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jupyter_core-5.5.1-py3-none-any.whl", hash = "sha256:220dfb00c45f0d780ce132bb7976b58263f81a3ada6e90a9b6823785a424f739"}, + {file = "jupyter_core-5.5.1.tar.gz", hash = "sha256:1553311a97ccd12936037f36b9ab4d6ae8ceea6ad2d5c90d94a909e752178e40"}, +] + +[package.dependencies] +platformdirs = ">=2.5" +pywin32 = {version = ">=300", markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} +traitlets = ">=5.3" + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] +test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] + [[package]] name = "kiwisolver" version = "1.4.5" @@ -660,6 +887,20 @@ pillow = ">=8" pyparsing = ">=2.3.1" python-dateutil = ">=2.7" +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +optional = false +python-versions = ">=3.5" +files = [ + {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, + {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, +] + +[package.dependencies] +traitlets = "*" + [[package]] name = "metpy" version = "1.5.1" @@ -687,6 +928,17 @@ doc = ["myst-parser", "netCDF4", "sphinx", "sphinx-gallery (>=0.4)"] examples = ["cartopy (>=0.17.0)", "geopandas (>=0.6.0)", "matplotlib (>=3.3.0)", "shapely (>=1.6.0)"] test = ["cartopy (>=0.17.0)", "netCDF4", "packaging (>=21.0)", "pytest (>=2.4)", "pytest-mpl", "shapely (>=1.6.0)"] +[[package]] +name = "nest-asyncio" +version = "1.5.8" +description = "Patch asyncio to allow nested event loops" +optional = false +python-versions = ">=3.5" +files = [ + {file = "nest_asyncio-1.5.8-py3-none-any.whl", hash = "sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d"}, + {file = "nest_asyncio-1.5.8.tar.gz", hash = "sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb"}, +] + [[package]] name = "netcdf4" version = "1.6.5" @@ -854,6 +1106,35 @@ sql-other = ["SQLAlchemy (>=1.4.36)"] test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] +[[package]] +name = "parso" +version = "0.8.3" +description = "A Python Parser" +optional = false +python-versions = ">=3.6" +files = [ + {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, + {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, +] + +[package.extras] +qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] +testing = ["docopt", "pytest (<6.0.0)"] + +[[package]] +name = "pexpect" +version = "4.9.0" +description = "Pexpect allows easy control of interactive console applications." +optional = false +python-versions = "*" +files = [ + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, +] + +[package.dependencies] +ptyprocess = ">=0.5" + [[package]] name = "pillow" version = "10.1.0" @@ -1010,6 +1291,73 @@ files = [ [package.extras] twisted = ["twisted"] +[[package]] +name = "prompt-toolkit" +version = "3.0.43" +description = "Library for building powerful interactive command lines in Python" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, +] + +[package.dependencies] +wcwidth = "*" + +[[package]] +name = "psutil" +version = "5.9.7" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"}, + {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"}, + {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"}, + {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"}, + {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"}, + {file = "psutil-5.9.7-cp27-none-win32.whl", hash = "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"}, + {file = "psutil-5.9.7-cp27-none-win_amd64.whl", hash = "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"}, + {file = "psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"}, + {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"}, + {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"}, + {file = "psutil-5.9.7-cp36-cp36m-win32.whl", hash = "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"}, + {file = "psutil-5.9.7-cp36-cp36m-win_amd64.whl", hash = "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"}, + {file = "psutil-5.9.7-cp37-abi3-win32.whl", hash = "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"}, + {file = "psutil-5.9.7-cp37-abi3-win_amd64.whl", hash = "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"}, + {file = "psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"}, + {file = "psutil-5.9.7.tar.gz", hash = "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +description = "Run a subprocess in a pseudo terminal" +optional = false +python-versions = "*" +files = [ + {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, + {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, +] + +[[package]] +name = "pure-eval" +version = "0.2.2" +description = "Safely evaluate AST nodes without side effects" +optional = false +python-versions = "*" +files = [ + {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, + {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, +] + +[package.extras] +tests = ["pytest"] + [[package]] name = "pycparser" version = "2.21" @@ -1021,6 +1369,21 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "pyparsing" version = "3.1.1" @@ -1119,6 +1482,29 @@ files = [ {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + [[package]] name = "pyyaml" version = "6.0.1" @@ -1131,6 +1517,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1138,8 +1525,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1156,6 +1550,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1163,11 +1558,117 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "pyzmq" +version = "25.1.2" +description = "Python bindings for 0MQ" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:e624c789359f1a16f83f35e2c705d07663ff2b4d4479bad35621178d8f0f6ea4"}, + {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49151b0efece79f6a79d41a461d78535356136ee70084a1c22532fc6383f4ad0"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a5f194cf730f2b24d6af1f833c14c10f41023da46a7f736f48b6d35061e76e"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faf79a302f834d9e8304fafdc11d0d042266667ac45209afa57e5efc998e3872"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f51a7b4ead28d3fca8dda53216314a553b0f7a91ee8fc46a72b402a78c3e43d"}, + {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0ddd6d71d4ef17ba5a87becf7ddf01b371eaba553c603477679ae817a8d84d75"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:246747b88917e4867e2367b005fc8eefbb4a54b7db363d6c92f89d69abfff4b6"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:00c48ae2fd81e2a50c3485de1b9d5c7c57cd85dc8ec55683eac16846e57ac979"}, + {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a68d491fc20762b630e5db2191dd07ff89834086740f70e978bb2ef2668be08"}, + {file = "pyzmq-25.1.2-cp310-cp310-win32.whl", hash = "sha256:09dfe949e83087da88c4a76767df04b22304a682d6154de2c572625c62ad6886"}, + {file = "pyzmq-25.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:fa99973d2ed20417744fca0073390ad65ce225b546febb0580358e36aa90dba6"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:82544e0e2d0c1811482d37eef297020a040c32e0687c1f6fc23a75b75db8062c"}, + {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:01171fc48542348cd1a360a4b6c3e7d8f46cdcf53a8d40f84db6707a6768acc1"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc69c96735ab501419c432110016329bf0dea8898ce16fab97c6d9106dc0b348"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e124e6b1dd3dfbeb695435dff0e383256655bb18082e094a8dd1f6293114642"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7598d2ba821caa37a0f9d54c25164a4fa351ce019d64d0b44b45540950458840"}, + {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d1299d7e964c13607efd148ca1f07dcbf27c3ab9e125d1d0ae1d580a1682399d"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e6f689880d5ad87918430957297c975203a082d9a036cc426648fcbedae769b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cc69949484171cc961e6ecd4a8911b9ce7a0d1f738fcae717177c231bf77437b"}, + {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9880078f683466b7f567b8624bfc16cad65077be046b6e8abb53bed4eeb82dd3"}, + {file = "pyzmq-25.1.2-cp311-cp311-win32.whl", hash = "sha256:4e5837af3e5aaa99a091302df5ee001149baff06ad22b722d34e30df5f0d9097"}, + {file = "pyzmq-25.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:25c2dbb97d38b5ac9fd15586e048ec5eb1e38f3d47fe7d92167b0c77bb3584e9"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:11e70516688190e9c2db14fcf93c04192b02d457b582a1f6190b154691b4c93a"}, + {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:313c3794d650d1fccaaab2df942af9f2c01d6217c846177cfcbc693c7410839e"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b3cbba2f47062b85fe0ef9de5b987612140a9ba3a9c6d2543c6dec9f7c2ab27"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc31baa0c32a2ca660784d5af3b9487e13b61b3032cb01a115fce6588e1bed30"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c9087b109070c5ab0b383079fa1b5f797f8d43e9a66c07a4b8b8bdecfd88ee"}, + {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f8429b17cbb746c3e043cb986328da023657e79d5ed258b711c06a70c2ea7537"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5074adeacede5f810b7ef39607ee59d94e948b4fd954495bdb072f8c54558181"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7ae8f354b895cbd85212da245f1a5ad8159e7840e37d78b476bb4f4c3f32a9fe"}, + {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b264bf2cc96b5bc43ce0e852be995e400376bd87ceb363822e2cb1964fcdc737"}, + {file = "pyzmq-25.1.2-cp312-cp312-win32.whl", hash = "sha256:02bbc1a87b76e04fd780b45e7f695471ae6de747769e540da909173d50ff8e2d"}, + {file = "pyzmq-25.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:ced111c2e81506abd1dc142e6cd7b68dd53747b3b7ae5edbea4578c5eeff96b7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b6d09a8962a91151f0976008eb7b29b433a560fde056ec7a3db9ec8f1075438"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967668420f36878a3c9ecb5ab33c9d0ff8d054f9c0233d995a6d25b0e95e1b6b"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5edac3f57c7ddaacdb4d40f6ef2f9e299471fc38d112f4bc6d60ab9365445fb0"}, + {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0dabfb10ef897f3b7e101cacba1437bd3a5032ee667b7ead32bbcdd1a8422fe7"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2c6441e0398c2baacfe5ba30c937d274cfc2dc5b55e82e3749e333aabffde561"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:16b726c1f6c2e7625706549f9dbe9b06004dfbec30dbed4bf50cbdfc73e5b32a"}, + {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:a86c2dd76ef71a773e70551a07318b8e52379f58dafa7ae1e0a4be78efd1ff16"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win32.whl", hash = "sha256:359f7f74b5d3c65dae137f33eb2bcfa7ad9ebefd1cab85c935f063f1dbb245cc"}, + {file = "pyzmq-25.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:55875492f820d0eb3417b51d96fea549cde77893ae3790fd25491c5754ea2f68"}, + {file = "pyzmq-25.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8c8a419dfb02e91b453615c69568442e897aaf77561ee0064d789705ff37a92"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8807c87fa893527ae8a524c15fc505d9950d5e856f03dae5921b5e9aa3b8783b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5e319ed7d6b8f5fad9b76daa0a68497bc6f129858ad956331a5835785761e003"}, + {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c53687dde4d9d473c587ae80cc328e5b102b517447456184b485587ebd18b62"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9add2e5b33d2cd765ad96d5eb734a5e795a0755f7fc49aa04f76d7ddda73fd70"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e690145a8c0c273c28d3b89d6fb32c45e0d9605b2293c10e650265bf5c11cfec"}, + {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00a06faa7165634f0cac1abb27e54d7a0b3b44eb9994530b8ec73cf52e15353b"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win32.whl", hash = "sha256:0f97bc2f1f13cb16905a5f3e1fbdf100e712d841482b2237484360f8bc4cb3d7"}, + {file = "pyzmq-25.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6cc0020b74b2e410287e5942e1e10886ff81ac77789eb20bec13f7ae681f0fdd"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bef02cfcbded83473bdd86dd8d3729cd82b2e569b75844fb4ea08fee3c26ae41"}, + {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e10a4b5a4b1192d74853cc71a5e9fd022594573926c2a3a4802020360aa719d8"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c5f80e578427d4695adac6fdf4370c14a2feafdc8cb35549c219b90652536ae"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5dde6751e857910c1339890f3524de74007958557593b9e7e8c5f01cd919f8a7"}, + {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea1608dd169da230a0ad602d5b1ebd39807ac96cae1845c3ceed39af08a5c6df"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0f513130c4c361201da9bc69df25a086487250e16b5571ead521b31ff6b02220"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:019744b99da30330798bb37df33549d59d380c78e516e3bab9c9b84f87a9592f"}, + {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e2713ef44be5d52dd8b8e2023d706bf66cb22072e97fc71b168e01d25192755"}, + {file = "pyzmq-25.1.2-cp38-cp38-win32.whl", hash = "sha256:07cd61a20a535524906595e09344505a9bd46f1da7a07e504b315d41cd42eb07"}, + {file = "pyzmq-25.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb7e49a17fb8c77d3119d41a4523e432eb0c6932187c37deb6fbb00cc3028088"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:94504ff66f278ab4b7e03e4cba7e7e400cb73bfa9d3d71f58d8972a8dc67e7a6"}, + {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd0d50bbf9dca1d0bdea219ae6b40f713a3fb477c06ca3714f208fd69e16fd8"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:004ff469d21e86f0ef0369717351073e0e577428e514c47c8480770d5e24a565"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0b5ca88a8928147b7b1e2dfa09f3b6c256bc1135a1338536cbc9ea13d3b7add"}, + {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9a79f1d2495b167119d02be7448bfba57fad2a4207c4f68abc0bab4b92925b"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:518efd91c3d8ac9f9b4f7dd0e2b7b8bf1a4fe82a308009016b07eaa48681af82"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1ec23bd7b3a893ae676d0e54ad47d18064e6c5ae1fadc2f195143fb27373f7f6"}, + {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db36c27baed588a5a8346b971477b718fdc66cf5b80cbfbd914b4d6d355e44e2"}, + {file = "pyzmq-25.1.2-cp39-cp39-win32.whl", hash = "sha256:39b1067f13aba39d794a24761e385e2eddc26295826530a8c7b6c6c341584289"}, + {file = "pyzmq-25.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:8e9f3fabc445d0ce320ea2c59a75fe3ea591fdbdeebec5db6de530dd4b09412e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a8c1d566344aee826b74e472e16edae0a02e2a044f14f7c24e123002dcff1c05"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:759cfd391a0996345ba94b6a5110fca9c557ad4166d86a6e81ea526c376a01e8"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c61e346ac34b74028ede1c6b4bcecf649d69b707b3ff9dc0fab453821b04d1e"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cb8fc1f8d69b411b8ec0b5f1ffbcaf14c1db95b6bccea21d83610987435f1a4"}, + {file = "pyzmq-25.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3c00c9b7d1ca8165c610437ca0c92e7b5607b2f9076f4eb4b095c85d6e680a1d"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:df0c7a16ebb94452d2909b9a7b3337940e9a87a824c4fc1c7c36bb4404cb0cde"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45999e7f7ed5c390f2e87ece7f6c56bf979fb213550229e711e45ecc7d42ccb8"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac170e9e048b40c605358667aca3d94e98f604a18c44bdb4c102e67070f3ac9b"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b604734bec94f05f81b360a272fc824334267426ae9905ff32dc2be433ab96"}, + {file = "pyzmq-25.1.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a793ac733e3d895d96f865f1806f160696422554e46d30105807fdc9841b9f7d"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0806175f2ae5ad4b835ecd87f5f85583316b69f17e97786f7443baaf54b9bb98"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef12e259e7bc317c7597d4f6ef59b97b913e162d83b421dd0db3d6410f17a244"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea253b368eb41116011add00f8d5726762320b1bda892f744c91997b65754d73"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b9b1f2ad6498445a941d9a4fee096d387fee436e45cc660e72e768d3d8ee611"}, + {file = "pyzmq-25.1.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8b14c75979ce932c53b79976a395cb2a8cd3aaf14aef75e8c2cb55a330b9b49d"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:889370d5174a741a62566c003ee8ddba4b04c3f09a97b8000092b7ca83ec9c49"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18fff090441a40ffda8a7f4f18f03dc56ae73f148f1832e109f9bffa85df15"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99a6b36f95c98839ad98f8c553d8507644c880cf1e0a57fe5e3a3f3969040882"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4345c9a27f4310afbb9c01750e9461ff33d6fb74cd2456b107525bbeebcb5be3"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3516e0b6224cf6e43e341d56da15fd33bdc37fa0c06af4f029f7d7dfceceabbc"}, + {file = "pyzmq-25.1.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:146b9b1f29ead41255387fb07be56dc29639262c0f7344f570eecdcd8d683314"}, + {file = "pyzmq-25.1.2.tar.gz", hash = "sha256:93f1aa311e8bb912e34f004cf186407a4e90eec4f0ecc0efd26056bf7eda0226"}, +] + +[package.dependencies] +cffi = {version = "*", markers = "implementation_name == \"pypy\""} + [[package]] name = "requests" version = "2.31.0" @@ -1268,6 +1769,45 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "stack-data" +version = "0.6.3" +description = "Extract data from python stack frames and tracebacks for informative displays" +optional = false +python-versions = "*" +files = [ + {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, + {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, +] + +[package.dependencies] +asttokens = ">=2.1.0" +executing = ">=1.2.0" +pure-eval = "*" + +[package.extras] +tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] + +[[package]] +name = "tornado" +version = "6.4" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">= 3.8" +files = [ + {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, + {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579"}, + {file = "tornado-6.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78"}, + {file = "tornado-6.4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f"}, + {file = "tornado-6.4-cp38-abi3-win32.whl", hash = "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052"}, + {file = "tornado-6.4-cp38-abi3-win_amd64.whl", hash = "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63"}, + {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, +] + [[package]] name = "traitlets" version = "5.14.0" @@ -1332,6 +1872,17 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "wcwidth" +version = "0.2.12" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.12-py2.py3-none-any.whl", hash = "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"}, + {file = "wcwidth-0.2.12.tar.gz", hash = "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02"}, +] + [[package]] name = "xarray" version = "2023.11.0" @@ -1358,4 +1909,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "77b941447f5222afe1c9b0912f313831ec5bb0408f69af4e733f837d85294474" +content-hash = "aa19369fd2254684620726dd1449351be360f28d24aecd49ab461282cc0d846a" diff --git a/pyproject.toml b/pyproject.toml index f74818ca..32e1f68f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ prometheus-client = "^0.19.0" pytest = "^7.4.3" types-pyyaml = "^6.0.12.12" ruff = "^0.1.6" +ipykernel = "^6.27.1" [build-system] requires = ["poetry-core"] diff --git a/scripts/admin/cluster/rbac_export_import.py b/scripts/admin/cluster/rbac_export_import.py index b9cfff7a..0c3c2c2c 100644 --- a/scripts/admin/cluster/rbac_export_import.py +++ b/scripts/admin/cluster/rbac_export_import.py @@ -12,9 +12,11 @@ # example restore: python rbac.py --restore import argparse -import requests import json +import requests + + def backup(): url = 'http://' + args.host + '/settings/rbac/users' print("my url : " + str(url)) @@ -35,7 +37,7 @@ def backup(): for entry in records: entry['password'] = "password" rbacInfo['userInfo'].append(entry) - with open(args.rbacFile, 'wt') as f: + with open(args.rbacFile, "w") as f: f.write(json.dumps(rbacInfo, indent=4, sort_keys=True)) print('Successfully backed up RBAC data to: [%s]' % args.rbacFile) except Exception as e: @@ -45,7 +47,7 @@ def backup(): def restore(): try: - with open(args.rbacFile,'r') as f: + with open(args.rbacFile) as f: rbacData = json.loads(f.read()) for cluster in rbacData['clusterInfo']['cluster']: print(cluster) @@ -69,7 +71,7 @@ def restore(): print(output + json.dumps(params)) else: print(params) - print('Http requests {} failed: {} {}'.format(url, response.status_code, response.text)) + print(f'Http requests {url} failed: {response.status_code} {response.text}') except Exception as e: print('Error:' + str(e.args)) diff --git a/scripts/admin/index/index-analysis.py b/scripts/admin/index/index-analysis.py index 8590783d..3cd568d0 100644 --- a/scripts/admin/index/index-analysis.py +++ b/scripts/admin/index/index-analysis.py @@ -1,5 +1,4 @@ #!/usr/bin/python -# -*- coding: utf-8 -*- # pylint: disable=C0303, C0325 diff --git a/scripts/admin/stats/StoreAndGetStats/getStats/bucket.py b/scripts/admin/stats/StoreAndGetStats/getStats/bucket.py index 8602aacc..4a916a43 100755 --- a/scripts/admin/stats/StoreAndGetStats/getStats/bucket.py +++ b/scripts/admin/stats/StoreAndGetStats/getStats/bucket.py @@ -1,5 +1,4 @@ -import json -class Bucket(object): +class Bucket: def __init__(self, attributes_list): self.attribute_map = {} self.attributes_list = attributes_list diff --git a/scripts/admin/stats/StoreAndGetStats/getStats/cbcluster.py b/scripts/admin/stats/StoreAndGetStats/getStats/cbcluster.py index c6683af6..d7612988 100644 --- a/scripts/admin/stats/StoreAndGetStats/getStats/cbcluster.py +++ b/scripts/admin/stats/StoreAndGetStats/getStats/cbcluster.py @@ -1,8 +1,8 @@ -from couchbase.cluster import Cluster -from couchbase.cluster import PasswordAuthenticator +from couchbase.cluster import Cluster, PasswordAuthenticator from couchbase.n1ql import N1QLQuery -class CBCluster(object): + +class CBCluster: def __init__(self, attributes_list, ipaddr): cluster = Cluster('couchbase://' + ipaddr) authenticator = PasswordAuthenticator(attributes_list[unicode("user")], attributes_list[unicode("pwd")]) diff --git a/scripts/admin/stats/StoreAndGetStats/getStats/fileOutput.py b/scripts/admin/stats/StoreAndGetStats/getStats/fileOutput.py index 96c8fff6..cea0376d 100644 --- a/scripts/admin/stats/StoreAndGetStats/getStats/fileOutput.py +++ b/scripts/admin/stats/StoreAndGetStats/getStats/fileOutput.py @@ -1,7 +1,8 @@ #!/usr/bin/env python from nodeOutput import NodeOutput -class FileOutput(object): + +class FileOutput: header_line = "\"Date Time\"" quota_used = "\"quotaPercentUsed\"" itemCount = "\"itemCount\"" diff --git a/scripts/admin/stats/StoreAndGetStats/getStats/node.py b/scripts/admin/stats/StoreAndGetStats/getStats/node.py index 276c66e3..ef4a672a 100755 --- a/scripts/admin/stats/StoreAndGetStats/getStats/node.py +++ b/scripts/admin/stats/StoreAndGetStats/getStats/node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -class Node(object): +class Node: def __init__(self, attributes_dict, timeslice): self.attributes_dict = attributes_dict diff --git a/scripts/admin/stats/StoreAndGetStats/getStats/nodeOutput.py b/scripts/admin/stats/StoreAndGetStats/getStats/nodeOutput.py index 181e5500..426654aa 100644 --- a/scripts/admin/stats/StoreAndGetStats/getStats/nodeOutput.py +++ b/scripts/admin/stats/StoreAndGetStats/getStats/nodeOutput.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -class NodeOutput(object): +class NodeOutput: def __init__(self, nodeName): self.nodeName = nodeName diff --git a/scripts/admin/stats/StoreAndGetStats/getStats/timeSlice.py b/scripts/admin/stats/StoreAndGetStats/getStats/timeSlice.py index 618af6c1..c401ca94 100644 --- a/scripts/admin/stats/StoreAndGetStats/getStats/timeSlice.py +++ b/scripts/admin/stats/StoreAndGetStats/getStats/timeSlice.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -class TimeSlice(object): +class TimeSlice: def __init__(self, id, timeslice): self.id = id diff --git a/scripts/admin/stats/StoreAndGetStats/storeStats/bucket.py b/scripts/admin/stats/StoreAndGetStats/storeStats/bucket.py index 340a6569..f1f95a45 100644 --- a/scripts/admin/stats/StoreAndGetStats/storeStats/bucket.py +++ b/scripts/admin/stats/StoreAndGetStats/storeStats/bucket.py @@ -1,5 +1,4 @@ -import json -class Bucket(object): +class Bucket: def __init__(self, attributes_list): self.attribute_map = {} self.attributes_list = attributes_list diff --git a/scripts/admin/stats/StoreAndGetStats/storeStats/cbcluster.py b/scripts/admin/stats/StoreAndGetStats/storeStats/cbcluster.py index 6bb0611b..bc6aeee3 100644 --- a/scripts/admin/stats/StoreAndGetStats/storeStats/cbcluster.py +++ b/scripts/admin/stats/StoreAndGetStats/storeStats/cbcluster.py @@ -1,6 +1,7 @@ -from couchbase.cluster import Cluster -from couchbase.cluster import PasswordAuthenticator -class CBCluster(object): +from couchbase.cluster import Cluster, PasswordAuthenticator + + +class CBCluster: def __init__(self, attributes_list, ipaddr): cluster = Cluster('couchbase://' + ipaddr) authenticator = PasswordAuthenticator(attributes_list[unicode("user")], attributes_list[unicode("pwd")]) diff --git a/scripts/backup/automate_backup.py b/scripts/backup/automate_backup.py index 70efdfda..5471eb9b 100755 --- a/scripts/backup/automate_backup.py +++ b/scripts/backup/automate_backup.py @@ -8,15 +8,17 @@ __maintainer__ = "tdenton" __version__ = "0.0.1" -import subprocess -from datetime import datetime -import shutil import argparse -import os import json +import os +import shutil +import subprocess +from datetime import datetime + from config import Config -class BackerUpper(object): + +class BackerUpper: '''This class is what actually does the backing up, archiving and cleaning of old backups''' def __init__(self, hostname="", port="", username="", password=""): self.instance = self.get_config() @@ -60,7 +62,7 @@ def get_repo_name(self): def archive_repo(self): '''Copys the existing current backup and turns it into an archive''' - shutil.copytree("{}/{}".format(self.instance.archive, self.instance.repo), + shutil.copytree(f"{self.instance.archive}/{self.instance.repo}", "{}/{}".format(self.instance.archive, datetime.now().strftime("%Y-%m-%d")), symlinks=False, ignore=None) @@ -77,15 +79,15 @@ def backup_repo(self): "-r", self.instance.repo, "-c", - "{}:{}".format(self.instance.hostname, self.instance.port), + f"{self.instance.hostname}:{self.instance.port}", "-u", self.instance.username, "-p", self.instance.password] if self.instance.value_compression != "unchanged": - backup_array.append("--value_compression {}".format(self.instance.value_compression)) + backup_array.append(f"--value_compression {self.instance.value_compression}") if self.instance.threads != 1: - backup_array.append("--threads {}".format(self.instance.threads)) + backup_array.append(f"--threads {self.instance.threads}") if self.instance.no_progress_bar: backup_array.append("--no-progress-bar") try: diff --git a/scripts/backup/config.py b/scripts/backup/config.py index 9cca6a22..89af9d4c 100644 --- a/scripts/backup/config.py +++ b/scripts/backup/config.py @@ -1,6 +1,6 @@ # pyline disable=C0303, C0325 -class Config(object): +class Config: def __init__(self, my_json): self.my_json = my_json self.weeks_to_keep = self.my_json['weeks_to_keep'] diff --git a/scripts/troubleshooting/sdk/orphans_threshold.py b/scripts/troubleshooting/sdk/orphans_threshold.py index a4ffe0f2..af40f766 100644 --- a/scripts/troubleshooting/sdk/orphans_threshold.py +++ b/scripts/troubleshooting/sdk/orphans_threshold.py @@ -64,12 +64,14 @@ If including a space between the date and time, enclose the datetime within quotes. """ -import json -import csv import argparse +import csv +import json from datetime import datetime + # pylint: disable=import-error import dictfier + # pylint: enable=import-error # pylint: disable=superfluous-parens @@ -82,7 +84,7 @@ timeouts = [] -class Orphans(): +class Orphans: """ This is the collection of orphan log entries that are extracted from the log file. @@ -108,7 +110,7 @@ def __init__(self, log_line): # pylint: disable=too-many-instance-attributes -class Orphan(): +class Orphan: """ This is a single orphan log entry from the RTO log file. @@ -125,7 +127,7 @@ def __init__(self, data_dict, log_time): # pylint: enable=too-many-instance-attributes -class Thresholds(): +class Thresholds: """ This is the collection of threshold log entries that are extracted from the log file. @@ -151,7 +153,7 @@ def __init__(self, log_line, latency_threshold): # pylint: disable=too-many-instance-attributes -class Threshold(): +class Threshold: """ This is a single threshold log entry from the RTO log file. @@ -195,7 +197,7 @@ def __init__(self, data_dict, log_time, latency_threshold): -class Timeout(): +class Timeout: """ This is the combined orphan and threshold log entries that will be written out to the .CSV file. @@ -299,7 +301,7 @@ def read_log_file(log_file, start_date, end_date, threshold): threshold entries. """ # Open and read in the file - input_file = open(log_file, "r") + input_file = open(log_file) # Iterate through the lines in the log file for line in input_file: # Extract the datetime from the line and compare it to diff --git a/src/vxingest/netcdf_to_cb/netcdf_builder.py b/src/vxingest/netcdf_to_cb/netcdf_builder.py index 9655b40f..7369f3cf 100644 --- a/src/vxingest/netcdf_to_cb/netcdf_builder.py +++ b/src/vxingest/netcdf_to_cb/netcdf_builder.py @@ -20,7 +20,7 @@ import netCDF4 as nc import numpy.ma as ma -from metpy.calc import wind_components +from metpy.calc import relative_humidity_from_dewpoint, wind_components from metpy.units import units from vxingest.builder_common.builder import Builder from vxingest.builder_common.builder_utilities import ( @@ -363,7 +363,7 @@ def build_document(self, queue_element): if self.do_profiling: with cProfile.Profile() as _pr: self.handle_document() - with open("profiling_stats.txt", "w") as stream: + with open("profiling_stats.txt", "w", encoding="utf-8" ) as stream: stats = Stats(_pr, stream=stream) stats.strip_dirs() stats.sort_stats("time") @@ -643,7 +643,26 @@ def umask_value_transform(self, params_dict): ) return None - def handle_wind_dir_U(self, params_dict): + def handle_rh(self, params_dict): + """Retrieves a RH value, if there is one, + and if not derives the RH from the dewpoint and temperature. + Args: + params_dict (dict): named function parameters + Returns: + float: the RH + """ + try: + # dewpoint = self.umask_value_transform({"recNum": params_dict["recNum"], "dewpoint":"dewpoint"}) + # temperature = self.umask_value_transform({"recNum": params_dict["recNum"], "temperature":"temperature"}) + dewpoint = self.umask_value_transform({"recNum": params_dict["recNum"], "dewpoint":params_dict["dewpoint"]}) + temperature = self.umask_value_transform({"recNum": params_dict["recNum"], "temperature":params_dict["temperature"]}) + _q = (relative_humidity_from_dewpoint(temperature * units.kelvin, dewpoint * units.kelvin).magnitude) * 100 + return _q + except Exception as _e: # pylint:disable=broad-except + # there must not have been one + return None + + def handle_wind_dir_u(self, params_dict): """Retrieves a wind direction U value, if there is one, and if not derives the U component from the wind direction and speed. expects wind speed and wind direction to be in the params_dict. @@ -653,19 +672,24 @@ def handle_wind_dir_U(self, params_dict): float: the wind direction """ try: - value = self.umask_value_transform(params_dict) - # value = self.meterspersecond_to_milesperhour({"recNum": rec_num, "windSpeed": ""}) - if value is not None: - value = 360 - value - return value - except Exception as _e: - # there must not have been one - windDir = self.umask_value_transform({}) - windSpeed = self.umask_value_transform({}) - u, _v = wind_components(windSpeed * units("m/s"), windDir * units.deg) + _wind_dir = self.umask_value_transform({"recNum": params_dict["recNum"], "windDir":params_dict["windDir"]}) + _wind_speed = self.umask_value_transform({"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]}) + if _wind_dir is None: + return None + if _wind_speed is None: + return None + # wind speed is in meters per second and windDir is in degrees from netcdf file + u = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[0].magnitude return u + except Exception as _e: # pylint:disable=broad-exception-caught + logger.error( + "%s handle_wind_dir_v: Exception in named function: error: %s", + self.__class__.__name__, + str(_e), + ) + return None - def handle_wind_dir_V(self, params_dict): + def handle_wind_dir_v(self, params_dict): """Retrieves a wind direction V value, if there is one, and if not derives the V component from the wind direction and speed. Args: @@ -674,16 +698,22 @@ def handle_wind_dir_V(self, params_dict): float: the wind direction """ try: - value = self.umask_value_transform({}) - if value is not None: - value = 360 - value - return value - except Exception as _e: - # there must not have been one - windDir = self.umask_value_transform({}) - windSpeed = self.umask_value_transform({}) - _u, v = wind_components(windSpeed * units("m/s"), windDir * units.deg) + _wind_dir = self.umask_value_transform({"recNum": params_dict["recNum"], "windDir":params_dict["windDir"]}) + _wind_speed = self.umask_value_transform({"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]}) + if _wind_dir is None: + return None + if _wind_speed is None: + return None + # wind speed is in meters per second and windDir is in degrees from netcdf file + v = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[1].magnitude return v + except Exception as _e: # pylint:disable=broad-exception-caught + logger.error( + "%s handle_wind_dir_v: Exception in named function: error: %s", + self.__class__.__name__, + str(_e), + ) + return None def handle_pressure(self, params_dict): """Retrieves a pressure value and converts it to millibars from pascals @@ -908,7 +938,7 @@ def handle_station(self, params_dict): "version": "V01", } # add the new station to the document map with the new id - if an_id not in self.document_map.keys(): + if an_id not in self.document_map: self.document_map[an_id] = new_station self.stations.append(new_station) else: diff --git a/src/vxingest/utilities/README.MD b/src/vxingest/utilities/README.MD new file mode 100644 index 00000000..605a83d2 --- /dev/null +++ b/src/vxingest/utilities/README.MD @@ -0,0 +1,3 @@ +# VxIngest utilities + +These are scripts that have been used for various tasks, like backfilling data, that can be useful examples for other tasks. diff --git a/src/vxingest/utilities/backfill_obs_with_rh.py b/src/vxingest/utilities/backfill_obs_with_rh.py new file mode 100644 index 00000000..a2afe63f --- /dev/null +++ b/src/vxingest/utilities/backfill_obs_with_rh.py @@ -0,0 +1,80 @@ +""" +Module to backfill observations with relative humidity, WindU, and WindV. +""" + +import os +from datetime import timedelta +from pathlib import Path + +import yaml +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.options import ClusterOptions, ClusterTimeoutOptions +from metpy.calc import relative_humidity_from_dewpoint, wind_components +from metpy.units import units + + +def setup_connection(): + """test setup""" + try: + credentials_file = os.environ["CREDENTIALS"] + assert Path(credentials_file).is_file(), "credentials_file Does not exist" + _f = Path.open(credentials_file, encoding="utf-8") + yaml_data = yaml.load(_f, yaml.SafeLoader) + _host = yaml_data["cb_host"] + _user = yaml_data["cb_user"] + _password = yaml_data["cb_password"] + _bucket = yaml_data["cb_bucket"] + _collection = yaml_data["cb_collection"] + _scope = yaml_data["cb_scope"] + _f.close() + + timeout_options = ClusterTimeoutOptions( + kv_timeout=timedelta(seconds=25), query_timeout=timedelta(seconds=120) + ) + options = ClusterOptions( + PasswordAuthenticator(_user, _password), timeout_options=timeout_options + ) + options = ClusterOptions(PasswordAuthenticator(_user, _password)) + connection = {} + connection['cluster'] = Cluster("couchbase://" + _host, options) + connection['bucket'] = connection['cluster'].bucket(_bucket) + connection['scope'] = connection['bucket'].scope(_scope) + connection['collection'] = connection['scope'].collection(_collection) + return connection + except Exception as _e: # pylint:disable=broad-except + print (f"test_credentials_and_load_spec Exception failure: {_e}") + +def calc_components(doc): + """Calculate RH, WindU, and WindV from Temperature, DewPoint, WS, and WD. + """ + # doc = {"data":{'station_name': {'Temperature'} {'DewPoint'} {'WS'} {'WD'} ... } + for _name, station in doc['data'].items(): + if "RH" not in station: + if station["Temperature"] is not None and station["DewPoint"] is not None: + station["RH"] = relative_humidity_from_dewpoint(station["Temperature"] * units.degC, station["DewPoint"] * units.degC).magnitude * 100 + else: + station["RH"] = None + if "WindU" not in station or "WindV" not in station.keys(): + if station["WS"] is not None and station["WD"] is not None: + _u, _v = wind_components(station["WS"] * units("m/s"), station["WD"] * units.deg) + station["WindU"] = _u.magnitude + station["WindV"] = _v.magnitude + else: + station["WindU"] = None + station["WindV"] = None + +def run_backfill() -> None: + """entrypoint""" + connection = setup_connection() + _query_result = connection['cluster'].query("select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR';") + _result = list(_query_result) + print(f"number of ids is {len(_result)}: starting id is {_result[0]} and ending id is {_result[-1]}") + for i,_id in enumerate(_result): + print (f"processing #{i} with id {_id}") + doc = connection['collection'].get(_id).content_as[dict] + calc_components(doc) + connection['collection'].upsert(_id, doc) + +if __name__ == "__main__": + run_backfill() diff --git a/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py b/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py index dbad00fd..38a10a05 100644 --- a/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py +++ b/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py @@ -117,6 +117,8 @@ def test_grib_builder_one_thread_file_pattern_hrrr_ops_conus(tmp_path): continue _statement = f"select METAR.* from `{connect_cb()['bucket']}`._default.METAR where meta().id = '{_id}'" qresult = connect_cb()["cluster"].query(_statement) + res_rows = list(qresult.rows()) + assert (len(res_rows) > 0 ), f"TestGribBuilderV01.test_gribBuilder_one_epoch_hrrr_ops_conus failure test document {_id} not found in couchbase" result = list(qresult.rows())[0] # assert top level fields keys = _json.keys() diff --git a/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py b/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py index 600e8a20..94f0c6d7 100644 --- a/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py +++ b/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py @@ -1,6 +1,20 @@ """" integration tests for netcdf + This test derives a METAR observation from a netcdf file and then compares the derived + observation to the observation that is stored in the couchbase database. + Special note on test data: + The test data is located in the directory /opt/data/netcdf_to_cb/input_files/20211108_0000 + and this data does exist in the couchbase database. The document id of the obs document + is "DD-TEST:V01:METAR:obs:1636329600" and the data set also includes many station documents. + The data set aslo includes a data file document "DF:METAR:netcdf:madis:20211108_0000" which is used + by the ingest manager to determine if the data has already been ingested. + If you want to re-import the + data ***you will need to delete the DF document from the couchbase database*** in order + for the data to be re-processed. + If you re-import the data and forget to delete the DF document you will get an error like... + "AssertionError: There are no output files". """ +import json import os from glob import glob from multiprocessing import Queue @@ -8,9 +22,9 @@ from vxingest.netcdf_to_cb.run_ingest_threads import VXIngest -def stub_worker_log_configurer(queue: Queue): +def stub_worker_log_configurer(queue: Queue): # pylint:disable=unused-argument """A stub to replace log_config.worker_log_configurer""" - pass + pass # pylint:disable=unnecessary-pass def setup_connection(): """test setup""" @@ -27,6 +41,14 @@ def setup_connection(): assert False, f"test_credentials_and_load_spec Exception failure: {_e}" return None +def ordered(obj): + """ Utliity function to sort a dictionary so that it can be compared to another dictionary""" + if isinstance(obj, dict): + return sorted((k, ordered(v)) for k, v in obj.items()) + if isinstance(obj, list): + return sorted(ordered(x) for x in obj) + else: + return obj def test_one_thread_specify_file_pattern(tmp_path): # pylint:disable=missing-function-docstring try: @@ -64,6 +86,30 @@ def test_one_thread_specify_file_pattern(tmp_path): # pylint:disable=missing-fu assert len(glob(f"{tmp_path}/20211108*.json")) == len( glob("/opt/data/netcdf_to_cb/input_files/20211108_0000") ), "number of output files is incorrect" + derived_data = json.load(open(f"{tmp_path}/20211108_0000.json", encoding="utf-8")) + station_id = "" + derived_station = {} + obs_id = "" + derived_obs = {} + for item in derived_data: + if item["docType"] == "station" and item["name"] == "KDEN": + station_id = item["id"] + derived_station = item + else: + if item["docType"] == "obs": + obs_id = item["id"] + derived_obs = item + if derived_station and derived_obs: + break + retrieved_station = vx_ingest.collection.get(station_id).content_as[dict] + retrieved_obs = vx_ingest.collection.get(obs_id).content_as[dict] + # make sure the updateTime is the same in both the derived and retrieved station + retrieved_station['updateTime'] = derived_station['updateTime'] + # make sure the firstTime and lastTime are the same in both the derived and retrieved station['geo'] + retrieved_station['geo'][0]['firstTime'] = derived_station['geo'][0]['firstTime'] + retrieved_station['geo'][0]['lastTime'] = derived_station['geo'][0]['lastTime'] + assert ordered(derived_station) == ordered(retrieved_station), "derived station does not match retrieved station" + assert ordered(derived_obs) == ordered(retrieved_obs), "derived obs does not match retrieved obs" except Exception as _e: # pylint: disable=broad-except assert False, f"TestGsdIngestManager Exception failure: {_e}" diff --git a/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py b/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py index a7c496ea..07b6d230 100644 --- a/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py +++ b/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py @@ -31,9 +31,9 @@ mysql_model_obs_data = [] stations = [] -def stub_worker_log_configurer(queue: Queue): +def stub_worker_log_configurer(queue: Queue): # pylint: disable=unused-argument """A stub to replace log_config.worker_log_configurer""" - pass + pass # pylint: disable=unnecessary-pass def test_check_fcst_valid_epoch_fcst_valid_iso(): """ diff --git a/tests/vxingest/utilities/test_backfill_obs_with_rh.py b/tests/vxingest/utilities/test_backfill_obs_with_rh.py new file mode 100644 index 00000000..47200716 --- /dev/null +++ b/tests/vxingest/utilities/test_backfill_obs_with_rh.py @@ -0,0 +1,82 @@ +""" + _test for VxIngest backfill_obs_with_rh.py +""" +from vxingest.utilities.backfill_obs_with_rh import calc_components + + +def test_calc_components_backfills_rh(): + """test calculating missing RH""" + doc = { + "data":{ + 'NZCM':{"Temperature": 25, "DewPoint": 20, "WS": 5, "WD": 180, "WindU":-6.123233995736766e-16 , "WindV": 5}, + 'SUMU':{"Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, "WindU": 10, "WindV": 1.8369701987210296e-15}, + } + } + for entries in doc["data"]: + assert "RH" not in entries + calc_components(doc) + assert doc["data"]['NZCM']["RH"] == 73.78055835142725 + assert doc["data"]['NZCM']["WindU"] == -6.123233995736766e-16 + assert doc["data"]['NZCM']["WindV"] == 5.0 + assert doc["data"]['NZCM']["Temperature"] == 25 + assert doc["data"]['NZCM']["DewPoint"] == 20 + assert doc["data"]['NZCM']["WS"] == 5 + assert doc["data"]['NZCM']["WD"] == 180 + + assert doc["data"]['SUMU']["RH"] == 40.1370668529669 + assert doc["data"]['SUMU']["WindU"] == 10.0 + assert doc["data"]['SUMU']["WindV"] == 1.8369701987210296e-15 + assert doc["data"]['SUMU']["Temperature"] == 30 + assert doc["data"]['SUMU']["DewPoint"] == 15 + assert doc["data"]['SUMU']["WS"] == 10 + assert doc["data"]['SUMU']["WD"] == 270 + +def test_calc_components_backfills_windu_and_windv(): + """test calculating missing WindU and WindV""" + doc = { + "data": { + "NZCM": {"Temperature": 25, "DewPoint": 20, "WS": 5, "WD": 180, "RH": 73.78055835142725}, + "SUMU": {"Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, "RH": 40.1370668529669}, + } + } + calc_components(doc) + assert doc["data"]['NZCM']["RH"] == 73.78055835142725, "RH wrong value" + assert doc["data"]['NZCM']["WindU"] == -6.123233995736766e-16, "WindU wrong value" + assert doc["data"]['NZCM']["WindV"] == 5.0, "WindV wrong value" + assert doc["data"]['NZCM']["Temperature"] == 25, "temperature wrong value" + assert doc["data"]['NZCM']["DewPoint"] == 20, "DewPoint wrong value" + assert doc["data"]['NZCM']["WS"] == 5, "WS wrong value" + assert doc["data"]['NZCM']["WD"] == 180, "WD wrong value" + + assert doc["data"]['SUMU']["RH"] == 40.1370668529669, "RH wrong value" + assert doc["data"]['SUMU']["WindU"] == 10.0, "WindU wrong value" + assert doc["data"]['SUMU']["WindV"] == 1.8369701987210296e-15, "WindV wrong value" + assert doc["data"]['SUMU']["Temperature"] == 30, "temperature wrong value" + assert doc["data"]['SUMU']["DewPoint"] == 15, "DewPoint wrong value" + assert doc["data"]['SUMU']["WS"] == 10, "WS wrong value" + assert doc["data"]['SUMU']["WD"] == 270, "WD wrong value" + +def test_calc_components_backfills_nochange(): + """test no change if RH, WindU, and WindV are already present""" + doc = { + "data": { + "NZCM": {"Temperature": 25, "DewPoint": 20, "WS": 5, "WD": 180, "WindU": 5, "WindV": -5, "RH": 75}, + "SUMU": {"Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, "WindU": 3, "WindV": -7, "RH": 70}, + } + } + calc_components(doc) + assert doc["data"]['NZCM']["Temperature"] == 25, "temperature wrong value" + assert doc["data"]['NZCM']["DewPoint"] == 20, "DewPoint wrong value" + assert doc["data"]['NZCM']["WS"] == 5, "WS wrong value" + assert doc["data"]['NZCM']["WD"] == 180, "WD wrong value" + assert doc["data"]['NZCM']["WindU"] == 5, "WindU wrong value" + assert doc["data"]['NZCM']["WindV"] == -5, "WindV wrong value" + assert doc["data"]['NZCM']["RH"] == 75, "RH wrong value" + + assert doc["data"]['SUMU']["Temperature"] == 30, "temperature wrong value" + assert doc["data"]['SUMU']["DewPoint"] == 15, "DewPoint wrong value" + assert doc["data"]['SUMU']["WS"] == 10, "WS wrong value" + assert doc["data"]['SUMU']["WD"] == 270, "WD wrong value" + assert doc["data"]['SUMU']["WindU"] == 3, "WindU wrong value" + assert doc["data"]['SUMU']["WindV"] == -7, "WindV wrong value" + assert doc["data"]['SUMU']["RH"] == 70, "RH wrong value" From 15be331857e8fc4de3cf2046b0cc65ef99a1e33f Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Fri, 22 Dec 2023 14:26:38 -0700 Subject: [PATCH 03/10] add retry for timeouts --- .../utilities/backfill_obs_with_rh.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) mode change 100644 => 100755 src/vxingest/utilities/backfill_obs_with_rh.py diff --git a/src/vxingest/utilities/backfill_obs_with_rh.py b/src/vxingest/utilities/backfill_obs_with_rh.py old mode 100644 new mode 100755 index a2afe63f..0d263946 --- a/src/vxingest/utilities/backfill_obs_with_rh.py +++ b/src/vxingest/utilities/backfill_obs_with_rh.py @@ -3,12 +3,14 @@ """ import os +import time from datetime import timedelta from pathlib import Path import yaml from couchbase.auth import PasswordAuthenticator from couchbase.cluster import Cluster +from couchbase.exceptions import TimeoutException from couchbase.options import ClusterOptions, ClusterTimeoutOptions from metpy.calc import relative_humidity_from_dewpoint, wind_components from metpy.units import units @@ -72,9 +74,20 @@ def run_backfill() -> None: print(f"number of ids is {len(_result)}: starting id is {_result[0]} and ending id is {_result[-1]}") for i,_id in enumerate(_result): print (f"processing #{i} with id {_id}") - doc = connection['collection'].get(_id).content_as[dict] - calc_components(doc) - connection['collection'].upsert(_id, doc) - + ri = 0 + for ri in range(5): + try: + doc = connection['collection'].get(_id).content_as[dict] + calc_components(doc) + connection['collection'].upsert(_id, doc) + break + except TimeoutException as _e: + print (f"TimeoutException failure: {_e} retrying id {_id} #{ri}") + if ri == 5: + raise RuntimeWarning (f"Failed to process id {_id} - too many retries") from _e + time.sleep(1) # give it a second to breathe + except Exception as _e1: # pylint:disable=broad-except + print (f"Exception failure: {_e1}") + break if __name__ == "__main__": run_backfill() From cc7b882e456e6bdc257a375f76089fa14cc557a1 Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Thu, 28 Dec 2023 09:55:41 -0700 Subject: [PATCH 04/10] add script for backfilling obs with RH, WindU,and WindV --- src/vxingest/utilities/backfill_obs_with_rh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vxingest/utilities/backfill_obs_with_rh.py b/src/vxingest/utilities/backfill_obs_with_rh.py index 0d263946..a1f61f89 100755 --- a/src/vxingest/utilities/backfill_obs_with_rh.py +++ b/src/vxingest/utilities/backfill_obs_with_rh.py @@ -69,7 +69,7 @@ def calc_components(doc): def run_backfill() -> None: """entrypoint""" connection = setup_connection() - _query_result = connection['cluster'].query("select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR';") + _query_result = connection['cluster'].query("select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR' AND id > 'DD:V01:METAR:obs:1699966800';") _result = list(_query_result) print(f"number of ids is {len(_result)}: starting id is {_result[0]} and ending id is {_result[-1]}") for i,_id in enumerate(_result): From 115196aea256483af4050250aab86d863b877cbb Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Thu, 28 Dec 2023 12:13:55 -0700 Subject: [PATCH 05/10] formatting --- src/vxingest/netcdf_to_cb/netcdf_builder.py | 49 +++++++++++----- .../utilities/backfill_obs_with_rh.py | 57 ++++++++++++------- 2 files changed, 73 insertions(+), 33 deletions(-) diff --git a/src/vxingest/netcdf_to_cb/netcdf_builder.py b/src/vxingest/netcdf_to_cb/netcdf_builder.py index a31e4b1e..b798769d 100644 --- a/src/vxingest/netcdf_to_cb/netcdf_builder.py +++ b/src/vxingest/netcdf_to_cb/netcdf_builder.py @@ -363,7 +363,7 @@ def build_document(self, queue_element): if self.do_profiling: with cProfile.Profile() as _pr: self.handle_document() - with open("profiling_stats.txt", "w", encoding="utf-8" ) as stream: + with open("profiling_stats.txt", "w", encoding="utf-8") as stream: stats = Stats(_pr, stream=stream) stats.strip_dirs() stats.sort_stats("time") @@ -650,11 +650,22 @@ def handle_rh(self, params_dict): try: # dewpoint = self.umask_value_transform({"recNum": params_dict["recNum"], "dewpoint":"dewpoint"}) # temperature = self.umask_value_transform({"recNum": params_dict["recNum"], "temperature":"temperature"}) - dewpoint = self.umask_value_transform({"recNum": params_dict["recNum"], "dewpoint":params_dict["dewpoint"]}) - temperature = self.umask_value_transform({"recNum": params_dict["recNum"], "temperature":params_dict["temperature"]}) - _q = (relative_humidity_from_dewpoint(temperature * units.kelvin, dewpoint * units.kelvin).magnitude) * 100 + dewpoint = self.umask_value_transform( + {"recNum": params_dict["recNum"], "dewpoint": params_dict["dewpoint"]} + ) + temperature = self.umask_value_transform( + { + "recNum": params_dict["recNum"], + "temperature": params_dict["temperature"], + } + ) + _q = ( + relative_humidity_from_dewpoint( + temperature * units.kelvin, dewpoint * units.kelvin + ).magnitude + ) * 100 return _q - except Exception as _e: # pylint:disable=broad-except + except Exception as _e: # pylint:disable=broad-except # there must not have been one return None @@ -668,16 +679,22 @@ def handle_wind_dir_u(self, params_dict): float: the wind direction """ try: - _wind_dir = self.umask_value_transform({"recNum": params_dict["recNum"], "windDir":params_dict["windDir"]}) - _wind_speed = self.umask_value_transform({"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]}) + _wind_dir = self.umask_value_transform( + {"recNum": params_dict["recNum"], "windDir": params_dict["windDir"]} + ) + _wind_speed = self.umask_value_transform( + {"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]} + ) if _wind_dir is None: return None if _wind_speed is None: return None # wind speed is in meters per second and windDir is in degrees from netcdf file - u = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[0].magnitude + u = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[ + 0 + ].magnitude return u - except Exception as _e: # pylint:disable=broad-exception-caught + except Exception as _e: # pylint:disable=broad-exception-caught logger.error( "%s handle_wind_dir_v: Exception in named function: error: %s", self.__class__.__name__, @@ -694,16 +711,22 @@ def handle_wind_dir_v(self, params_dict): float: the wind direction """ try: - _wind_dir = self.umask_value_transform({"recNum": params_dict["recNum"], "windDir":params_dict["windDir"]}) - _wind_speed = self.umask_value_transform({"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]}) + _wind_dir = self.umask_value_transform( + {"recNum": params_dict["recNum"], "windDir": params_dict["windDir"]} + ) + _wind_speed = self.umask_value_transform( + {"recNum": params_dict["recNum"], "windSpeed": params_dict["windSpeed"]} + ) if _wind_dir is None: return None if _wind_speed is None: return None # wind speed is in meters per second and windDir is in degrees from netcdf file - v = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[1].magnitude + v = wind_components(_wind_speed * units("m/s"), _wind_dir * units.deg)[ + 1 + ].magnitude return v - except Exception as _e: # pylint:disable=broad-exception-caught + except Exception as _e: # pylint:disable=broad-exception-caught logger.error( "%s handle_wind_dir_v: Exception in named function: error: %s", self.__class__.__name__, diff --git a/src/vxingest/utilities/backfill_obs_with_rh.py b/src/vxingest/utilities/backfill_obs_with_rh.py index a1f61f89..8f56b2fb 100755 --- a/src/vxingest/utilities/backfill_obs_with_rh.py +++ b/src/vxingest/utilities/backfill_obs_with_rh.py @@ -39,55 +39,72 @@ def setup_connection(): ) options = ClusterOptions(PasswordAuthenticator(_user, _password)) connection = {} - connection['cluster'] = Cluster("couchbase://" + _host, options) - connection['bucket'] = connection['cluster'].bucket(_bucket) - connection['scope'] = connection['bucket'].scope(_scope) - connection['collection'] = connection['scope'].collection(_collection) + connection["cluster"] = Cluster("couchbase://" + _host, options) + connection["bucket"] = connection["cluster"].bucket(_bucket) + connection["scope"] = connection["bucket"].scope(_scope) + connection["collection"] = connection["scope"].collection(_collection) return connection except Exception as _e: # pylint:disable=broad-except - print (f"test_credentials_and_load_spec Exception failure: {_e}") + print(f"test_credentials_and_load_spec Exception failure: {_e}") + def calc_components(doc): - """Calculate RH, WindU, and WindV from Temperature, DewPoint, WS, and WD. - """ + """Calculate RH, WindU, and WindV from Temperature, DewPoint, WS, and WD.""" # doc = {"data":{'station_name': {'Temperature'} {'DewPoint'} {'WS'} {'WD'} ... } - for _name, station in doc['data'].items(): + for _name, station in doc["data"].items(): if "RH" not in station: if station["Temperature"] is not None and station["DewPoint"] is not None: - station["RH"] = relative_humidity_from_dewpoint(station["Temperature"] * units.degC, station["DewPoint"] * units.degC).magnitude * 100 + station["RH"] = ( + relative_humidity_from_dewpoint( + station["Temperature"] * units.degC, + station["DewPoint"] * units.degC, + ).magnitude + * 100 + ) else: station["RH"] = None if "WindU" not in station or "WindV" not in station.keys(): if station["WS"] is not None and station["WD"] is not None: - _u, _v = wind_components(station["WS"] * units("m/s"), station["WD"] * units.deg) + _u, _v = wind_components( + station["WS"] * units("m/s"), station["WD"] * units.deg + ) station["WindU"] = _u.magnitude station["WindV"] = _v.magnitude else: station["WindU"] = None station["WindV"] = None + def run_backfill() -> None: """entrypoint""" connection = setup_connection() - _query_result = connection['cluster'].query("select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR' AND id > 'DD:V01:METAR:obs:1699966800';") + _query_result = connection["cluster"].query( + "select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR' AND id > 'DD:V01:METAR:obs:1699966800';" + ) _result = list(_query_result) - print(f"number of ids is {len(_result)}: starting id is {_result[0]} and ending id is {_result[-1]}") - for i,_id in enumerate(_result): - print (f"processing #{i} with id {_id}") + print( + f"number of ids is {len(_result)}: starting id is {_result[0]} and ending id is {_result[-1]}" + ) + for i, _id in enumerate(_result): + print(f"processing #{i} with id {_id}") ri = 0 for ri in range(5): try: - doc = connection['collection'].get(_id).content_as[dict] + doc = connection["collection"].get(_id).content_as[dict] calc_components(doc) - connection['collection'].upsert(_id, doc) + connection["collection"].upsert(_id, doc) break except TimeoutException as _e: - print (f"TimeoutException failure: {_e} retrying id {_id} #{ri}") + print(f"TimeoutException failure: {_e} retrying id {_id} #{ri}") if ri == 5: - raise RuntimeWarning (f"Failed to process id {_id} - too many retries") from _e - time.sleep(1) # give it a second to breathe + raise RuntimeWarning( + f"Failed to process id {_id} - too many retries" + ) from _e + time.sleep(1) # give it a second to breathe except Exception as _e1: # pylint:disable=broad-except - print (f"Exception failure: {_e1}") + print(f"Exception failure: {_e1}") break + + if __name__ == "__main__": run_backfill() From a3b18fb613f7e6565314618bc423bc5273be0cbc Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Thu, 28 Dec 2023 12:17:52 -0700 Subject: [PATCH 06/10] formatting --- .../grib2_to_cb/test_int_metar_model_grib.py | 4 +- .../netcdf_to_cb/test_int_metar_obs_netcdf.py | 28 ++-- .../test_int_metar_partial_sums.py | 6 +- .../utilities/test_backfill_obs_with_rh.py | 142 ++++++++++++------ 4 files changed, 119 insertions(+), 61 deletions(-) diff --git a/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py b/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py index 661f474e..231a7241 100644 --- a/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py +++ b/tests/vxingest/grib2_to_cb/test_int_metar_model_grib.py @@ -125,7 +125,9 @@ def test_grib_builder_one_thread_file_pattern_hrrr_ops_conus(tmp_path): _statement = f"select METAR.* from `{connect_cb()['bucket']}`._default.METAR where meta().id = '{_id}'" qresult = connect_cb()["cluster"].query(_statement) res_rows = list(qresult.rows()) - assert (len(res_rows) > 0 ), f"TestGribBuilderV01.test_gribBuilder_one_epoch_hrrr_ops_conus failure test document {_id} not found in couchbase" + assert ( + len(res_rows) > 0 + ), f"TestGribBuilderV01.test_gribBuilder_one_epoch_hrrr_ops_conus failure test document {_id} not found in couchbase" result = list(qresult.rows())[0] # assert top level fields keys = _json.keys() diff --git a/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py b/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py index 0341b330..f532e47a 100644 --- a/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py +++ b/tests/vxingest/netcdf_to_cb/test_int_metar_obs_netcdf.py @@ -22,9 +22,9 @@ from vxingest.netcdf_to_cb.run_ingest_threads import VXIngest -def stub_worker_log_configurer(queue: Queue): # pylint:disable=unused-argument +def stub_worker_log_configurer(queue: Queue): # pylint:disable=unused-argument """A stub to replace log_config.worker_log_configurer""" - pass # pylint:disable=unnecessary-pass + pass # pylint:disable=unnecessary-pass def setup_connection(): @@ -42,8 +42,9 @@ def setup_connection(): assert False, f"test_credentials_and_load_spec Exception failure: {_e}" return None + def ordered(obj): - """ Utliity function to sort a dictionary so that it can be compared to another dictionary""" + """Utliity function to sort a dictionary so that it can be compared to another dictionary""" if isinstance(obj, dict): return sorted((k, ordered(v)) for k, v in obj.items()) if isinstance(obj, list): @@ -51,6 +52,7 @@ def ordered(obj): else: return obj + def test_one_thread_specify_file_pattern(tmp_path): # pylint:disable=missing-function-docstring try: log_queue = Queue() @@ -84,7 +86,9 @@ def test_one_thread_specify_file_pattern(tmp_path): # pylint:disable=missing-fu assert len(glob(f"{tmp_path}/20211108*.json")) == len( glob("/opt/data/netcdf_to_cb/input_files/20211108_0000") ), "number of output files is incorrect" - derived_data = json.load(open(f"{tmp_path}/20211108_0000.json", encoding="utf-8")) + derived_data = json.load( + open(f"{tmp_path}/20211108_0000.json", encoding="utf-8") + ) station_id = "" derived_station = {} obs_id = "" @@ -102,12 +106,18 @@ def test_one_thread_specify_file_pattern(tmp_path): # pylint:disable=missing-fu retrieved_station = vx_ingest.collection.get(station_id).content_as[dict] retrieved_obs = vx_ingest.collection.get(obs_id).content_as[dict] # make sure the updateTime is the same in both the derived and retrieved station - retrieved_station['updateTime'] = derived_station['updateTime'] + retrieved_station["updateTime"] = derived_station["updateTime"] # make sure the firstTime and lastTime are the same in both the derived and retrieved station['geo'] - retrieved_station['geo'][0]['firstTime'] = derived_station['geo'][0]['firstTime'] - retrieved_station['geo'][0]['lastTime'] = derived_station['geo'][0]['lastTime'] - assert ordered(derived_station) == ordered(retrieved_station), "derived station does not match retrieved station" - assert ordered(derived_obs) == ordered(retrieved_obs), "derived obs does not match retrieved obs" + retrieved_station["geo"][0]["firstTime"] = derived_station["geo"][0][ + "firstTime" + ] + retrieved_station["geo"][0]["lastTime"] = derived_station["geo"][0]["lastTime"] + assert ordered(derived_station) == ordered( + retrieved_station + ), "derived station does not match retrieved station" + assert ordered(derived_obs) == ordered( + retrieved_obs + ), "derived obs does not match retrieved obs" except Exception as _e: # pylint: disable=broad-except assert False, f"TestGsdIngestManager Exception failure: {_e}" diff --git a/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py b/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py index 79c98f0e..6f07e13c 100644 --- a/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py +++ b/tests/vxingest/partial_sums_to_cb/test_int_metar_partial_sums.py @@ -31,9 +31,11 @@ mysql_model_obs_data = [] stations = [] -def stub_worker_log_configurer(queue: Queue): # pylint: disable=unused-argument + +def stub_worker_log_configurer(queue: Queue): # pylint: disable=unused-argument """A stub to replace log_config.worker_log_configurer""" - pass # pylint: disable=unnecessary-pass + pass # pylint: disable=unnecessary-pass + def test_check_fcst_valid_epoch_fcst_valid_iso(): """ diff --git a/tests/vxingest/utilities/test_backfill_obs_with_rh.py b/tests/vxingest/utilities/test_backfill_obs_with_rh.py index 47200716..73703b9c 100644 --- a/tests/vxingest/utilities/test_backfill_obs_with_rh.py +++ b/tests/vxingest/utilities/test_backfill_obs_with_rh.py @@ -7,76 +7,120 @@ def test_calc_components_backfills_rh(): """test calculating missing RH""" doc = { - "data":{ - 'NZCM':{"Temperature": 25, "DewPoint": 20, "WS": 5, "WD": 180, "WindU":-6.123233995736766e-16 , "WindV": 5}, - 'SUMU':{"Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, "WindU": 10, "WindV": 1.8369701987210296e-15}, + "data": { + "NZCM": { + "Temperature": 25, + "DewPoint": 20, + "WS": 5, + "WD": 180, + "WindU": -6.123233995736766e-16, + "WindV": 5, + }, + "SUMU": { + "Temperature": 30, + "DewPoint": 15, + "WS": 10, + "WD": 270, + "WindU": 10, + "WindV": 1.8369701987210296e-15, + }, } } for entries in doc["data"]: assert "RH" not in entries calc_components(doc) - assert doc["data"]['NZCM']["RH"] == 73.78055835142725 - assert doc["data"]['NZCM']["WindU"] == -6.123233995736766e-16 - assert doc["data"]['NZCM']["WindV"] == 5.0 - assert doc["data"]['NZCM']["Temperature"] == 25 - assert doc["data"]['NZCM']["DewPoint"] == 20 - assert doc["data"]['NZCM']["WS"] == 5 - assert doc["data"]['NZCM']["WD"] == 180 + assert doc["data"]["NZCM"]["RH"] == 73.78055835142725 + assert doc["data"]["NZCM"]["WindU"] == -6.123233995736766e-16 + assert doc["data"]["NZCM"]["WindV"] == 5.0 + assert doc["data"]["NZCM"]["Temperature"] == 25 + assert doc["data"]["NZCM"]["DewPoint"] == 20 + assert doc["data"]["NZCM"]["WS"] == 5 + assert doc["data"]["NZCM"]["WD"] == 180 + + assert doc["data"]["SUMU"]["RH"] == 40.1370668529669 + assert doc["data"]["SUMU"]["WindU"] == 10.0 + assert doc["data"]["SUMU"]["WindV"] == 1.8369701987210296e-15 + assert doc["data"]["SUMU"]["Temperature"] == 30 + assert doc["data"]["SUMU"]["DewPoint"] == 15 + assert doc["data"]["SUMU"]["WS"] == 10 + assert doc["data"]["SUMU"]["WD"] == 270 - assert doc["data"]['SUMU']["RH"] == 40.1370668529669 - assert doc["data"]['SUMU']["WindU"] == 10.0 - assert doc["data"]['SUMU']["WindV"] == 1.8369701987210296e-15 - assert doc["data"]['SUMU']["Temperature"] == 30 - assert doc["data"]['SUMU']["DewPoint"] == 15 - assert doc["data"]['SUMU']["WS"] == 10 - assert doc["data"]['SUMU']["WD"] == 270 def test_calc_components_backfills_windu_and_windv(): """test calculating missing WindU and WindV""" doc = { "data": { - "NZCM": {"Temperature": 25, "DewPoint": 20, "WS": 5, "WD": 180, "RH": 73.78055835142725}, - "SUMU": {"Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, "RH": 40.1370668529669}, + "NZCM": { + "Temperature": 25, + "DewPoint": 20, + "WS": 5, + "WD": 180, + "RH": 73.78055835142725, + }, + "SUMU": { + "Temperature": 30, + "DewPoint": 15, + "WS": 10, + "WD": 270, + "RH": 40.1370668529669, + }, } } calc_components(doc) - assert doc["data"]['NZCM']["RH"] == 73.78055835142725, "RH wrong value" - assert doc["data"]['NZCM']["WindU"] == -6.123233995736766e-16, "WindU wrong value" - assert doc["data"]['NZCM']["WindV"] == 5.0, "WindV wrong value" - assert doc["data"]['NZCM']["Temperature"] == 25, "temperature wrong value" - assert doc["data"]['NZCM']["DewPoint"] == 20, "DewPoint wrong value" - assert doc["data"]['NZCM']["WS"] == 5, "WS wrong value" - assert doc["data"]['NZCM']["WD"] == 180, "WD wrong value" + assert doc["data"]["NZCM"]["RH"] == 73.78055835142725, "RH wrong value" + assert doc["data"]["NZCM"]["WindU"] == -6.123233995736766e-16, "WindU wrong value" + assert doc["data"]["NZCM"]["WindV"] == 5.0, "WindV wrong value" + assert doc["data"]["NZCM"]["Temperature"] == 25, "temperature wrong value" + assert doc["data"]["NZCM"]["DewPoint"] == 20, "DewPoint wrong value" + assert doc["data"]["NZCM"]["WS"] == 5, "WS wrong value" + assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" + + assert doc["data"]["SUMU"]["RH"] == 40.1370668529669, "RH wrong value" + assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" + assert doc["data"]["SUMU"]["WindV"] == 1.8369701987210296e-15, "WindV wrong value" + assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" + assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" + assert doc["data"]["SUMU"]["WS"] == 10, "WS wrong value" + assert doc["data"]["SUMU"]["WD"] == 270, "WD wrong value" - assert doc["data"]['SUMU']["RH"] == 40.1370668529669, "RH wrong value" - assert doc["data"]['SUMU']["WindU"] == 10.0, "WindU wrong value" - assert doc["data"]['SUMU']["WindV"] == 1.8369701987210296e-15, "WindV wrong value" - assert doc["data"]['SUMU']["Temperature"] == 30, "temperature wrong value" - assert doc["data"]['SUMU']["DewPoint"] == 15, "DewPoint wrong value" - assert doc["data"]['SUMU']["WS"] == 10, "WS wrong value" - assert doc["data"]['SUMU']["WD"] == 270, "WD wrong value" def test_calc_components_backfills_nochange(): """test no change if RH, WindU, and WindV are already present""" doc = { "data": { - "NZCM": {"Temperature": 25, "DewPoint": 20, "WS": 5, "WD": 180, "WindU": 5, "WindV": -5, "RH": 75}, - "SUMU": {"Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, "WindU": 3, "WindV": -7, "RH": 70}, + "NZCM": { + "Temperature": 25, + "DewPoint": 20, + "WS": 5, + "WD": 180, + "WindU": 5, + "WindV": -5, + "RH": 75, + }, + "SUMU": { + "Temperature": 30, + "DewPoint": 15, + "WS": 10, + "WD": 270, + "WindU": 3, + "WindV": -7, + "RH": 70, + }, } } calc_components(doc) - assert doc["data"]['NZCM']["Temperature"] == 25, "temperature wrong value" - assert doc["data"]['NZCM']["DewPoint"] == 20, "DewPoint wrong value" - assert doc["data"]['NZCM']["WS"] == 5, "WS wrong value" - assert doc["data"]['NZCM']["WD"] == 180, "WD wrong value" - assert doc["data"]['NZCM']["WindU"] == 5, "WindU wrong value" - assert doc["data"]['NZCM']["WindV"] == -5, "WindV wrong value" - assert doc["data"]['NZCM']["RH"] == 75, "RH wrong value" + assert doc["data"]["NZCM"]["Temperature"] == 25, "temperature wrong value" + assert doc["data"]["NZCM"]["DewPoint"] == 20, "DewPoint wrong value" + assert doc["data"]["NZCM"]["WS"] == 5, "WS wrong value" + assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" + assert doc["data"]["NZCM"]["WindU"] == 5, "WindU wrong value" + assert doc["data"]["NZCM"]["WindV"] == -5, "WindV wrong value" + assert doc["data"]["NZCM"]["RH"] == 75, "RH wrong value" - assert doc["data"]['SUMU']["Temperature"] == 30, "temperature wrong value" - assert doc["data"]['SUMU']["DewPoint"] == 15, "DewPoint wrong value" - assert doc["data"]['SUMU']["WS"] == 10, "WS wrong value" - assert doc["data"]['SUMU']["WD"] == 270, "WD wrong value" - assert doc["data"]['SUMU']["WindU"] == 3, "WindU wrong value" - assert doc["data"]['SUMU']["WindV"] == -7, "WindV wrong value" - assert doc["data"]['SUMU']["RH"] == 70, "RH wrong value" + assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" + assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" + assert doc["data"]["SUMU"]["WS"] == 10, "WS wrong value" + assert doc["data"]["SUMU"]["WD"] == 270, "WD wrong value" + assert doc["data"]["SUMU"]["WindU"] == 3, "WindU wrong value" + assert doc["data"]["SUMU"]["WindV"] == -7, "WindV wrong value" + assert doc["data"]["SUMU"]["RH"] == 70, "RH wrong value" From 24c80dcd5e0055c67d600695d8a5190bcbaa2c9d Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Thu, 28 Dec 2023 14:34:22 -0700 Subject: [PATCH 07/10] corrections per review --- poetry.lock | 543 +----------------- pyproject.toml | 1 - src/vxingest/netcdf_to_cb/netcdf_builder.py | 9 +- .../utilities/test_backfill_obs_with_rh.py | 38 +- 4 files changed, 24 insertions(+), 567 deletions(-) diff --git a/poetry.lock b/poetry.lock index ea621794..e5a09979 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,34 +1,5 @@ # This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. -[[package]] -name = "appnope" -version = "0.1.3" -description = "Disable App Nap on macOS >= 10.9" -optional = false -python-versions = "*" -files = [ - {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, - {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, -] - -[[package]] -name = "asttokens" -version = "2.4.1" -description = "Annotate AST trees with source code positions" -optional = false -python-versions = "*" -files = [ - {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, - {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, -] - -[package.dependencies] -six = ">=1.12.0" - -[package.extras] -astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] -test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] - [[package]] name = "attrs" version = "23.1.0" @@ -303,23 +274,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "comm" -version = "0.2.0" -description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." -optional = false -python-versions = ">=3.8" -files = [ - {file = "comm-0.2.0-py3-none-any.whl", hash = "sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001"}, - {file = "comm-0.2.0.tar.gz", hash = "sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be"}, -] - -[package.dependencies] -traitlets = ">=4" - -[package.extras] -test = ["pytest"] - [[package]] name = "contourpy" version = "1.2.0" @@ -432,44 +386,6 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] -[[package]] -name = "debugpy" -version = "1.8.0" -description = "An implementation of the Debug Adapter Protocol for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "debugpy-1.8.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:7fb95ca78f7ac43393cd0e0f2b6deda438ec7c5e47fa5d38553340897d2fbdfb"}, - {file = "debugpy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada"}, - {file = "debugpy-1.8.0-cp310-cp310-win32.whl", hash = "sha256:a8b7a2fd27cd9f3553ac112f356ad4ca93338feadd8910277aff71ab24d8775f"}, - {file = "debugpy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:5d9de202f5d42e62f932507ee8b21e30d49aae7e46d5b1dd5c908db1d7068637"}, - {file = "debugpy-1.8.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e"}, - {file = "debugpy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60009b132c91951354f54363f8ebdf7457aeb150e84abba5ae251b8e9f29a8a6"}, - {file = "debugpy-1.8.0-cp311-cp311-win32.whl", hash = "sha256:8cd0197141eb9e8a4566794550cfdcdb8b3db0818bdf8c49a8e8f8053e56e38b"}, - {file = "debugpy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:a64093656c4c64dc6a438e11d59369875d200bd5abb8f9b26c1f5f723622e153"}, - {file = "debugpy-1.8.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:b05a6b503ed520ad58c8dc682749113d2fd9f41ffd45daec16e558ca884008cd"}, - {file = "debugpy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c6fb41c98ec51dd010d7ed650accfd07a87fe5e93eca9d5f584d0578f28f35f"}, - {file = "debugpy-1.8.0-cp38-cp38-win32.whl", hash = "sha256:46ab6780159eeabb43c1495d9c84cf85d62975e48b6ec21ee10c95767c0590aa"}, - {file = "debugpy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:bdc5ef99d14b9c0fcb35351b4fbfc06ac0ee576aeab6b2511702e5a648a2e595"}, - {file = "debugpy-1.8.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:61eab4a4c8b6125d41a34bad4e5fe3d2cc145caecd63c3fe953be4cc53e65bf8"}, - {file = "debugpy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:125b9a637e013f9faac0a3d6a82bd17c8b5d2c875fb6b7e2772c5aba6d082332"}, - {file = "debugpy-1.8.0-cp39-cp39-win32.whl", hash = "sha256:57161629133113c97b387382045649a2b985a348f0c9366e22217c87b68b73c6"}, - {file = "debugpy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:e3412f9faa9ade82aa64a50b602544efcba848c91384e9f93497a458767e6926"}, - {file = "debugpy-1.8.0-py2.py3-none-any.whl", hash = "sha256:9c9b0ac1ce2a42888199df1a1906e45e6f3c9555497643a85e0bf2406e3ffbc4"}, - {file = "debugpy-1.8.0.zip", hash = "sha256:12af2c55b419521e33d5fb21bd022df0b5eb267c3e178f1d374a63a2a6bdccd0"}, -] - -[[package]] -name = "decorator" -version = "5.1.1" -description = "Decorators for Humans" -optional = false -python-versions = ">=3.5" -files = [ - {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, - {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, -] - [[package]] name = "eccodes" version = "1.6.1" @@ -486,20 +402,6 @@ cffi = "*" findlibs = "*" numpy = "*" -[[package]] -name = "executing" -version = "2.0.1" -description = "Get the currently executing AST node of a frame, and other information" -optional = false -python-versions = ">=3.5" -files = [ - {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, - {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, -] - -[package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] - [[package]] name = "findlibs" version = "0.0.5" @@ -597,135 +499,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "ipykernel" -version = "6.27.1" -description = "IPython Kernel for Jupyter" -optional = false -python-versions = ">=3.8" -files = [ - {file = "ipykernel-6.27.1-py3-none-any.whl", hash = "sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686"}, - {file = "ipykernel-6.27.1.tar.gz", hash = "sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6"}, -] - -[package.dependencies] -appnope = {version = "*", markers = "platform_system == \"Darwin\""} -comm = ">=0.1.1" -debugpy = ">=1.6.5" -ipython = ">=7.23.1" -jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" -matplotlib-inline = ">=0.1" -nest-asyncio = "*" -packaging = "*" -psutil = "*" -pyzmq = ">=20" -tornado = ">=6.1" -traitlets = ">=5.4.0" - -[package.extras] -cov = ["coverage[toml]", "curio", "matplotlib", "pytest-cov", "trio"] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "trio"] -pyqt5 = ["pyqt5"] -pyside6 = ["pyside6"] -test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "ipython" -version = "8.19.0" -description = "IPython: Productive Interactive Computing" -optional = false -python-versions = ">=3.10" -files = [ - {file = "ipython-8.19.0-py3-none-any.whl", hash = "sha256:2f55d59370f59d0d2b2212109fe0e6035cfea436b1c0e6150ad2244746272ec5"}, - {file = "ipython-8.19.0.tar.gz", hash = "sha256:ac4da4ecf0042fb4e0ce57c60430c2db3c719fa8bdf92f8631d6bd8a5785d1f0"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -decorator = "*" -jedi = ">=0.16" -matplotlib-inline = "*" -pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} -prompt-toolkit = ">=3.0.41,<3.1.0" -pygments = ">=2.4.0" -stack-data = "*" -traitlets = ">=5" - -[package.extras] -all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.23)", "pandas", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] -black = ["black"] -doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] -kernel = ["ipykernel"] -nbconvert = ["nbconvert"] -nbformat = ["nbformat"] -notebook = ["ipywidgets", "notebook"] -parallel = ["ipyparallel"] -qtconsole = ["qtconsole"] -test = ["pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath"] -test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "pickleshare", "pytest", "pytest-asyncio (<0.22)", "testpath", "trio"] - -[[package]] -name = "jedi" -version = "0.19.1" -description = "An autocompletion tool for Python that can be used for text editors." -optional = false -python-versions = ">=3.6" -files = [ - {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, - {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, -] - -[package.dependencies] -parso = ">=0.8.3,<0.9.0" - -[package.extras] -docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] -qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] - -[[package]] -name = "jupyter-client" -version = "8.6.0" -description = "Jupyter protocol implementation and client libraries" -optional = false -python-versions = ">=3.8" -files = [ - {file = "jupyter_client-8.6.0-py3-none-any.whl", hash = "sha256:909c474dbe62582ae62b758bca86d6518c85234bdee2d908c778db6d72f39d99"}, - {file = "jupyter_client-8.6.0.tar.gz", hash = "sha256:0642244bb83b4764ae60d07e010e15f0e2d275ec4e918a8f7b80fbbef3ca60c7"}, -] - -[package.dependencies] -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" -python-dateutil = ">=2.8.2" -pyzmq = ">=23.0" -tornado = ">=6.2" -traitlets = ">=5.3" - -[package.extras] -docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] - -[[package]] -name = "jupyter-core" -version = "5.5.1" -description = "Jupyter core package. A base package on which Jupyter projects rely." -optional = false -python-versions = ">=3.8" -files = [ - {file = "jupyter_core-5.5.1-py3-none-any.whl", hash = "sha256:220dfb00c45f0d780ce132bb7976b58263f81a3ada6e90a9b6823785a424f739"}, - {file = "jupyter_core-5.5.1.tar.gz", hash = "sha256:1553311a97ccd12936037f36b9ab4d6ae8ceea6ad2d5c90d94a909e752178e40"}, -] - -[package.dependencies] -platformdirs = ">=2.5" -pywin32 = {version = ">=300", markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} -traitlets = ">=5.3" - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] -test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] - [[package]] name = "kiwisolver" version = "1.4.5" @@ -887,20 +660,6 @@ pillow = ">=8" pyparsing = ">=2.3.1" python-dateutil = ">=2.7" -[[package]] -name = "matplotlib-inline" -version = "0.1.6" -description = "Inline Matplotlib backend for Jupyter" -optional = false -python-versions = ">=3.5" -files = [ - {file = "matplotlib-inline-0.1.6.tar.gz", hash = "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304"}, - {file = "matplotlib_inline-0.1.6-py3-none-any.whl", hash = "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311"}, -] - -[package.dependencies] -traitlets = "*" - [[package]] name = "metpy" version = "1.5.1" @@ -928,17 +687,6 @@ doc = ["myst-parser", "netCDF4", "sphinx", "sphinx-gallery (>=0.4)"] examples = ["cartopy (>=0.17.0)", "geopandas (>=0.6.0)", "matplotlib (>=3.3.0)", "shapely (>=1.6.0)"] test = ["cartopy (>=0.17.0)", "netCDF4", "packaging (>=21.0)", "pytest (>=2.4)", "pytest-mpl", "shapely (>=1.6.0)"] -[[package]] -name = "nest-asyncio" -version = "1.5.8" -description = "Patch asyncio to allow nested event loops" -optional = false -python-versions = ">=3.5" -files = [ - {file = "nest_asyncio-1.5.8-py3-none-any.whl", hash = "sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d"}, - {file = "nest_asyncio-1.5.8.tar.gz", hash = "sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb"}, -] - [[package]] name = "netcdf4" version = "1.6.5" @@ -1106,35 +854,6 @@ sql-other = ["SQLAlchemy (>=1.4.36)"] test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] -[[package]] -name = "parso" -version = "0.8.3" -description = "A Python Parser" -optional = false -python-versions = ">=3.6" -files = [ - {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, - {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, -] - -[package.extras] -qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["docopt", "pytest (<6.0.0)"] - -[[package]] -name = "pexpect" -version = "4.9.0" -description = "Pexpect allows easy control of interactive console applications." -optional = false -python-versions = "*" -files = [ - {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, - {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, -] - -[package.dependencies] -ptyprocess = ">=0.5" - [[package]] name = "pillow" version = "10.1.0" @@ -1291,73 +1010,6 @@ files = [ [package.extras] twisted = ["twisted"] -[[package]] -name = "prompt-toolkit" -version = "3.0.43" -description = "Library for building powerful interactive command lines in Python" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, - {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, -] - -[package.dependencies] -wcwidth = "*" - -[[package]] -name = "psutil" -version = "5.9.7" -description = "Cross-platform lib for process and system monitoring in Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"}, - {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"}, - {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"}, - {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"}, - {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"}, - {file = "psutil-5.9.7-cp27-none-win32.whl", hash = "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"}, - {file = "psutil-5.9.7-cp27-none-win_amd64.whl", hash = "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"}, - {file = "psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"}, - {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"}, - {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"}, - {file = "psutil-5.9.7-cp36-cp36m-win32.whl", hash = "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"}, - {file = "psutil-5.9.7-cp36-cp36m-win_amd64.whl", hash = "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"}, - {file = "psutil-5.9.7-cp37-abi3-win32.whl", hash = "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"}, - {file = "psutil-5.9.7-cp37-abi3-win_amd64.whl", hash = "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"}, - {file = "psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"}, - {file = "psutil-5.9.7.tar.gz", hash = "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"}, -] - -[package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] - -[[package]] -name = "ptyprocess" -version = "0.7.0" -description = "Run a subprocess in a pseudo terminal" -optional = false -python-versions = "*" -files = [ - {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, - {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, -] - -[[package]] -name = "pure-eval" -version = "0.2.2" -description = "Safely evaluate AST nodes without side effects" -optional = false -python-versions = "*" -files = [ - {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, - {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, -] - -[package.extras] -tests = ["pytest"] - [[package]] name = "pycparser" version = "2.21" @@ -1369,21 +1021,6 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -[[package]] -name = "pygments" -version = "2.17.2" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, - {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, -] - -[package.extras] -plugins = ["importlib-metadata"] -windows-terminal = ["colorama (>=0.4.6)"] - [[package]] name = "pyparsing" version = "3.1.1" @@ -1482,29 +1119,6 @@ files = [ {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] -[[package]] -name = "pywin32" -version = "306" -description = "Python for Window Extensions" -optional = false -python-versions = "*" -files = [ - {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, - {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, - {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, - {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, - {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, - {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, - {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, - {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, - {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, - {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, - {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, - {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, - {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, - {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, -] - [[package]] name = "pyyaml" version = "6.0.1" @@ -1564,111 +1178,6 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] -[[package]] -name = "pyzmq" -version = "25.1.2" -description = "Python bindings for 0MQ" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:e624c789359f1a16f83f35e2c705d07663ff2b4d4479bad35621178d8f0f6ea4"}, - {file = "pyzmq-25.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49151b0efece79f6a79d41a461d78535356136ee70084a1c22532fc6383f4ad0"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9a5f194cf730f2b24d6af1f833c14c10f41023da46a7f736f48b6d35061e76e"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:faf79a302f834d9e8304fafdc11d0d042266667ac45209afa57e5efc998e3872"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f51a7b4ead28d3fca8dda53216314a553b0f7a91ee8fc46a72b402a78c3e43d"}, - {file = "pyzmq-25.1.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0ddd6d71d4ef17ba5a87becf7ddf01b371eaba553c603477679ae817a8d84d75"}, - {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:246747b88917e4867e2367b005fc8eefbb4a54b7db363d6c92f89d69abfff4b6"}, - {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:00c48ae2fd81e2a50c3485de1b9d5c7c57cd85dc8ec55683eac16846e57ac979"}, - {file = "pyzmq-25.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a68d491fc20762b630e5db2191dd07ff89834086740f70e978bb2ef2668be08"}, - {file = "pyzmq-25.1.2-cp310-cp310-win32.whl", hash = "sha256:09dfe949e83087da88c4a76767df04b22304a682d6154de2c572625c62ad6886"}, - {file = "pyzmq-25.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:fa99973d2ed20417744fca0073390ad65ce225b546febb0580358e36aa90dba6"}, - {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:82544e0e2d0c1811482d37eef297020a040c32e0687c1f6fc23a75b75db8062c"}, - {file = "pyzmq-25.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:01171fc48542348cd1a360a4b6c3e7d8f46cdcf53a8d40f84db6707a6768acc1"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc69c96735ab501419c432110016329bf0dea8898ce16fab97c6d9106dc0b348"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3e124e6b1dd3dfbeb695435dff0e383256655bb18082e094a8dd1f6293114642"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7598d2ba821caa37a0f9d54c25164a4fa351ce019d64d0b44b45540950458840"}, - {file = "pyzmq-25.1.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d1299d7e964c13607efd148ca1f07dcbf27c3ab9e125d1d0ae1d580a1682399d"}, - {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4e6f689880d5ad87918430957297c975203a082d9a036cc426648fcbedae769b"}, - {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cc69949484171cc961e6ecd4a8911b9ce7a0d1f738fcae717177c231bf77437b"}, - {file = "pyzmq-25.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9880078f683466b7f567b8624bfc16cad65077be046b6e8abb53bed4eeb82dd3"}, - {file = "pyzmq-25.1.2-cp311-cp311-win32.whl", hash = "sha256:4e5837af3e5aaa99a091302df5ee001149baff06ad22b722d34e30df5f0d9097"}, - {file = "pyzmq-25.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:25c2dbb97d38b5ac9fd15586e048ec5eb1e38f3d47fe7d92167b0c77bb3584e9"}, - {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:11e70516688190e9c2db14fcf93c04192b02d457b582a1f6190b154691b4c93a"}, - {file = "pyzmq-25.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:313c3794d650d1fccaaab2df942af9f2c01d6217c846177cfcbc693c7410839e"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b3cbba2f47062b85fe0ef9de5b987612140a9ba3a9c6d2543c6dec9f7c2ab27"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fc31baa0c32a2ca660784d5af3b9487e13b61b3032cb01a115fce6588e1bed30"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c9087b109070c5ab0b383079fa1b5f797f8d43e9a66c07a4b8b8bdecfd88ee"}, - {file = "pyzmq-25.1.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f8429b17cbb746c3e043cb986328da023657e79d5ed258b711c06a70c2ea7537"}, - {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5074adeacede5f810b7ef39607ee59d94e948b4fd954495bdb072f8c54558181"}, - {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7ae8f354b895cbd85212da245f1a5ad8159e7840e37d78b476bb4f4c3f32a9fe"}, - {file = "pyzmq-25.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b264bf2cc96b5bc43ce0e852be995e400376bd87ceb363822e2cb1964fcdc737"}, - {file = "pyzmq-25.1.2-cp312-cp312-win32.whl", hash = "sha256:02bbc1a87b76e04fd780b45e7f695471ae6de747769e540da909173d50ff8e2d"}, - {file = "pyzmq-25.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:ced111c2e81506abd1dc142e6cd7b68dd53747b3b7ae5edbea4578c5eeff96b7"}, - {file = "pyzmq-25.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b6d09a8962a91151f0976008eb7b29b433a560fde056ec7a3db9ec8f1075438"}, - {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967668420f36878a3c9ecb5ab33c9d0ff8d054f9c0233d995a6d25b0e95e1b6b"}, - {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5edac3f57c7ddaacdb4d40f6ef2f9e299471fc38d112f4bc6d60ab9365445fb0"}, - {file = "pyzmq-25.1.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:0dabfb10ef897f3b7e101cacba1437bd3a5032ee667b7ead32bbcdd1a8422fe7"}, - {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2c6441e0398c2baacfe5ba30c937d274cfc2dc5b55e82e3749e333aabffde561"}, - {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:16b726c1f6c2e7625706549f9dbe9b06004dfbec30dbed4bf50cbdfc73e5b32a"}, - {file = "pyzmq-25.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:a86c2dd76ef71a773e70551a07318b8e52379f58dafa7ae1e0a4be78efd1ff16"}, - {file = "pyzmq-25.1.2-cp36-cp36m-win32.whl", hash = "sha256:359f7f74b5d3c65dae137f33eb2bcfa7ad9ebefd1cab85c935f063f1dbb245cc"}, - {file = "pyzmq-25.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:55875492f820d0eb3417b51d96fea549cde77893ae3790fd25491c5754ea2f68"}, - {file = "pyzmq-25.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8c8a419dfb02e91b453615c69568442e897aaf77561ee0064d789705ff37a92"}, - {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8807c87fa893527ae8a524c15fc505d9950d5e856f03dae5921b5e9aa3b8783b"}, - {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5e319ed7d6b8f5fad9b76daa0a68497bc6f129858ad956331a5835785761e003"}, - {file = "pyzmq-25.1.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:3c53687dde4d9d473c587ae80cc328e5b102b517447456184b485587ebd18b62"}, - {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9add2e5b33d2cd765ad96d5eb734a5e795a0755f7fc49aa04f76d7ddda73fd70"}, - {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e690145a8c0c273c28d3b89d6fb32c45e0d9605b2293c10e650265bf5c11cfec"}, - {file = "pyzmq-25.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:00a06faa7165634f0cac1abb27e54d7a0b3b44eb9994530b8ec73cf52e15353b"}, - {file = "pyzmq-25.1.2-cp37-cp37m-win32.whl", hash = "sha256:0f97bc2f1f13cb16905a5f3e1fbdf100e712d841482b2237484360f8bc4cb3d7"}, - {file = "pyzmq-25.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6cc0020b74b2e410287e5942e1e10886ff81ac77789eb20bec13f7ae681f0fdd"}, - {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:bef02cfcbded83473bdd86dd8d3729cd82b2e569b75844fb4ea08fee3c26ae41"}, - {file = "pyzmq-25.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e10a4b5a4b1192d74853cc71a5e9fd022594573926c2a3a4802020360aa719d8"}, - {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8c5f80e578427d4695adac6fdf4370c14a2feafdc8cb35549c219b90652536ae"}, - {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5dde6751e857910c1339890f3524de74007958557593b9e7e8c5f01cd919f8a7"}, - {file = "pyzmq-25.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea1608dd169da230a0ad602d5b1ebd39807ac96cae1845c3ceed39af08a5c6df"}, - {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0f513130c4c361201da9bc69df25a086487250e16b5571ead521b31ff6b02220"}, - {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:019744b99da30330798bb37df33549d59d380c78e516e3bab9c9b84f87a9592f"}, - {file = "pyzmq-25.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2e2713ef44be5d52dd8b8e2023d706bf66cb22072e97fc71b168e01d25192755"}, - {file = "pyzmq-25.1.2-cp38-cp38-win32.whl", hash = "sha256:07cd61a20a535524906595e09344505a9bd46f1da7a07e504b315d41cd42eb07"}, - {file = "pyzmq-25.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb7e49a17fb8c77d3119d41a4523e432eb0c6932187c37deb6fbb00cc3028088"}, - {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:94504ff66f278ab4b7e03e4cba7e7e400cb73bfa9d3d71f58d8972a8dc67e7a6"}, - {file = "pyzmq-25.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6dd0d50bbf9dca1d0bdea219ae6b40f713a3fb477c06ca3714f208fd69e16fd8"}, - {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:004ff469d21e86f0ef0369717351073e0e577428e514c47c8480770d5e24a565"}, - {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c0b5ca88a8928147b7b1e2dfa09f3b6c256bc1135a1338536cbc9ea13d3b7add"}, - {file = "pyzmq-25.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c9a79f1d2495b167119d02be7448bfba57fad2a4207c4f68abc0bab4b92925b"}, - {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:518efd91c3d8ac9f9b4f7dd0e2b7b8bf1a4fe82a308009016b07eaa48681af82"}, - {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1ec23bd7b3a893ae676d0e54ad47d18064e6c5ae1fadc2f195143fb27373f7f6"}, - {file = "pyzmq-25.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db36c27baed588a5a8346b971477b718fdc66cf5b80cbfbd914b4d6d355e44e2"}, - {file = "pyzmq-25.1.2-cp39-cp39-win32.whl", hash = "sha256:39b1067f13aba39d794a24761e385e2eddc26295826530a8c7b6c6c341584289"}, - {file = "pyzmq-25.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:8e9f3fabc445d0ce320ea2c59a75fe3ea591fdbdeebec5db6de530dd4b09412e"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a8c1d566344aee826b74e472e16edae0a02e2a044f14f7c24e123002dcff1c05"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:759cfd391a0996345ba94b6a5110fca9c557ad4166d86a6e81ea526c376a01e8"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c61e346ac34b74028ede1c6b4bcecf649d69b707b3ff9dc0fab453821b04d1e"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cb8fc1f8d69b411b8ec0b5f1ffbcaf14c1db95b6bccea21d83610987435f1a4"}, - {file = "pyzmq-25.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3c00c9b7d1ca8165c610437ca0c92e7b5607b2f9076f4eb4b095c85d6e680a1d"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:df0c7a16ebb94452d2909b9a7b3337940e9a87a824c4fc1c7c36bb4404cb0cde"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45999e7f7ed5c390f2e87ece7f6c56bf979fb213550229e711e45ecc7d42ccb8"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ac170e9e048b40c605358667aca3d94e98f604a18c44bdb4c102e67070f3ac9b"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1b604734bec94f05f81b360a272fc824334267426ae9905ff32dc2be433ab96"}, - {file = "pyzmq-25.1.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:a793ac733e3d895d96f865f1806f160696422554e46d30105807fdc9841b9f7d"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0806175f2ae5ad4b835ecd87f5f85583316b69f17e97786f7443baaf54b9bb98"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ef12e259e7bc317c7597d4f6ef59b97b913e162d83b421dd0db3d6410f17a244"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea253b368eb41116011add00f8d5726762320b1bda892f744c91997b65754d73"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b9b1f2ad6498445a941d9a4fee096d387fee436e45cc660e72e768d3d8ee611"}, - {file = "pyzmq-25.1.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8b14c75979ce932c53b79976a395cb2a8cd3aaf14aef75e8c2cb55a330b9b49d"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:889370d5174a741a62566c003ee8ddba4b04c3f09a97b8000092b7ca83ec9c49"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18fff090441a40ffda8a7f4f18f03dc56ae73f148f1832e109f9bffa85df15"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99a6b36f95c98839ad98f8c553d8507644c880cf1e0a57fe5e3a3f3969040882"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4345c9a27f4310afbb9c01750e9461ff33d6fb74cd2456b107525bbeebcb5be3"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3516e0b6224cf6e43e341d56da15fd33bdc37fa0c06af4f029f7d7dfceceabbc"}, - {file = "pyzmq-25.1.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:146b9b1f29ead41255387fb07be56dc29639262c0f7344f570eecdcd8d683314"}, - {file = "pyzmq-25.1.2.tar.gz", hash = "sha256:93f1aa311e8bb912e34f004cf186407a4e90eec4f0ecc0efd26056bf7eda0226"}, -] - -[package.dependencies] -cffi = {version = "*", markers = "implementation_name == \"pypy\""} - [[package]] name = "requests" version = "2.31.0" @@ -1769,45 +1278,6 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -[[package]] -name = "stack-data" -version = "0.6.3" -description = "Extract data from python stack frames and tracebacks for informative displays" -optional = false -python-versions = "*" -files = [ - {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, - {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, -] - -[package.dependencies] -asttokens = ">=2.1.0" -executing = ">=1.2.0" -pure-eval = "*" - -[package.extras] -tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] - -[[package]] -name = "tornado" -version = "6.4" -description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -optional = false -python-versions = ">= 3.8" -files = [ - {file = "tornado-6.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0"}, - {file = "tornado-6.4-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:27787de946a9cffd63ce5814c33f734c627a87072ec7eed71f7fc4417bb16263"}, - {file = "tornado-6.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e"}, - {file = "tornado-6.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e43bc2e5370a6a8e413e1e1cd0c91bedc5bd62a74a532371042a18ef19e10579"}, - {file = "tornado-6.4-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0251554cdd50b4b44362f73ad5ba7126fc5b2c2895cc62b14a1c2d7ea32f212"}, - {file = "tornado-6.4-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"}, - {file = "tornado-6.4-cp38-abi3-musllinux_1_1_i686.whl", hash = "sha256:88b84956273fbd73420e6d4b8d5ccbe913c65d31351b4c004ae362eba06e1f78"}, - {file = "tornado-6.4-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:71ddfc23a0e03ef2df1c1397d859868d158c8276a0603b96cf86892bff58149f"}, - {file = "tornado-6.4-cp38-abi3-win32.whl", hash = "sha256:6f8a6c77900f5ae93d8b4ae1196472d0ccc2775cc1dfdc9e7727889145c45052"}, - {file = "tornado-6.4-cp38-abi3-win_amd64.whl", hash = "sha256:10aeaa8006333433da48dec9fe417877f8bcc21f48dda8d661ae79da357b2a63"}, - {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, -] - [[package]] name = "traitlets" version = "5.14.0" @@ -1872,17 +1342,6 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] -[[package]] -name = "wcwidth" -version = "0.2.12" -description = "Measures the displayed width of unicode strings in a terminal" -optional = false -python-versions = "*" -files = [ - {file = "wcwidth-0.2.12-py2.py3-none-any.whl", hash = "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"}, - {file = "wcwidth-0.2.12.tar.gz", hash = "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02"}, -] - [[package]] name = "xarray" version = "2023.11.0" @@ -1909,4 +1368,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "aa19369fd2254684620726dd1449351be360f28d24aecd49ab461282cc0d846a" +content-hash = "77b941447f5222afe1c9b0912f313831ec5bb0408f69af4e733f837d85294474" diff --git a/pyproject.toml b/pyproject.toml index 32e1f68f..f74818ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ prometheus-client = "^0.19.0" pytest = "^7.4.3" types-pyyaml = "^6.0.12.12" ruff = "^0.1.6" -ipykernel = "^6.27.1" [build-system] requires = ["poetry-core"] diff --git a/src/vxingest/netcdf_to_cb/netcdf_builder.py b/src/vxingest/netcdf_to_cb/netcdf_builder.py index b798769d..45e72734 100644 --- a/src/vxingest/netcdf_to_cb/netcdf_builder.py +++ b/src/vxingest/netcdf_to_cb/netcdf_builder.py @@ -640,8 +640,7 @@ def umask_value_transform(self, params_dict): return None def handle_rh(self, params_dict): - """Retrieves a RH value, if there is one, - and if not derives the RH from the dewpoint and temperature. + """Derives the RH from the dewpoint and temperature. Args: params_dict (dict): named function parameters Returns: @@ -670,8 +669,7 @@ def handle_rh(self, params_dict): return None def handle_wind_dir_u(self, params_dict): - """Retrieves a wind direction U value, if there is one, - and if not derives the U component from the wind direction and speed. + """Derives the U component from the wind direction and speed. expects wind speed and wind direction to be in the params_dict. Args: params_dict (dict): named function parameters @@ -703,8 +701,7 @@ def handle_wind_dir_u(self, params_dict): return None def handle_wind_dir_v(self, params_dict): - """Retrieves a wind direction V value, if there is one, - and if not derives the V component from the wind direction and speed. + """Derives the V component from the wind direction and speed. Args: params_dict (dict): named function parameters Returns: diff --git a/tests/vxingest/utilities/test_backfill_obs_with_rh.py b/tests/vxingest/utilities/test_backfill_obs_with_rh.py index 73703b9c..448930ce 100644 --- a/tests/vxingest/utilities/test_backfill_obs_with_rh.py +++ b/tests/vxingest/utilities/test_backfill_obs_with_rh.py @@ -1,6 +1,8 @@ """ _test for VxIngest backfill_obs_with_rh.py """ +import math + from vxingest.utilities.backfill_obs_with_rh import calc_components @@ -29,21 +31,21 @@ def test_calc_components_backfills_rh(): for entries in doc["data"]: assert "RH" not in entries calc_components(doc) - assert doc["data"]["NZCM"]["RH"] == 73.78055835142725 - assert doc["data"]["NZCM"]["WindU"] == -6.123233995736766e-16 - assert doc["data"]["NZCM"]["WindV"] == 5.0 - assert doc["data"]["NZCM"]["Temperature"] == 25 - assert doc["data"]["NZCM"]["DewPoint"] == 20 - assert doc["data"]["NZCM"]["WS"] == 5 - assert doc["data"]["NZCM"]["WD"] == 180 + assert math.isclose(doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001), "RH wrong value - not within 0.001" + assert math.isclose(doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001), "WindU wrong value - not within 0.001" + assert doc["data"]["NZCM"]["WindV"] == 5.0, "WindV wrong value" + assert doc["data"]["NZCM"]["Temperature"] == 25, "temperature wrong value" + assert doc["data"]["NZCM"]["DewPoint"] == 20, "DewPoint wrong value" + assert doc["data"]["NZCM"]["WS"] == 5, "WS wrong value" + assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" - assert doc["data"]["SUMU"]["RH"] == 40.1370668529669 - assert doc["data"]["SUMU"]["WindU"] == 10.0 - assert doc["data"]["SUMU"]["WindV"] == 1.8369701987210296e-15 - assert doc["data"]["SUMU"]["Temperature"] == 30 - assert doc["data"]["SUMU"]["DewPoint"] == 15 - assert doc["data"]["SUMU"]["WS"] == 10 - assert doc["data"]["SUMU"]["WD"] == 270 + assert math.isclose(doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001), "RH wrong value - not within 0.001" + assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" + assert math.isclose(doc["data"]["SUMU"]["WindV"], 1.8369701987210296e-15, abs_tol=0.001), "WindV wrong value - not within 0.001" + assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" + assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" + assert doc["data"]["SUMU"]["WS"] == 10, "WS wrong value" + assert doc["data"]["SUMU"]["WD"] == 270, "WD wrong value" def test_calc_components_backfills_windu_and_windv(): @@ -67,17 +69,17 @@ def test_calc_components_backfills_windu_and_windv(): } } calc_components(doc) - assert doc["data"]["NZCM"]["RH"] == 73.78055835142725, "RH wrong value" - assert doc["data"]["NZCM"]["WindU"] == -6.123233995736766e-16, "WindU wrong value" + assert math.isclose(doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001), "RH wrong value - not within 0.001" + assert math.isclose(doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001), "WindU wrong value - not within 0.001" assert doc["data"]["NZCM"]["WindV"] == 5.0, "WindV wrong value" assert doc["data"]["NZCM"]["Temperature"] == 25, "temperature wrong value" assert doc["data"]["NZCM"]["DewPoint"] == 20, "DewPoint wrong value" assert doc["data"]["NZCM"]["WS"] == 5, "WS wrong value" assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" - assert doc["data"]["SUMU"]["RH"] == 40.1370668529669, "RH wrong value" + assert math.isclose(doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001), "RH wrong value - not within 0.001" assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" - assert doc["data"]["SUMU"]["WindV"] == 1.8369701987210296e-15, "WindV wrong value" + assert math.isclose(doc["data"]["SUMU"]["WindV"], 1.8369701987210296e-15, abs_tol=0.001), "WindV wrong value - not within 0.001" assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" assert doc["data"]["SUMU"]["WS"] == 10, "WS wrong value" From dae4ca569b667e597814df0a4f501056ac8d9b18 Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Thu, 28 Dec 2023 14:49:46 -0700 Subject: [PATCH 08/10] formatting --- .../utilities/test_backfill_obs_with_rh.py | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/vxingest/utilities/test_backfill_obs_with_rh.py b/tests/vxingest/utilities/test_backfill_obs_with_rh.py index 448930ce..df317bcd 100644 --- a/tests/vxingest/utilities/test_backfill_obs_with_rh.py +++ b/tests/vxingest/utilities/test_backfill_obs_with_rh.py @@ -31,17 +31,25 @@ def test_calc_components_backfills_rh(): for entries in doc["data"]: assert "RH" not in entries calc_components(doc) - assert math.isclose(doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001), "RH wrong value - not within 0.001" - assert math.isclose(doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001), "WindU wrong value - not within 0.001" + assert math.isclose( + doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001 + ), "RH wrong value - not within 0.001" + assert math.isclose( + doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001 + ), "WindU wrong value - not within 0.001" assert doc["data"]["NZCM"]["WindV"] == 5.0, "WindV wrong value" assert doc["data"]["NZCM"]["Temperature"] == 25, "temperature wrong value" assert doc["data"]["NZCM"]["DewPoint"] == 20, "DewPoint wrong value" assert doc["data"]["NZCM"]["WS"] == 5, "WS wrong value" assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" - assert math.isclose(doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001), "RH wrong value - not within 0.001" + assert math.isclose( + doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001 + ), "RH wrong value - not within 0.001" assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" - assert math.isclose(doc["data"]["SUMU"]["WindV"], 1.8369701987210296e-15, abs_tol=0.001), "WindV wrong value - not within 0.001" + assert math.isclose( + doc["data"]["SUMU"]["WindV"], 1.8369701987210296e-15, abs_tol=0.001 + ), "WindV wrong value - not within 0.001" assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" assert doc["data"]["SUMU"]["WS"] == 10, "WS wrong value" @@ -69,17 +77,25 @@ def test_calc_components_backfills_windu_and_windv(): } } calc_components(doc) - assert math.isclose(doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001), "RH wrong value - not within 0.001" - assert math.isclose(doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001), "WindU wrong value - not within 0.001" + assert math.isclose( + doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001 + ), "RH wrong value - not within 0.001" + assert math.isclose( + doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001 + ), "WindU wrong value - not within 0.001" assert doc["data"]["NZCM"]["WindV"] == 5.0, "WindV wrong value" assert doc["data"]["NZCM"]["Temperature"] == 25, "temperature wrong value" assert doc["data"]["NZCM"]["DewPoint"] == 20, "DewPoint wrong value" assert doc["data"]["NZCM"]["WS"] == 5, "WS wrong value" assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" - assert math.isclose(doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001), "RH wrong value - not within 0.001" + assert math.isclose( + doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001 + ), "RH wrong value - not within 0.001" assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" - assert math.isclose(doc["data"]["SUMU"]["WindV"], 1.8369701987210296e-15, abs_tol=0.001), "WindV wrong value - not within 0.001" + assert math.isclose( + doc["data"]["SUMU"]["WindV"], 1.8369701987210296e-15, abs_tol=0.001 + ), "WindV wrong value - not within 0.001" assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" assert doc["data"]["SUMU"]["WS"] == 10, "WS wrong value" From c5b21b942f6a6b9c591bf06918914226abc3aa04 Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Fri, 29 Dec 2023 12:47:10 -0700 Subject: [PATCH 09/10] correct RH calculation and add start id parameter --- .../utilities/backfill_obs_with_rh.py | 38 +++++++++++-------- .../utilities/test_backfill_obs_with_rh.py | 16 ++++---- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/vxingest/utilities/backfill_obs_with_rh.py b/src/vxingest/utilities/backfill_obs_with_rh.py index 8f56b2fb..a0343a0c 100755 --- a/src/vxingest/utilities/backfill_obs_with_rh.py +++ b/src/vxingest/utilities/backfill_obs_with_rh.py @@ -3,6 +3,7 @@ """ import os +import sys import time from datetime import timedelta from pathlib import Path @@ -52,17 +53,17 @@ def calc_components(doc): """Calculate RH, WindU, and WindV from Temperature, DewPoint, WS, and WD.""" # doc = {"data":{'station_name': {'Temperature'} {'DewPoint'} {'WS'} {'WD'} ... } for _name, station in doc["data"].items(): - if "RH" not in station: - if station["Temperature"] is not None and station["DewPoint"] is not None: - station["RH"] = ( - relative_humidity_from_dewpoint( - station["Temperature"] * units.degC, - station["DewPoint"] * units.degC, - ).magnitude - * 100 - ) - else: - station["RH"] = None + # always calculate RH to correct incorrect values that were previously calculated with DegC + if station["Temperature"] is not None and station["DewPoint"] is not None: + station["RH"] = ( + relative_humidity_from_dewpoint( + station["Temperature"] * units.degF, + station["DewPoint"] * units.degF, + ).magnitude + * 100 + ) + else: + station["RH"] = None if "WindU" not in station or "WindV" not in station.keys(): if station["WS"] is not None and station["WD"] is not None: _u, _v = wind_components( @@ -75,11 +76,15 @@ def calc_components(doc): station["WindV"] = None -def run_backfill() -> None: +def run_backfill(start_id) -> None: """entrypoint""" + start_clause = "" + if start_id is not None: + print(f"starting id is {start_id}") + start_clause = f"AND meta().id >= '{start_id}'" connection = setup_connection() _query_result = connection["cluster"].query( - "select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR' AND id > 'DD:V01:METAR:obs:1699966800';" + f"select raw meta().id FROM `vxdata`._default.METAR WHERE type='DD' AND docType ='obs' AND version='V01' AND subset='METAR' {start_clause};" ) _result = list(_query_result) print( @@ -88,7 +93,7 @@ def run_backfill() -> None: for i, _id in enumerate(_result): print(f"processing #{i} with id {_id}") ri = 0 - for ri in range(5): + for ri in range(5): # retry up to 5 times try: doc = connection["collection"].get(_id).content_as[dict] calc_components(doc) @@ -107,4 +112,7 @@ def run_backfill() -> None: if __name__ == "__main__": - run_backfill() + STARTID= None + if sys.argv[1]: + STARTID = sys.argv[1] + run_backfill(STARTID) diff --git a/tests/vxingest/utilities/test_backfill_obs_with_rh.py b/tests/vxingest/utilities/test_backfill_obs_with_rh.py index df317bcd..95f7da94 100644 --- a/tests/vxingest/utilities/test_backfill_obs_with_rh.py +++ b/tests/vxingest/utilities/test_backfill_obs_with_rh.py @@ -32,7 +32,7 @@ def test_calc_components_backfills_rh(): assert "RH" not in entries calc_components(doc) assert math.isclose( - doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001 + doc["data"]["NZCM"]["RH"], 81.008, abs_tol=0.001 ), "RH wrong value - not within 0.001" assert math.isclose( doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001 @@ -44,7 +44,7 @@ def test_calc_components_backfills_rh(): assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" assert math.isclose( - doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001 + doc["data"]["SUMU"]["RH"], 53.152, abs_tol=0.001 ), "RH wrong value - not within 0.001" assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" assert math.isclose( @@ -65,20 +65,20 @@ def test_calc_components_backfills_windu_and_windv(): "DewPoint": 20, "WS": 5, "WD": 180, - "RH": 73.78055835142725, + "RH": 81.008, }, "SUMU": { "Temperature": 30, "DewPoint": 15, "WS": 10, "WD": 270, - "RH": 40.1370668529669, + "RH": 53.152, }, } } calc_components(doc) assert math.isclose( - doc["data"]["NZCM"]["RH"], 73.78055835142725, abs_tol=0.001 + doc["data"]["NZCM"]["RH"], 81.008, abs_tol=0.001 ), "RH wrong value - not within 0.001" assert math.isclose( doc["data"]["NZCM"]["WindU"], -6.123233995736766e-16, abs_tol=0.001 @@ -90,7 +90,7 @@ def test_calc_components_backfills_windu_and_windv(): assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" assert math.isclose( - doc["data"]["SUMU"]["RH"], 40.1370668529669, abs_tol=0.001 + doc["data"]["SUMU"]["RH"], 53.152, abs_tol=0.001 ), "RH wrong value - not within 0.001" assert doc["data"]["SUMU"]["WindU"] == 10.0, "WindU wrong value" assert math.isclose( @@ -133,7 +133,7 @@ def test_calc_components_backfills_nochange(): assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" assert doc["data"]["NZCM"]["WindU"] == 5, "WindU wrong value" assert doc["data"]["NZCM"]["WindV"] == -5, "WindV wrong value" - assert doc["data"]["NZCM"]["RH"] == 75, "RH wrong value" + assert math.isclose(doc["data"]["NZCM"]["RH"], 81.008, abs_tol=0.001), "RH wrong value" assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" @@ -141,4 +141,4 @@ def test_calc_components_backfills_nochange(): assert doc["data"]["SUMU"]["WD"] == 270, "WD wrong value" assert doc["data"]["SUMU"]["WindU"] == 3, "WindU wrong value" assert doc["data"]["SUMU"]["WindV"] == -7, "WindV wrong value" - assert doc["data"]["SUMU"]["RH"] == 70, "RH wrong value" + assert math.isclose(doc["data"]["SUMU"]["RH"], 53.152, abs_tol=0.001), "RH wrong value" From 2ea6d7329b9447618638ba38db2e615fac1a0979 Mon Sep 17 00:00:00 2001 From: "randy.pierce" Date: Fri, 29 Dec 2023 13:03:15 -0700 Subject: [PATCH 10/10] formatting --- src/vxingest/utilities/backfill_obs_with_rh.py | 2 +- tests/vxingest/utilities/test_backfill_obs_with_rh.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/vxingest/utilities/backfill_obs_with_rh.py b/src/vxingest/utilities/backfill_obs_with_rh.py index a0343a0c..93b0b1ec 100755 --- a/src/vxingest/utilities/backfill_obs_with_rh.py +++ b/src/vxingest/utilities/backfill_obs_with_rh.py @@ -112,7 +112,7 @@ def run_backfill(start_id) -> None: if __name__ == "__main__": - STARTID= None + STARTID = None if sys.argv[1]: STARTID = sys.argv[1] run_backfill(STARTID) diff --git a/tests/vxingest/utilities/test_backfill_obs_with_rh.py b/tests/vxingest/utilities/test_backfill_obs_with_rh.py index 95f7da94..dd84a03a 100644 --- a/tests/vxingest/utilities/test_backfill_obs_with_rh.py +++ b/tests/vxingest/utilities/test_backfill_obs_with_rh.py @@ -133,7 +133,9 @@ def test_calc_components_backfills_nochange(): assert doc["data"]["NZCM"]["WD"] == 180, "WD wrong value" assert doc["data"]["NZCM"]["WindU"] == 5, "WindU wrong value" assert doc["data"]["NZCM"]["WindV"] == -5, "WindV wrong value" - assert math.isclose(doc["data"]["NZCM"]["RH"], 81.008, abs_tol=0.001), "RH wrong value" + assert math.isclose( + doc["data"]["NZCM"]["RH"], 81.008, abs_tol=0.001 + ), "RH wrong value" assert doc["data"]["SUMU"]["Temperature"] == 30, "temperature wrong value" assert doc["data"]["SUMU"]["DewPoint"] == 15, "DewPoint wrong value" @@ -141,4 +143,6 @@ def test_calc_components_backfills_nochange(): assert doc["data"]["SUMU"]["WD"] == 270, "WD wrong value" assert doc["data"]["SUMU"]["WindU"] == 3, "WindU wrong value" assert doc["data"]["SUMU"]["WindV"] == -7, "WindV wrong value" - assert math.isclose(doc["data"]["SUMU"]["RH"], 53.152, abs_tol=0.001), "RH wrong value" + assert math.isclose( + doc["data"]["SUMU"]["RH"], 53.152, abs_tol=0.001 + ), "RH wrong value"