From 734f8a3a682ed0865ff6e9e4229dcec2e987ea80 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 12 Oct 2023 14:45:38 -0600 Subject: [PATCH 01/14] CI: Add 3.12 to CI jobs In order to avoid bloating the Conda test matrix, cut down the "middle" versions to split across Windows/MacOS. Also bump anything using 3.11 for 'latest' to now use 3.12. --- .github/workflows/docs-conda.yml | 2 ++ .github/workflows/docs.yml | 5 ++++- .github/workflows/tests-conda.yml | 7 ++++++- .github/workflows/tests-pypi.yml | 5 +++-- .github/workflows/unstable-builds.yml | 4 ++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs-conda.yml b/.github/workflows/docs-conda.yml index 2cfe2dff471..469245dc9e4 100644 --- a/.github/workflows/docs-conda.yml +++ b/.github/workflows/docs-conda.yml @@ -28,6 +28,8 @@ jobs: os: Windows - python-version: 3.11 os: macOS + - python-version: 3.12 + os: macOS steps: - name: Checkout source diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a9a95b0ccfe..0ed767efeea 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,6 +35,9 @@ jobs: check-links: false dep-versions: requirements.txt - python-version: 3.11 + check-links: false + dep-versions: requirements.txt + - python-version: 3.12 check-links: true dep-versions: requirements.txt outputs: @@ -73,7 +76,7 @@ jobs: - name: Download doc build uses: actions/download-artifact@v3 with: - name: Linux-3.11-docs + name: Linux-3.12-docs path: ./docs/build/html # This overrides the version "dev" with the proper version if we're building off a diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml index 781521a0a15..2f05a6c343f 100644 --- a/.github/workflows/tests-conda.yml +++ b/.github/workflows/tests-conda.yml @@ -28,8 +28,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9, '3.10', 3.11] + python-version: [3.9, 3.12] os: [macOS, Windows] + include: + - python-version: '3.10' + os: macOS + - python-version: 3.11 + os: Windows steps: - name: Checkout source diff --git a/.github/workflows/tests-pypi.yml b/.github/workflows/tests-pypi.yml index fb9ca741eff..9116f5f697c 100644 --- a/.github/workflows/tests-pypi.yml +++ b/.github/workflows/tests-pypi.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.9, '3.10', 3.11] + python-version: [3.9, '3.10', 3.11, 3.12] dep-versions: [Latest] no-extras: [''] include: @@ -34,7 +34,8 @@ jobs: - python-version: 3.9 dep-versions: Minimum no-extras: 'No Extras' - - python-version: 3.11 + - python-version: 3.12 + dep-versions: Latest no-extras: 'No Extras' steps: diff --git a/.github/workflows/unstable-builds.yml b/.github/workflows/unstable-builds.yml index c10e65da733..ef67567e6c5 100644 --- a/.github/workflows/unstable-builds.yml +++ b/.github/workflows/unstable-builds.yml @@ -33,7 +33,7 @@ jobs: need-extras: true type: test version-file: Prerelease - python-version: 3.11 + python-version: 3.12 - name: Run tests id: tests @@ -72,7 +72,7 @@ jobs: with: type: doc version-file: Prerelease - python-version: 3.11 + python-version: 3.12 - name: Build docs id: build From 177edbd492f704d55f371f70248cab83469cb479 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 12 Oct 2023 14:49:42 -0600 Subject: [PATCH 02/14] MNT: Add Python 3.12 support to package metadata --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f6b032aaee1..c50ce34a386 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Atmospheric Science", "Intended Audience :: Science/Research", From 16d015045c99189283ae8a23110781f8f6d8ab91 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Tue, 24 Oct 2023 14:56:05 -0600 Subject: [PATCH 03/14] MNT: Avoid subclassing Struct Python 3.12 broke this and we can't subclass Struct and modify the __init__ method. Get around this by switching to use composition instead of inheritance. --- src/metpy/io/_tools.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/metpy/io/_tools.py b/src/metpy/io/_tools.py index ef2e5ee858f..174566cc615 100644 --- a/src/metpy/io/_tools.py +++ b/src/metpy/io/_tools.py @@ -57,7 +57,7 @@ def open_as_needed(filename, mode='rb'): return open(filename, mode, **kwargs) # noqa: SIM115 -class NamedStruct(Struct): +class NamedStruct: """Parse bytes using :class:`Struct` but provide named fields.""" def __init__(self, info, prefmt='', tuple_name=None): @@ -73,7 +73,12 @@ def __init__(self, info, prefmt='', tuple_name=None): elif not i[0]: # Skip items with no name conv_off += 1 self._tuple = namedtuple(tuple_name, ' '.join(n for n in names if n)) - super().__init__(prefmt + ''.join(f for f in fmts if f)) + self._struct = Struct(prefmt + ''.join(f for f in fmts if f)) + + @property + def size(self): + """Return the size of the struct in bytes.""" + return self._struct.size def _create(self, items): if self.converters: @@ -90,11 +95,11 @@ def make_tuple(self, *args, **kwargs): def unpack(self, s): """Parse bytes and return a namedtuple.""" - return self._create(super().unpack(s)) + return self._create(self._struct.unpack(s)) def unpack_from(self, buff, offset=0): """Read bytes from a buffer and return as a namedtuple.""" - return self._create(super().unpack_from(buff, offset)) + return self._create(self._struct.unpack_from(buff, offset)) def unpack_file(self, fobj): """Unpack the next bytes from a file object.""" @@ -103,12 +108,12 @@ def unpack_file(self, fobj): def pack(self, **kwargs): """Pack the arguments into bytes using the structure.""" t = self.make_tuple(**kwargs) - return super().pack(*t) + return self._struct.pack(*t) # This works around times when we have more than 255 items and can't use # NamedStruct. This is a CPython limit for arguments. -class DictStruct(Struct): +class DictStruct: """Parse bytes using :class:`Struct` but provide named fields using dictionary access.""" def __init__(self, info, prefmt=''): @@ -118,18 +123,23 @@ def __init__(self, info, prefmt=''): # Remove empty names self._names = [n for n in names if n] - super().__init__(prefmt + ''.join(f for f in formats if f)) + self._struct = Struct(prefmt + ''.join(f for f in formats if f)) + + @property + def size(self): + """Return the size of the struct in bytes.""" + return self._struct.size def _create(self, items): return dict(zip(self._names, items)) def unpack(self, s): """Parse bytes and return a dict.""" - return self._create(super().unpack(s)) + return self._create(self._struct.unpack(s)) def unpack_from(self, buff, offset=0): """Unpack the next bytes from a file object.""" - return self._create(super().unpack_from(buff, offset)) + return self._create(self._struct.unpack_from(buff, offset)) class Enum: From ac62213368067ebcb811927c20f75282851dd9c6 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 25 Oct 2023 15:43:11 -0600 Subject: [PATCH 04/14] CI: Make sure the job fails if coverage upload fails Apparently this isn't the default behavior. Otherwise, the only way to know is to notice slight coverage drops due to missing reports. --- .github/workflows/tests-conda.yml | 1 + .github/workflows/tests-pypi.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml index 2f05a6c343f..bf6d1e50098 100644 --- a/.github/workflows/tests-conda.yml +++ b/.github/workflows/tests-conda.yml @@ -71,3 +71,4 @@ jobs: uses: codecov/codecov-action@v3 with: name: ${{ github.workflow }} + fail_ci_if_error: true diff --git a/.github/workflows/tests-pypi.yml b/.github/workflows/tests-pypi.yml index 9116f5f697c..2f1df015f0f 100644 --- a/.github/workflows/tests-pypi.yml +++ b/.github/workflows/tests-pypi.yml @@ -95,3 +95,4 @@ jobs: uses: codecov/codecov-action@v3 with: name: ${{ github.workflow }} + fail_ci_if_error: true From ef556285a15c9a72047c6efe8344bb758d1caa7c Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 25 Oct 2023 15:51:28 -0600 Subject: [PATCH 05/14] CI: Simplify matrix specification for Linux doc builds --- .github/workflows/docs.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 0ed767efeea..9893138398b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,24 +22,16 @@ jobs: # Build our docs on Linux against multiple Pythons # Docs: - name: ${{ matrix.python-version }} ${{ matrix.dep-versions }} + name: "Linux ${{ matrix.python-version }}" runs-on: ubuntu-latest strategy: fail-fast: false matrix: + python-version: [3.9, '3.10', 3.11] + check-links: [false] include: - - python-version: 3.9 - check-links: false - dep-versions: requirements.txt - - python-version: '3.10' - check-links: false - dep-versions: requirements.txt - - python-version: 3.11 - check-links: false - dep-versions: requirements.txt - python-version: 3.12 check-links: true - dep-versions: requirements.txt outputs: doc-version: ${{ steps.build-docs.outputs.doc-version }} From 215a871ddf4efba09241483c9645803e37009026 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 25 Oct 2023 17:05:13 -0600 Subject: [PATCH 06/14] CI: Run nightly on PR if "unstable" workflow changes --- .github/workflows/nightly-builds.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nightly-builds.yml b/.github/workflows/nightly-builds.yml index ec0fa569a2d..7112dca7098 100644 --- a/.github/workflows/nightly-builds.yml +++ b/.github/workflows/nightly-builds.yml @@ -14,9 +14,11 @@ on: - main paths: - .github/workflows/nightly-builds.yml + - .github/workflows/unstable-builds.yml pull_request: paths: - .github/workflows/nightly-builds.yml + - .github/workflows/unstable-builds.yml jobs: Builds: From bbd892e5262bd11ae2606dd1cc18d29d258463d6 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 26 Oct 2023 12:25:23 -0600 Subject: [PATCH 07/14] DOC: Update URLs to WMO reports These are supposed to be the "permalinks", so let's use those. This is attempting to address some failures on linkchecking. --- docs/api/references.rst | 9 ++++----- docs/conf.py | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/api/references.rst b/docs/api/references.rst index e4df24baa3e..c99e3d7d753 100644 --- a/docs/api/references.rst +++ b/docs/api/references.rst @@ -198,11 +198,10 @@ References .. [WMO8] WMO, 2020: Guide to Meteorological Instruments and Methods of Observation, Volume 1: Measurement of Meteorological Variables. - `WMO No.8 `_. + `WMO No.8 `_. .. [WMO306] WMO, 2011: Manual on Codes - International Codes, Volume I.1, Annex II to the WMO - Technical Regulations: Part A - Alphanumeric Codes. `WMO No.306 - `_. + Technical Regulations: Part A - Alphanumeric Codes. `WMO No.306 `_. -.. [WMO1966] WMO, 1966: International Meteorological Tables, `WMO-No. 188.TP.94 - `_. +.. [WMO1966] WMO, 1966: International Meteorological Tables, + `WMO-No. 188.TP.94 `_. diff --git a/docs/conf.py b/docs/conf.py index e8de320fa84..f92d44e0f88 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -440,7 +440,8 @@ r'https://conda.io/docs/': r'https://conda.io/en/latest/', r'https://github.com/Unidata/MetPy/issues/new/choose': r'https://github.com/login.*choose', r'https://doi.org/.*': r'https://.*', - r'https://gitter.im/Unidata/MetPy': r'https://app.gitter.im/.*MetPy.*' + r'https://gitter.im/Unidata/MetPy': r'https://app.gitter.im/.*MetPy.*', + r'https://library.wmo.int/idurl/.*': r'https://library.wmo.int/.*' } # Domain-specific HTTP headers for requests From 93043c7f18f4bf679152272a60a327fd1666dfb9 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 26 Oct 2023 15:10:23 -0600 Subject: [PATCH 08/14] DOC: Make sure everything is cleared between area plots This helps with memory usage when running generating all these plots. --- docs/make_areas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/make_areas.py b/docs/make_areas.py index 5d565409b24..3df13db1921 100644 --- a/docs/make_areas.py +++ b/docs/make_areas.py @@ -52,6 +52,7 @@ def generate_area_file(app): if area in states_provinces: code = textwrap.dedent(f""" .. plot:: + :context: reset import matplotlib.pyplot as plt import cartopy.crs as ccrs @@ -75,6 +76,7 @@ def generate_area_file(app): else: code = textwrap.dedent(f""" .. plot:: + :context: reset import matplotlib.pyplot as plt import cartopy.crs as ccrs From c420586866377ed6aff987d75926c047c41d1c3b Mon Sep 17 00:00:00 2001 From: Ryan May Date: Mon, 30 Oct 2023 15:32:15 -0600 Subject: [PATCH 09/14] CI: Only build shapely from src on Minimum runs --- .github/actions/install-pypi/action.yml | 10 +++++++--- .github/workflows/tests-pypi.yml | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/actions/install-pypi/action.yml b/.github/actions/install-pypi/action.yml index 54cd9fe4b05..9347aa60d81 100644 --- a/.github/actions/install-pypi/action.yml +++ b/.github/actions/install-pypi/action.yml @@ -15,6 +15,10 @@ inputs: python-version: description: 'What version of Python to use' required: true + old-build: + description: 'Whether to enable old builds for shapely and cartopy' + required: false + default: 'false' runs: using: composite @@ -30,13 +34,13 @@ runs: ci/extra_requirements.txt # This installs the stuff needed to build and install Shapely and CartoPy from source. - - name: Install CartoPy build dependencies - if: ${{ inputs.need-extras == 'true' }} + - name: Install CartoPy/Shapely build dependencies + if: ${{ inputs.old-build == 'true' }} shell: bash run: sudo apt-get install libgeos-dev - name: Disable Shapely Wheels - if: ${{ inputs.need-extras == 'true' }} + if: ${{ inputs.old-build == 'true' }} shell: bash run: echo "PIP_NO_BINARY=shapely" >> $GITHUB_ENV diff --git a/.github/workflows/tests-pypi.yml b/.github/workflows/tests-pypi.yml index 2f1df015f0f..d9c5dab440d 100644 --- a/.github/workflows/tests-pypi.yml +++ b/.github/workflows/tests-pypi.yml @@ -72,6 +72,7 @@ jobs: need-extras: ${{ matrix.no-extras != 'No Extras' }} type: 'test' python-version: ${{ matrix.python-version }} + old-build: ${{ matrix.no-extras != 'No Extras' && matrix.dep-versions == 'Minimum' }} - name: Run tests uses: ./.github/actions/run-tests From 36cfffe4abc6647e18a57e1f1a5539c4a1d4c946 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Mon, 30 Oct 2023 17:24:16 -0600 Subject: [PATCH 10/14] CI: Only install additional test tools for test builds --- .github/actions/install-pypi/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/install-pypi/action.yml b/.github/actions/install-pypi/action.yml index 9347aa60d81..cc687e8dd39 100644 --- a/.github/actions/install-pypi/action.yml +++ b/.github/actions/install-pypi/action.yml @@ -61,6 +61,7 @@ runs: -c ci/${{ inputs.version-file }} -c ci/${{ inputs.type }}_requirements.txt -c ci/extra_requirements.txt - name: Install additional test tools + if: ${{ inputs.type == 'test' }} shell: bash run: > python -m pip install coverage From e9ce6a340689ec69add5f7c26f4f44e7a0737e2e Mon Sep 17 00:00:00 2001 From: Ryan May Date: Mon, 30 Oct 2023 17:58:38 -0600 Subject: [PATCH 11/14] CI: Specify that doc builds need extras While we were already using the right dependencies, this didn't do any of the extra steps (e.g. downloading maps). --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9893138398b..10964da8398 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -47,6 +47,7 @@ jobs: with: type: 'doc' python-version: ${{ matrix.python-version }} + need-extras: true - name: Build docs id: build-docs From 0c597224f92d1b8f5df22e91f9a2f847fda24007 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 1 Nov 2023 15:05:41 -0600 Subject: [PATCH 12/14] CI: Reduce load from generating all the maps Due to the extra 10 minutes required, avoid running as part of PR CI jobs. --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f92d44e0f88..bf9f8bb5a68 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,8 +61,8 @@ 'reset_modules': [lambda conf, fname: sys.modules.pop('pint', None)] } -# By default, only generate all the areas when running in CI -metpy_generate_all_areas = 'GITHUB_ACTIONS' in os.environ +# By default, only generate all the areas when running on a release CI job +metpy_generate_all_areas = not os.environ.get('GITHUB_REF', '').startswith('refs/pull') # Turn off code and image links for embedded mpl plots plot_html_show_source_link = False From 1256b14d9ec03bc10505fda88323536fb7592d76 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 1 Nov 2023 15:09:56 -0600 Subject: [PATCH 13/14] CI: Work around problems with generating maps on 3.12 For now just continue using Python 3.11 for our release docs and avoid running the generation on 3.12. I'm assuming right now that this problem will go away when we cease building Cartopy ourselves. --- .github/workflows/docs.yml | 2 +- docs/conf.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 10964da8398..946e8925f3c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -69,7 +69,7 @@ jobs: - name: Download doc build uses: actions/download-artifact@v3 with: - name: Linux-3.12-docs + name: Linux-3.11-docs path: ./docs/build/html # This overrides the version "dev" with the proper version if we're building off a diff --git a/docs/conf.py b/docs/conf.py index bf9f8bb5a68..dad0256d64b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -62,7 +62,8 @@ } # By default, only generate all the areas when running on a release CI job -metpy_generate_all_areas = not os.environ.get('GITHUB_REF', '').startswith('refs/pull') +metpy_generate_all_areas = (not os.environ.get('GITHUB_REF', '').startswith('refs/pull') + and sys.version_info < (3, 12)) # Turn off code and image links for embedded mpl plots plot_html_show_source_link = False From 84ccdb5bf3a3461295033fa82158345bc0f02705 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 1 Nov 2023 15:40:32 -0600 Subject: [PATCH 14/14] CI: Skip WMO URLs on linkchecker These are currently triggering certificate errors on GitHub. --- docs/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index dad0256d64b..87493d09c4d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -432,7 +432,9 @@ r'https://doi\.org/10\.1289/ehp\.1206273', # Couldn't fix these 403's with user agents r'https://doi\.org/10\.1029/2010GL045777', - r'https://doi\.org/10\.1098/rspa\.2004\.1430' + r'https://doi\.org/10\.1098/rspa\.2004\.1430', + # Currently giving certificate errors on GitHub + r'https://library.wmo.int/.*' ] # Dictionary of URL redirects allowed