diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..fb64dd1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +sphinx: + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index db03a79..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: 2 - -build: - image: latest - -python: - version: 3.7 - install: - - method: pip - path: . - extra_requirements: - - docs - -# Don't build any extra formats -formats: [] diff --git a/CHANGELOG.md b/CHANGELOG.md index ea122c5..2ac1ed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,23 @@ +# toasty 0.19.0 (2023-12-14) + +- Implement a `--tiling-method` argument for `toasty view` (#97, @pkgw). This + allows you to force the choice of a specific method. In particular, sometimes + it is helpful to force the use of TOAST to get the most accurate display over + large angular sizes. +- Fix an outdated usage that broke processing of all-sky plate carrée ecliptic + images with Astropy 6.x (#99, @pkgw) +- Fix TOASTing of multi-plane (e.g, RGB) images that don't cover the whole sky + (#99, @pkgw). This fixes processing of RGB images that have lots of pixels but + don't necessarily cover huge solid angles. +- In `toasty tile-study`, if we're getting WCS information from a FITS file but + its dimensions disagree with the input image, try rescaling the pixel size if + the overall image shape is the same (#99, @pkgw). This reproduces behavior + already implemented for AVM processing, and helps if, say, you want to process + a large image where you've submitted a scaled-down version of it to + Astrometry.Net. +- Various fixes to the test suite and CI system. + + # toasty 0.18.1 (2022-09-07) - Fix tiling on macOS+Py3.7 due to an unexpectedly unimplemented API (#94, diff --git a/README.md b/README.md index 0497acd..131865e 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,15 @@ [toasty] is a Python library that helps you create “tile pyramids” from astronomical image data as used in the [TOAST] format. These multi-resolution -maps can be viewed in software such as the [AAS] [WorldWide Telescope]. +maps can be viewed in software such as [WorldWide Telescope]. [toasty]: https://toasty.readthedocs.io/ [TOAST]: https://doi.org/10.3847/1538-4365/aaf79e -[AAS]: https://aas.org/ [WorldWide Telescope]: http://www.worldwidetelescope.org/ [toasty] was originally written by [Chris Beaumont], benefited from contributions by Clara Brasseur (Space Telescope Science Institute), and is -currently maintained as part of the AAS [WorldWide Telescope] project. +currently maintained as part of the [WorldWide Telescope] project. [Chris Beaumont]: https://chrisbeaumont.org/ @@ -100,13 +99,13 @@ and [PyPI](https://pypi.org/project/toasty/#history). ## Legalities -[toasty] is copyright Chris Beaumont, Clara Brasseur, and the AAS WorldWide +[toasty] is copyright Chris Beaumont, Clara Brasseur, and the WorldWide Telescope Team. It is licensed under the [MIT License](./LICENSE). ## Acknowledgments -[toasty] is part of the AAS WorldWide Telescope system, a [.NET Foundation] +[toasty] is part of the WorldWide Telescope system, a [.NET Foundation] project managed by the non-profit [American Astronomical Society] (AAS). Work on WWT has been supported by the AAS, the US [National Science Foundation] (grants [1550701], [1642446], and [2004840]), the [Gordon and Betty Moore Foundation], and diff --git a/ci/azure-build-and-test.yml b/ci/azure-build-and-test.yml index fc08668..fde849f 100644 --- a/ci/azure-build-and-test.yml +++ b/ci/azure-build-and-test.yml @@ -5,33 +5,33 @@ parameters: type: object default: - - name: linux_38 + - name: linux_310 vmImage: ubuntu-20.04 vars: - PYTHON_SERIES: "3.8" + PYTHON_SERIES: "3.10" - name: linux_39 vmImage: ubuntu-20.04 vars: PYTHON_SERIES: "3.9" - - name: macos_38 - vmImage: macos-10.15 + - name: macos_310 + vmImage: macos-11 vars: - PYTHON_SERIES: "3.8" + PYTHON_SERIES: "3.10" - name: macos_39 - vmImage: macos-10.15 + vmImage: macos-11 vars: PYTHON_SERIES: "3.9" - - name: windows_38 - vmImage: windows-2019 + - name: windows_310 + vmImage: windows-2022 vars: - PYTHON_SERIES: "3.8" + PYTHON_SERIES: "3.10" - name: windows_39 - vmImage: windows-2019 + vmImage: windows-2022 vars: PYTHON_SERIES: "3.9" diff --git a/ci/azure-deployment.yml b/ci/azure-deployment.yml index 3c7ebe0..f145531 100644 --- a/ci/azure-deployment.yml +++ b/ci/azure-deployment.yml @@ -18,14 +18,14 @@ jobs: - ${{ if parameters.isMainDev }}: - job: dummy_setup_only pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest steps: - template: azure-job-setup.yml - ${{ if parameters.isRelease }}: - job: branch_and_tag pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest variables: - group: Deployment Credentials steps: @@ -44,7 +44,7 @@ jobs: - job: github_releases dependsOn: branch_and_tag # otherwise, GitHub creates the tag itself pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest variables: - group: Deployment Credentials steps: @@ -61,7 +61,7 @@ jobs: - job: python_publish pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest variables: - group: Deployment Credentials steps: @@ -88,7 +88,7 @@ jobs: - job: zenodo_publish pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest variables: - group: Deployment Credentials diff --git a/ci/azure-sdist.yml b/ci/azure-sdist.yml index 92591aa..c848657 100644 --- a/ci/azure-sdist.yml +++ b/ci/azure-sdist.yml @@ -7,7 +7,7 @@ jobs: - job: sdist pool: - vmImage: ubuntu-20.04 + vmImage: ubuntu-latest # Need Zenodo credentials to generate DOIs during formal releases. But make # sure not to provide credentials otherwise (although Azure has its own checks diff --git a/ci/zenodo.json5 b/ci/zenodo.json5 index b24aa0c..950e233 100644 --- a/ci/zenodo.json5 +++ b/ci/zenodo.json5 @@ -42,13 +42,13 @@ ], "language": "eng", "license": "MIT", - "publication_date": "2022-09-07", - "title": "toasty 0.18.1", + "publication_date": "2023-12-14", + "title": "toasty 0.19.0", "upload_type": "software", - "version": "0.18.1" + "version": "0.19.0" }, "conceptdoi": "10.5281/zenodo.7055476", - "record_id": "7058238", - "doi": "10.5281/zenodo.7058238", - "bucket_link": "https://zenodo.org/api/files/e5ca7ed0-557d-4271-b77e-d9fe78addae0" + "record_id": "10383112", + "doi": "10.5281/zenodo.10383112", + "bucket_link": "https://zenodo.org/api/files/0f1838fd-9749-4f07-a18c-79180e5ffd9f" } diff --git a/docs/cli/view.rst b/docs/cli/view.rst index 114d9f6..cc54181 100644 --- a/docs/cli/view.rst +++ b/docs/cli/view.rst @@ -33,6 +33,7 @@ Detailed Usage [--hdu-index INDEX[,INDEX,...]] [--parallelism COUNT, -j COUNT] [--tile-only] + [--tiling-method METHOD] {FITS [FITS ...]} The ``FITS`` argument(s) give the path(s) of one or more input FITS files. These @@ -67,6 +68,17 @@ parallism to use in the tiling and downsampling process. On operating systems that support parallel processing, the default is to use all CPUs. To disable parallel processing, explicitly specify a factor of 1. +The ``--tiling-method METHOD`` argument indicates the target projection to use +when tiling the data. The default value, ``auto``, causes Toasty to use a +heuristic to automatically determine the best method. Other valid values are +``tan`` (to use a tangential/gnomonic projection, only valid for relatively +small areas on the sky), ``toast`` (for `TOAST`_, best in WWT for large-area +images), or ``hips`` (for `HiPS`_). + +.. _TOAST: https://docs.worldwidetelescope.org/data-guide/1/spherical-projections/toast-projection/ + +.. _HiPS: https://www.ivoa.net/documents/HiPS/ + The ``--appurl`` option can be used to override the base URL for the preview app that will be used. This can be helpful when developing new features in one of these apps. @@ -109,7 +121,7 @@ The basic usage of this mode is .. code-block:: shell - toasty view -t HOST FITS1 [FITS2...] + toasty view -t HOST [other arguments...] FITS1 [FITS2...] where ``HOST`` is the hostname of the machine with the image(s), and the ``FITSn`` values are the paths of the images on the machine relative to the diff --git a/docs/conf.py b/docs/conf.py index 2d51132..24306b5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,7 +4,7 @@ author = "Chris Beaumont and the AAS WorldWide Telescope Team" copyright = "2014-2020, " + author -release = "0.18.1" # cranko project-version +release = "0.19.0" # cranko project-version version = ".".join(release.split(".")[:2]) extensions = [ diff --git a/setup.py b/setup.py index f730a24..54a9ff9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # -*- mode: python; coding: utf-8 -*- -# Copyright 2013-2022 Chris Beaumont and the AAS WorldWide Telescope team +# Copyright 2013-2023 Chris Beaumont and the WorldWide Telescope team # Licensed under the MIT License from Cython.Distutils import build_ext # in pyproject.toml @@ -36,15 +36,15 @@ def get_long_desc(): setup_args = dict( name="toasty", # cranko project-name - version="0.18.1", # cranko project-version - description="Generate TOAST image tile pyramids from existing image data", + version="0.19.0", # cranko project-version + description="Generate image tile pyramids from existing image data", long_description=get_long_desc(), long_description_content_type="text/markdown", url="https://toasty.readthedocs.io/", license="MIT", platforms="Linux, Mac OS X", - author="Chris Beaumont, AAS WorldWide Telescope Team", - author_email="wwt@aas.org", + author="Chris Beaumont, WorldWide Telescope Team", + author_email="hello@worldwidetelescope.org", classifiers=[ "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", diff --git a/toasty/cli.py b/toasty/cli.py index dee0ebf..640a453 100644 --- a/toasty/cli.py +++ b/toasty/cli.py @@ -249,11 +249,11 @@ def show_impl(settings): print(doi) elif settings.show_command == "version": # This string constant will be rewritten by Cranko during releases: - version = "0.18.1" # cranko project-version + version = "0.19.0" # cranko project-version print(version) elif settings.show_command == "version-doi": # This string constant will be rewritten by Cranko during releases: - doi = "10.5281/zenodo.7058238" + doi = "10.5281/zenodo.10383112" if not doi.startswith("10."): warn("this DOI is a fake value used for development builds") print(doi) @@ -641,10 +641,33 @@ def tile_study_impl(settings): elif img.shape[:2] != (wcs_height, wcs_width): warn( f"image `{settings.imgpath}` has shape {img.shape}, but " - f"WCS reference file `{settings.fits_wcs}` has shape ({wcs_height}, {wcs_width}); " - f"astrometry may not transfer correctly" + f"WCS reference file `{settings.fits_wcs}` has shape ({wcs_height}, {wcs_width})" ) + # Cribbing some code from pyavm ... + scale_x = img.shape[1] / wcs_width + scale_y = img.shape[0] / wcs_height + + if abs(scale_x - scale_y) / (scale_x + scale_y) >= 0.005: + warn( + "the aspect ratios are not compatible; I'll proceed, but astrometry is unlikely transfer correctly" + ) + scale = 1.0 + else: + warn("the aspect ratios are compatible, so I'll try rescaling") + scale = scale_x + + wcs.wcs.crpix *= scale + + if hasattr(wcs.wcs, "cd"): + wcs.wcs.cd /= scale + else: + wcs.wcs.cdelt /= scale + + if hasattr(wcs, "naxis1"): + wcs.naxis1 = img.shape[1] + wcs.naxis2 = img.shape[0] + img._wcs = wcs # <= hack alert, but I think this is OK img.ensure_negative_parity() builder.apply_wcs_info(img.wcs, img.width, img.height) @@ -884,6 +907,12 @@ def view_getparser(parser): action="store_true", help="Tile the data but do not open for viewing", ) + parser.add_argument( + "--tiling-method", + default="auto", + choices=["auto", "tan", "toast", "hips"], + help="The target projection when tiling: `auto`, `tan`, `toast`, or `hips`", + ) parser.add_argument( "paths", metavar="PATHS", @@ -895,15 +924,28 @@ def view_getparser(parser): def view_locally(settings): from wwt_data_formats.server import preview_wtml + from . import TilingMethod from .collection import CollectionLoader from .fits_tiler import FitsTiler + if settings.tiling_method == "auto": + tiling_method = TilingMethod.AUTO_DETECT + elif settings.tiling_method == "tan": + tiling_method = TilingMethod.TAN + elif settings.tiling_method == "toast": + tiling_method = TilingMethod.TOAST + elif settings.tiling_method == "hips": + tiling_method = TilingMethod.HIPS + else: + # This shouldn't happen since argparse should validate the input + die(f"unhandled tiling method `{settings.tiling_method}`") + coll = CollectionLoader.create_from_args(settings).load_paths(settings.paths) # Ignore any astropy WCS/FITS warnings, which can spew a lot of annoying output. with warnings.catch_warnings(): warnings.simplefilter("ignore") - tiler = FitsTiler(coll) + tiler = FitsTiler(coll, tiling_method=tiling_method) tiler.tile(cli_progress=True, parallel=settings.parallelism) rel_wtml_path = os.path.join(tiler.out_dir, "index_rel.wtml") @@ -941,7 +983,14 @@ def view_tunneled(settings): # We give SSH `-T` to prevent warnings about not allocating pseudo-TTYs. ssh_argv = ["ssh", "-T", settings.tunnel] - toasty_argv = ["exec", "toasty", "view", "--tile-only"] + toasty_argv = [ + "exec", + "toasty", + "view", + "--tile-only", + f"--tiling-method={settings.tiling_method}", + ] + if settings.parallelism: toasty_argv += ["--parallelism", str(settings.parallelism)] toasty_argv += [shlex.quote(p) for p in settings.paths] diff --git a/toasty/multi_wcs.py b/toasty/multi_wcs.py index 6fdfd14..165ffdd 100644 --- a/toasty/multi_wcs.py +++ b/toasty/multi_wcs.py @@ -71,7 +71,7 @@ def create_mwcs_descriptor(coll_desc): # ImageDescription helper to ensure we get that. wcs, shape = find_optimal_celestial_wcs( - ((desc.in_shape, desc.in_wcs) for desc in self._descs), + [(desc.in_shape, desc.in_wcs) for desc in self._descs], auto_rotate=True, projection="TAN", ) diff --git a/toasty/samplers.py b/toasty/samplers.py index 4520f0a..b68e633 100644 --- a/toasty/samplers.py +++ b/toasty/samplers.py @@ -285,7 +285,7 @@ def plate_carree_ecliptic_sampler(data): lat0 = HALFPI - 0.5 / dy # latitudes of the centers of the pixels with iy = 0 def vec2pix(lon, lat): - ecl = ICRS(lon * u.rad, lat * u.rad).transform_to(Ecliptic) + ecl = ICRS(lon * u.rad, lat * u.rad).transform_to(Ecliptic()) lon, lat = ecl.lon.rad, ecl.lat.rad lon = lon % TWOPI - np.pi # ensure in range [-pi, pi] @@ -478,7 +478,7 @@ def _image_bounds(self): D2R = np.pi / 180 N_COARSE = 32 - naxis2, naxis1 = self._image.shape + naxis2, naxis1 = self._image.shape[:2] coarse_pix = np.empty((N_COARSE, N_COARSE, 2)) coarse_idx1 = np.linspace(0.5, naxis1 + 0.5, N_COARSE) coarse_idx2 = np.linspace(0.5, naxis2 + 0.5, N_COARSE) diff --git a/toasty/tests/__init__.py b/toasty/tests/__init__.py index c51e18f..0dc6ca4 100644 --- a/toasty/tests/__init__.py +++ b/toasty/tests/__init__.py @@ -2,28 +2,31 @@ # Copyright 2013-2020 Chris Beaumont and the AAS WorldWide Telescope project # Licensed under the MIT License. -__all__ = ''' +__all__ = """ assert_xml_elements_equal check_xml_elements_equal HAS_AVM -test_path -'''.split() +mk_test_path +""".split() import os.path TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) -def test_path(*pieces): + +def mk_test_path(*pieces): return os.path.join(TESTS_DIR, *pieces) try: import pyavm + HAS_AVM = True except ImportError: HAS_AVM = False + def check_xml_elements_equal(observed, expected): """See if two XML elements are equal through recursive comparison. We do *not* check the "tail" text item, and we strip whitespace in "text". @@ -32,24 +35,30 @@ def check_xml_elements_equal(observed, expected): """ if observed.tag != expected.tag: - return 'expected tag {}, observed tag {}'.format(expected.tag, observed.tag) + return "expected tag {}, observed tag {}".format(expected.tag, observed.tag) if observed.text is None: - otext = '' + otext = "" else: otext = observed.text.strip() if expected.text is None: - etext = '' + etext = "" else: etext = expected.text.strip() if otext != etext: - return 'expected text {!r} in tag {}, observed {!r}'.format(etext, expected.tag, otext) + return "expected text {!r} in tag {}, observed {!r}".format( + etext, expected.tag, otext + ) if observed.attrib != expected.attrib: - return 'expected attrs {!r} in tag {}, observed {!r}'.format(expected.attrib, expected.tag, observed.attrib) + return "expected attrs {!r} in tag {}, observed {!r}".format( + expected.attrib, expected.tag, observed.attrib + ) if len(observed) != len(expected): - return 'expected {} children of in tag {}, observed {}'.format(len(expected), expected.tag, len(observed)) + return "expected {} children of in tag {}, observed {}".format( + len(expected), expected.tag, len(observed) + ) for c1, c2 in zip(observed, expected): reason = check_xml_elements_equal(c1, c2) @@ -58,7 +67,8 @@ def check_xml_elements_equal(observed, expected): return None + def assert_xml_elements_equal(observed, expected): reason = check_xml_elements_equal(observed, expected) if reason is not None: - raise Exception('unequal XML elements: {}'.format(reason)) + raise Exception("unequal XML elements: {}".format(reason)) diff --git a/toasty/tests/test_avm.py b/toasty/tests/test_avm.py index 72c5f2c..54bf5a6 100644 --- a/toasty/tests/test_avm.py +++ b/toasty/tests/test_avm.py @@ -5,7 +5,7 @@ import os.path import pytest -from . import HAS_AVM, test_path +from . import HAS_AVM, mk_test_path from .. import cli @@ -29,7 +29,7 @@ def test_check_cli_bad(self): "check-avm", "--print", "--exitcode", - test_path("badavm-type.png"), + mk_test_path("badavm-type.png"), ] try: @@ -44,7 +44,7 @@ def test_check_cli_good(self): args = [ "check-avm", "--print", - test_path("geminiann11015a.jpg"), + mk_test_path("geminiann11015a.jpg"), ] try: diff --git a/toasty/tests/test_collection.py b/toasty/tests/test_collection.py index ffbe21b..c089c02 100644 --- a/toasty/tests/test_collection.py +++ b/toasty/tests/test_collection.py @@ -5,27 +5,32 @@ from __future__ import absolute_import, division, print_function import pytest -from . import test_path +from . import mk_test_path from .. import collection try: from astropy.io import fits + HAS_ASTRO = True except ImportError: HAS_ASTRO = False class TestCollection(object): - - @pytest.mark.skipif('not HAS_ASTRO') + @pytest.mark.skipif("not HAS_ASTRO") def test_is_multi_tan(self): - coll = collection.SimpleFitsCollection([test_path('wcs512.fits.gz')]) + coll = collection.SimpleFitsCollection([mk_test_path("wcs512.fits.gz")]) assert coll._is_multi_tan() - coll = collection.SimpleFitsCollection([test_path('herschel_spire.fits.gz'), - test_path('herschel_spire.fits.gz')]) + coll = collection.SimpleFitsCollection( + [ + mk_test_path("herschel_spire.fits.gz"), + mk_test_path("herschel_spire.fits.gz"), + ] + ) assert coll._is_multi_tan() - coll = collection.SimpleFitsCollection([test_path('wcs512.fits.gz'), - test_path('herschel_spire.fits.gz')]) + coll = collection.SimpleFitsCollection( + [mk_test_path("wcs512.fits.gz"), mk_test_path("herschel_spire.fits.gz")] + ) assert not coll._is_multi_tan() diff --git a/toasty/tests/test_merge.py b/toasty/tests/test_merge.py index c59a21e..6cc91c3 100644 --- a/toasty/tests/test_merge.py +++ b/toasty/tests/test_merge.py @@ -6,7 +6,7 @@ import numpy.testing as nt import os.path -from . import test_path +from . import mk_test_path from .. import cli from .. import merge @@ -45,7 +45,7 @@ def test_basic_cli(self): args += [ "--outdir", self.work_path("basic_cli"), - test_path("Equirectangular_projection_SW-tweaked.jpg"), + mk_test_path("Equirectangular_projection_SW-tweaked.jpg"), "1", ] cli.entrypoint(args) diff --git a/toasty/tests/test_misc.py b/toasty/tests/test_misc.py index c553431..bd6f60e 100644 --- a/toasty/tests/test_misc.py +++ b/toasty/tests/test_misc.py @@ -8,7 +8,7 @@ import os.path import pytest -from . import test_path +from . import mk_test_path from .. import cli @@ -32,7 +32,7 @@ def test_make_thumbnail(self): """ args = [ "make-thumbnail", - test_path("Equirectangular_projection_SW-tweaked.jpg"), + mk_test_path("Equirectangular_projection_SW-tweaked.jpg"), self.work_path("basic_cli"), ] cli.entrypoint(args) @@ -47,7 +47,7 @@ def test_crop(self): ImageLoader.add_arguments(parser) settings = parser.parse_args(["--crop=1,2,3,4"]) img = ImageLoader.create_from_args(settings).load_path( - test_path("crop_input.png") + mk_test_path("crop_input.png") ) arr = img.asarray() assert arr.shape == (256, 256, 3) diff --git a/toasty/tests/test_multi_tan.py b/toasty/tests/test_multi_tan.py index c933c0f..5273576 100644 --- a/toasty/tests/test_multi_tan.py +++ b/toasty/tests/test_multi_tan.py @@ -11,7 +11,7 @@ import sys from xml.etree import ElementTree as etree -from . import assert_xml_elements_equal, test_path +from . import assert_xml_elements_equal, mk_test_path from ..builder import Builder from .. import cli from .. import collection @@ -102,7 +102,7 @@ def work_path(self, *pieces): return os.path.join(self.work_dir, *pieces) def test_basic(self): - coll = collection.SimpleFitsCollection([test_path("wcs512.fits.gz")]) + coll = collection.SimpleFitsCollection([mk_test_path("wcs512.fits.gz")]) proc = multi_tan.MultiTanProcessor(coll) @@ -173,7 +173,7 @@ def test_basic_cli(self): "0", "--outdir", self.work_path("basic_cli"), - test_path("wcs512.fits.gz"), + mk_test_path("wcs512.fits.gz"), ] cli.entrypoint(args) @@ -208,7 +208,7 @@ def test_study_cli(self): "--placeholder-thumbnail", "--outdir", self.work_path("study_cli"), - test_path("wcs512.fits.gz"), + mk_test_path("wcs512.fits.gz"), ] cli.entrypoint(args) @@ -246,7 +246,7 @@ def test_as_multi_wcs(self): pio = pyramid.PyramidIO(outdir, default_format="fits") bld = builder.Builder(pio) coll = collection.SimpleFitsCollection( - [test_path("wcs512.fits.gz")], hdu_index=0 + [mk_test_path("wcs512.fits.gz")], hdu_index=0 ) proc = multi_wcs.MultiWcsProcessor(coll) proc.compute_global_pixelization(bld) diff --git a/toasty/tests/test_pipeline.py b/toasty/tests/test_pipeline.py index 28a0bdd..18866cd 100644 --- a/toasty/tests/test_pipeline.py +++ b/toasty/tests/test_pipeline.py @@ -11,7 +11,7 @@ import shutil import sys -from . import assert_xml_elements_equal, test_path +from . import assert_xml_elements_equal, mk_test_path from .. import cli from .. import pipeline from ..pipeline import astropix @@ -47,26 +47,30 @@ def query_candidates(self): "image_width": "7416", "image_height": "4320", "image_max_boundry": "7416", - "astropix_id": 21642 + "astropix_id": 21642, } yield astropix.AstroPixCandidateInput(item) def fetch_candidate(self, unique_id, cand_data_stream, cachedir): - shutil.copy(test_path('NGC253ALMA.jpg'), os.path.join(cachedir, 'image.jpg')) + shutil.copy(mk_test_path("NGC253ALMA.jpg"), os.path.join(cachedir, "image.jpg")) class TestPipeline(object): def setup_method(self, method): from tempfile import mkdtemp + self.work_dir = mkdtemp() - pipeline.IMAGE_SOURCE_CLASS_LOADERS['_local_test_astropix'] = lambda: LocalTestAstroPixImageSource + pipeline.IMAGE_SOURCE_CLASS_LOADERS[ + "_local_test_astropix" + ] = lambda: LocalTestAstroPixImageSource - os.makedirs(self.work_path('repo')) - shutil.copy(test_path('toasty-pipeline-config.yaml'), self.work_path('repo')) + os.makedirs(self.work_path("repo")) + shutil.copy(mk_test_path("toasty-pipeline-config.yaml"), self.work_path("repo")) def teardown_method(self, method): from shutil import rmtree + rmtree(self.work_dir) def work_path(self, *pieces): @@ -74,72 +78,93 @@ def work_path(self, *pieces): def test_workflow(self): args = [ - 'pipeline', 'init', - '--local', self.work_path('repo'), - self.work_path('work'), + "pipeline", + "init", + "--local", + self.work_path("repo"), + self.work_path("work"), ] cli.entrypoint(args) args = [ - 'pipeline', 'refresh', - '--workdir', self.work_path('work'), + "pipeline", + "refresh", + "--workdir", + self.work_path("work"), ] cli.entrypoint(args) args = [ - 'pipeline', 'fetch', - '--workdir', self.work_path('work'), - 'fake_test1', '*nomatchisok*', + "pipeline", + "fetch", + "--workdir", + self.work_path("work"), + "fake_test1", + "*nomatchisok*", ] cli.entrypoint(args) args = [ - 'pipeline', 'process-todos', - '--workdir', self.work_path('work'), + "pipeline", + "process-todos", + "--workdir", + self.work_path("work"), ] cli.entrypoint(args) args = [ - 'pipeline', 'approve', - '--workdir', self.work_path('work'), - 'fake_test1', 'fake_test?', + "pipeline", + "approve", + "--workdir", + self.work_path("work"), + "fake_test1", + "fake_test?", ] cli.entrypoint(args) args = [ - 'pipeline', 'publish', - '--workdir', self.work_path('work'), + "pipeline", + "publish", + "--workdir", + self.work_path("work"), ] cli.entrypoint(args) args = [ - 'pipeline', 'ignore-rejects', - '--workdir', self.work_path('work'), + "pipeline", + "ignore-rejects", + "--workdir", + self.work_path("work"), ] cli.entrypoint(args) def test_args(self): with pytest.raises(SystemExit): args = [ - 'pipeline', 'init', - self.work_path('work'), + "pipeline", + "init", + self.work_path("work"), ] cli.entrypoint(args) with pytest.raises(SystemExit): args = [ - 'pipeline', 'init', - '--azure-conn-env', 'NOTAVARIABLE', - self.work_path('work'), + "pipeline", + "init", + "--azure-conn-env", + "NOTAVARIABLE", + self.work_path("work"), ] cli.entrypoint(args) - os.environ['FAKECONNSTRING'] = 'fake' + os.environ["FAKECONNSTRING"] = "fake" with pytest.raises(SystemExit): args = [ - 'pipeline', 'init', - '--azure-conn-env', 'FAKECONNSTRING', - self.work_path('work'), + "pipeline", + "init", + "--azure-conn-env", + "FAKECONNSTRING", + self.work_path("work"), ] cli.entrypoint(args) diff --git a/toasty/tests/test_samplers.py b/toasty/tests/test_samplers.py index f846c4a..1531bad 100644 --- a/toasty/tests/test_samplers.py +++ b/toasty/tests/test_samplers.py @@ -7,13 +7,14 @@ import os.path import pytest -from . import test_path +from . import mk_test_path from .. import cli from .. import samplers try: import healpy as hp from astropy.io import fits + HAS_ASTRO = True except ImportError: HAS_ASTRO = False @@ -22,16 +23,18 @@ class TestSamplers(object): def setup_method(self, method): from tempfile import mkdtemp + self.work_dir = mkdtemp() def teardown_method(self, method): from shutil import rmtree + rmtree(self.work_dir) def work_path(self, *pieces): return os.path.join(self.work_dir, *pieces) - @pytest.mark.skipif('not HAS_ASTRO') + @pytest.mark.skipif("not HAS_ASTRO") def test_basic_cli(self): """Test some CLI interfaces. We don't go out of our way to validate the computations in detail -- that's for the unit tests that probe the @@ -39,9 +42,10 @@ def test_basic_cli(self): """ args = [ - 'tile-healpix', - '--outdir', self.work_path('basic_cli'), - test_path('earth_healpix_equ.fits'), - '1', + "tile-healpix", + "--outdir", + self.work_path("basic_cli"), + mk_test_path("earth_healpix_equ.fits"), + "1", ] cli.entrypoint(args) diff --git a/toasty/tests/test_study.py b/toasty/tests/test_study.py index 34c981b..ab0941c 100644 --- a/toasty/tests/test_study.py +++ b/toasty/tests/test_study.py @@ -9,7 +9,7 @@ import pytest import sys -from . import HAS_AVM, assert_xml_elements_equal, test_path +from . import HAS_AVM, assert_xml_elements_equal, mk_test_path from .. import cli from .. import study @@ -113,7 +113,7 @@ def test_sample_cli(self): for variants in ([], ["--placeholder-thumbnail"]): args = ["tile-study"] args += variants - args += ["--outdir", self.work_path(), test_path("NGC253ALMA.jpg")] + args += ["--outdir", self.work_path(), mk_test_path("NGC253ALMA.jpg")] cli.entrypoint(args) with open(self.work_path("index_rel.wtml"), "rt", encoding="utf8") as f: @@ -197,7 +197,7 @@ def test_avm(self): "--avm", "--outdir", self.work_path(), - test_path("geminiann11015a.jpg"), + mk_test_path("geminiann11015a.jpg"), ] ) @@ -208,10 +208,10 @@ def test_avm_from(self): [ "tile-study", "--avm-from", - test_path("geminiann11015a.jpg"), + mk_test_path("geminiann11015a.jpg"), "--outdir", self.work_path(), - test_path("geminiann11015a.jpg"), + mk_test_path("geminiann11015a.jpg"), ] ) @@ -279,10 +279,10 @@ def test_fits_wcs(self): [ "tile-study", "--fits-wcs", - test_path("geminiann11015a_wcs.fits"), + mk_test_path("geminiann11015a_wcs.fits"), "--outdir", self.work_path(), - test_path("geminiann11015a.jpg"), + mk_test_path("geminiann11015a.jpg"), ] ) diff --git a/toasty/tests/test_toast.py b/toasty/tests/test_toast.py index f0b9188..8228aab 100644 --- a/toasty/tests/test_toast.py +++ b/toasty/tests/test_toast.py @@ -26,7 +26,7 @@ except ImportError: HAS_OPENEXR = False -from . import test_path +from . import mk_test_path from .. import cli, toast from ..image import ImageLoader from ..jpeg2000 import ChunkedJPEG2000Reader, HAS_JPEG2000 @@ -178,7 +178,7 @@ def verify_level1( ref_x = x warn = "" - ref_path = test_path(ref, str(n), str(y), "%i_%i.png" % (y, ref_x)) + ref_path = mk_test_path(ref, str(n), str(y), "%i_%i.png" % (y, ref_x)) expected = ImageLoader().load_path(ref_path).asarray() if planetary: @@ -202,7 +202,7 @@ def test_plate_carree(self): from ..samplers import plate_carree_sampler img = ImageLoader().load_path( - test_path("Equirectangular_projection_SW-tweaked.jpg") + mk_test_path("Equirectangular_projection_SW-tweaked.jpg") ) sampler = plate_carree_sampler(img.asarray()) sample_layer(self.pio, sampler, 1, format="png") @@ -211,7 +211,7 @@ def test_plate_carree(self): def test_plate_carree_ecliptic(self): from ..samplers import plate_carree_ecliptic_sampler - img = ImageLoader().load_path(test_path("tess_platecarree_ecliptic_512.jpg")) + img = ImageLoader().load_path(mk_test_path("tess_platecarree_ecliptic_512.jpg")) sampler = plate_carree_ecliptic_sampler(img.asarray()) sample_layer(self.pio, sampler, 1, format="png") self.verify_level1(ref="tess") @@ -221,7 +221,7 @@ def test_earth_plate_carree_exr(self): from ..samplers import plate_carree_sampler img = ImageLoader().load_path( - test_path("Equirectangular_projection_SW-tweaked.exr") + mk_test_path("Equirectangular_projection_SW-tweaked.exr") ) sampler = plate_carree_sampler(img.asarray()) sample_layer(self.pio, sampler, 1, format="npy") @@ -234,7 +234,7 @@ def test_earth_plate_carree_jpeg2000_chunked_planetary(self): from ..toast import ToastCoordinateSystem img = ChunkedJPEG2000Reader( - test_path("Equirectangular_projection_SW-tweaked.jp2") + mk_test_path("Equirectangular_projection_SW-tweaked.jp2") ) # this currently (2021 Oct) only supports planetary coordinates: chunker = ChunkedPlateCarreeSampler(img, planetary=True) @@ -257,7 +257,7 @@ def test_earth_plate_carree_jpeg2000_chunked_planetary(self): def test_healpix_equ(self): from ..samplers import healpix_fits_file_sampler - sampler = healpix_fits_file_sampler(test_path("earth_healpix_equ.fits")) + sampler = healpix_fits_file_sampler(mk_test_path("earth_healpix_equ.fits")) sample_layer(self.pio, sampler, 1, format="npy") self.verify_level1(format="npy", expected_2d=True) @@ -265,7 +265,7 @@ def test_healpix_equ(self): def test_healpix_gal(self): from ..samplers import healpix_fits_file_sampler - sampler = healpix_fits_file_sampler(test_path("earth_healpix_gal.fits")) + sampler = healpix_fits_file_sampler(mk_test_path("earth_healpix_gal.fits")) sample_layer(self.pio, sampler, 1, format="fits") self.verify_level1(format="fits", expected_2d=True) @@ -335,7 +335,7 @@ def inner_test(self, image_path, projection): def test_planet(self): self.inner_test( - test_path("Equirectangular_projection_SW-tweaked.jpg"), + mk_test_path("Equirectangular_projection_SW-tweaked.jpg"), "plate-carree-planet", ) @@ -346,7 +346,7 @@ def test_planet_zeroleft(self): need to swap the left and right halves. """ img = ImageLoader().load_path( - test_path("Equirectangular_projection_SW-tweaked.jpg") + mk_test_path("Equirectangular_projection_SW-tweaked.jpg") ) hw = img.width // 2 @@ -368,7 +368,7 @@ def test_planet_zeroright(self): the right edge of the image is longitude 0. """ img = ImageLoader().load_path( - test_path("Equirectangular_projection_SW-tweaked.jpg") + mk_test_path("Equirectangular_projection_SW-tweaked.jpg") ) hw = img.width // 2 diff --git a/toasty/tests/test_toasty.py b/toasty/tests/test_toasty.py index 0f3ce17..0b4bb45 100644 --- a/toasty/tests/test_toasty.py +++ b/toasty/tests/test_toasty.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function import pytest -from . import test_path +from . import mk_test_path from .. import TilingMethod, tile_fits from wwt_data_formats.enums import ProjectionType from shutil import rmtree @@ -24,7 +24,7 @@ class TestToasty(object): def test_tile_fits_tan(self): out_dir_input = "test_tiled" out_dir, bld = tile_fits( - fits=test_path("herschel_spire.fits.gz"), + fits=mk_test_path("herschel_spire.fits.gz"), out_dir=out_dir_input, tiling_method=TilingMethod.TAN, cli_progress=True, @@ -41,12 +41,12 @@ def test_tile_fits_tan(self): # Only testing with a multi tan collection, since multi WCS # collections take a significant time to process out_dir, bld = tile_fits( - fits=[test_path("wcs512.fits.gz"), test_path("wcs512.fits.gz")], + fits=[mk_test_path("wcs512.fits.gz"), mk_test_path("wcs512.fits.gz")], tiling_method=TilingMethod.TAN, override=True, ) assert bld.imgset.projection == ProjectionType.TAN - assert out_dir == test_path("wcs512_tiled") + assert out_dir == mk_test_path("wcs512_tiled") assert Path(out_dir, "index_rel.wtml").is_file() assert Path(out_dir, "0", "0", "0_0.fits").is_file() rmtree(out_dir) @@ -55,7 +55,7 @@ def test_tile_fits_tan(self): def test_tile_fits_toast(self): out_dir_input = "test_tiled" out_dir, bld = tile_fits( - fits=test_path("herschel_spire.fits.gz"), + fits=mk_test_path("herschel_spire.fits.gz"), out_dir=out_dir_input, tiling_method=TilingMethod.TOAST, cli_progress=True, @@ -69,12 +69,12 @@ def test_tile_fits_toast(self): rmtree(out_dir) out_dir, bld = tile_fits( - fits=[test_path("wcs512.fits.gz")], + fits=[mk_test_path("wcs512.fits.gz")], tiling_method=TilingMethod.TOAST, override=True, ) assert bld.imgset.projection == ProjectionType.TOAST - assert out_dir == test_path("wcs512_tiled_TOAST") + assert out_dir == mk_test_path("wcs512_tiled_TOAST") assert Path(out_dir, "index_rel.wtml").is_file() assert Path(out_dir, "0", "0", "0_0.fits").is_file() rmtree(out_dir) diff --git a/toasty/tests/test_wwtl.py b/toasty/tests/test_wwtl.py index e41dd9b..253c9ab 100644 --- a/toasty/tests/test_wwtl.py +++ b/toasty/tests/test_wwtl.py @@ -12,7 +12,7 @@ from wwt_data_formats.filecabinet import FileCabinetWriter -from . import assert_xml_elements_equal, test_path +from . import assert_xml_elements_equal, mk_test_path from .. import cli from .. import study @@ -20,10 +20,12 @@ class TestStudy(object): def setup_method(self, method): from tempfile import mkdtemp + self.work_dir = mkdtemp() def teardown_method(self, method): from shutil import rmtree + rmtree(self.work_dir) def work_path(self, *pieces): @@ -35,26 +37,26 @@ def test_basic_cli(self): fw = FileCabinetWriter() - with open(test_path('layercontainer.wwtxml'), 'rb') as f: + with open(mk_test_path("layercontainer.wwtxml"), "rb") as f: b = f.read() - fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5.wwtxml', b) + fw.add_file_with_data("55cb0cce-c44a-4a44-a509-ea66fce643a5.wwtxml", b) - with open(test_path('NGC253ALMA.jpg'), 'rb') as f: + with open(mk_test_path("NGC253ALMA.jpg"), "rb") as f: b = f.read() - fw.add_file_with_data('55cb0cce-c44a-4a44-a509-ea66fce643a5\\7ecb6411-e4ee-4dfa-90ef-77d6f486c7d2.jpg', b) + fw.add_file_with_data( + "55cb0cce-c44a-4a44-a509-ea66fce643a5\\7ecb6411-e4ee-4dfa-90ef-77d6f486c7d2.jpg", + b, + ) - with open(self.work_path('image.wwtl'), 'wb') as f: + with open(self.work_path("image.wwtl"), "wb") as f: fw.emit(f) # Now run it through the CLI. - for variants in ([], ['--placeholder-thumbnail'], ['--name=Custom Name']): - args = ['tile-wwtl'] + for variants in ([], ["--placeholder-thumbnail"], ["--name=Custom Name"]): + args = ["tile-wwtl"] args += variants - args += [ - '--outdir', self.work_path('tiles'), - self.work_path('image.wwtl') - ] + args += ["--outdir", self.work_path("tiles"), self.work_path("image.wwtl")] cli.entrypoint(args)