Release commit created with Cranko.

+++ cranko-release-info-v1
[[projects]]
qnames = ["toasty", "pypa"]
version = "0.10.0"
age = 0

+++

CHANGELOG.md

@@ -1,3 +1,27 @@
+# toasty 0.10.0 (2021-09-10)
+
+- Add `toasty check-avm`, which opens up an image file and reports whether it
+  contains AVM (Astronomy Visualization Metadata) tags. This requires that the
+  `pyavm` module is installed (#59, @pkgw).
+- Add the `--avm` option to the `toasty tile-study` command (#59, @pkgw). When
+  specified, spatial positioning information for the input image will be loaded
+  from AVM tags in the input image and preserved in the resulting WTML file.
+  This option doesn't "just work" automatically (for now) because it requires
+  the `pyavm` module to be present, and we don't want to make that a hard
+  requirement of installing Toasty.
+- Fix `toasty tile-wwtl` to emit correct WTML files once again (#58, @pkgw).
+- Increase the ability of the "multi-WCS" FITS tiling functionality to handle
+  huge images by reprojecting them in (large) chunks. This shouldn't affect
+  performance with reasonable-sized images, but makes it possible to handle
+  large ones. Here "large" means that the image consumes something like 10-25%
+  of the available system memory.
+- Silence various unhelpful Python warnings
+- Enable FITS processing to work when the input image has more than two axes,
+  if the other axes are only one element long (#57, @pkgw).
+- Write out `DATAMIN` and `DATAMAX` headers in output FITS files, which helps
+  WWT set the correct scaling for FITS visualization (#57, @pkgw).
+
+
 # toasty 0.9.0 (2021-08-25)
 
 - Add a `plate-caree-panorama` projection mode to the `tile-allsky` command

ci/azure-build-and-test.yml

@@ -66,7 +66,7 @@ jobs:
         if [[ $AGENT_OS == Windows_NT ]] ; then
           \conda install -y astropy pytest
         else
-          \conda install -y astropy healpy pytest
+          \conda install -y astropy healpy pyavm pytest
         fi
 
         pytest --pyargs toasty
@@ -101,6 +101,7 @@ jobs:
         numpy \
         openexr-python \
         pillow \
+        pyavm \
         pytest-cov \
         pyyaml \
         tqdm

docs/cli.rst

@@ -6,6 +6,7 @@ CLI Reference
 
    cli/std-image-options
    cli/cascade
+   cli/check-avm
    cli/make-thumbnail
    cli/pipeline-approve
    cli/pipeline-fetch

docs/cli/check-avm.rst

@@ -0,0 +1,52 @@
+.. _cli-check-avm:
+
+====================
+``toasty check-avm``
+====================
+
+The ``check-avm`` command checks whether an image file contains AVM (`Astronomy
+Visualization Metadata`_) tags in its metadata. AVM metadata often include
+spatial positioning data that specify how an astronomical image maps onto the
+sky, similar to `WCS`_ headers in FITS files.
+
+.. _Astronomy Visualization Metadata: https://virtualastronomy.org/avm_metadata.php
+
+Usage
+=====
+
+.. code-block:: shell
+
+   toasty check-avm
+      [--print] [-p]
+      [--exitcode]
+      IMAGE-PATH
+
+The ``IMAGE-PATH`` argument gives the filesystem path of an image to check. This
+should typically be an RGB bitmap image such as a JPEG, PNG, or TIFF file. The
+tool will try to load AVM data from the file and report whether it is
+successful.
+
+.. _WCS: https://fits.gsfc.nasa.gov/fits_wcs.html
+
+By default, only the presence of AVM data is reported. If the ``--print`` or
+``-p`` option is given, the AVM data will be dumped out.
+
+If the ``--exitcode`` option is given, the exit code of the command line tool
+will reflect whether AVM data were found. Zero (success) means that data were
+found, while nonzero (failure) indicates that no data were present, the data
+were corrupt, or that some other failure occurred. Without this option, the exit
+code will be zero if the image can be opened but contains no AVM data.
+
+
+Notes
+=====
+
+This functionality requires that the `pyavm`_ Python package has been installed.
+
+.. _pyavm: https://astrofrog.github.io/pyavm/
+
+
+See Also
+========
+
+- :ref:`cli-tile-study`

docs/cli/tile-study.rst

@@ -17,6 +17,7 @@ Usage
       [--placeholder-thumbnail]
       [--outdir DIR]
       [--name NAME]
+      [--avm]
       IMAGE-PATH
 
 See the :ref:`cli-std-image-options` section for documentation on those options.
@@ -38,6 +39,15 @@ memory-intensive part of the process, and can yield poor results with
 mostly-empty images. You can avoid this by using this argument and then invoking
 :ref:`cli-make-thumbnail` with a better-suited input image.
 
+If the ``--avm`` argument is given, Toasty will attempt to load world-coordinate
+information from AVM (`Astronomy Visualization Metadata`_) tags in the input
+image’s metadata. This isn’t checked automatically because this functionality
+requires the `pyavm`_ package to be installed on your system.
+
+.. _Astronomy Visualization Metadata: https://virtualastronomy.org/avm_metadata.php
+
+.. _pyavm: https://astrofrog.github.io/pyavm/
+
 
 Notes
 =====
@@ -49,3 +59,9 @@ be able to get away with fudging the projection type.
 If the input image does not contain any useful astrometric information, the
 emited ``index_rel.wtml`` file will contain generic information that makes the
 image 1° wide and places it at RA = Dec = 0.
+
+
+See Also
+========
+
+- :ref:`cli-check-avm`

setup.py

@@ -36,7 +36,7 @@ def get_long_desc():
 
 setup_args = dict(
     name = 'toasty',  # cranko project-name
-    version = '0.9.0',  # cranko project-version
+    version = '0.10.0',  # cranko project-version
     description = 'Generate TOAST image tile pyramids from existing image data',
     long_description = get_long_desc(),
     long_description_content_type = 'text/markdown',

toasty/builder.py

@@ -112,9 +112,15 @@ def load_from_wwtl(self, cli_settings, wwtl_path, cli_progress=False):
             img_data = lc.read_layer_file(layer, layer.extension)
             img = loader.load_stream(BytesIO(img_data))
 
+        # (Re-)initialize with the imageset info extracted from the WWTL.
+
         self.imgset = imgset
         self.place.foreground_image_set = self.imgset
 
+        self.imgset.file_type = '.' + self.pio.get_default_format()
+        self.imgset.url = self.pio.get_path_scheme() + self.imgset.file_type
+        self.place.name = self.imgset.name
+
         # Transmogrify untiled image info to tiled image info. We reuse the
         # existing imageset as much as possible, but update the parameters that
         # change in the tiling process.
@@ -188,7 +194,22 @@ def apply_wcs_info(self, wcs, width, height):
 
 
     def apply_avm_info(self, avm, width, height):
-        self.apply_wcs_info(avm.to_wcs(target_shape=(width, height)), width, height)
+        # So. The AVM standard discusses how parity should be expressed and how
+        # it should be translated into WCS data, but in practice things are a
+        # bit wonky: the AVM data that we've seen in the wild basically express
+        # FITS-like (positive parity) WCS, while the actual associated image
+        # data have a JPEG-like (negative parity) data layout. WCS can express
+        # either parity so it would arguably be more correct for the generated
+        # WCS to have negative parity. Based on the current state of knowledge,
+        # I think the best option for now is to always flip the parity of the
+        # WCS that pyavm hands us. We might need to change the heuristic or
+        # allow the user to change the behavior.
+
+        wcs = avm.to_wcs(target_shape=(width, height))
+        from .image import _flip_wcs_parity
+        wcs = _flip_wcs_parity(wcs, height)
+
+        self.apply_wcs_info(wcs, width, height)
 
         if avm.Title:
             self.imgset.name = avm.Title

toasty/cli.py

@@ -17,6 +17,8 @@
 import argparse
 import os.path
 import sys
+import warnings
+
 from wwt_data_formats.cli import EnsureGlobsExpandedAction
 
 
@@ -79,6 +81,61 @@ def cascade_impl(settings):
     )
 
 
+# "check-avm" subcommand
+
+def check_avm_getparser(parser):
+    parser.add_argument(
+        '--print', '-p',
+        dest = 'print_avm',
+        action = 'store_true',
+        help = 'Print the AVM data if present',
+    )
+    parser.add_argument(
+        '--exitcode',
+        action = 'store_true',
+        help = 'Exit with code 1 if no AVM tags are present',
+    )
+    parser.add_argument(
+        'image',
+        metavar = 'PATH',
+        help = 'An image to check for AVM tags',
+    )
+
+
+def check_avm_impl(settings):
+    try:
+        from pyavm import AVM, exceptions
+
+        SYNTAX_EXCEPTIONS = (
+            exceptions.AVMListLengthError,
+            exceptions.AVMItemNotInControlledVocabularyError,
+            exceptions.AVMEmptyValueError,
+        )
+    except ImportError:
+        die('cannot check AVM tags: you must install the `pyavm` package')
+
+    try:
+        # PyAVM can issue a lot of warnings that aren't helpful to the user.
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            avm = AVM.from_image(settings.image)
+    except exceptions.NoXMPPacketFound:
+        print(f'{settings.image}: no AVM tags')
+        sys.exit(1 if settings.exitcode else 0)
+    except SYNTAX_EXCEPTIONS as e:
+        print(f'{settings.image}: AVM data are present but malformed: {e} ({e.__class__.__name__})')
+        sys.exit(1)
+    except IOError as e:
+        die(f'error probing AVM tags of `{settings.image}`: {e}')
+
+    if settings.print_avm:
+        print(f'{settings.image}: AVM tags are present')
+        print()
+        print(avm)
+    else:
+        print(f'{settings.image}: AVM tags are present (use `--print` to display them)')
+
+
 # "make_thumbnail" subcommand
 
 def make_thumbnail_getparser(parser):
@@ -329,6 +386,11 @@ def tile_study_getparser(parser):
         default = 'Toasty',
         help = 'The image name to embed in the output WTML file (default: %(default)s)',
     )
+    parser.add_argument(
+        '--avm',
+        action = 'store_true',
+        help = 'Expect the input image to have AVM positioning tags',
+    )
     parser.add_argument(
         '--placeholder-thumbnail',
         action = 'store_true',
@@ -356,10 +418,23 @@ def tile_study_impl(settings):
     pio = PyramidIO(settings.outdir, default_format=img.default_format)
     builder = Builder(pio)
 
-    if img.wcs is None:
-        builder.default_tiled_study_astrometry()
+    if settings.avm:
+        # We don't *always* check for AVM because pyavm prints out a lot of junk
+        # and may not be installed.
+        try:
+            from pyavm import AVM
+        except ImportError:
+            die('cannot use AVM data: you must install the `pyavm` package')
+
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                avm = AVM.from_image(settings.imgpath)
+        except Exception:
+            print(f'error: failed to read AVM tags of input `{settings.imgpath}`', file=sys.stderr)
+            raise
     else:
-        builder.apply_wcs_info(img.wcs, img.width, img.height)
+        builder.default_tiled_study_astrometry()
 
     # Do the thumbnail first since for large inputs it can be the memory high-water mark!
     if settings.placeholder_thumbnail:
@@ -368,6 +443,12 @@ def tile_study_impl(settings):
         builder.make_thumbnail_from_other(img)
 
     builder.tile_base_as_study(img, cli_progress=True)
+
+    if settings.avm:
+        builder.apply_avm_info(avm, img.width, img.height)
+    elif img.wcs is not None:
+        builder.apply_wcs_info(img.wcs, img.width, img.height)
+
     builder.set_name(settings.name)
     builder.write_index_rel_wtml()
 

toasty/collection.py

@@ -56,9 +56,10 @@ def images(self):
 
 
 class SimpleFitsCollection(ImageCollection):
-    def __init__(self, paths, hdu_index=None):
+    def __init__(self, paths, hdu_index=None, blankval=None):
         self._paths = list(paths)
         self._hdu_index = hdu_index
+        self._blankval = blankval
 
     def _load(self, actually_load_data):
         from astropy.io import fits
@@ -74,11 +75,43 @@ def _load(self, actually_load_data):
                             break
 
                 wcs = WCS(hdu.header)
+                shape = hdu.shape
+
+                # We need to make sure the data are 2D celestial, since that's
+                # what our image code and `reproject` (if it's being used) expect.
+
+                full_wcs = None
+                keep_axes = None
+
+                if wcs.naxis != 2:
+                    if not wcs.has_celestial:
+                        raise Exception(f'cannot process input `{fits_path}`: WCS cannot be reduced to 2D celestial')
+
+                    full_wcs = wcs
+                    wcs = full_wcs.celestial
+
+                    # note: get_axis_types returns axes in FITS order, innermost first
+                    keep_axes = [t.get('coordinate_type') == 'celestial' for t in full_wcs.get_axis_types()[::-1]]
+
+                    for keep, axlen in zip(keep_axes, shape):
+                        if not keep and axlen != 1:
+                            raise Exception(f'cannot process input `{fits_path}`: found a non-celestial axis with size other than 1')
+
+                # OK, almost there. Are we loading data or just the descriptor?
 
                 if actually_load_data:
-                    result = Image.from_array(hdu.data, wcs=wcs, default_format='fits')
+                    data = hdu.data
+
+                    if full_wcs is not None:  # need to subset?
+                        data = data[tuple(slice(None) if k else 0 for k in keep_axes)]
+
+                    if self._blankval is not None:
+                        data[data == self._blankval] = np.nan
+
+                    result = Image.from_array(data, wcs=wcs, default_format='fits')
                 else:
-                    shape = hdu.shape
+                    if full_wcs is not None:  # need to subset?
+                        shape = tuple(t[1] for t in zip(keep_axes, shape) if t[0])
 
                     if hasattr(hdu, 'dtype'):
                         mode = ImageMode.from_array_info(shape, hdu.dtype)