Merge branch 'main' into revised-sky-matching-notebook

spacetelescope · Aug 5, 2024 · c7ce2cc · c7ce2cc
2 parents 1b014c0 + 3f3454f
commit c7ce2cc
Show file tree

Hide file tree

Showing 29 changed files with 591 additions and 622 deletions.
diff --git a/.github/workflows/ci_buildondemand.yml b/.github/workflows/ci_buildondemand.yml
@@ -4,6 +4,6 @@ on:
 
 jobs:
   ExecuteNotebooks:
-   uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_scheduled.yml@v4
+   uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_scheduled.yml@multi_dev
    with:
       python-version: ${{ vars.PYTHON_VERSION }}
diff --git a/.github/workflows/ci_execute_single.yml b/.github/workflows/ci_execute_single.yml
@@ -0,0 +1,20 @@
+name: Manual Single File Execute w/OS matrix
+on:
+  workflow_dispatch:
+    inputs:
+      filename:
+        description: 'Notebook file name:'
+        required: true
+        default: 'notebook.ipynb'
+
+jobs:
+  GenerateHTML:
+   uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_execute_single.yml@multi_dev
+   with:
+      python-version: ${{ vars.PYTHON_VERSION }}
+      filename: ${{ github.event.inputs.filename }}
+   secrets:
+      CASJOBS_PW: ${{ secrets.CASJOBS_PW }}
+      CASJOBS_USERID: ${{ secrets.CASJOBS_USERID }}
+   permissions:
+     contents: write
diff --git a/.github/workflows/ci_security_scan.yml b/.github/workflows/ci_security_scan.yml
@@ -0,0 +1,10 @@
+name: Manual Security Scan
+on:
+  workflow_dispatch:
+  #schedule:
+    #- cron: '0 3 * * *' # run at 2 AM UTC
+   # - cron: '0 0 * * 0' # midnight sunday UTC
+
+jobs:
+  Scheduled:
+   uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_security_scan.yml@v3
diff --git a/_config.yml b/_config.yml
@@ -49,5 +49,4 @@ html:
   use_repository_button: true
 # Exclude notebooks that have as-yet unresolved build failures (see tickets SPB-1153SPB-1160, SPB-1168)
 exclude_patterns: [notebooks/DrizzlePac/drizzle_wfpc2/drizzle_wfpc2.ipynb,
-                   notebooks/DrizzlePac/Initialization/Initialization.ipynb,
                    notebooks/WFC3/dash/dash.ipynb]
diff --git a/_toc.yml b/_toc.yml
@@ -30,18 +30,16 @@ parts:
   - caption: DrizzlePac
     chapters:
       - file: notebooks/DrizzlePac/README.md
-      #- file: notebooks/DrizzlePac/Initialization/Initialization.ipynb
+      - file: notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.ipynb
       - file: notebooks/DrizzlePac/align_to_catalogs/align_to_catalogs.ipynb
-# Notebook excluded from build until build failures are fixed (See SPB-1168)
-      - file: notebooks/DrizzlePac/align_mosaics/align_mosaics.ipynb
-      - file: notebooks/DrizzlePac/align_multiple_visits/align_multiple_visits.ipynb
       - file: notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb
-      #- file: notebooks/DrizzlePac/drizzle_wfpc2/drizzle_wfpc2.ipynb
-      - file: notebooks/DrizzlePac/mask_satellite/mask_satellite.ipynb
+      - file: notebooks/DrizzlePac/align_multiple_visits/align_multiple_visits.ipynb
+      - file: notebooks/DrizzlePac/use_ds9_regions_in_tweakreg/use_ds9_regions_in_tweakreg.ipynb
       - file: notebooks/DrizzlePac/optimize_image_sampling/optimize_image_sampling.ipynb
+      - file: notebooks/DrizzlePac/align_mosaics/align_mosaics.ipynb
       - file: notebooks/DrizzlePac/sky_matching/sky_matching.ipynb
-      - file: notebooks/DrizzlePac/use_ds9_regions_in_tweakreg/use_ds9_regions_in_tweakreg.ipynb
-      - file: notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.ipynb
+      - file: notebooks/DrizzlePac/mask_satellite/mask_satellite.ipynb
+      #- file: notebooks/DrizzlePac/drizzle_wfpc2/drizzle_wfpc2.ipynb
   - caption: HASP
     chapters:
     - file: notebooks/HASP/Setup/Setup.ipynb

diff --git a/notebooks/ACS/acs_reduction/requirements.txt b/notebooks/ACS/acs_reduction/requirements.txt
@@ -1,5 +1,5 @@
-astropy==5.3.3
-astroquery==0.4.6
-stwcs==1.7.2
-crds==11.17.0
-matplotlib==3.7.0
+astropy>=5.3.3
+astroquery>=0.4.6
+stwcs>=1.7.2
+crds>=11.17.0
+matplotlib>=3.7.0
diff --git a/notebooks/DrizzlePac/Initialization/Initialization.ipynb b/notebooks/DrizzlePac/Initialization/Initialization.ipynb
diff --git a/notebooks/DrizzlePac/Initialization/requirements.txt b/notebooks/DrizzlePac/Initialization/requirements.txt
diff --git a/notebooks/DrizzlePac/README.md b/notebooks/DrizzlePac/README.md
@@ -1,25 +1,55 @@
-# DrizzlePac Jupyter Notebook Tutorials
+DrizzlePac Notebooks
+=================
 
+An updated set of HST drizzling and alignment tutorials are now available and compatible with the latest STScI distributed software environment [stenv](https://stenv.readthedocs.io/en/latest/). These notebooks include a new recommended workflow for MAST data retrieved after December 2019, which includes updated astrometric information included as additional FITS extensions.  Alternatively, the new World Coordinate System (WCS) solutions may be downloaded directly from MAST as small 'headerlet' files and applied to existing data. For example, the Hubble Advanced Product 'Single Visit Mosaics' may have improved relative alignment for different filters acquired in the same visit. These headerlets may be used to update the WCS in the FITS images prior to drizzling. For details on the alignment of HST data in MAST, see Section 4.5 [Absolute Astrometry](https://hst-docs.stsci.edu/drizzpac/chapter-4-astrometric-information-in-the-header/4-5-absolute-astrometry) in the DrizzlePac Handbook.  
 
-Improved drizzling tutorials are now available as Jupyter Notebooks and compatible with the latest STScI distributed software as part of AstroConda. Prior drizzling examples were written for the DrizzlePac Handbook in 2012, just after MultiDrizzle was replaced, and supplemental examples were posted to the DrizzlePac Webpage in 2015 to support enhanced features in DrizzlePac 2.0. The new interactive notebooks consolidate information from these prior examples to form a more cohesive set, and any references to outdated software, such as PyRAF, have been removed and replaced with python functionality.
+In each notebook, a sample WFC3 or ACS dataset is used to demonstrate how to download the calibrated data, inspect the quality of the alignment, and test whether the observations need to be realigned before combining the data with `AstroDrizzle`. Different workflows are illustrated to enhance the scientific value of the drizzled data products using advanced reprocessing techniques. These notebooks highlight different use cases, e.g. images acquired using small sub-pixel dithers to optimally sample the PSF versus those acquired in multiple pointings to generate large mosaics on the sky.
 
-The notebooks contain live code and visualizations, along with the usual narrative text, making them an ideal training exercise for new users. Each tutorial includes blocks of code demonstrating how to download the calibrated data from the MAST archive, how to align frames and update the image world coordinate system, and how to enhance the scientific value of the drizzled data products using advanced reprocessing techniques.
+The notebooks available in this repository include:
 
-The eleven notebooks available in this repository include the following topics:
+Alignment Workflows:
+- [Improving Absolute and Relative Astrometry Using Alternate WCS Solutions](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.html)
+- [Aligning HST images to an Absolute Reference Catalog](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/align_to_catalogs/align_to_catalogs.html)
+- [Aligning Deep Exposures of Sparse Fields](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.html)
+- [Aligning Multiple HST Visits](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/align_multiple_visits/align_multiple_visits.html)
+- [Using DS9 Regions for Source Inclusion/Exclusion](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/use_ds9_regions_in_tweakreg/use_ds9_regions_in_tweakreg.html)
 
-* Initializing DrizzlePac
-* Aligning observations obtained in multiple HST visits
-* Aligning HST images to an absolute reference catalog (e.g. GAIA, SDSS)
-* Aligning sparse fields
-* Improving alignment with DS9 exclusion regions
-* Masking satellite trails in DQ arrays prior to drizzling
-* Optimizing the image sampling for dithered datasets
-* Drizzling WFPC2 data to use a single zeropoint
-* Sky matching features for HST mosaics
-* Aligning HST mosaics observed with multiple detectors
-* Using the updated astrometry solutions based on _Gaia_ positions
+Drizzling Features:
+- [Optimizing the Image Sampling for Sub-pixel dithers](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/optimize_image_sampling/optimize_image_sampling.html)
+- [Creating HST mosaics observed with multiple detectors](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/align_mosaics/align_mosaics.html)
+- [Using Sky Matching features for HST mosaics](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/sky_matching/sky_matching.html)
+- [Masking satellite trails prior to drizzling](https://spacetelescope.github.io/hst_notebooks/notebooks/DrizzlePac/mask_satellite/mask_satellite.html)
 
-Additional tutorials will be added to the repository as new software functionality becomes available, especially for advanced use cases. For additional assistance with DrizzlePac tools, users may submit a ticket to the [STScI Help Desk](https://stsci.service-now.com/hst?id=hst_index) or send an email to [email protected].
+For more information, see the [DrizzlePac Handbook](https://hst-docs.stsci.edu/drizzpac) and the [readthedocs](https://drizzlepac.readthedocs.io/en/latest/) software documentation. For additional assistance with DrizzlePac tools, users may submit a ticket to the [STScI Help Desk](https://stsci.service-now.com/hst?id=hst_index) and should select the DrizzlePac category.
 
 
-Special thanks to the authors of these notebooks: J. Mack, S. Hoffmann, R. Avila, V. Bajaj, M. Cara, T. Desjardins, K. Huynh, B. Kuhn, C. Martlin, A. O’Connor, and C. Shanahan
+Installation
+------------
+
+It is recommended to clone the entire repository. To do so, run the following command in a terminal:
+
+```
+git clone https://github.com/spacetelescope/hst_notebooks
+```
+
+`stenv` is the preferred base virtual environment for running WFC3 Noteboks since 
+it contains libraries necessary for processing and analyzing data from the Hubble 
+Space Telescope (HST) and the James Webb Space Telescope (JWST). To install, see 
+[stenv readthedocs](https://stenv.readthedocs.io/en/latest/) or 
+[stenv GitHub](https://github.com/spacetelescope/stenv). 
+
+`hst_notebooks/notebooks_env` is the default virtual environment for HST Notebooks, 
+which contains the same scientific computing libraries in `stenv`, but not the HST and
+JWST libraries. This environment can also be used as a base, but is not recommended.
+
+In addition, each notebook contains a `requirements.txt` file that needs to be
+installed before running the notebooks. Here is a common set of commands to run 
+before executing the notebooks (assuming your virtual environment is activated):
+
+```
+pip install -r requirements.txt
+pip install notebook
+```
+
+With the environment activated and additional libraries installed based on the
+individual requirement files, you will be able to complete the notebooks. 
diff --git a/notebooks/DrizzlePac/align_mosaics/align_mosaics.ipynb b/notebooks/DrizzlePac/align_mosaics/align_mosaics.ipynb
@@ -13,15 +13,15 @@
     "\n",
     "***\n",
     "\n",
-    "<div class=\"alert alert-block alert-warning\" style=\"color:black\" > <b> This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository. Please make sure you have read the contents of the README file before continuing the notebook. Note that the text file \"align_mosaics_uvis_skyfile.txt\" is one of the downloads expected for this notebook.</b> </div> \n",
+    "<div class=\"alert alert-block alert-warning\" style=\"color:black\" > <b> This notebook requires creating and activating a virtual environment using the requirements file in this notebook's repository. Please also review the README file before using the notebook. Note that the text file \"align_mosaics_uvis_skyfile.txt\" is one of the downloads expected for this notebook.</b> </div> \n",
     "\n",
     "## Learning Goals\n",
     "\n",
     "By the end of this notebook tutorial, you will:\n",
     "\n",
     "- Download WFC3 UVIS & IR images with `astroquery`\n",
     "- Check the active WCS (world coordinate system) solution in the FITS images\n",
-    "- Create a Gaia reference catalog and align the IR images using `TweakReg`.\n",
+    "- Create a Gaia reference catalog and align the IR images using `TweakReg`\n",
     "- Verify the quality of the alignment results and compare to the MAST alignment solutions\n",
     "- Update the WCS and use `AstroDrizzle` to combine the IR mosaic \n",
     "- Align the UVIS data to the IR drizzled mosaic using `TweakReg`\n",
@@ -155,6 +155,7 @@
     "\n",
     "from astroquery.gaia import Gaia\n",
     "from astroquery.mast import Observations\n",
+    "%config InlineBackend.figure_format = 'retina' # Improves the resolution of figures rendered in notebooks.\n",
     "Gaia.MAIN_GAIA_TABLE = 'gaiadr3.gaia_source'   # Change if different data release is desired\n",
     "Gaia.ROW_LIMIT = 100000"
    ]
@@ -1243,7 +1244,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.13"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/DrizzlePac/align_mosaics/requirements.txt b/notebooks/DrizzlePac/align_mosaics/requirements.txt
@@ -4,3 +4,4 @@ drizzlepac==3.5.1
 ipython==8.11.0
 matplotlib==3.7.0
 numpy==1.23.4
+photutils==1.12.0
diff --git a/notebooks/DrizzlePac/align_multiple_visits/requirements.txt b/notebooks/DrizzlePac/align_multiple_visits/requirements.txt
@@ -3,3 +3,4 @@ astroquery==0.4.6
 drizzlepac==3.5.1
 ipython==8.11.0
 matplotlib==3.7.0
+photutils==1.12.0
diff --git a/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb b/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb
@@ -106,7 +106,7 @@
     "from drizzlepac import tweakreg, astrodrizzle\n",
     "\n",
     "%matplotlib inline\n",
-    "%config InlineBackend.figure_format = 'retina'"
+    "%config InlineBackend.figure_format = 'retina' # Improves the resolution of figures rendered in notebooks."
    ]
   },
   {

diff --git a/notebooks/DrizzlePac/align_sparse_fields/requirements.txt b/notebooks/DrizzlePac/align_sparse_fields/requirements.txt
@@ -5,4 +5,5 @@ drizzlepac==3.6.2
 ipython==8.21.0
 matplotlib==3.8.2
 numpy==1.26.3
+photutils==1.12.0
 jupyter==1.0.0