Drizzlepac: Preparation and Tagging of Release Candidate RC5 (#1830)

Co-authored-by: Steve Goldman <[email protected]>

CHANGELOG.rst

@@ -23,6 +23,16 @@ number of the code change for that issue.  These PRs can be viewed at:
 - Modify the dependencies portion of the project.toml file to specify
   numpy<2.0 [#1813]
 
+- Reverted #1798 until further testing is done with Photutils.
+
+- Corrected the way that the number of constituent images are accumulated
+  per pixel by ensuring each contributing pixel has a finite value and 
+  is not zero. [#1820]
+
+- Within the HAP configuration files, increased the minimum number of matches 
+  for a successful "rscale" fit from 6 to 10, and removed "shift" as a fit geometry 
+  option. [#1823].
+
 - Removed the use of a custom smoothing kernel based upon actual image
   data as a poorly determined kernel can ultimately cause poor source
   position determination.  The default kernel has been set to a
@@ -78,8 +88,7 @@ number of the code change for that issue.  These PRs can be viewed at:
 - Improved calculation of S_REGION using dialation and erosion. [#1762]
 
 - Skycell added to flt(c) and drz(c) science headers for the pipeline and svm products. [#1729]
-
-- resolved ``AstropyDeprecationWarning`` s [#1798]
+
 
 3.7.0 (02-Apr-2024)
 ===================

JenkinsfileRT

@@ -61,7 +61,7 @@ bc3.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory',
                 'CFLAGS=-std=gnu99']
 bc3.name = '3.11'
 bc3.conda_packages = ['python=3.11']
-bc3.build_cmds = ["pip install numpy astropy ci-watson",
+bc3.build_cmds = ["pip install numpy astropy pytest-cov ci-watson",
 		 "pip install --upgrade -e '.[test]'",
                  "pip freeze"]
 bc3.test_cmds = ["pytest --env=${artifactory_env} --cov=./ --basetemp=tests_output --junitxml=results.xml --bigdata",

drizzlepac/haputils/astrometric_utils.py

@@ -61,8 +61,8 @@
 
 from photutils.background import (Background2D, MMMBackground,
                                   SExtractorBackground, StdBackgroundRMS)
-from photutils.psf import (IntegratedGaussianPRF, SourceGrouper,
-                           IterativePSFPhotometry)
+from photutils.psf import (IntegratedGaussianPRF, DAOGroup,
+                           IterativelySubtractedPSFPhotometry)
 
 from tweakwcs.correctors import FITSWCSCorrector
 from stwcs.distortion import utils
@@ -847,8 +847,8 @@ def find_fwhm(psf, default_fwhm):
     """Determine FWHM for auto-kernel PSF
 
     This function iteratively fits a Gaussian model to the extracted PSF
-    using `photutils.psf.IterativePSFPhotometry
-    <https://photutils.readthedocs.io/en/stable/api/photutils.psf.IterativePSFPhotometry.html>`_
+    using `photutils.psf.IterativelySubtractedPSFPhotometry
+    <https://photutils.readthedocs.io/en/stable/api/photutils.psf.IterativelySubtractedPSFPhotometry.html>`_
     to determine the FWHM of the PSF.
 
     Parameters
@@ -865,21 +865,21 @@ def find_fwhm(psf, default_fwhm):
         Value of the computed Gaussian FWHM for the PSF
 
     """
-    daogroup = SourceGrouper(min_separation=8)
+    daogroup = DAOGroup(crit_separation=8)
     mmm_bkg = MMMBackground()
     iraffind = DAOStarFinder(threshold=2.5 * mmm_bkg(psf), fwhm=default_fwhm)
     fitter = LevMarLSQFitter()
     sigma_psf = gaussian_fwhm_to_sigma * default_fwhm
     gaussian_prf = IntegratedGaussianPRF(sigma=sigma_psf)
     gaussian_prf.sigma.fixed = False
     try:
-        itr_phot_obj = IterativePSFPhotometry(finder=iraffind,
-                                              group_maker=daogroup,
-                                              bkg_estimator=mmm_bkg,
-                                              psf_model=gaussian_prf,
-                                              fitter=fitter,
-                                              fitshape=(11, 11),
-                                              niters=2)
+        itr_phot_obj = IterativelySubtractedPSFPhotometry(finder=iraffind,
+                                                          group_maker=daogroup,
+                                                          bkg_estimator=mmm_bkg,
+                                                          psf_model=gaussian_prf,
+                                                          fitter=fitter,
+                                                          fitshape=(11, 11),
+                                                          niters=2)
         phot_results = itr_phot_obj(psf)
     except Exception:
         log.error("The find_fwhm() failed due to problem with fitting.")

drizzlepac/haputils/hla_flag_filter.py

@@ -1244,7 +1244,9 @@ def hla_nexp_flags(drizzled_image, flt_list, param_dict, plate_scale, catalog_na
     nexp_array = numpy.zeros((nx, ny), dtype=numpy.int32)
 
     for comp_drz_img in component_drz_img_list:
-        comp_drz_data = (fits.getdata(comp_drz_img) != 0).astype(numpy.int32)
+        imdata = fits.getdata(comp_drz_img)
+        comp_drz_data = (numpy.isfinite(imdata) & (imdata != 0)).astype(numpy.int32)
+        del imdata
         try:
             nexp_array += comp_drz_data
         except ValueError:
@@ -1632,7 +1634,8 @@ def make_mask_array(drz_image):
     mask : numpy.ndarray object
         mask array
     """
-    mask = fits.open(drz_image)[1].data != 0
+    with fits.open(drz_image) as fh:
+        mask = numpy.isfinite(fh[1].data) & (fh[1].data != 0)
     dilate = scipy.ndimage.morphology.binary_dilation
     erode = scipy.ndimage.morphology.binary_erosion
     kernel1 = numpy.ones((25, 25), dtype=int)

drizzlepac/pars/hap_pars/svm_parameters/acs/hrc/acs_hrc_alignment_all.json

@@ -16,7 +16,7 @@
       "MAX_FIT_LIMIT": 150,
       "mosaic_catalog_list": ["GAIAeDR3", "GSC242", "2MASS"],
       "mosaic_fit_list": ["match_relative_fit", "match_2dhist_fit", "match_default_fit"],
-        "mosaic_fitgeom_list": {"shift": 3, "rshift": 4, "rscale": 6, "general": 6}
+        "mosaic_fitgeom_list": {"rshift": 4, "rscale": 10, "general": 6}
     },
   "generate_source_catalogs":
     {

drizzlepac/pars/hap_pars/svm_parameters/acs/sbc/acs_sbc_alignment_all.json

@@ -16,7 +16,7 @@
       "MAX_FIT_LIMIT": 150,
       "mosaic_catalog_list": ["GAIAeDR3", "GSC242", "2MASS"],
       "mosaic_fit_list": ["match_relative_fit", "match_2dhist_fit", "match_default_fit"],
-        "mosaic_fitgeom_list": {"shift": 3, "rshift": 4, "rscale": 6, "general": 6}
+        "mosaic_fitgeom_list": {"rshift": 4, "rscale": 10, "general": 6}
     },
   "generate_source_catalogs":
     {

drizzlepac/pars/hap_pars/svm_parameters/acs/wfc/acs_wfc_alignment_all.json

@@ -16,7 +16,7 @@
         "MAX_FIT_LIMIT": 150,
         "mosaic_catalog_list": ["GAIAeDR3", "GSC242", "2MASS"],
         "mosaic_fit_list": ["match_relative_fit", "match_2dhist_fit", "match_default_fit"],
-        "mosaic_fitgeom_list": {"shift": 3, "rshift": 4, "rscale": 6, "general": 6}
+        "mosaic_fitgeom_list": {"rshift": 4, "rscale": 10, "general": 6}
       },
     "generate_source_catalogs":
       {

drizzlepac/pars/hap_pars/svm_parameters/wfc3/ir/wfc3_ir_alignment_all.json

@@ -16,7 +16,7 @@
         "MAX_FIT_LIMIT": 150,
         "mosaic_catalog_list": ["GAIAeDR3", "GSC242", "2MASS"],
         "mosaic_fit_list": ["match_relative_fit", "match_2dhist_fit", "match_default_fit"],
-        "mosaic_fitgeom_list": {"shift": 3, "rshift": 4, "rscale": 6, "general": 6}
+        "mosaic_fitgeom_list": {"rshift": 4, "rscale": 10, "general": 6}
       },
     "generate_source_catalogs":
       {

drizzlepac/pars/hap_pars/svm_parameters/wfc3/uvis/wfc3_uvis_alignment_all.json

@@ -16,7 +16,7 @@
         "MAX_FIT_LIMIT": 150,
         "mosaic_catalog_list": ["GAIAeDR3", "GSC242", "2MASS"],
         "mosaic_fit_list": ["match_relative_fit", "match_2dhist_fit", "match_default_fit"],
-        "mosaic_fitgeom_list": {"shift": 3, "rshift": 4, "rscale": 6, "general": 6}
+        "mosaic_fitgeom_list": {"rshift": 4, "rscale": 10, "general": 6}
       },
     "generate_source_catalogs":
       {

drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_alignment_all.json

@@ -16,7 +16,7 @@
         "MAX_FIT_LIMIT": 150,
         "mosaic_catalog_list": ["GAIAeDR3", "GSC242", "2MASS"],
         "mosaic_fit_list": ["match_relative_fit", "match_2dhist_fit", "match_default_fit"],
-        "mosaic_fitgeom_list": {"shift": 3, "rshift": 4, "rscale": 6, "general": 6}
+        "mosaic_fitgeom_list": {"rshift": 4, "rscale": 10, "general": 6}
       },
     "generate_source_catalogs":
       {

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     "spherical_geometry>=1.2.22",
     "astroquery>=0.4",
     "astrocut<=0.9",
-    "photutils>1.5.0",
+    "photutils>1.5.0,<1.13.0",
     "lxml",
     "PyPDF2",
     "scikit-image>=0.14.2",

tests/hap/test_svm_ibqk07.py

@@ -29,7 +29,8 @@
 """
 
 POLLER_FILE = "wfc3_bqk_07_input.out"
-WCS_SUB_NAME = "HSC30"
+WCS_IR_SUB_NAME = "HSC30"
+WCS_UVIS_SUB_NAME = "FIT_SVM_GAIA"
 expected_total_point_sources = {
 "hst_12557_07_wfc3_ir_total_ibqk07_point-cat.ecsv": 2,
 "hst_12557_07_wfc3_uvis_total_ibqk07_point-cat.ecsv": 13}
@@ -183,7 +184,7 @@ def test_svm_wcs_ir(gather_output_data):
     # Check the WCS solution is as expected
     wcsname = fits.getval(tdp_files[0], "WCSNAME", ext=1).upper()
     print("\ntest_svm_wcs_ir.  WCSNAME: {} Output file: {}".format(wcsname, tdp_files[0]))
-    assert WCS_SUB_NAME in wcsname, f"WCSNAME is not as expected for file {tdp_files[0]}."
+    assert WCS_IR_SUB_NAME in wcsname, f"WCSNAME is not as expected for file {tdp_files[0]}."
 
 
 def test_svm_wcs_ir_all(gather_output_data):
@@ -203,7 +204,7 @@ def test_svm_wcs_uvis(gather_output_data):
     # Check the WCS solution is as expected
     wcsname = fits.getval(tdp_files[0], "WCSNAME", ext=1).upper()
     print("\ntest_svm_wcs_uvis.  WCSNAME: {} Output file: {}".format(wcsname, tdp_files[0]))
-    assert WCS_SUB_NAME in wcsname, f"WCSNAME is not as expected for file {tdp_files[0]}."
+    assert WCS_UVIS_SUB_NAME in wcsname, f"WCSNAME is not as expected for file {tdp_files[0]}."
 
 
 def test_svm_wcs_uvis_all(gather_output_data):

tests/hap/test_svm_ibyt50.py

@@ -30,13 +30,13 @@
 
 POLLER_FILE = "wfc3_byt_50_input.out"
 WCS_UVIS_SUB_NAME = "FIT_SVM_GAIA"
-WCS_IR_SUB_NAME = "FIT_SVM_GSC242"
+WCS_IR_SUB_NAME = "FIT_SVM_GAIA"
 expected_total_point_sources = {
-"hst_13023_50_wfc3_ir_total_ibyt50_point-cat.ecsv": 122,
-"hst_13023_50_wfc3_uvis_total_ibyt50_point-cat.ecsv": 105}
+"hst_13023_50_wfc3_ir_total_ibyt50_point-cat.ecsv": 118,
+"hst_13023_50_wfc3_uvis_total_ibyt50_point-cat.ecsv": 100}
 expected_total_segment_sources = {
-"hst_13023_50_wfc3_ir_total_ibyt50_segment-cat.ecsv": 107,
-"hst_13023_50_wfc3_uvis_total_ibyt50_segment-cat.ecsv": 415}
+"hst_13023_50_wfc3_ir_total_ibyt50_segment-cat.ecsv": 120,
+"hst_13023_50_wfc3_uvis_total_ibyt50_segment-cat.ecsv": 300}
 tolerance = 0.25