Update release candidate to define v3.5.0rc1 (#1435)

* Insure correct filetype gets defined for filter and total products (#1433)

* Insure consistent filetypes are used to create HAP products

* Enforce FLC over FLT

* Update change log

* Update azure Mac usage

* Update to fully support numpy v1.23 (#1434)

* Update code to support numpy v1.23

* Fix typo in parameter for align_utils

CHANGELOG.rst

@@ -30,6 +30,9 @@ number of the code change for that issue.  These PRs can be viewed at:
 
 - Refactored code to work with changes in ``tweakwcs`` version 0.8.0. [#1430]
 
+- Ignore non-CTE-corrected exposures when SVM or MVM products also include
+  CTE-corrected exposures as inputs. [#1433]
+
 
 3.4.3 (24-Aug-2022)
 ===================

azure-templates.yml

@@ -4,7 +4,7 @@ jobs:
     ${{ if eq(parameters.os, 'windows') }}:
       vmImage: windows-latest
     ${{ if eq(parameters.os, 'macos') }}:
-      vmImage: macOS-10.15
+      vmImage: macOS-latest
     ${{ if eq(parameters.os, 'linux') }}:
       vmImage: ubuntu-latest
 

drizzlepac/haputils/align_utils.py

@@ -916,7 +916,7 @@ def match_relative_fit(imglist, reference_catalog, **fit_pars):
             image.meta["group_id"] = 1234567
         # 2: Perform absolute alignment
         matched_cat = align_wcs(imglist, reference_catalog, match=match,
-                                inobj=common_pars['minobj'][fitgeom],
+                                minobj=common_pars['minobj'][fitgeom],
                                 fitgeom=fitgeom, nclip=nclip)
         # Insure the expanded reference catalog has all the information needed
         # to complete processing.

drizzlepac/haputils/cell_utils.py

@@ -337,8 +337,8 @@ def build(self, expnames, scale=False, scale_kw='EXPTIME'):
                 meta_x = np.clip(meta_x, 0, self.meta_wcs.array_shape[1] - 1)
 
                 # define subarray spanned by this chip on the SkyCell
-                scell_slice = [slice(meta_y.min(), meta_y.max()),
-                               slice(meta_x.min(), meta_x.max())]
+                scell_slice = (slice(meta_y.min(), meta_y.max()),
+                               slice(meta_x.min(), meta_x.max()))
                 scell_ltm = [meta_x.min(), meta_y.min()]
                 # Reset range of pixels to be relative to starting pixel position in SkyCell
                 meta_x -= scell_ltm[0]

drizzlepac/haputils/poller_utils.py

@@ -442,9 +442,6 @@ def parse_mvm_tree(det_tree, all_mvm_exposures, log_level):
 
     tdp_list = []
 
-    # Determine if the individual files being processed are flt or flc and
-    # set the filetype accordingly (flt->drz or flc->drc).
-    filetype = ''
     # Setup products for each detector used
     #
     # det_tree = {'UVIS': {'f200lp': [('skycell_p1234_x01y01 WFC3 UVIS IACS01T9Q F200LP 1', 'iacs01t9q_flt.fits'),
@@ -466,14 +463,17 @@ def parse_mvm_tree(det_tree, all_mvm_exposures, log_level):
             # Populate single exposure dictionary entry now as well
             # filename = ('skycell_p1234_x01y01 WFC3 UVIS IACS01T9Q F200LP 1', 'iacs01t9q_flt.fits')
             #
-            for filename in filter_files:
-                # Parse the first filename[1] to determine if the products are flt or flc
+            filter_members, filetype = select_common_filetype(filter_files)
+
+            for filename, is_member in zip(filter_files, filter_members):
+
                 if det_indx != prev_det_indx:
-                    filetype = "drc"
-                    if filename[1][-8:-5].lower().endswith("flt"):
-                        filetype = "drz"
                     prev_det_indx = det_indx
 
+                # If this input exposure does not have the same filetype (FLT/FLC) as the
+                # rest of the images in this layer, ignore it.
+                if not is_member:
+                    continue
                 # Generate the full product dictionary information string:
                 #
                 # [row['proposal_id'], row['obset_id'], row['instrument'],
@@ -624,9 +624,6 @@ def parse_obset_tree(det_tree, log_level):
 
     tdp_list = []
 
-    # Determine if the individual files being processed are flt or flc and
-    # set the filetype accordingly (flt->drz or flc->drc).
-    filetype = ''
     # Setup products for each detector used
     for filt_tree in det_tree.values():
         totprod = TDP_STR.format(det_indx)
@@ -639,23 +636,30 @@ def parse_obset_tree(det_tree, log_level):
             obset_products[fprod] = {'info': "", 'files': []}
             filt_indx += 1
             # Populate single exposure dictionary entry now as well
-            for filename in filter_files:
-                # Parse the first filename[1] to determine if the products are flt or flc
+            filter_members, filetype = select_common_filetype(filter_files)
+
+            for filename, is_member in zip(filter_files, filter_members):
                 if det_indx != prev_det_indx:
-                    filetype = "drc"
-                    if filename[1][10:13].lower().endswith("flt"):
-                        filetype = "drz"
                     prev_det_indx = det_indx
 
+                if not is_member:
+                    # This logic should get the opposite filetype from the
+                    # value returned for the entire filter (at least that is the plan)
+                    exp_filetype = 'drz' if filetype == 'drc' else 'drc'
+                else:
+                    exp_filetype = filetype
+
                 # Generate the full product dictionary information string:
                 # proposal_id, obset_id, instrument, detector, ipppssoot, filter, and filetype
                 # filename = ('11515 01 WFC3 UVIS IACS01T9Q F200LP', 'iacs01t9q_flt.fits')
-                prod_info = (filename[0] + " " + filetype).lower()
+                exp_prod_info = (filename[0] + " " + exp_filetype).lower()
 
                 # Set up the single exposure product dictionary
                 sep = SEP_STR.format(sep_indx)
-                obset_products[sep] = {'info': prod_info,
+                obset_products[sep] = {'info': exp_prod_info,
                                        'files': [filename[1]]}
+                # Increment single exposure index
+                sep_indx += 1
 
                 # Create a single exposure product object
                 #
@@ -666,7 +670,7 @@ def parse_obset_tree(det_tree, log_level):
                 # Grism/Prism products.
                 #
                 # The GrismExposureProduct is only an attribute of the TotalProduct.
-                prod_list = prod_info.split(" ")
+                prod_list = exp_prod_info.split(" ")
 
                 # Determine if this image is a Grism/Prism or a nominal direct exposure
                 is_grism = False
@@ -679,6 +683,16 @@ def parse_obset_tree(det_tree, log_level):
                 else:
                     sep_obj = ExposureProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3],
                                               filename[1], prod_list[5], prod_list[6], log_level)
+                # Now that we have defined the ExposureProduct for this input exposure,
+                # do not include it any total or filter product.
+                if not is_member:
+                    continue
+
+                prod_info = (filename[0] + " " + filetype).lower()
+
+                # Redefine for the filter and total product definitions
+                # This will insure the correct product type (DRZ vs DRC) gets passed along
+                prod_list = prod_info.split(" ")
 
                 # Set up the filter product dictionary and create a filter product object
                 # Initialize `info` key for this filter product dictionary
@@ -707,9 +721,6 @@ def parse_obset_tree(det_tree, log_level):
                     # Append exposure object to the list of exposure objects for this specific total detection product
                     tdp_obj.add_member(sep_obj)
 
-                    # Increment single exposure index
-                    sep_indx += 1
-
                     # Populate total detection product dictionary with input filename
                     obset_products[totprod]['files'].append(filename[1])
 
@@ -723,8 +734,6 @@ def parse_obset_tree(det_tree, log_level):
                         for e in filt_obj.edp_list:
                             e.crclean = True
 
-                    #tdp_obj.add_product(filt_obj)
-
                 elif is_grism:
                     tdp_obj.add_grism_member(grism_sep_obj)
 
@@ -750,9 +759,46 @@ def parse_obset_tree(det_tree, log_level):
     # Done... return dict and object product list
     return obset_products, tdp_list
 
+
 # ------------------------------------------------------------------------------
+def select_common_filetype(filter_files):
+    """Select common filter type and set of filter_files for input list of filenames.
 
+    Parameters
+    -----------
+    filter_files : list
+        List of tuples describing input files for the filter product
+
+    Returns
+    -------
+    filter_members : list
+        Mask indicating which input files are members correspond to the output product type
+
+    prod_type : str
+        String corresponding to the suffix (DRZ vs DRC) to be used for the output product based
+        on the common filter type (FLT vs FLC)
+
+    """
+    filetypes = np.array([f[1][:-5].split("_")[-1] for f in filter_files])
+    # Check whether or not all input files for this filter product have the same filetype (suffix)
+    filetypes_flt = filetypes == 'flt'
+    filetypes_flc = filetypes == 'flc'
+    num_flc = len(np.where(filetypes_flc)[0])
+
+    # determine whether we have FLC images in set or not
+    # and set the filter_type accordingly
+    filter_type = 'flc' if num_flc > 0 else 'flt'
+
+    filter_members = filetypes_flc if filter_type == 'flc' else filetypes_flt
+
+    # Return the updated/cleaned list of input files,
+    # along with the suffix for the files in the input list for this filter product
+    prod_type = 'drc' if filter_type == 'flc' else 'drz'
 
+    return filter_members, prod_type
+
+
+# ------------------------------------------------------------------------------
 def define_exp_layers(obset_table, method='hard', exp_limit=None):
     """Determine what exposures will be grouped into the same layer of a sky cell"""
     # Add 'exp_layer' column to table