Add runastrodriz manifests (#1447) (#1449)

* Add manifest file to runastrodriz

* Discover runastrodriz manifest files in hapsequencer

* Add new command-line parameter

* Remove temp d2im ref file for WFPC2

* Properly initialize lists for manifest file

drizzlepac/hapsequencer.py

@@ -54,6 +54,7 @@
 import shutil
 import sys
 import traceback
+import glob
 
 import numpy as np
 from astropy.io import ascii, fits
@@ -514,11 +515,15 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_
     ramp_product_list = []
     manifest_name = ""
     found_data = False
+    # The product_list is a list of all the output products which will be put into the manifest file
+    product_list = []
+    mfile_list = []
+
     try:
-        # Parse the poller file and generate the the obs_info_dict, as well as the total detection
+        # Parse the poller file and generate the obs_info_dict, as well as the total detection
         # product lists which contain the ExposureProduct, FilterProduct, and TotalProduct objects
         # A poller file contains visit data for a single instrument.  The TotalProduct discriminant
-        # is the detector.  A TotalProduct object is comprised of FilterProducts and ExposureProducts
+        # is the detector.  A TotalProduct object is composed of FilterProducts and ExposureProducts
         # where its FilterProduct is distinguished by the filter in use, and the ExposureProduct
         # is the atomic exposure data. Note: the TotalProduct was enhanced to also be comprised
         # of an GrismExposureProduct list which is exclusive to the TotalProduct.
@@ -533,9 +538,6 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_
         log.info("Generate the manifest name for this visit.")
         log.info("The manifest will contain the names of all the output products.")
 
-        # The product_list is a list of all the output products which will be put into the manifest file
-        product_list = []
-
         # It is possible the total_obj_list output from the poller_utils contains only Grism/Prism
         # data and no direct images, so no further processing should be done.  If this is the case,
         # there is actually nothing to be done for the visit, except write out an empty manifest file.
@@ -557,6 +559,17 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_
 
         # Update all of the product objects with their associated configuration information.
         for total_item in total_obj_list:
+            for edp_file in total_item.edp_list:
+                # Pull in any manifest files from previous processing and include in product_list
+                mfiles = glob.glob(f"{edp_file.exposure_name}*_manifest.txt")
+                # If nothing is found, no nothing gets added to the product_list
+                for mfile in mfiles:
+                    with open(mfile, 'r') as finput:
+                        _ = [product_list.append(fname.strip("\n")) for fname in finput.readlines()]
+                    # Once it has been merged in with product_list for output to the manifest file,
+                    # record it's name so it can be deleted upon successful completion of this processing.
+                    mfile_list.append(mfile)
+
             log.info("Preparing configuration parameter values for total product {}".format(total_item.drizzle_filename))
 
             total_item.configobj_pars = config_utils.HapConfig(total_item,
@@ -692,6 +705,12 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_
             with open(manifest_name, mode='w') as catfile:
                 if total_obj_list:
                     [catfile.write("{}\n".format(name)) for name in product_list]
+        # Remove additional manifest files from previous processing now as well,
+        # Keep those manifest files, though, if running in diagnotic mode
+        if not diagnostic_mode:
+            for mfile in mfile_list:
+                if os.path.exists(mfile):
+                    os.remove(mfile)
 
         end_dt = datetime.datetime.now()
         log.info('Processing completed at {}'.format(str(end_dt)))

drizzlepac/runastrodriz.py

@@ -188,7 +188,8 @@
 
 # Primary user interface
 def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
-            headerlets=True, align_to_gaia=True, force_alignment=False, do_verify_guiding=False, debug=False):
+            headerlets=True, align_to_gaia=True, force_alignment=False,
+            do_verify_guiding=True, debug=False, make_manifest=False):
     """ Run astrodrizzle on input file/ASN table
         using default values for astrodrizzle parameters.
     """
@@ -201,6 +202,7 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
     trlmsg += "    numpy version {}".format(np.__version__)
     pipeline_pars = PIPELINE_PARS.copy()
     _verify = True  # Switch to control whether to verify alignment or not
+    manifest_list = []
 
     # interpret envvar variable, if specified
     if envvar_compute_name in os.environ:
@@ -342,6 +344,7 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
 
     _trlfile = _trlroot + '.tra'
     _alignlog = _trlroot + '_align.log'
+    _manifest_filename = _trlroot + '_manifest.txt'
     _calfiles_flc = []
 
     # Write message out to temp file and append it to full trailer file
@@ -662,7 +665,13 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
         # names
         for drz_product in drz_products:
             _plower = drz_product.lower()
-            if drz_product != _plower: os.rename(drz_product, _plower)
+            if drz_product != _plower:
+                os.rename(drz_product, _plower)
+                drz_output = _plower
+            else:
+                drz_output = drz_product
+            # keep track of output files
+            manifest_list.append(drz_output)
 
     else:
         # Create default trailer file messages when astrodrizzle is not
@@ -679,7 +688,8 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
 
     # If we created a new ASN table, we need to remove it
     if _new_asn is not None:
-        for _name in _new_asn: fileutil.removeFile(_name)
+        for _name in _new_asn:
+            fileutil.removeFile(_name)
 
     # Insure all input FLC/FLT files have updated WCSTYPE* keywords
     for fname in _calfiles + _calfiles_flc:
@@ -699,6 +709,8 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
                         scihdr[wname] = wval
 
                     scihdr[wtype] = updatehdr.interpret_wcsname_type(wval)
+    # Keep track of input files that have been updated
+    manifest_list.extend(_calfiles + _calfiles_flc)
 
     # If headerlets have already been written out by alignment code,
     # do NOT write out this version of the headerlets
@@ -724,6 +736,8 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
                                               attach=False,
                                               clobber=True,
                                               logging=False)
+                    # Keep track of headerlet files written out to disk
+                    manifest_list.append(hdrname)
                 except ValueError:
                     hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n" % fname)
                     # update trailer file to log creation of headerlet files
@@ -732,9 +746,21 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
         ftrl.write(hlet_msg)
         ftrl.close()
 
+    # add trailer file to list of output products
+    manifest_list.append(_trlfile)
+
     # Remove secondary log files for good...
     logging.shutdown()
 
+    # write out manifest file, if requested
+    if make_manifest:
+        if os.path.exists(_manifest_filename):
+            os.remove(_manifest_filename)
+        with open(_manifest_filename, 'w') as fout:
+            _ = [fout.write(f"{fname}\n") for fname in manifest_list]
+        print(f"Created manifest file: {_manifest_filename}")
+
+    # delete log files generated by alignment code
     for _olog in [_alignlog]:
         if os.path.exists(_olog):
             os.remove(_olog)
@@ -751,10 +777,16 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True,
         print([ofile.path for ofile in Process().open_files()])
 
     if not debug:
+        # clean up any temporary ref files created for WFPC2 data
+        if wfpc2_input:
+            d2im_file = f"{inFilename.split('_')[0]}_flt_d2im.fits"
+            if os.path.exists(d2im_file):
+                os.remove(d2im_file)
         try:
             # Remove all temp sub-directories now that we are done
             for sd in sub_dirs:
-                if os.path.exists(sd): rmtree2(sd)
+                if os.path.exists(sd):
+                    rmtree2(sd)
         except Exception:
             # If we are unable to remove any of these sub-directories,
             # leave them for the user or calling routine/pipeline to clean up.
@@ -1986,7 +2018,7 @@ def main():
     import getopt
 
     try:
-        optlist, args = getopt.getopt(sys.argv[1:], 'bdahfgin:v:')
+        optlist, args = getopt.getopt(sys.argv[1:], 'bdahfgimn:v:')
     except getopt.error as e:
         print(str(e))
         print(__doc__)
@@ -2003,6 +2035,7 @@ def main():
     debug = False
     force_alignment = False
     do_verify_guiding = False
+    make_manifest = False
 
     # read options
     for opt, value in optlist:
@@ -2018,6 +2051,8 @@ def main():
             force = True
         if opt == "-i":
             inmemory = True
+        if opt == "-m":
+            make_manifest = True
         if opt == "-v":
             do_verify_guiding = True
         if opt == '-n':
@@ -2041,7 +2076,8 @@ def main():
             process(args[0], force=force, newpath=newdir, num_cores=num_cores,
                     inmemory=inmemory, headerlets=headerlets,
                     align_to_gaia=align_to_gaia, force_alignment=force_alignment,
-                    do_verify_guiding=do_verify_guiding, debug=debug)
+                    do_verify_guiding=do_verify_guiding, debug=debug,
+                    make_manifest=make_manifest)
 
         except Exception as errorobj:
             print(str(errorobj))