From d1196c098e3fb0e2aff2b0da129375e3f624cfe4 Mon Sep 17 00:00:00 2001 From: "Warren J. Hack" Date: Wed, 5 Oct 2022 10:32:30 -0700 Subject: [PATCH] Add runastrodriz manifests (#1447) (#1449) * Add manifest file to runastrodriz * Discover runastrodriz manifest files in hapsequencer * Add new command-line parameter * Remove temp d2im ref file for WFPC2 * Properly initialize lists for manifest file --- drizzlepac/hapsequencer.py | 29 +++++++++++++++++++---- drizzlepac/runastrodriz.py | 48 +++++++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 11 deletions(-) mode change 100755 => 100644 drizzlepac/runastrodriz.py diff --git a/drizzlepac/hapsequencer.py b/drizzlepac/hapsequencer.py index 1fb58051d..6e2713086 100755 --- a/drizzlepac/hapsequencer.py +++ b/drizzlepac/hapsequencer.py @@ -54,6 +54,7 @@ import shutil import sys import traceback +import glob import numpy as np from astropy.io import ascii, fits @@ -514,11 +515,15 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_ ramp_product_list = [] manifest_name = "" found_data = False + # The product_list is a list of all the output products which will be put into the manifest file + product_list = [] + mfile_list = [] + try: - # Parse the poller file and generate the the obs_info_dict, as well as the total detection + # Parse the poller file and generate the obs_info_dict, as well as the total detection # product lists which contain the ExposureProduct, FilterProduct, and TotalProduct objects # A poller file contains visit data for a single instrument. The TotalProduct discriminant - # is the detector. A TotalProduct object is comprised of FilterProducts and ExposureProducts + # is the detector. A TotalProduct object is composed of FilterProducts and ExposureProducts # where its FilterProduct is distinguished by the filter in use, and the ExposureProduct # is the atomic exposure data. Note: the TotalProduct was enhanced to also be comprised # of an GrismExposureProduct list which is exclusive to the TotalProduct. @@ -533,9 +538,6 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_ log.info("Generate the manifest name for this visit.") log.info("The manifest will contain the names of all the output products.") - # The product_list is a list of all the output products which will be put into the manifest file - product_list = [] - # It is possible the total_obj_list output from the poller_utils contains only Grism/Prism # data and no direct images, so no further processing should be done. If this is the case, # there is actually nothing to be done for the visit, except write out an empty manifest file. @@ -557,6 +559,17 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_ # Update all of the product objects with their associated configuration information. for total_item in total_obj_list: + for edp_file in total_item.edp_list: + # Pull in any manifest files from previous processing and include in product_list + mfiles = glob.glob(f"{edp_file.exposure_name}*_manifest.txt") + # If nothing is found, no nothing gets added to the product_list + for mfile in mfiles: + with open(mfile, 'r') as finput: + _ = [product_list.append(fname.strip("\n")) for fname in finput.readlines()] + # Once it has been merged in with product_list for output to the manifest file, + # record it's name so it can be deleted upon successful completion of this processing. + mfile_list.append(mfile) + log.info("Preparing configuration parameter values for total product {}".format(total_item.drizzle_filename)) total_item.configobj_pars = config_utils.HapConfig(total_item, @@ -692,6 +705,12 @@ def run_hap_processing(input_filename, diagnostic_mode=False, input_custom_pars_ with open(manifest_name, mode='w') as catfile: if total_obj_list: [catfile.write("{}\n".format(name)) for name in product_list] + # Remove additional manifest files from previous processing now as well, + # Keep those manifest files, though, if running in diagnotic mode + if not diagnostic_mode: + for mfile in mfile_list: + if os.path.exists(mfile): + os.remove(mfile) end_dt = datetime.datetime.now() log.info('Processing completed at {}'.format(str(end_dt))) diff --git a/drizzlepac/runastrodriz.py b/drizzlepac/runastrodriz.py old mode 100755 new mode 100644 index 99319a635..e83f4cb91 --- a/drizzlepac/runastrodriz.py +++ b/drizzlepac/runastrodriz.py @@ -188,7 +188,8 @@ # Primary user interface def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, - headerlets=True, align_to_gaia=True, force_alignment=False, do_verify_guiding=False, debug=False): + headerlets=True, align_to_gaia=True, force_alignment=False, + do_verify_guiding=True, debug=False, make_manifest=False): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ @@ -201,6 +202,7 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, trlmsg += " numpy version {}".format(np.__version__) pipeline_pars = PIPELINE_PARS.copy() _verify = True # Switch to control whether to verify alignment or not + manifest_list = [] # interpret envvar variable, if specified if envvar_compute_name in os.environ: @@ -342,6 +344,7 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, _trlfile = _trlroot + '.tra' _alignlog = _trlroot + '_align.log' + _manifest_filename = _trlroot + '_manifest.txt' _calfiles_flc = [] # Write message out to temp file and append it to full trailer file @@ -662,7 +665,13 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, # names for drz_product in drz_products: _plower = drz_product.lower() - if drz_product != _plower: os.rename(drz_product, _plower) + if drz_product != _plower: + os.rename(drz_product, _plower) + drz_output = _plower + else: + drz_output = drz_product + # keep track of output files + manifest_list.append(drz_output) else: # Create default trailer file messages when astrodrizzle is not @@ -679,7 +688,8 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, # If we created a new ASN table, we need to remove it if _new_asn is not None: - for _name in _new_asn: fileutil.removeFile(_name) + for _name in _new_asn: + fileutil.removeFile(_name) # Insure all input FLC/FLT files have updated WCSTYPE* keywords for fname in _calfiles + _calfiles_flc: @@ -699,6 +709,8 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, scihdr[wname] = wval scihdr[wtype] = updatehdr.interpret_wcsname_type(wval) + # Keep track of input files that have been updated + manifest_list.extend(_calfiles + _calfiles_flc) # If headerlets have already been written out by alignment code, # do NOT write out this version of the headerlets @@ -724,6 +736,8 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, attach=False, clobber=True, logging=False) + # Keep track of headerlet files written out to disk + manifest_list.append(hdrname) except ValueError: hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n" % fname) # update trailer file to log creation of headerlet files @@ -732,9 +746,21 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, ftrl.write(hlet_msg) ftrl.close() + # add trailer file to list of output products + manifest_list.append(_trlfile) + # Remove secondary log files for good... logging.shutdown() + # write out manifest file, if requested + if make_manifest: + if os.path.exists(_manifest_filename): + os.remove(_manifest_filename) + with open(_manifest_filename, 'w') as fout: + _ = [fout.write(f"{fname}\n") for fname in manifest_list] + print(f"Created manifest file: {_manifest_filename}") + + # delete log files generated by alignment code for _olog in [_alignlog]: if os.path.exists(_olog): os.remove(_olog) @@ -751,10 +777,16 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, print([ofile.path for ofile in Process().open_files()]) if not debug: + # clean up any temporary ref files created for WFPC2 data + if wfpc2_input: + d2im_file = f"{inFilename.split('_')[0]}_flt_d2im.fits" + if os.path.exists(d2im_file): + os.remove(d2im_file) try: # Remove all temp sub-directories now that we are done for sd in sub_dirs: - if os.path.exists(sd): rmtree2(sd) + if os.path.exists(sd): + rmtree2(sd) except Exception: # If we are unable to remove any of these sub-directories, # leave them for the user or calling routine/pipeline to clean up. @@ -1986,7 +2018,7 @@ def main(): import getopt try: - optlist, args = getopt.getopt(sys.argv[1:], 'bdahfgin:v:') + optlist, args = getopt.getopt(sys.argv[1:], 'bdahfgimn:v:') except getopt.error as e: print(str(e)) print(__doc__) @@ -2003,6 +2035,7 @@ def main(): debug = False force_alignment = False do_verify_guiding = False + make_manifest = False # read options for opt, value in optlist: @@ -2018,6 +2051,8 @@ def main(): force = True if opt == "-i": inmemory = True + if opt == "-m": + make_manifest = True if opt == "-v": do_verify_guiding = True if opt == '-n': @@ -2041,7 +2076,8 @@ def main(): process(args[0], force=force, newpath=newdir, num_cores=num_cores, inmemory=inmemory, headerlets=headerlets, align_to_gaia=align_to_gaia, force_alignment=force_alignment, - do_verify_guiding=do_verify_guiding, debug=debug) + do_verify_guiding=do_verify_guiding, debug=debug, + make_manifest=make_manifest) except Exception as errorobj: print(str(errorobj))