diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py index d2482e47f..fd3e16c2a 100755 --- a/jwql/preview_image/generate_preview_images.py +++ b/jwql/preview_image/generate_preview_images.py @@ -12,7 +12,8 @@ Authors ------- - Matthew Bourque + - Matthew Bourque + - Bryan Hilbert Use @@ -25,57 +26,658 @@ python generate_preview_images.py """ -import glob +from glob import glob +import logging import os +import re +import numpy as np + +from jwql.logging.logging_functions import configure_logging +from jwql.logging.logging_functions import log_info +from jwql.logging.logging_functions import log_fail from jwql.permissions import permissions from jwql.preview_image.preview_image import PreviewImage from jwql.utils.utils import get_config from jwql.utils.utils import filename_parser +from jwql.utils.utils import NIRCAM_LONGWAVE_DETECTORS +from jwql.utils.utils import NIRCAM_SHORTWAVE_DETECTORS + +# Size of NIRCam inter- and intra-module chip gaps +SW_MOD_GAP = 1387 # pixels = int(43 arcsec / 0.031 arcsec/pixel) +LW_MOD_GAP = 741 # pixels = int(46 arcsec / 0.062 arcsec/pixel) +SW_DET_GAP = 145 # pixels = int(4.5 arcsec / 0.031 arcsec/pixel) +FULLX = 2048 # Width of the full detector +FULLY = 2048 # Height of the full detector + + +def array_coordinates(channelmod, detector_list, lowerleft_list): + """Create an appropriately sized ``numpy`` array to contain the + mosaic image given the channel and module of the data. + + Parameters + ---------- + channelmod : str + Indicator of the NIRCam channel/module of the data. + Options are: + ``LW`` - for longwave channel data + ``SWA`` - for shortwave A module only (4 detectors) data + ``SWB`` - for shortwave B module only (4 detectors) data + ``SW`` - for shortwave both module data (8 detectors) + + detector_list : list + List of detectors used in data to be simulated + + lowerleft_list : list + Each element is a tuple giving the (x, y) coordinates + corresponding to the lower left corner of the aperture within + the full frame detector. These values come from the + ``SUBSTRT1`` and 2 values in the file headers. + + Returns + ------- + xdim : int + Length of the output array needed to contain all detectors' data + + ydim : int + Height of the output array needed to contain all detectors' data + + module_lowerlefts : dict + Dictionary giving the ``(x, y)`` coordinate in the coordinate + system of the full module(s) where the lower left corner of the + data from a given detector will be placed. (e.g. + ``NRCA1: (1888, 1888)`` means that the data from detector + ``NRCA1`` should be placed into + ``[1888: 1888+y_dim_of_data, 1888: 1888+x_dim_of_data]`` within + the final array (which has total dimensions of ``(xdim, ydim)`` + """ + + # Create dictionary of lower left pixel values for each + # detector as it sits in the MODULE. B1-B4 values here will + # need to have sw_mod_gap added to their x coordinates in + # order to translate to the full A and B module coordinate system. + # The only case where LW data will be in this function is where + # both detectors are used, so set A5/B5 coordinates to be in + # the full module A and B coordinate system. Note that these + # tuples are (x,y), NOT (y,x) + ashort = ["NRCA1", "NRCA2", "NRCA3", "NRCA4"] + bshort = ["NRCB1", "NRCB2", "NRCB3", "NRCB4"] + module_lowerlefts = {"NRCA1": (0, 0), + "NRCA2": (0, FULLY + SW_DET_GAP), + "NRCA3": (FULLX + SW_DET_GAP, 0), + "NRCA4": (FULLX + SW_DET_GAP, FULLY + SW_DET_GAP), + "NRCB1": (FULLX + SW_DET_GAP, FULLY + SW_DET_GAP), + "NRCB2": (FULLX + SW_DET_GAP, 0), + "NRCB3": (0, FULLY + SW_DET_GAP), + "NRCB4": (0, 0), + "NRCA5": (0, 0), + "NRCB5": (FULLX + LW_MOD_GAP, 0)} + + # If both module A and B are used, then shift the B module + # coordinates to account for the intermodule gap + if channelmod == "SW": + mod_delta = (FULLX * 2 + SW_DET_GAP + SW_MOD_GAP, 0) + for b_detector in bshort: + module_lowerlefts[b_detector] = tuple([sum(x) for x in + zip(module_lowerlefts[b_detector], + mod_delta)]) + + # The only subarrays we need to worry about are the SW SUBXXX + # All other subarrays are on a single detector. + # All we need to do is find the lower left values for NRCA1 and/or NRCB4. + # From that we can calculate the rest. Also, if observing with both + # A and B modules, only full frame is allowed, so no need to worry about + # subarrays in both modules simultaneously. + subx = 1 + suby = 1 + if 'SW' in channelmod: + detector_list = np.array(detector_list) + a1 = detector_list == 'NRCA1' + b4 = detector_list == 'NRCB4' + + lowerleft_list = np.array(lowerleft_list) + if np.sum(a1) == 1: + subx, suby = lowerleft_list[a1][0] + elif np.sum(b4) == 1: + subx, suby = lowerleft_list[b4][0] + else: + missing_a14 = "SW data provided, but neither NRCA1 nor NRCB4 are present." + logging.error(missing_a14) + raise ValueError(missing_a14) + + # Adjust the lower left positions of the apertures within + # the module(s) in the case of subarrays + if ((subx != 1) | (suby != 1)): + subarr_delta = {"NRCA1": (0, 0), + "NRCA2": (0, 0 - suby), + "NRCA3": (0 - subx, 0), + "NRCA4": (0 - subx, 0 - suby), + "NRCB1": (0 - subx, 0 - suby), + "NRCB2": (0 - subx, 0), + "NRCB3": (0, 0 - suby), + "NRCB4": (0, 0)} + + for det in ashort + bshort: + module_lowerlefts[det] = tuple([sum(x) for x in zip(module_lowerlefts[det], + subarr_delta[det])]) + + # Dimensions of array to hold all data + # Adjust dimensions of individual detector for subarray if necessary + aperturex = FULLX - (subx - 1) + aperturey = FULLY - (suby - 1) + + # Full module(s) dimensions + if channelmod in ['SWA', 'SWB']: + xdim = 2 * aperturex + SW_DET_GAP + ydim = 2 * aperturey + SW_DET_GAP + elif channelmod == 'SW': + xdim = 4 * aperturex + 2 * SW_DET_GAP + SW_MOD_GAP + ydim = 2 * aperturey + SW_DET_GAP + elif channelmod == 'LW': + xdim = 2 * aperturex + LW_MOD_GAP + ydim = aperturey + + return xdim, ydim, module_lowerlefts + + +def check_existence(file_list, outdir): + """Given a list of fits files, determine if a preview image has + already been created in ``outdir``. + + Parameters + ---------- + file_list : list + List of fits filenames from which preview image will be + generated + + outdir : str + Directory that will contain the preview image if it exists + + Returns + ------- + exists : bool + ``True`` if preview image exists, ``False`` if it does not + """ + + # If file_list contains only a single file, then we need to search + # for a preview image name that contains the detector name + if len(file_list) == 1: + filename = os.path.split(file_list[0])[1] + search_string = filename.split('.fits')[0] + '_*.jpg' + else: + # If file_list contains multiple files, then we need to search + # for the appropriately named jpg of the mosaic, which depends + # on the specific detectors in the file_list + file_parts = filename_parser(file_list[0]) + if file_parts['detector'].upper() in NIRCAM_SHORTWAVE_DETECTORS: + mosaic_str = "NRC_SW*_MOSAIC_" + elif file_parts['detector'].upper() in NIRCAM_LONGWAVE_DETECTORS: + mosaic_str = "NRC_LW*_MOSAIC_" + search_string = 'jw{}{}{}_{}{}{}_{}_{}{}*.jpg'.format( + file_parts['program_id'], file_parts['observation'], + file_parts['visit'], file_parts['visit_group'], + file_parts['parallel_seq_id'], file_parts['activity'], + file_parts['exposure_id'], mosaic_str, file_parts['suffix']) + + current_files = glob(os.path.join(outdir, search_string)) + if len(current_files) > 0: + return True + else: + return False + + +def create_dummy_filename(filelist): + """Create a dummy filename indicating the detectors used to create + the mosaic. Check the list of detectors used to determine the proper + text to substitute into the initial filename. + + Parameters + ---------- + filelist : list + List of filenames containing the data used to create the mosaic. + It is assumed these filenames follow JWST filenaming + conventions. + + Returns + ------- + dummy_name : str + The first filename in ``filelist`` is modified, such that the + detector name is replaced with text indicating the source of the + mosaic data. + """ + + det_string_list = [] + modules = [] + for filename in filelist: + indir, infile = os.path.split(filename) + det_string = filename_parser(infile)['detector'] + det_string_list.append(det_string) + modules.append(det_string[3].upper()) + + # Previous sorting means that either all of the + # input files are LW, or all are SW. So we can check any + # file to determine LW vs SW + if '5' in det_string_list[0]: + suffix = "NRC_LW_MOSAIC" + else: + moda = modules.count('A') + modb = modules.count('B') + if moda > 0: + if modb > 0: + suffix = "NRC_SWALL_MOSAIC" + else: + suffix = "NRC_SWA_MOSAIC" + else: + if modb > 0: + suffix = "NRC_SWB_MOSAIC" + dummy_name = filelist[0].replace(det_string_list[0], suffix) + + return dummy_name + + +def create_mosaic(filenames): + """If an exposure comprises data from multiple detectors read in all + the appropriate files and create a mosaic so that the preview image + will show all the data together. + + Parameters + ---------- + filenames : list + List of filenames to be combined into a mosaic + + Returns + ------- + mosaic_filename : str + Name of fits file containing the mosaicked data + """ + + # Use preview_image to load data and create difference image + # for each detector. Save in a list + data = [] + detector = [] + data_lower_left = [] + for filename in filenames: + image = PreviewImage(filename, "SCI") # Now have image.data, image.dq + data_dim = len(image.data.shape) + if data_dim == 4: + diff_im = image.difference_image(image.data) + else: + diff_im = image.data + data.append(diff_im) + detector.append(filename_parser(filename)['detector'].upper()) + data_lower_left.append((image.xstart, image.ystart)) + + # Make sure SW and LW data are not being mixed. Create the + # appropriately sized numpy array to hold all the data based + # on the channel, module, and subarray size + mosaic_channel = find_data_channel(detector) + full_xdim, full_ydim, full_lower_left = array_coordinates(mosaic_channel, detector, + data_lower_left) + + # Create the array to hold all the data + datashape = data[0].shape + datadim = len(datashape) + if datadim == 2: + full_array = np.zeros((1, full_ydim, full_xdim)) * np.nan + elif datadim == 3: + full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan + else: + raise ValueError((f'Difference image for {filenames[0]} must be either 2D or 3D.')) + + # Place the data from the individual detectors in the appropriate + # places in the final image + for pixdata, detect in zip(data, detector): + x0, y0 = full_lower_left[detect] + if datadim == 2: + yd, xd = pixdata.shape + full_array[0, y0: y0 + yd, x0: x0 + xd] = pixdata + elif datadim == 3: + ints, yd, xd = pixdata.shape + full_array[:, y0: y0 + yd, x0: x0 + xd] = pixdata + + # Create associated DQ array and set unpopulated pixels to be skipped + # in preview image scaling + full_dq = create_dq_array(full_xdim, full_ydim, full_array[0, :, :], mosaic_channel) + + return full_array, full_dq + + +def create_dq_array(xd, yd, mosaic, module): + """Create DQ array that goes with the mosaic image. Set unpopulated + pixels to be skipped in preview image scaling. Same for the + reference pixels for all detectors + + Parameters + ---------- + xd : int + X-coordinate dimension of the DQ array + + yd : int + Y-coordinate dimension of the DQ array + + mosaic : obj + 2D ``numpy`` array containing the mosaic image + + module : str + Module used for mosaic. Options are ``LW``,`` SW``, ``SWA``, + ``SWB`` + + Returns + ------- + dq : obj + 2D ``numpy`` array containing the DQ array. Pixels that are + ``True`` are considered science pixels and are used when + scaling the preview image. Pixels that are ``False`` are + skipped. + """ + # Create array + dq = np.ones((yd, xd), dtype="bool") + # Flag inter-chip and inter-module pixels as False + dq[np.isnan(mosaic)] = 0 + + # Flag reference pixels as False + + # Present in all cases other than subarrays + if xd >= FULLX: + dq[0:4, :] = 0 + dq[:, 0:4] = 0 + dq[2044:2048, :] = 0 + dq[:, 2044:2048] = 0 + + if module == "LW": + lwb_lower = FULLX + LW_MOD_GAP + dq[:, lwb_lower:lwb_lower + 4] = 0 + dq[:, lwb_lower + 2044:lwb_lower + 2048] = 0 + + if module in ["SWA", "SWB", "SW"]: + # Present for full frame single module or both modules + lowerval = FULLX + SW_DET_GAP + dq[lowerval: lowerval + 4, :] = 0 + dq[(lowerval + 2044):, :] = 0 + dq[:, lowerval: lowerval + 4] = 0 + dq[:, (lowerval + 2044):(lowerval + 2048)] = 0 + + if module == "SW": + # Present only if both modules are used (full frame) + modb_lower = lowerval + FULLX + SW_MOD_GAP + dq[:, modb_lower:(modb_lower + 4)] = 0 + dq[:, (modb_lower + 2044):(modb_lower + 2048)] = 0 + modb_upper = modb_lower + FULLX + SW_DET_GAP + dq[:, modb_upper:(modb_upper + 4)] = 0 + dq[:, (modb_upper + 2044):(modb_upper + 2048)] = 0 + + else: + # Subarrays: expand the pixels flagged due to chip gaps + # by one row and column + nan_indexes = np.where(np.isnan(mosaic)) + match = nan_indexes[0] == yd - 1 + vert_xmin = np.min(nan_indexes[1][match]) + vert_xmax = vert_xmin + SW_DET_GAP - 1 + + match2 = nan_indexes[1] == xd - 1 + horiz_ymin = np.min(nan_indexes[0][match2]) + horiz_ymax = horiz_ymin + SW_DET_GAP - 1 + + dq[:, vert_xmin - 4:vert_xmin] = 0 + dq[:, vert_xmax + 1:vert_xmax + 5] = 0 + dq[horiz_ymin - 4:horiz_ymin, :] = 0 + dq[horiz_ymax + 1:horiz_ymax + 5, :] = 0 + + return dq + + +def detector_check(detector_list, search_string): + """Search a given list of detector names for the provided regular + expression sting. + + Parameters + ---------- + detector_list : list + List of detector names (e.g. ``NRCA5``) + + search_string : str + Regular expression string to use for search + + Returns + ------- + total : int + Number of detectors in ``detector_list`` that match + ``search_string`` + """ + + pattern = re.compile(search_string, re.IGNORECASE) + match = [pattern.match(detector) for detector in detector_list] + total = np.sum(np.array([m is not None for m in match])) + + return total + + +def find_data_channel(detectors): + """Using a list of detectors, identify the channel(s) that the data + are from. + + Parameters + ---------- + detectors : list + List of detector names + + Returns + ------- + channel : str + Identifier noting which channels the given detectors are in. + Can be ``SWA`` for shortwave, module A only, ``SWB`` for + shortwave, module B only, ``SW``, for shortwave modules A and B, + and ``LW`` for longwave. + """ + + # Check the detector names for all files. + nrc_swa_total = detector_check(detectors, "NRCA[1-4]") + nrc_swb_total = detector_check(detectors, "NRCB[1-4]") + nrc_lw_total = detector_check(detectors, "NRC[AB]5") + + both_channels = "Can't mix NIRCam SW and LW data in same mosaic." + if nrc_swa_total != 0: + if nrc_lw_total != 0: + raise ValueError(both_channels) + else: + if nrc_swb_total != 0: + channel = "SW" + else: + channel = "SWA" + else: + if nrc_swb_total != 0: + if nrc_lw_total != 0: + raise ValueError(both_channels) + else: + channel = "SWB" + else: + if nrc_lw_total != 0: + channel = "LW" + else: + raise ValueError("No NIRCam SW nor LW data") + return channel + + +@log_fail +@log_info def generate_preview_images(): - """The main function of the generate_preview_image module.""" + """The main function of the ``generate_preview_image`` module.""" + + # Begin logging + logging.info("Beginning the script run") filesystem = get_config()['filesystem'] preview_image_filesystem = get_config()['preview_image_filesystem'] thumbnail_filesystem = get_config()['thumbnail_filesystem'] - filenames = glob.glob(os.path.join(filesystem, '*/*.fits')) - for filename in filenames: + filenames = glob(os.path.join(filesystem, '*/*.fits')) + grouped_filenames = group_filenames(filenames) + logging.info(f"Found {len(filenames)} filenames") + for file_list in grouped_filenames: + filename = file_list[0] # Determine the save location try: identifier = 'jw{}'.format(filename_parser(filename)['program_id']) except ValueError as error: identifier = os.path.basename(filename).split('.fits')[0] - output_directory = os.path.join(preview_image_filesystem, identifier) + preview_output_directory = os.path.join(preview_image_filesystem, identifier) thumbnail_output_directory = os.path.join(thumbnail_filesystem, identifier) + # Check to see if the preview images already exist and skip + # if they do + file_exists = check_existence(file_list, preview_output_directory) + if file_exists: + logging.info("JPG already exists for {}, skipping.".format(filename)) + continue + # Create the output directories if necessary - if not os.path.exists(output_directory): - os.makedirs(output_directory) - permissions.set_permissions(output_directory) + if not os.path.exists(preview_output_directory): + os.makedirs(preview_output_directory) + permissions.set_permissions(preview_output_directory) + logging.info(f'Created directory {preview_output_directory}') if not os.path.exists(thumbnail_output_directory): os.makedirs(thumbnail_output_directory) permissions.set_permissions(thumbnail_output_directory) + logging.info(f'Created directory {thumbnail_output_directory}') - # Create and save the preview image and thumbnail - args = zip((False, True), (output_directory, thumbnail_output_directory)) - for thumbnail_bool, directory in args: + # If the exposure contains more than one file (because more + # than one detector was used), then create a mosaic + max_size = 8 + numfiles = len(file_list) + if numfiles != 1: try: - im = PreviewImage(filename, "SCI") - im.clip_percent = 0.01 - im.scaling = 'log' - im.cmap = 'viridis' - im.output_format = 'jpg' - im.thumbnail = thumbnail_bool - im.output_directory = directory - im.make_image() - except ValueError as error: - print(error) + mosaic_image, mosaic_dq = create_mosaic(file_list) + logging.info('Created mosiac for:') + for item in file_list: + logging.info(f'\t{item}') + except (ValueError, FileNotFoundError) as error: + logging.error(error) + dummy_file = create_dummy_filename(file_list) + if numfiles in [2, 4]: + max_size = 16 + elif numfiles in [8]: + max_size = 32 + + # Create the nominal preview image and thumbnail + try: + im = PreviewImage(filename, "SCI") + im.clip_percent = 0.01 + im.scaling = 'log' + im.cmap = 'viridis' + im.output_format = 'jpg' + im.preview_output_directory = preview_output_directory + im.thumbnail_output_directory = thumbnail_output_directory + + # If a mosaic was made from more than one file + # insert it and it's associated DQ array into the + # instance of PreviewImage. Also set the input + # filename to indicate that we have mosaicked data + if numfiles != 1: + im.data = mosaic_image + im.dq = mosaic_dq + im.file = dummy_file + + im.make_image(max_img_size=max_size) + except ValueError as error: + logging.warning(error) + + # Complete logging: + logging.info("Completed.") + + +def group_filenames(input_files): + """Given a list of JWST filenames, group together files from the + same exposure. These files will share the same ``program_id``, + ``observation``, ``visit``, ``visit_group``, ``parallel_seq_id``, + ``activity``, ``exposure``, and ``suffix``. Only the ``detector`` + will be different. Currently only NIRCam files for a given exposure + will be grouped together. For other instruments multiple files for + a given exposure will be kept separate from one another and no + mosaic will be made. + + Parameters + ---------- + input_files : list + list of filenames + + Returns + ------- + grouped : list + grouped list of filenames where each element is a list and + contains the names of filenames with matching exposure + information. + """ + + grouped = [] + + # Sort files first + input_files.sort() + + goodindex = np.arange(len(input_files)) + input_files = np.array(input_files) + + # Loop over each file in the list of good files + for index, full_filename in enumerate(input_files[goodindex]): + file_directory, filename = os.path.split(full_filename) + + # Generate string to be matched with other filenames + filename_parts = filename_parser(filename) + program = filename_parts['program_id'] + observation = filename_parts['observation'] + visit = filename_parts['visit'] + visit_group = filename_parts['visit_group'] + parallel = filename_parts['parallel_seq_id'] + activity = filename_parts['activity'] + exposure = filename_parts['exposure_id'] + detector = filename_parts['detector'].upper() + suffix = filename_parts['suffix'] + + observation_base = f'jw{program}{observation}{visit}_{visit_group}{parallel}{activity}_{exposure}_' + + if detector in NIRCAM_SHORTWAVE_DETECTORS: + detector_str = 'NRC[AB][1234]' + elif detector in NIRCAM_LONGWAVE_DETECTORS: + detector_str = 'NRC[AB]5' + else: # non-NIRCam detectors - should never be used I think?? + detector_str = detector + match_str = f'{observation_base}{detector_str}_{suffix}.fits' + match_str = os.path.join(file_directory, match_str) + pattern = re.compile(match_str, re.IGNORECASE) + + # Try to match the substring to each good file + matches = [] + matched_name = [] + for index2, file2match in enumerate(input_files[goodindex]): + match = pattern.match(file2match) + + # Add any files that match the string + if match is not None: + matched_name.append(file2match) + matches.append(goodindex[index2]) + # For any matched files, remove from goodindex so we don't + # use them as a basis for matching later + all_locs = [] + for num in matches: + loc = np.where(goodindex == num) + all_locs.append(loc[0][0]) + if len(all_locs) != 0: + # Delete matched file indexes from the list of + # files to search + goodindex = np.delete(goodindex, all_locs) + + # Add the list of matched files to the overall list of files + grouped.append(matched_name) + + return grouped + if __name__ == '__main__': + module = os.path.basename(__file__).strip('.py') + configure_logging(module) + generate_preview_images() diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py index de407cdc2..44dd7c516 100755 --- a/jwql/preview_image/preview_image.py +++ b/jwql/preview_image/preview_image.py @@ -3,16 +3,16 @@ """ Create a preview image from a fits file containing an observation. -This module creates and saves a "preview image" from a fits file -that contains a JWST observation. Data from the user-supplied -``extension`` of the file are read in, along with the ``PIXELDQ`` -extension if present. For each integration in the exposure, the -first group is subtracted from the final group in order to create -a difference image. The lower and upper limits to be displayed are -defined as the ``clip_percent`` and (1. - ``clip_percent``) percentile -signals. ``matplotlib`` is then used to display a linear- or -log-stretched version of the image, with accompanying colorbar. The -image is then saved. +This module creates and saves a "preview image" from a fits file that +contains a JWST observation. Data from the user-supplied ``extension`` +of the file are read in, along with the ``PIXELDQ`` extension if +present. For each integration in the exposure, the first group is +subtracted from the final group in order to create a difference image. +The lower and upper limits to be displayed are defined as the +``clip_percent`` and ``(1. - clip_percent)`` percentile signals. +``matplotlib`` is then used to display a linear- or log-stretched +version of the image, with accompanying colorbar. The image is then +saved. Authors: -------- @@ -34,8 +34,8 @@ im.make_image() """ +import logging import os -import sys from astropy.io import fits from jwst.datamodels import dqflags @@ -50,8 +50,50 @@ import matplotlib.colors as colors - class PreviewImage(): + """An object for generating and saving preview images, used by + ``generate_preview_images``. + + Attributes + ---------- + clip_percent : float + The amount to sigma clip the input data by when scaling the + preview image. Default is 0.01. + cmap : str + The colormap used by ``matplotlib`` in the preview image. + Default value is ``viridis``. + data : obj + The data used to generate the preview image. + dq : obj + The DQ data used to generate the preview image. + file : str + The filename to generate the preview image from. + output_format : str + The format to which the preview image is saved. Options are + ``jpg`` and ``thumb`` + preview_output_directory : str or None + The output directory to which the preview image is saved. + scaling : str + The scaling used in the preview image. Default is ``log``. + thumbnail_output_directory : str or None + The output directory to which the thumbnail is saved. + + Methods + ------- + difference_image(data) + Create a difference image from the data + find_limits(data, pixmap, clipperc) + Find the min and max signal levels after clipping by + ``clipperc`` + get_data(filename, ext) + Read in data from the given ``filename`` and ``ext`` + make_figure(image, integration_number, min_value, max_value, scale, maxsize, thumbnail) + Create the ``matplotlib`` figure + make_image(max_img_size) + Main function + save_image(fname, thumbnail) + Save the figure + """ def __init__(self, filename, extension): """Initialize the class. @@ -67,9 +109,9 @@ def __init__(self, filename, extension): self.cmap = 'viridis' self.file = filename self.output_format = 'jpg' - self.output_directory = None + self.preview_output_directory = None self.scaling = 'log' - self.thumbnail = False + self.thumbnail_output_directory = None # Read in file self.data, self.dq = self.get_data(self.file, extension) @@ -152,23 +194,34 @@ def get_data(self, filename, ext): except: pass if ext in extnames: - data = hdulist[ext].data.astype(np.float) + dimensions = len(hdulist[ext].data.shape) + if dimensions == 4: + data = hdulist[ext].data[:, [0, -1], :, :].astype(np.float) + else: + data = hdulist[ext].data.astype(np.float) else: - raise ValueError(("WARNING: no {} extension in {}!" - .format(ext, filename))) + raise ValueError((f'WARNING: no {ext} extension in {filename}!')) if 'PIXELDQ' in extnames: dq = hdulist['PIXELDQ'].data dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0) else: yd, xd = data.shape[-2:] dq = np.ones((yd, xd), dtype="bool") + + # Collect information on aperture location within the + # full detector. This is needed for mosaicking NIRCam + # detectors later. + self.xstart = hdulist[0].header['SUBSTRT1'] + self.ystart = hdulist[0].header['SUBSTRT2'] + self.xlen = hdulist[0].header['SUBSIZE1'] + self.ylen = hdulist[0].header['SUBSIZE2'] else: - raise FileNotFoundError(("WARNING: {} does not exist!" - .format(filename))) + raise FileNotFoundError((f'WARNING: {filename} does not exist!')) + return data, dq def make_figure(self, image, integration_number, min_value, max_value, - scale, maxsize=8): + scale, maxsize=8, thumbnail=False): """ Create the matplotlib figure of the image @@ -176,15 +229,26 @@ def make_figure(self, image, integration_number, min_value, max_value, ---------- image : obj 2D ``numpy`` ``ndarray`` of floats + integration_number : int Integration number within exposure + min_value : float Minimum value for display + max_value : float Maximum value for display + scale : str Image scaling (``log``, ``linear``) + maxsize : int + Size of the longest dimension of the output figure (inches) + + thumbnail : bool + True to create a thumbnail image, False to create the full + preview image + Returns ------- result : obj @@ -192,9 +256,8 @@ def make_figure(self, image, integration_number, min_value, max_value, """ # Check the input scaling - if scale not in ['linear','log']: - raise ValueError(("WARNING: scaling option {} not supported." - .format(scale))) + if scale not in ['linear', 'log']: + raise ValueError((f'WARNING: scaling option {scale} not supported.')) # Set the figure size yd, xd = image.shape @@ -214,10 +277,14 @@ def make_figure(self, image, integration_number, min_value, max_value, shiftmax = max_value - min_value + 1 # If making a thumbnail, make a figure with no axes - if self.thumbnail: + if thumbnail: fig = plt.imshow(shiftdata, - norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax), - cmap=self.cmap) + norm=colors.LogNorm(vmin=shiftmin, + vmax=shiftmax), + cmap=self.cmap) + # Invert y axis + plt.gca().invert_yaxis() + plt.axis('off') fig.axes.get_xaxis().set_visible(False) fig.axes.get_yaxis().set_visible(False) @@ -226,8 +293,11 @@ def make_figure(self, image, integration_number, min_value, max_value, else: fig, ax = plt.subplots(figsize=(xsize, ysize)) cax = ax.imshow(shiftdata, - norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax), + norm=colors.LogNorm(vmin=shiftmin, + vmax=shiftmax), cmap=self.cmap) + # Invert y axis + plt.gca().invert_yaxis() # Add colorbar, with original data values tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5) @@ -248,28 +318,34 @@ def make_figure(self, image, integration_number, min_value, max_value, tlabelstr = [format_string % number for number in tlabelflt] cbar = fig.colorbar(cax, ticks=tickvals) cbar.ax.set_yticklabels(tlabelstr) - ax.set_xlabel('Pixels') - ax.set_ylabel('Pixels') + cbar.ax.tick_params(labelsize=maxsize * 5./4) + # cbar.ax.set_ylabel('Signal', rotation=270, fontsize=maxsize*5./4) + ax.set_xlabel('Pixels', fontsize=maxsize * 5./4) + ax.set_ylabel('Pixels', fontsize=maxsize * 5./4) + ax.tick_params(labelsize=maxsize) plt.rcParams.update({'axes.titlesize': 'small'}) + plt.rcParams.update({'font.size': maxsize * 5./4}) + plt.rcParams.update({'axes.labelsize': maxsize * 5./4}) + plt.rcParams.update({'ytick.labelsize': maxsize * 5./4}) + plt.rcParams.update({'xtick.labelsize': maxsize * 5./4}) elif scale == 'linear': fig, ax = plt.subplots(figsize=(xsize, ysize)) cax = ax.imshow(image, clim=(min_value, max_value), cmap=self.cmap) - if not self.thumbnail: + if not thumbnail: cbar = fig.colorbar(cax) ax.set_xlabel('Pixels') ax.set_ylabel('Pixels') # If preview image, set a title - if not self.thumbnail: + if not thumbnail: filename = os.path.split(self.file)[-1] ax.set_title(filename + ' Int: {}'.format(np.int(integration_number))) - def make_image(self): - """ - MAIN FUNCTION - """ + def make_image(self, max_img_size=8): + """The main function of the ``PreviewImage`` class.""" + shape = self.data.shape if len(shape) == 4: @@ -292,20 +368,31 @@ def make_image(self): minval, maxval = self.find_limits(frame, self.dq, self.clip_percent) - # Create matplotlib object + # Create preview image matplotlib object indir, infile = os.path.split(self.file) suffix = '_integ{}.{}'.format(i, self.output_format) - if self.output_directory is None: + if self.preview_output_directory is None: outdir = indir else: - outdir = self.output_directory + outdir = self.preview_output_directory outfile = os.path.join(outdir, infile.split('.')[0] + suffix) - self.make_figure(frame, i, minval, maxval, self.scaling.lower()) - self.save_image(outfile) + self.make_figure(frame, i, minval, maxval, self.scaling.lower(), + maxsize=max_img_size, thumbnail=False) + self.save_image(outfile, thumbnail=False) plt.close() + # Create thumbnail image matplotlib object + if self.thumbnail_output_directory is None: + outdir = indir + else: + outdir = self.thumbnail_output_directory + outfile = os.path.join(outdir, infile.split('.')[0] + suffix) + self.make_figure(frame, i, minval, maxval, self.scaling.lower(), + maxsize=max_img_size, thumbnail=True) + self.save_image(outfile, thumbnail=True) + plt.close() - def save_image(self, outfile): + def save_image(self, fname, thumbnail=False): """ Save an image in the requested output format and sets the appropriate permissions @@ -314,17 +401,22 @@ def save_image(self, outfile): ---------- image : obj A ``matplotlib`` figure object + fname : str Output filename + + thumbnail : bool + True if saving a thumbnail image, false for the full + preview image. """ - plt.savefig(outfile, bbox_inches='tight', pad_inches=0) - permissions.set_permissions(outfile) + plt.savefig(fname, bbox_inches='tight', pad_inches=0) + permissions.set_permissions(fname) # If the image is a thumbnail, rename to '.thumb' - if self.thumbnail: - new_outfile = outfile.replace('.jpg', '.thumb') - os.rename(outfile, new_outfile) - print('Saved image to {}'.format(new_outfile)) + if thumbnail: + thumb_fname = fname.replace('.jpg', '.thumb') + os.rename(fname, thumb_fname) + logging.info(f'Saved image to {thumb_fname}') else: - print('Saved image to {}'.format(outfile)) + logging.info(f'Saved image to {fname}') diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 3009e038a..f6ddf89bd 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -47,6 +47,10 @@ 'Detector Health Monitor', 'Ref Pix Monitor', 'Internal Lamp Monitor', 'Instrument Model Updates', 'Failed-open Shutter Monitor']} +NIRCAM_SHORTWAVE_DETECTORS = ['NRCA1', 'NRCA2', 'NRCA3', 'NRCA4', + 'NRCB1', 'NRCB2', 'NRCB3', 'NRCB4'] +NIRCAM_LONGWAVE_DETECTORS = ['NRCA5', 'NRCB5'] + def ensure_dir_exists(fullpath):