diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index fe392cfba..9a1968921 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -101,8 +101,9 @@ from jwql.instrument_monitors import pipeline_tools from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline from jwql.utils import calculations, instrument_properties, mast_utils, monitor_utils -from jwql.utils.constants import ASIC_TEMPLATES, DARK_MONITOR_MAX_BADPOINTS_TO_PLOT, JWST_INSTRUMENT_NAMES, FULL_FRAME_APERTURES -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS +from jwql.utils.constants import ASIC_TEMPLATES, DARK_MONITOR_BETWEEN_EPOCH_THRESHOLD_TIME, DARK_MONITOR_MAX_BADPOINTS_TO_PLOT +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import JWST_DATAPRODUCTS, MINIMUM_DARK_CURRENT_GROUPS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png @@ -233,7 +234,8 @@ def add_bad_pix(self, coordinates, pixel_type, files, mean_filename, baseline_fi with engine.begin() as connection: connection.execute(self.pixel_table.__table__.insert(), entry) - def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, noisyxy=None, baseline_file=None): + def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, noisyxy=None, baseline_file=None, + min_time='', max_time=''): """Create and save a png containing the mean dark slope image, to be displayed in the web app @@ -257,10 +259,17 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no baseline_file : str Name of fits file containing the mean slope image to which ``image`` was compared when looking for new hot/dead/noisy pixels + + min_time : str + Earliest observation time, in MJD, used in the creation of ``image``. + + max_time : str + Latest observation time, in MJD, used in the creation of ``image``. + """ output_filename = '{}_{}_{}_to_{}_mean_slope_image.png'.format(self.instrument.lower(), self.aperture.lower(), - self.query_start, self.query_end) + min_time, max_time) mean_slope_dir = os.path.join(get_config()['outputs'], 'dark_monitor', 'mean_slope_images') @@ -274,8 +283,8 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) # Create figure - start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y") - end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") + start_time = Time(float(min_time), format='mjd').tt.datetime.strftime("%m/%d/%Y") + end_time = Time(float(max_time), format='mjd').tt.datetime.strftime("%m/%d/%Y") self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='') # tools='pan,box_zoom,reset,wheel_zoom,save') @@ -432,14 +441,30 @@ def exclude_existing_badpix(self, badpix, pixel_type): new_pixels_y.append(y) logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type)) -# pixel = (x, y) -# if pixel not in already_found: -# new_pixels_x.append(x) -# new_pixels_y.append(y) session.close() return (new_pixels_x, new_pixels_y) + def exclude_too_few_groups(self, result_list): + """Given a list of mast query results, go through and exlclude + files that have too few groups to be useful + + Parameters + ---------- + result_list : list + List of dictionaries containing a MAST query result + + Returns + ------- + filtered_results : list + List of dictionaries with files containing too few groups excluded + """ + filtered_results = [] + for result in result_list: + if result['ngroups'] >= MINIMUM_DARK_CURRENT_GROUPS: + filtered_results.append(result) + return filtered_results + def find_hot_dead_pixels(self, mean_image, comparison_image, hot_threshold=2., dead_threshold=0.1): """Create the ratio of the slope image to a baseline slope image. Pixels in the ratio image with values above @@ -675,14 +700,19 @@ def process(self, file_list): # Basic metadata that will be needed later self.get_metadata(file_list[0]) + # For MIRI, save the rateints files. For other instruments save the rate files. + if self.instrument == 'miri': + output_suffix = 'rateints' + else: + output_suffix = 'rate' + # Run pipeline steps on files, generating slope files pipeline_files = [] slope_files = [] for filename in file_list: + logging.info(f'\tWorking on file: {filename}') - logging.info('\tWorking on file: {}'.format(filename)) - - rate_file = filename.replace("dark", "rate") + rate_file = filename.replace("dark", output_suffix) rate_file_name = os.path.basename(rate_file) local_rate_file = os.path.join(self.working_data_dir, rate_file_name) @@ -697,15 +727,16 @@ def process(self, file_list): step_args = {'dark_current': {'skip': True}} # Call the pipeline - outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate"], self.instrument, step_args=step_args) + outputs = run_parallel_pipeline(pipeline_files, "dark", [output_suffix], self.instrument, step_args=step_args) + for filename in file_list: - processed_file = filename.replace("_dark", "_rate") + processed_file = filename.replace("_dark", f"_{output_suffix}") if processed_file not in slope_files and os.path.isfile(processed_file): slope_files.append(processed_file) os.remove(filename) obs_times = [] - logging.info('\tSlope images to use in the dark monitor for {}, {}:'.format(self.instrument, self.aperture)) + logging.info(f'\tSlope images to use in the dark monitor for {self.instrument}, {self.aperture}:') for item in slope_files: logging.info('\t\t{}'.format(item)) # Get the observation time for each file @@ -719,13 +750,23 @@ def process(self, file_list): mid_time = instrument_properties.mean_time(obs_times) try: - - # Read in all slope images and place into a list - slope_image_stack, slope_exptimes = pipeline_tools.image_stack(slope_files) + # Read in all slope images and create a stack of ints (from rateints files) + # or mean ints (from rate files) + slope_image_stack, slope_exptimes = pipeline_tools.image_stack(slope_files, skipped_initial_ints=self.skipped_initial_ints) + logging.info(f'Shape of slope image stack: {slope_image_stack.shape}') # Calculate a mean slope image from the inputs slope_image, stdev_image = calculations.mean_image(slope_image_stack, sigma_threshold=3) - mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image, slope_files) + + # Use the min and max observation time of the input files to create the slope file name + min_time_str = min_time.strftime('%Y-%m-%dT%H:%m:%S') + min_time_mjd = Time(min_time_str, format='isot', scale='utc').mjd + min_time_mjd_trunc = "{:.4f}".format(min_time_mjd) + max_time_str = max_time.strftime('%Y-%m-%dT%H:%m:%S') + max_time_mjd = Time(max_time_str, format='isot', scale='utc').mjd + max_time_mjd_trunc = "{:.4f}".format(max_time_mjd) + mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image, slope_files, + min_time_mjd_trunc, max_time_mjd_trunc) # Free up memory del slope_image_stack @@ -753,7 +794,15 @@ def process(self, file_list): baseline_stdev = deepcopy(stdev_image) else: logging.info('\tBaseline file is {}'.format(baseline_file)) - baseline_mean, baseline_stdev = self.read_baseline_slope_image(baseline_file) + + if not os.path.isfile(baseline_file): + logging.warning((f'\tBaseline file {baseline_file} does not exist. Setting ' + 'the current mean slope image to be the new baseline.')) + baseline_file = mean_slope_file + baseline_mean = deepcopy(slope_image) + baseline_stdev = deepcopy(stdev_image) + else: + baseline_mean, baseline_stdev = self.read_baseline_slope_image(baseline_file) # Check the hot/dead pixel population for changes logging.info("\tFinding new hot/dead pixels") @@ -796,10 +845,10 @@ def process(self, file_list): logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) - logging.info("Creating Mean Slope Image") # Create png file of mean slope image. Add bad pixels only for full frame apertures self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix, - noisyxy=new_noisy_pixels, baseline_file=baseline_file) + noisyxy=new_noisy_pixels, baseline_file=baseline_file, + min_time=min_time_mjd_trunc, max_time=max_time_mjd_trunc) logging.info('\tSigma-clipped mean of the slope images saved to: {}'.format(mean_slope_file)) # ----- Calculate image statistics ----- @@ -825,7 +874,10 @@ def process(self, file_list): # Construct new entry for dark database table source_files = [os.path.basename(item) for item in file_list] for key in amp_mean.keys(): - dark_db_entry = {'aperture': self.aperture, 'amplifier': key, 'mean': amp_mean[key], + dark_db_entry = {'aperture': self.aperture, + 'amplifier': key, + 'readpattern': self.readpatt, + 'mean': amp_mean[key], 'stdev': amp_stdev[key], 'source_files': source_files, 'obs_start_time': min_time, @@ -899,119 +951,138 @@ def run(self): self.query_end = Time.now().mjd # Loop over all instruments - for instrument in JWST_INSTRUMENT_NAMES: + for instrument in ['miri', 'nircam']: # JWST_INSTRUMENT_NAMES: self.instrument = instrument + logging.info(f'\n\nWorking on {instrument}') # Identify which database tables to use self.identify_tables() - # Get a list of all possible apertures from pysiaf - possible_apertures = list(Siaf(instrument).apernames) - possible_apertures = [ap for ap in possible_apertures if ap not in apertures_to_skip] + # Run the monitor only on the apertures listed in the threshold file. Skip all others. + instrument_entries = limits['Instrument'] == instrument + possible_apertures = limits['Aperture'][instrument_entries] # Get a list of all possible readout patterns associated with the aperture possible_readpatts = RAPID_READPATTERNS[instrument] for aperture in possible_apertures: logging.info('') - logging.info('Working on aperture {} in {}'.format(aperture, instrument)) + logging.info(f'Working on aperture {aperture} in {instrument}') # Find appropriate threshold for the number of new files needed match = aperture == limits['Aperture'] - - # If the aperture is not listed in the threshold file, we need - # a default - if not np.any(match): - file_count_threshold = 30 - logging.warning(('\tAperture {} is not present in the threshold file. Continuing ' - 'with the default threshold of 30 files.'.format(aperture))) - else: - file_count_threshold = limits['Threshold'][match][0] + integration_count_threshold = limits['Threshold'][match][0] + self.skipped_initial_ints = limits['N_skipped_integs'][match][0] self.aperture = aperture - # We need a separate search for each readout pattern for readpatt in possible_readpatts: self.readpatt = readpatt - logging.info('\tWorking on readout pattern: {}'.format(self.readpatt)) + logging.info(f'\tWorking on readout pattern: {self.readpatt}') # Locate the record of the most recent MAST search self.query_start = self.most_recent_search() - logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end)) + logging.info(f'\tQuery times: {self.query_start} {self.query_end}') # Query MAST using the aperture and the time of the # most recent previous search as the starting time - new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt) + new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, + self.query_end, readpatt=self.readpatt) # Exclude ASIC tuning data len_new_darks = len(new_entries) new_entries = monitor_utils.exclude_asic_tuning(new_entries) len_no_asic = len(new_entries) num_asic = len_new_darks - len_no_asic - logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic)) - logging.info('\tAperture: {}, Readpattern: {}, new entries: {}'.format(self.aperture, self.readpatt, - len(new_entries))) + # Exclude files that don't have enough groups to be useful + new_entries = self.exclude_too_few_groups(new_entries) + len_new_darks = len(new_entries) - # Check to see if there are enough new files to meet the - # monitor's signal-to-noise requirements - if len(new_entries) >= file_count_threshold: - logging.info('\tMAST query has returned sufficient new dark files for {}, {}, {} to run the dark monitor.' - .format(self.instrument, self.aperture, self.readpatt)) - - # Get full paths to the files - new_filenames = [] - for file_entry in new_entries: - try: - new_filenames.append(filesystem_path(file_entry['filename'])) - except FileNotFoundError: - logging.warning('\t\tUnable to locate {} in filesystem. Not including in processing.' - .format(file_entry['filename'])) - - # In some (unusual) cases, there are files in MAST with the correct aperture name - # but incorrect array sizes. Make sure that the new files all have the expected - # aperture size - temp_filenames = [] - bad_size_filenames = [] - expected_ap = Siaf(instrument)[aperture] - expected_xsize = expected_ap.XSciSize - expected_ysize = expected_ap.YSciSize - for new_file in new_filenames: - with fits.open(new_file) as hdulist: - xsize = hdulist[0].header['SUBSIZE1'] - ysize = hdulist[0].header['SUBSIZE2'] - if xsize == expected_xsize and ysize == expected_ysize: - temp_filenames.append(new_file) - else: - bad_size_filenames.append(new_file) - if len(temp_filenames) != len(new_filenames): - logging.info('\tSome files returned by MAST have unexpected aperture sizes. These files will be ignored: ') - for badfile in bad_size_filenames: - logging.info('\t\t{}'.format(badfile)) - new_filenames = deepcopy(temp_filenames) - - # If it turns out that the monitor doesn't find enough - # of the files returned by the MAST query to meet the threshold, - # then the monitor will not be run - if len(new_filenames) < file_count_threshold: - logging.info(("\tFilesystem search for the files identified by MAST has returned {} files. " - "This is less than the required minimum number of files ({}) necessary to run " - "the monitor. Quitting.").format(len(new_filenames), file_count_threshold)) - monitor_run = False + logging.info(f'\tAperture: {self.aperture}, Readpattern: {self.readpatt}, new entries: {len(new_entries)}') + + # Get full paths to the files + new_filenames = [] + for file_entry in new_entries: + try: + new_filenames.append(filesystem_path(file_entry['filename'])) + except FileNotFoundError: + logging.warning((f"\t\tUnable to locate {file_entry['filename']} in filesystem. " + "Not including in processing.")) + + # Generate a count of the total number of integrations across the files. This number will + # be compared to the threshold value to determine if the monitor is run. + # Also, in some (unusual) cases, there are files in MAST with the correct aperture name + # but incorrect array sizes. Make sure that the new files all have the expected + # aperture size + total_integrations = 0 + integrations = [] + starting_times = [] + ending_times = [] + temp_filenames = [] + bad_size_filenames = [] + expected_ap = Siaf(instrument)[aperture] + expected_xsize = expected_ap.XSciSize + expected_ysize = expected_ap.YSciSize + for new_file in new_filenames: + with fits.open(new_file) as hdulist: + xsize = hdulist[0].header['SUBSIZE1'] + ysize = hdulist[0].header['SUBSIZE2'] + nints = hdulist[0].header['NINTS'] + # If the array size matches expectataions, or if Siaf doesn't give an expected size, then + # keep the file. Also, make sure there is at leasat one integration, after ignoring any user-input + # number of integrations. + keep_ints = int(nints) - self.skipped_initial_ints + if ((keep_ints > 0) and ((xsize == expected_xsize and ysize == expected_ysize) + or expected_xsize is None or expected_ysize is None)): + temp_filenames.append(new_file) + total_integrations += int(nints) + integrations.append(int(nints) - self.skipped_initial_ints) + starting_times.append(hdulist[0].header['EXPSTART']) + ending_times.append(hdulist[0].header['EXPEND']) else: - logging.info(("\tFilesystem search for the files identified by MAST has returned {} files.") - .format(len(new_filenames))) - monitor_run = True - - if monitor_run: - # Set up directories for the copied data - ensure_dir_exists(os.path.join(self.working_dir, 'data')) - self.working_data_dir = os.path.join(self.working_dir, - 'data/{}_{}'.format(self.instrument.lower(), - self.aperture.lower())) - ensure_dir_exists(self.working_data_dir) - + bad_size_filenames.append(new_file) + logging.info((f'\t\t{new_file} has unexpected aperture size. Expecting ' + f'{expected_xsize}x{expected_ysize}. Got {xsize}x{ysize}')) + + if len(temp_filenames) != len(new_filenames): + logging.info(('\t\tSome files returned by MAST have unexpected aperture sizes. These files ' + 'will be ignored: ')) + for badfile in bad_size_filenames: + logging.info('\t\t\t{}'.format(badfile)) + new_filenames = deepcopy(temp_filenames) + + # Check to see if there are enough new integrations to meet the + # monitor's signal-to-noise requirements + if len(new_filenames) > 0: + logging.info((f'\t\tFilesystem search for new dark integrations for {self.instrument}, {self.aperture}, ' + f'{self.readpatt} has found {total_integrations} integrations spread ' + f'across {len(new_filenames)} files.')) + if total_integrations >= integration_count_threshold: + logging.info(f'\tThis meets the threshold of {integration_count_threshold} integrations.') + monitor_run = True + + # Set up directories for the copied data + ensure_dir_exists(os.path.join(self.working_dir, 'data')) + self.working_data_dir = os.path.join(self.working_dir, + 'data/{}_{}'.format(self.instrument.lower(), + self.aperture.lower())) + ensure_dir_exists(self.working_data_dir) + + # Split the list of good files into sub-lists based on the integration + # threshold. The monitor will then be run on each sub-list independently, + # in order to produce results with roughly the same signal-to-noise. This + # also prevents the monitor running on a huge chunk of files in the case + # where it hasn't been run in a while and data have piled up in the meantime. + self.split_files_into_sub_lists(new_filenames, starting_times, ending_times, + integrations, integration_count_threshold) + + # Run the monitor once on each list + for new_file_list, batch_start_time, batch_end_time, batch_integrations in zip(self.file_batches, + self.start_time_batches, + self.end_time_batches, + self.integration_batches): # Copy files from filesystem - dark_files, not_copied = copy_files(new_filenames, self.working_data_dir) + dark_files, not_copied = copy_files(new_file_list, self.working_data_dir) # Check that there were no problems with the file copying. If any of the copied # files have different sizes between the MAST filesystem and the JWQL filesystem, @@ -1020,43 +1091,73 @@ def run(self): copied_size = os.stat(dark_file).st_size orig_size = os.stat(filesystem_path(os.path.basename(dark_file))).st_size if orig_size != copied_size: - logging.info(f"\tProblem copying {os.path.basename(dark_file)} from the filesystem.") - logging.info(f"Size in filesystem: {orig_size}, size of copy: {copied_size}. Skipping file.") + logging.error(f"\tProblem copying {os.path.basename(dark_file)} from the filesystem!") + logging.error(f"Size in filesystem: {orig_size}, size of copy: {copied_size}. Skipping file.") not_copied.append(dark_file) dark_files.remove(dark_file) os.remove(dark_file) - logging.info('\tNew_filenames: {}'.format(new_filenames)) + logging.info('\tNew_filenames: {}'.format(new_file_list)) logging.info('\tData dir: {}'.format(self.working_data_dir)) - logging.info('\tCopied to working dir: {}'.format(dark_files)) + logging.info('\tCopied to data dir: {}'.format(dark_files)) logging.info('\tNot copied: {}'.format(not_copied)) - # Run the dark monitor - self.process(dark_files) + # Get the starting and ending time of the files in this monitor run + batch_start_time = np.min(np.array(batch_start_time)) + batch_end_time = np.max(np.array(batch_end_time)) + + if len(dark_files) > 0: + # Run the dark monitor + logging.info(f'\tRunning process for {instrument}, {aperture}, {readpatt} with:') + for dkfile in dark_files: + logging.info(f'\t{dkfile}') + self.process(dark_files) + else: + logging.info('\tNo files remaining to process. Skipping monitor.') + monitor_run = False + + # Update the query history once for each group of files + new_entry = {'instrument': instrument, + 'aperture': aperture, + 'readpattern': self.readpatt, + 'start_time_mjd': batch_start_time, + 'end_time_mjd': batch_end_time, + 'files_found': len(dark_files), + 'run_monitor': monitor_run, + 'entry_date': datetime.datetime.now()} + + with engine.begin() as connection: + connection.execute( + self.query_table.__table__.insert(), new_entry) + logging.info('\tUpdated the query history table') + logging.info('NEW ENTRY: ') + logging.info(new_entry) else: - logging.info(('\tDark monitor skipped. MAST query has returned {} new dark files for ' - '{}, {}, {}. {} new files are required to run dark current monitor.') - .format(len(new_entries), instrument, aperture, self.readpatt, file_count_threshold)) + logging.info((f'\tThis is below the threshold of {integration_count_threshold} ' + 'integrations. Monitor not run.')) monitor_run = False - # Update the query history - new_entry = {'instrument': instrument, - 'aperture': aperture, - 'readpattern': self.readpatt, - 'start_time_mjd': self.query_start, - 'end_time_mjd': self.query_end, - 'files_found': len(new_entries), - 'run_monitor': monitor_run, - 'entry_date': datetime.datetime.now()} - with engine.begin() as connection: - connection.execute( - self.query_table.__table__.insert(), new_entry) - logging.info('\tUpdated the query history table') + # Update the query history + new_entry = {'instrument': instrument, + 'aperture': aperture, + 'readpattern': self.readpatt, + 'start_time_mjd': self.query_start, + 'end_time_mjd': self.query_end, + 'files_found': len(new_entries), + 'run_monitor': monitor_run, + 'entry_date': datetime.datetime.now()} + + with engine.begin() as connection: + connection.execute( + self.query_table.__table__.insert(), new_entry) + logging.info('\tUpdated the query history table') + logging.info('NEW ENTRY: ') + logging.info(new_entry) logging.info('Dark Monitor completed successfully.') - def save_mean_slope_image(self, slope_img, stdev_img, files): + def save_mean_slope_image(self, slope_img, stdev_img, files, min_time, max_time): """Save the mean slope image and associated stdev image to a file @@ -1072,6 +1173,12 @@ def save_mean_slope_image(self, slope_img, stdev_img, files): files : list List of input files used to construct the mean slope image + min_time : str + Earliest observation time, in MJD, corresponding to ``files``. + + max_time : str + Latest observation time, in MJD, corresponding to ``files``. + Returns ------- output_filename : str @@ -1080,7 +1187,7 @@ def save_mean_slope_image(self, slope_img, stdev_img, files): output_filename = '{}_{}_{}_to_{}_mean_slope_image.fits'.format(self.instrument.lower(), self.aperture.lower(), - self.query_start, self.query_end) + min_time, max_time) mean_slope_dir = os.path.join(get_config()['outputs'], 'dark_monitor', 'mean_slope_images') ensure_dir_exists(mean_slope_dir) @@ -1092,6 +1199,8 @@ def save_mean_slope_image(self, slope_img, stdev_img, files): primary_hdu.header['APERTURE'] = (self.aperture, 'Aperture name') primary_hdu.header['QRY_STRT'] = (self.query_start, 'MAST Query start time (MJD)') primary_hdu.header['QRY_END'] = (self.query_end, 'MAST Query end time (MJD)') + primary_hdu.header['MIN_TIME'] = (min_time, 'Beginning obs time (MJD)') + primary_hdu.header['MAX_TIME'] = (max_time, 'Ending obs time (MJD)') files_string = 'FILES USED: ' for filename in files: @@ -1128,6 +1237,202 @@ def shift_to_full_frame(self, coords): return (x, y) + def split_files_into_sub_lists(self, files, start_times, end_times, integration_list, threshold): + """Given a list of filenames and a list of the number of integrations + within each, split the files into sub-lists, where the files in each + list have a total number of integrations that is just over the given + threshold value. + + General assumption: Keeping files in different epochs separate is probably more + important than rigidly enforcing that the required number of integrations is reached. + + When dividing up the input files into separate lists, we first divide up by + epoch, where the start/end of epochs are defined as times where + DARK_MONITOR_BETWEEN_EPOCH_THRESHOLD_TIME days pass without any new data appearing. + Each epoch is then potentially subdivided further based on the threshold number + of integrations (not exposures). The splitting does not operate within files. + For example, if the threshold is 2 integrations, and a particular file contains 5 + integrations, then the dark monitor will be called once on that file, working on + all 5 integrations. + + At the end of the epoch, if the final group of file(s) do not have enough + integrations to reach the threshold, they are ignored since there is no way + to know if there are more files in the same epoch that have not yet been taken. So + the files are ignored, and the query end time will be adjusted such that these files + will be found in the next run of the monitor. + + Dark calibration plans per instrument: + NIRCam - for full frame, takes only 2 integrations (150 groups) once per ~30-50 days. + for subarrays, takes 5-10 integrations once per 30-50 days + team response - + NIRISS - full frame - 2 exps of 5 ints within each 2 week period. No requirement for + the 2 exps to be taken at the same time though. Could be separated + by almost 2 weeks, and be closer to the darks from the previous or + following 2 week period. + subarrays - 30 ints in each month-long span + MIRI - 2 ints every 2 hours-5 days for a while, then 2 ints every 14-21 days + team response - monitor should run on each exp separately. It should also throw out + the first integration of each exp. + + NIRSpec - full frame 5-6 integrations spread over each month + subarray - 12 ints spread over each 2 month period + FGS - N/A + + Parameters + ---------- + files : list + List of filenames + + integration_list : list + List of integers describing how many integrations are in each file + + start_times : list + List of MJD dates corresponding to the exposure start time of each file in ``files`` + + end_times : list + List of MJD dates corresponding to the exposures end time of each file in ``files`` + + integration_list : list + List of the number of integrations for each file in ``files`` + + threshold : int + Threshold number of integrations needed to trigger a run of the + dark monitor + """ + + logging.info('\t\tSplitting into sub-lists. Inputs at the beginning: (file, start time, end time, nints, threshold)') + for f, st, et, inte in zip(files, start_times, end_times, integration_list): + logging.info(f'\t\t {f}, {st}, {et}, {inte}, {threshold}') + logging.info('\n') + + # Eventual return parameters + self.file_batches = [] + self.start_time_batches = [] + self.end_time_batches = [] + self.integration_batches = [] + + # Add the current time onto the end of start_times + start_times = np.array(start_times) + + # Get the delta t between each pair of files. Insert 0 as the initial + # delta_t, to make the coding easier + delta_t = start_times[1:] - start_times[0:-1] # units are days + delta_t = np.insert(delta_t, 0, 0) + + # Divide up the list such that you don't cross large delta t values. We want to measure + # dark current during each "epoch" within a calibration proposal + dividers = np.where(delta_t >= DARK_MONITOR_BETWEEN_EPOCH_THRESHOLD_TIME[self.instrument])[0] + + # Add dividers at the beginning index to make the coding easier + dividers = np.insert(dividers, 0, 0) + + # If there is no divider at the end of the list of files, then add one + if dividers[-1] < len(delta_t): + dividers = np.insert(dividers, len(dividers), len(delta_t)) + + logging.info(f'\t\t\tThreshold delta time used to divide epochs: {DARK_MONITOR_BETWEEN_EPOCH_THRESHOLD_TIME[self.instrument]} days') + logging.info(f'\t\t\tdelta_t between files: {delta_t} days.') + logging.info(f'\t\t\tFinal dividers (divide data based on time gaps between files): {dividers}') + logging.info('\n') + + # Loop over epochs. + # Within each batch, divide up the exposures into multiple batches if the total + # number of integrations are above 2*threshold. + for i in range(len(dividers) - 1): + batch_ints = integration_list[dividers[i]:dividers[i + 1]] + batch_files = files[dividers[i]:dividers[i + 1]] + batch_start_times = start_times[dividers[i]:dividers[i + 1]] + batch_end_times = end_times[dividers[i]:dividers[i + 1]] + batch_int_sum = np.sum(batch_ints) + + logging.info(f'\t\t\tLoop over time-based batches. Working on batch {i}') + logging.info(f'\t\t\tBatch Files, Batch integrations') + for bi, bf in zip(batch_ints, batch_files): + logging.info(f'\t\t\t{bf}, {bi}') + + # Calculate the total number of integrations up to each file + batch_int_sums = np.array([np.sum(batch_ints[0:jj]) for jj in range(1, len(batch_ints) + 1)]) + + base = 0 + startidx = 0 + endidx = 0 + complete = False + + # Divide into sublists + while True: + + endidx = np.where(batch_int_sums >= (base + threshold))[0] + + # Check if we reach the end of the file list + if len(endidx) == 0: + endidx = len(batch_int_sums) - 1 + complete = True + else: + endidx = endidx[0] + if endidx == (len(batch_int_sums) - 1): + complete = True + + logging.debug(f'\t\t\tstartidx: {startidx}') + logging.debug(f'\t\t\tendidx: {endidx}') + logging.debug(f'\t\t\tcomplete: {complete}') + + subgroup_ints = batch_ints[startidx: endidx + 1] + subgroup_files = batch_files[startidx: endidx + 1] + subgroup_start_times = batch_start_times[startidx: endidx + 1] + subgroup_end_times = batch_end_times[startidx: endidx + 1] + subgroup_int_sum = np.sum(subgroup_ints) + + logging.debug(f'\t\t\tsubgroup_ints: {subgroup_ints}') + logging.debug(f'\t\t\tsubgroup_files: {subgroup_files}') + logging.debug(f'\t\t\tsubgroup_int_sum: {subgroup_int_sum}') + + # Add to output lists. The exception is if we are in the + # final subgroup of the final epoch. In that case, we don't know + # if more data are coming soon that may be able to be combined. So + # in that case, we ignore the files for this run of the monitor. + if (i == len(dividers) - 2) and endidx == len(batch_files) - 1: + # Here we are in the final subgroup of the final epoch, where we + # do not necessarily know if there will be future data to combine + # with these data + logging.debug(f'\t\t\tShould be final epoch and final subgroup. epoch number: {i}') + + if np.sum(subgroup_ints) >= threshold: + logging.debug('\t\t\tADDED - final subgroup of final epoch') + self.file_batches.append(subgroup_files) + self.start_time_batches.append(subgroup_start_times) + self.end_time_batches.append(subgroup_end_times) + self.integration_batches.append(subgroup_ints) + else: + # Here the final subgroup does not have enough integrations to reach the threshold + # and we're not sure if the epoch is complete, so we skip these files and save them + # for a future dark monitor run + logging.info('\t\t\tSkipping final subgroup. Not clear if the epoch is complete') + pass + + else: + self.file_batches.append(subgroup_files) + self.start_time_batches.append(subgroup_start_times) + self.end_time_batches.append(subgroup_end_times) + self.integration_batches.append(subgroup_ints) + + if not complete: + startidx = deepcopy(endidx + 1) + base = batch_int_sums[endidx] + else: + # If we reach the end of the list before the expected number of + # subgroups, then we quit. + break + + logging.info(f'\n\t\t\tEpoch number: {i}') + logging.info('\t\t\tFiles, integrations in file batch:') + for bi, bf in zip(batch_ints, batch_files): + logging.info(f'\t\t\t{bf}, {bi}') + logging.info(f'\n\t\t\tSplit into separate subgroups for processing:') + logging.info('\t\t\tFiles and number of integrations in each subgroup:') + for fb, ib in zip(self.file_batches, self.integration_batches): + logging.info(f'\t\t\t{fb}, {ib}') + logging.info(f'\t\t\tDONE WITH SUBGROUPS\n\n\n\n') + def stats_by_amp(self, image, amps): """Calculate statistics in the input image for each amplifier as well as the full image @@ -1203,7 +1508,7 @@ def stats_by_amp(self, image, amps): maxx = copy(mxx) if mxy > maxy: maxy = copy(mxy) - amps['5'] = [(0, maxx, 1), (0, maxy, 1)] + amps['5'] = [(4, maxx, 1), (4, maxy, 1)] logging.info(('\tFull frame exposure detected. Adding the full frame to the list ' 'of amplifiers upon which to calculate statistics.')) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt b/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt index d423bbbdb..010831446 100644 --- a/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt +++ b/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt @@ -1,624 +1,89 @@ -Instrument Aperture Threshold -nircam NRCA1_FULL_OSS 10 -nircam NRCA2_FULL_OSS 10 -nircam NRCA3_FULL_OSS 10 -nircam NRCA4_FULL_OSS 10 -nircam NRCA5_FULL_OSS 10 -nircam NRCB1_FULL_OSS 10 -nircam NRCB2_FULL_OSS 10 -nircam NRCB3_FULL_OSS 10 -nircam NRCB4_FULL_OSS 10 -nircam NRCB5_FULL_OSS 10 -nircam NRCALL_FULL 10 -nircam NRCAS_FULL 10 -nircam NRCA1_FULL 10 -nircam NRCA2_FULL 10 -nircam NRCA3_FULL 10 -nircam NRCA4_FULL 10 -nircam NRCA5_FULL 10 -nircam NRCBS_FULL 10 -nircam NRCB1_FULL 10 -nircam NRCB2_FULL 10 -nircam NRCB3_FULL 10 -nircam NRCB4_FULL 10 -nircam NRCB5_FULL 10 -nircam NRCB1_FULLP 10 -nircam NRCB5_FULLP 10 -nircam NRCA1_SUB160 30 -nircam NRCA2_SUB160 30 -nircam NRCA3_SUB160 30 -nircam NRCA4_SUB160 30 -nircam NRCA5_SUB160 30 -nircam NRCB1_SUB160 30 -nircam NRCB2_SUB160 30 -nircam NRCB3_SUB160 30 -nircam NRCB4_SUB160 30 -nircam NRCB5_SUB160 30 -nircam NRCA1_SUB320 30 -nircam NRCA2_SUB320 30 -nircam NRCA3_SUB320 30 -nircam NRCA4_SUB320 30 -nircam NRCA5_SUB320 30 -nircam NRCB1_SUB320 30 -nircam NRCB2_SUB320 30 -nircam NRCB3_SUB320 30 -nircam NRCB4_SUB320 30 -nircam NRCB5_SUB320 30 -nircam NRCA1_SUB640 30 -nircam NRCA2_SUB640 30 -nircam NRCA3_SUB640 30 -nircam NRCA4_SUB640 30 -nircam NRCA5_SUB640 30 -nircam NRCB1_SUB640 30 -nircam NRCB2_SUB640 30 -nircam NRCB3_SUB640 30 -nircam NRCB4_SUB640 30 -nircam NRCB5_SUB640 30 -nircam NRCA5_GRISM256_F322W2 30 -nircam NRCA5_GRISM128_F322W2 30 -nircam NRCA5_GRISM64_F322W2 30 -nircam NRCA5_GRISM256_F277W 30 -nircam NRCA5_GRISM128_F277W 30 -nircam NRCA5_GRISM64_F277W 30 -nircam NRCA5_GRISM256_F356W 30 -nircam NRCA5_GRISM128_F356W 30 -nircam NRCA5_GRISM64_F356W 30 -nircam NRCA5_GRISM256_F444W 30 -nircam NRCA5_GRISM128_F444W 30 -nircam NRCA5_GRISM64_F444W 30 -nircam NRCA5_GRISM_F322W2 30 -nircam NRCA5_GRISM_F277W 30 -nircam NRCA5_GRISM_F356W 30 -nircam NRCA5_GRISM_F444W 30 -nircam NRCA1_GRISMTS 30 -nircam NRCA1_GRISMTS256 30 -nircam NRCA1_GRISMTS128 30 -nircam NRCA1_GRISMTS64 30 -nircam NRCA3_GRISMTS 30 -nircam NRCA3_GRISMTS256 30 -nircam NRCA3_GRISMTS128 30 -nircam NRCA3_GRISMTS64 30 -nircam NRCA5_TAGRISMTS32 30 -nircam NRCA5_TAGRISMTS32_F405N 30 -nircam NRCA5_TAGRISMTS_SCI_F322W2 30 -nircam NRCA5_TAGRISMTS_SCI_F444W 30 -nircam NRCA3_DHSPIL 30 -nircam NRCA3_DHSPIL_SUB96 30 -nircam NRCA3_DHSPIL_WEDGES 30 -nircam NRCB4_DHSPIL 30 -nircam NRCB4_DHSPIL_SUB96 30 -nircam NRCB4_DHSPIL_WEDGES 30 -nircam NRCA3_FP1 30 -nircam NRCA3_FP1_SUB8 30 -nircam NRCA3_FP1_SUB64 30 -nircam NRCA3_FP2MIMF 30 -nircam NRCA1_FP3MIMF 30 -nircam NRCA2_FP4MIMF 30 -nircam NRCA4_FP5MIMF 30 -nircam NRCB4_FP1 30 -nircam NRCB4_FP1_SUB8 30 -nircam NRCB4_FP1_SUB64 30 -nircam NRCB4_FP2MIMF 30 -nircam NRCB2_FP3MIMF 30 -nircam NRCB1_FP4MIMF 30 -nircam NRCB3_FP5MIMF 30 -nircam NRCA3_SUB64P 30 -nircam NRCA3_SUB160P 30 -nircam NRCA3_SUB400P 30 -nircam NRCA5_SUB64P 30 -nircam NRCA5_SUB160P 30 -nircam NRCA5_SUB400P 30 -nircam NRCB1_SUB64P 30 -nircam NRCB1_SUB160P 30 -nircam NRCB1_SUB400P 30 -nircam NRCB5_SUB64P 30 -nircam NRCB5_SUB160P 30 -nircam NRCB5_SUB400P 30 -nircam NRCB5_TAPSIMG32 30 -nircam NRCA5_GRISMC_WFSS 30 -nircam NRCA5_GRISMR_WFSS 30 -nircam NRCALL_GRISMC_WFSS 30 -nircam NRCALL_GRISMR_WFSS 30 -nircam NRCB5_GRISMC_WFSS 30 -nircam NRCB5_GRISMR_WFSS 30 -nircam NRCA2_MASK210R 30 -nircam NRCA5_MASK335R 30 -nircam NRCA5_MASK430R 30 -nircam NRCA4_MASKSWB 30 -nircam NRCA5_MASKLWB 30 -nircam NRCA2_TAMASK210R 30 -nircam NRCA5_TAMASK335R 30 -nircam NRCA5_TAMASK430R 30 -nircam NRCA4_TAMASKSWB 30 -nircam NRCA5_TAMASKLWB 30 -nircam NRCA5_TAMASKLWBL 30 -nircam NRCA4_TAMASKSWBS 30 -nircam NRCB1_MASK210R 30 -nircam NRCB5_MASK335R 30 -nircam NRCB5_MASK430R 30 -nircam NRCB3_MASKSWB 30 -nircam NRCB5_MASKLWB 30 -nircam NRCB1_TAMASK210R 30 -nircam NRCB5_TAMASK335R 30 -nircam NRCB5_TAMASK430R 30 -nircam NRCB3_TAMASKSWB 30 -nircam NRCB5_TAMASKLWB 30 -nircam NRCB5_TAMASKLWBL 30 -nircam NRCB3_TAMASKSWBS 30 -nircam NRCA2_FSTAMASK210R 30 -nircam NRCA4_FSTAMASKSWB 30 -nircam NRCA5_FSTAMASKLWB 30 -nircam NRCA5_FSTAMASK335R 30 -nircam NRCA5_FSTAMASK430R 30 -nircam NRCA4_MASKSWB_F182M 30 -nircam NRCA4_MASKSWB_F187N 30 -nircam NRCA4_MASKSWB_F210M 30 -nircam NRCA4_MASKSWB_F212N 30 -nircam NRCA4_MASKSWB_F200W 30 -nircam NRCA4_MASKSWB_NARROW 30 -nircam NRCA5_MASKLWB_F250M 30 -nircam NRCA5_MASKLWB_F300M 30 -nircam NRCA5_MASKLWB_F277W 30 -nircam NRCA5_MASKLWB_F335M 30 -nircam NRCA5_MASKLWB_F360M 30 -nircam NRCA5_MASKLWB_F356W 30 -nircam NRCA5_MASKLWB_F410M 30 -nircam NRCA5_MASKLWB_F430M 30 -nircam NRCA5_MASKLWB_F460M 30 -nircam NRCA5_MASKLWB_F480M 30 -nircam NRCA5_MASKLWB_F444W 30 -nircam NRCA5_MASKLWB_NARROW 30 -nircam NRCA2_FULL_MASK210R 10 -nircam NRCA5_FULL_MASK335R 10 -nircam NRCA5_FULL_MASK430R 10 -nircam NRCA4_FULL_MASKSWB 10 -nircam NRCA4_FULL_MASKSWB_F182M 10 -nircam NRCA4_FULL_MASKSWB_F187N 10 -nircam NRCA4_FULL_MASKSWB_F210M 10 -nircam NRCA4_FULL_MASKSWB_F212N 10 -nircam NRCA4_FULL_MASKSWB_F200W 10 -nircam NRCA5_FULL_MASKLWB 10 -nircam NRCA5_FULL_MASKLWB_F250M 10 -nircam NRCA5_FULL_MASKLWB_F300M 10 -nircam NRCA5_FULL_MASKLWB_F277W 10 -nircam NRCA5_FULL_MASKLWB_F335M 10 -nircam NRCA5_FULL_MASKLWB_F360M 10 -nircam NRCA5_FULL_MASKLWB_F356W 10 -nircam NRCA5_FULL_MASKLWB_F410M 10 -nircam NRCA5_FULL_MASKLWB_F430M 10 -nircam NRCA5_FULL_MASKLWB_F460M 10 -nircam NRCA5_FULL_MASKLWB_F480M 10 -nircam NRCA5_FULL_MASKLWB_F444W 10 -nircam NRCA2_FULL_WEDGE_RND 10 -nircam NRCA4_FULL_WEDGE_BAR 10 -nircam NRCA5_FULL_WEDGE_RND 10 -nircam NRCA5_FULL_WEDGE_BAR 10 -nircam NRCA2_FULL_TAMASK210R 10 -nircam NRCA5_FULL_TAMASK335R 10 -nircam NRCA5_FULL_TAMASK430R 10 -nircam NRCA4_FULL_TAMASKSWB 10 -nircam NRCA5_FULL_TAMASKLWB 10 -nircam NRCA5_FULL_TAMASKLWBL 10 -nircam NRCA4_FULL_TAMASKSWBS 10 -nircam NRCA2_FULL_FSTAMASK210R 10 -nircam NRCA4_FULL_FSTAMASKSWB 10 -nircam NRCA5_FULL_FSTAMASKLWB 10 -nircam NRCA5_FULL_FSTAMASK335R 10 -nircam NRCA5_FULL_FSTAMASK430R 10 -niriss NIS_CEN_OSS 10 -niriss NIS_CEN 10 -niriss NIS_AMI1 30 -niriss NIS_AMI2 30 -niriss NIS_AMI3 30 -niriss NIS_AMI4 30 -niriss NIS_AMITA 30 -niriss NIS_SOSSTA 30 -niriss NIS_WFSS_OFFSET 30 -niriss NIS_WFSS64 30 -niriss NIS_WFSS64R 30 -niriss NIS_WFSS64R3 30 -niriss NIS_WFSS64C 30 -niriss NIS_WFSS64C3 30 -niriss NIS_WFSS128 30 -niriss NIS_WFSS128R 30 -niriss NIS_WFSS128R3 30 -niriss NIS_WFSS128C 30 -niriss NIS_WFSS128C3 30 -niriss NIS_SUB64 30 -niriss NIS_SUB128 30 -niriss NIS_SUB256 30 -niriss NIS_SUBAMPCAL 30 -niriss NIS_SUBSTRIP96 30 -niriss NIS_SUBSTRIP256 30 -niriss NIS_FP1MIMF 30 -niriss NIS_FP2MIMF 30 -niriss NIS_FP3MIMF 30 -niriss NIS_FP4MIMF 30 -niriss NIS_FP5MIMF 30 -niriss NIS_AMIFULL 10 -niriss NIS_SOSSFULL 10 -niriss NIS_WFSS 10 -miri MIRIM_FULL_OSS 10 -miri MIRIM_FULL 10 -miri MIRIM_ILLUM 30 -miri MIRIM_BRIGHTSKY 30 -miri MIRIM_SUB256 30 -miri MIRIM_SUB128 30 -miri MIRIM_SUB64 30 -miri MIRIM_SLITLESSPRISM 30 -miri MIRIM_SLITLESSUPPER 30 -miri MIRIM_SLITLESSLOWER 30 -miri MIRIM_MASK1065 30 -miri MIRIM_MASK1140 30 -miri MIRIM_MASK1550 30 -miri MIRIM_MASKLYOT 30 -miri MIRIM_TAMRS 30 -miri MIRIM_TALRS 30 -miri MIRIM_TABLOCK 30 -miri MIRIM_TALYOT_UL 30 -miri MIRIM_TALYOT_UR 30 -miri MIRIM_TALYOT_LL 30 -miri MIRIM_TALYOT_LR 30 -miri MIRIM_TALYOT_CUL 30 -miri MIRIM_TALYOT_CUR 30 -miri MIRIM_TALYOT_CLL 30 -miri MIRIM_TALYOT_CLR 30 -miri MIRIM_TA1550_UL 30 -miri MIRIM_TA1550_UR 30 -miri MIRIM_TA1550_LL 30 -miri MIRIM_TA1550_LR 30 -miri MIRIM_TA1550_CUL 30 -miri MIRIM_TA1550_CUR 30 -miri MIRIM_TA1550_CLL 30 -miri MIRIM_TA1550_CLR 30 -miri MIRIM_TA1140_UL 30 -miri MIRIM_TA1140_UR 30 -miri MIRIM_TA1140_LL 30 -miri MIRIM_TA1140_LR 30 -miri MIRIM_TA1140_CUL 30 -miri MIRIM_TA1140_CUR 30 -miri MIRIM_TA1140_CLL 30 -miri MIRIM_TA1140_CLR 30 -miri MIRIM_TA1065_UL 30 -miri MIRIM_TA1065_UR 30 -miri MIRIM_TA1065_LL 30 -miri MIRIM_TA1065_LR 30 -miri MIRIM_TA1065_CUL 30 -miri MIRIM_TA1065_CUR 30 -miri MIRIM_TA1065_CLL 30 -miri MIRIM_TA1065_CLR 30 -miri MIRIM_TAFULL 10 -miri MIRIM_TAILLUM 30 -miri MIRIM_TABRIGHTSKY 30 -miri MIRIM_TASUB256 30 -miri MIRIM_TASUB128 30 -miri MIRIM_TASUB64 30 -miri MIRIM_TASLITLESSPRISM 30 -miri MIRIM_CORON1065 30 -miri MIRIM_CORON1140 30 -miri MIRIM_CORON1550 30 -miri MIRIM_CORONLYOT 30 -miri MIRIM_KNIFE 30 -miri MIRIM_FP1MIMF 30 -miri MIRIM_FP2MIMF 30 -miri MIRIM_FP3MIMF 30 -miri MIRIM_FP4MIMF 30 -miri MIRIM_FP5MIMF 30 -miri MIRIM_SLIT 30 -miri MIRIFU_CHANNEL1A 30 -miri MIRIFU_1ASLICE01 30 -miri MIRIFU_1ASLICE02 30 -miri MIRIFU_1ASLICE03 30 -miri MIRIFU_1ASLICE04 30 -miri MIRIFU_1ASLICE05 30 -miri MIRIFU_1ASLICE06 30 -miri MIRIFU_1ASLICE07 30 -miri MIRIFU_1ASLICE08 30 -miri MIRIFU_1ASLICE09 30 -miri MIRIFU_1ASLICE10 30 -miri MIRIFU_1ASLICE11 30 -miri MIRIFU_1ASLICE12 30 -miri MIRIFU_1ASLICE13 30 -miri MIRIFU_1ASLICE14 30 -miri MIRIFU_1ASLICE15 30 -miri MIRIFU_1ASLICE16 30 -miri MIRIFU_1ASLICE17 30 -miri MIRIFU_1ASLICE18 30 -miri MIRIFU_1ASLICE19 30 -miri MIRIFU_1ASLICE20 30 -miri MIRIFU_1ASLICE21 30 -miri MIRIFU_CHANNEL1B 30 -miri MIRIFU_1BSLICE01 30 -miri MIRIFU_1BSLICE02 30 -miri MIRIFU_1BSLICE03 30 -miri MIRIFU_1BSLICE04 30 -miri MIRIFU_1BSLICE05 30 -miri MIRIFU_1BSLICE06 30 -miri MIRIFU_1BSLICE07 30 -miri MIRIFU_1BSLICE08 30 -miri MIRIFU_1BSLICE09 30 -miri MIRIFU_1BSLICE10 30 -miri MIRIFU_1BSLICE11 30 -miri MIRIFU_1BSLICE12 30 -miri MIRIFU_1BSLICE13 30 -miri MIRIFU_1BSLICE14 30 -miri MIRIFU_1BSLICE15 30 -miri MIRIFU_1BSLICE16 30 -miri MIRIFU_1BSLICE17 30 -miri MIRIFU_1BSLICE18 30 -miri MIRIFU_1BSLICE19 30 -miri MIRIFU_1BSLICE20 30 -miri MIRIFU_1BSLICE21 30 -miri MIRIFU_CHANNEL1C 30 -miri MIRIFU_1CSLICE01 30 -miri MIRIFU_1CSLICE02 30 -miri MIRIFU_1CSLICE03 30 -miri MIRIFU_1CSLICE04 30 -miri MIRIFU_1CSLICE05 30 -miri MIRIFU_1CSLICE06 30 -miri MIRIFU_1CSLICE07 30 -miri MIRIFU_1CSLICE08 30 -miri MIRIFU_1CSLICE09 30 -miri MIRIFU_1CSLICE10 30 -miri MIRIFU_1CSLICE11 30 -miri MIRIFU_1CSLICE12 30 -miri MIRIFU_1CSLICE13 30 -miri MIRIFU_1CSLICE14 30 -miri MIRIFU_1CSLICE15 30 -miri MIRIFU_1CSLICE16 30 -miri MIRIFU_1CSLICE17 30 -miri MIRIFU_1CSLICE18 30 -miri MIRIFU_1CSLICE19 30 -miri MIRIFU_1CSLICE20 30 -miri MIRIFU_1CSLICE21 30 -miri MIRIFU_CHANNEL2A 30 -miri MIRIFU_2ASLICE01 30 -miri MIRIFU_2ASLICE02 30 -miri MIRIFU_2ASLICE03 30 -miri MIRIFU_2ASLICE04 30 -miri MIRIFU_2ASLICE05 30 -miri MIRIFU_2ASLICE06 30 -miri MIRIFU_2ASLICE07 30 -miri MIRIFU_2ASLICE08 30 -miri MIRIFU_2ASLICE09 30 -miri MIRIFU_2ASLICE10 30 -miri MIRIFU_2ASLICE11 30 -miri MIRIFU_2ASLICE12 30 -miri MIRIFU_2ASLICE13 30 -miri MIRIFU_2ASLICE14 30 -miri MIRIFU_2ASLICE15 30 -miri MIRIFU_2ASLICE16 30 -miri MIRIFU_2ASLICE17 30 -miri MIRIFU_CHANNEL2B 30 -miri MIRIFU_2BSLICE01 30 -miri MIRIFU_2BSLICE02 30 -miri MIRIFU_2BSLICE03 30 -miri MIRIFU_2BSLICE04 30 -miri MIRIFU_2BSLICE05 30 -miri MIRIFU_2BSLICE06 30 -miri MIRIFU_2BSLICE07 30 -miri MIRIFU_2BSLICE08 30 -miri MIRIFU_2BSLICE09 30 -miri MIRIFU_2BSLICE10 30 -miri MIRIFU_2BSLICE11 30 -miri MIRIFU_2BSLICE12 30 -miri MIRIFU_2BSLICE13 30 -miri MIRIFU_2BSLICE14 30 -miri MIRIFU_2BSLICE15 30 -miri MIRIFU_2BSLICE16 30 -miri MIRIFU_2BSLICE17 30 -miri MIRIFU_CHANNEL2C 30 -miri MIRIFU_2CSLICE01 30 -miri MIRIFU_2CSLICE02 30 -miri MIRIFU_2CSLICE03 30 -miri MIRIFU_2CSLICE04 30 -miri MIRIFU_2CSLICE05 30 -miri MIRIFU_2CSLICE06 30 -miri MIRIFU_2CSLICE07 30 -miri MIRIFU_2CSLICE08 30 -miri MIRIFU_2CSLICE09 30 -miri MIRIFU_2CSLICE10 30 -miri MIRIFU_2CSLICE11 30 -miri MIRIFU_2CSLICE12 30 -miri MIRIFU_2CSLICE13 30 -miri MIRIFU_2CSLICE14 30 -miri MIRIFU_2CSLICE15 30 -miri MIRIFU_2CSLICE16 30 -miri MIRIFU_2CSLICE17 30 -miri MIRIFU_CHANNEL3A 30 -miri MIRIFU_3ASLICE01 30 -miri MIRIFU_3ASLICE02 30 -miri MIRIFU_3ASLICE03 30 -miri MIRIFU_3ASLICE04 30 -miri MIRIFU_3ASLICE05 30 -miri MIRIFU_3ASLICE06 30 -miri MIRIFU_3ASLICE07 30 -miri MIRIFU_3ASLICE08 30 -miri MIRIFU_3ASLICE09 30 -miri MIRIFU_3ASLICE10 30 -miri MIRIFU_3ASLICE11 30 -miri MIRIFU_3ASLICE12 30 -miri MIRIFU_3ASLICE13 30 -miri MIRIFU_3ASLICE14 30 -miri MIRIFU_3ASLICE15 30 -miri MIRIFU_3ASLICE16 30 -miri MIRIFU_CHANNEL3B 30 -miri MIRIFU_3BSLICE01 30 -miri MIRIFU_3BSLICE02 30 -miri MIRIFU_3BSLICE03 30 -miri MIRIFU_3BSLICE04 30 -miri MIRIFU_3BSLICE05 30 -miri MIRIFU_3BSLICE06 30 -miri MIRIFU_3BSLICE07 30 -miri MIRIFU_3BSLICE08 30 -miri MIRIFU_3BSLICE09 30 -miri MIRIFU_3BSLICE10 30 -miri MIRIFU_3BSLICE11 30 -miri MIRIFU_3BSLICE12 30 -miri MIRIFU_3BSLICE13 30 -miri MIRIFU_3BSLICE14 30 -miri MIRIFU_3BSLICE15 30 -miri MIRIFU_3BSLICE16 30 -miri MIRIFU_CHANNEL3C 30 -miri MIRIFU_3CSLICE01 30 -miri MIRIFU_3CSLICE02 30 -miri MIRIFU_3CSLICE03 30 -miri MIRIFU_3CSLICE04 30 -miri MIRIFU_3CSLICE05 30 -miri MIRIFU_3CSLICE06 30 -miri MIRIFU_3CSLICE07 30 -miri MIRIFU_3CSLICE08 30 -miri MIRIFU_3CSLICE09 30 -miri MIRIFU_3CSLICE10 30 -miri MIRIFU_3CSLICE11 30 -miri MIRIFU_3CSLICE12 30 -miri MIRIFU_3CSLICE13 30 -miri MIRIFU_3CSLICE14 30 -miri MIRIFU_3CSLICE15 30 -miri MIRIFU_3CSLICE16 30 -miri MIRIFU_CHANNEL4A 30 -miri MIRIFU_4ASLICE01 30 -miri MIRIFU_4ASLICE02 30 -miri MIRIFU_4ASLICE03 30 -miri MIRIFU_4ASLICE04 30 -miri MIRIFU_4ASLICE05 30 -miri MIRIFU_4ASLICE06 30 -miri MIRIFU_4ASLICE07 30 -miri MIRIFU_4ASLICE08 30 -miri MIRIFU_4ASLICE09 30 -miri MIRIFU_4ASLICE10 30 -miri MIRIFU_4ASLICE11 30 -miri MIRIFU_4ASLICE12 30 -miri MIRIFU_CHANNEL4B 30 -miri MIRIFU_4BSLICE01 30 -miri MIRIFU_4BSLICE02 30 -miri MIRIFU_4BSLICE03 30 -miri MIRIFU_4BSLICE04 30 -miri MIRIFU_4BSLICE05 30 -miri MIRIFU_4BSLICE06 30 -miri MIRIFU_4BSLICE07 30 -miri MIRIFU_4BSLICE08 30 -miri MIRIFU_4BSLICE09 30 -miri MIRIFU_4BSLICE10 30 -miri MIRIFU_4BSLICE11 30 -miri MIRIFU_4BSLICE12 30 -miri MIRIFU_CHANNEL4C 30 -miri MIRIFU_4CSLICE01 30 -miri MIRIFU_4CSLICE02 30 -miri MIRIFU_4CSLICE03 30 -miri MIRIFU_4CSLICE04 30 -miri MIRIFU_4CSLICE05 30 -miri MIRIFU_4CSLICE06 30 -miri MIRIFU_4CSLICE07 30 -miri MIRIFU_4CSLICE08 30 -miri MIRIFU_4CSLICE09 30 -miri MIRIFU_4CSLICE10 30 -miri MIRIFU_4CSLICE11 30 -miri MIRIFU_4CSLICE12 30 -nirspec NRS1_FULL_OSS 10 -nirspec NRS1_FULL 10 -nirspec NRS2_FULL_OSS 10 -nirspec NRS2_FULL 10 -nirspec NRS_S200A1_SLIT 30 -nirspec NRS_S200A2_SLIT 30 -nirspec NRS_S400A1_SLIT 30 -nirspec NRS_S1600A1_SLIT 30 -nirspec NRS_S200B1_SLIT 30 -nirspec NRS_FULL_IFU 10 -nirspec NRS_IFU_SLICE00 30 -nirspec NRS_IFU_SLICE01 30 -nirspec NRS_IFU_SLICE02 30 -nirspec NRS_IFU_SLICE03 30 -nirspec NRS_IFU_SLICE04 30 -nirspec NRS_IFU_SLICE05 30 -nirspec NRS_IFU_SLICE06 30 -nirspec NRS_IFU_SLICE07 30 -nirspec NRS_IFU_SLICE08 30 -nirspec NRS_IFU_SLICE09 30 -nirspec NRS_IFU_SLICE10 30 -nirspec NRS_IFU_SLICE11 30 -nirspec NRS_IFU_SLICE12 30 -nirspec NRS_IFU_SLICE13 30 -nirspec NRS_IFU_SLICE14 30 -nirspec NRS_IFU_SLICE15 30 -nirspec NRS_IFU_SLICE16 30 -nirspec NRS_IFU_SLICE17 30 -nirspec NRS_IFU_SLICE18 30 -nirspec NRS_IFU_SLICE19 30 -nirspec NRS_IFU_SLICE20 30 -nirspec NRS_IFU_SLICE21 30 -nirspec NRS_IFU_SLICE22 30 -nirspec NRS_IFU_SLICE23 30 -nirspec NRS_IFU_SLICE24 30 -nirspec NRS_IFU_SLICE25 30 -nirspec NRS_IFU_SLICE26 30 -nirspec NRS_IFU_SLICE27 30 -nirspec NRS_IFU_SLICE28 30 -nirspec NRS_IFU_SLICE29 30 -nirspec NRS_FULL_MSA 10 -nirspec NRS_FULL_MSA1 10 -nirspec NRS_FULL_MSA2 10 -nirspec NRS_FULL_MSA3 10 -nirspec NRS_FULL_MSA4 10 -nirspec NRS_VIGNETTED_MSA 30 -nirspec NRS_VIGNETTED_MSA1 30 -nirspec NRS_VIGNETTED_MSA2 30 -nirspec NRS_VIGNETTED_MSA3 30 -nirspec NRS_VIGNETTED_MSA4 30 -nirspec NRS_FIELD1_MSA4 30 -nirspec NRS_FIELD2_MSA4 30 -nirspec NRS1_FP1MIMF 30 -nirspec NRS1_FP2MIMF 30 -nirspec NRS1_FP3MIMF 30 -nirspec NRS2_FP4MIMF 30 -nirspec NRS2_FP5MIMF 30 -nirspec CLEAR_GWA_OTE 30 -nirspec F110W_GWA_OTE 30 -nirspec F140X_GWA_OTE 30 -nirspec NRS_SKY_OTEIP 30 -nirspec NRS_CLEAR_OTEIP_MSA_L0 30 -nirspec NRS_CLEAR_OTEIP_MSA_L1 30 -nirspec NRS_F070LP_OTEIP_MSA_L0 30 -nirspec NRS_F070LP_OTEIP_MSA_L1 30 -nirspec NRS_F100LP_OTEIP_MSA_L0 30 -nirspec NRS_F100LP_OTEIP_MSA_L1 30 -nirspec NRS_F170LP_OTEIP_MSA_L0 30 -nirspec NRS_F170LP_OTEIP_MSA_L1 30 -nirspec NRS_F290LP_OTEIP_MSA_L0 30 -nirspec NRS_F290LP_OTEIP_MSA_L1 30 -nirspec NRS_F110W_OTEIP_MSA_L0 30 -nirspec NRS_F110W_OTEIP_MSA_L1 30 -nirspec NRS_F140X_OTEIP_MSA_L0 30 -nirspec NRS_F140X_OTEIP_MSA_L1 30 -fgs FGS1_FULL_OSS 10 -fgs FGS1_FULL 10 -fgs FGS2_FULL_OSS 10 -fgs FGS2_FULL 10 -fgs FGS1_SUB128LL 30 -fgs FGS1_SUB128DIAG 30 -fgs FGS1_SUB128CNTR 30 -fgs FGS1_SUB32LL 30 -fgs FGS1_SUB32DIAG 30 -fgs FGS1_SUB32CNTR 30 -fgs FGS1_SUB8LL 30 -fgs FGS1_SUB8DIAG 30 -fgs FGS1_SUB8CNTR 30 -fgs FGS2_SUB128LL 30 -fgs FGS2_SUB128DIAG 30 -fgs FGS2_SUB128CNTR 30 -fgs FGS2_SUB32LL 30 -fgs FGS2_SUB32DIAG 30 -fgs FGS2_SUB32CNTR 30 -fgs FGS2_SUB8LL 30 -fgs FGS2_SUB8DIAG 30 -fgs FGS2_SUB8CNTR 30 -fgs FGS1_FP1MIMF 30 -fgs FGS1_FP2MIMF 30 -fgs FGS1_FP3MIMF 30 -fgs FGS1_FP4MIMF 30 -fgs FGS1_FP5MIMF 30 -fgs FGS2_FP1MIMF 30 -fgs FGS2_FP2MIMF 30 -fgs FGS2_FP3MIMF 30 -fgs FGS2_FP4MIMF 30 -fgs FGS2_FP5MIMF 30 \ No newline at end of file +Instrument Aperture Threshold N_skipped_integs +nircam NRCA1_FULL 1 0 +nircam NRCA2_FULL 1 0 +nircam NRCA3_FULL 1 0 +nircam NRCA4_FULL 1 0 +nircam NRCA5_FULL 1 0 +nircam NRCB1_FULL 1 0 +nircam NRCB2_FULL 1 0 +nircam NRCB3_FULL 1 0 +nircam NRCB4_FULL 1 0 +nircam NRCB5_FULL 1 0 +nircam NRCA1_SUB160 4 0 +nircam NRCA2_SUB160 4 0 +nircam NRCA3_SUB160 4 0 +nircam NRCA4_SUB160 4 0 +nircam NRCA5_SUB160 4 0 +nircam NRCB1_SUB160 4 0 +nircam NRCB2_SUB160 4 0 +nircam NRCB3_SUB160 4 0 +nircam NRCB4_SUB160 4 0 +nircam NRCB5_SUB160 4 0 +nircam NRCA1_SUB320 4 0 +nircam NRCA2_SUB320 4 0 +nircam NRCA3_SUB320 4 0 +nircam NRCA4_SUB320 4 0 +nircam NRCA5_SUB320 4 0 +nircam NRCB1_SUB320 4 0 +nircam NRCB2_SUB320 4 0 +nircam NRCB3_SUB320 4 0 +nircam NRCB4_SUB320 4 0 +nircam NRCB5_SUB320 4 0 +nircam NRCA1_SUB640 4 0 +nircam NRCA2_SUB640 4 0 +nircam NRCA3_SUB640 4 0 +nircam NRCA4_SUB640 4 0 +nircam NRCA5_SUB640 4 0 +nircam NRCB1_SUB640 4 0 +nircam NRCB2_SUB640 4 0 +nircam NRCB3_SUB640 4 0 +nircam NRCB4_SUB640 4 0 +nircam NRCB5_SUB640 4 0 +niriss NIS_CEN 1 0 +niriss NIS_AMI1 1 0 +niriss NIS_AMI2 1 0 +niriss NIS_AMI3 1 0 +niriss NIS_AMI4 1 0 +niriss NIS_SUB64 1 0 +niriss NIS_SUB128 1 0 +niriss NIS_SUB256 1 0 +miri MIRIM_FULL 1 1 +miri MIRIM_BRIGHTSKY 1 0 +miri MIRIM_SUB256 1 0 +miri MIRIM_SUB128 1 0 +miri MIRIM_SUB64 1 0 +miri MIRIM_SLITLESSPRISM 1 0 +miri MIRIM_MASK1065 1 0 +miri MIRIM_MASK1140 1 0 +miri MIRIM_MASK1550 1 0 +miri MIRIM_MASKLYOT 1 0 +miri MIRIM_CORON1065 1 0 +miri MIRIM_CORON1140 1 0 +miri MIRIM_CORON1550 1 0 +miri MIRIM_CORONLYOT 1 0 +miri MIRIM_SLIT 1 0 +miri MIRIFU_CHANNEL1A 1 0 +miri MIRIFU_CHANNEL1B 1 0 +miri MIRIFU_CHANNEL1C 1 0 +miri MIRIFU_CHANNEL2A 1 0 +miri MIRIFU_CHANNEL2B 1 0 +miri MIRIFU_CHANNEL2C 1 0 +miri MIRIFU_CHANNEL3A 1 0 +miri MIRIFU_CHANNEL3B 1 0 +miri MIRIFU_CHANNEL3C 1 0 +miri MIRIFU_CHANNEL4A 1 0 +miri MIRIFU_CHANNEL4B 1 0 +miri MIRIFU_CHANNEL4C 1 0 +nirspec NRS1_FULL 1 0 +nirspec NRS2_FULL 1 0 +nirspec NRS_S200A1_SLIT 1 0 +nirspec NRS_S200A2_SLIT 1 0 +nirspec NRS_S400A1_SLIT 1 0 +nirspec NRS_S1600A1_SLIT 1 0 +nirspec NRS_S200B1_SLIT 1 0 +nirspec NRS_FULL_IFU 1 0 +nirspec NRS_FULL_MSA 1 0 +fgs FGS1_FULL 1 0 +fgs FGS2_FULL 1 0 +fgs FGS1_SUB128CNTR 1 0 +fgs FGS2_SUB128CNTR 1 0 diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py index b5388309d..b73d916c0 100644 --- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py +++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py @@ -138,7 +138,7 @@ def make_background_plots(self, plot_type='bkg'): """ columns = ['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd', 'entry_date', 'mean', 'median', - 'stddev', 'frac_masked'] # , 'total_bkg'] + 'stddev', 'frac_masked', 'total_bkg'] # Get all of the background data. background_data = NIRCamClawStats.objects.all().values(*columns) @@ -192,8 +192,7 @@ def make_background_plots(self, plot_type='bkg'): df = df[df['stddev'] != 0] # older data has no accurate stddev measures plot_data = df['stddev'].values if plot_type == 'model': - total_bkg = [1. for x in df['median'].values] - plot_data = df['median'].values # / df['total_bkg'].values + plot_data = df['median'].values / df['total_bkg'].values plot_expstarts = df['expstart_mjd'].values # Plot the background data over time @@ -331,8 +330,8 @@ def process(self): 'stddev': float(stddev), 'frac_masked': len(segmap_orig[(segmap_orig != 0) | (dq & 1 != 0)]) / (segmap_orig.shape[0] * segmap_orig.shape[1]), 'skyflat_filename': os.path.basename(self.outfile), - # 'doy': float(doy), - # 'total_bkg': float(total_bkg), + 'doy': float(doy), + 'total_bkg': float(total_bkg), 'entry_date': datetime.datetime.now() } entry = self.stats_table(**claw_db_entry) diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py index c39e3bda4..e3ab88433 100644 --- a/jwql/instrument_monitors/pipeline_tools.py +++ b/jwql/instrument_monitors/pipeline_tools.py @@ -193,8 +193,8 @@ def get_pipeline_steps(instrument): return required_steps -def image_stack(file_list): - """Given a list of fits files containing 2D images, read in all data +def image_stack(file_list, skipped_initial_ints=0): + """Given a list of fits files containing 2D or 3D images, read in all data and place into a 3D stack Parameters @@ -202,6 +202,13 @@ def image_stack(file_list): file_list : list List of fits file names + skipped_initial_ints : int + Number of initial integrations from each file to skip over and + not include in the stack. Only works with files containing 3D + arrays (e.g. rateints files). This is primarily for MIRI, where + we want to skip the first N integrations due to dark current + instability. + Returns ------- cube : numpy.ndarray @@ -219,7 +226,8 @@ def image_stack(file_list): if i == 0: ndim_base = image.shape if len(ndim_base) == 3: - cube = copy.deepcopy(image) + cube = copy.deepcopy(image[skipped_initial_ints:, :, :]) + num_ints -= skipped_initial_ints elif len(ndim_base) == 2: cube = np.expand_dims(image, 0) else: @@ -227,9 +235,12 @@ def image_stack(file_list): if ndim_base[-2:] == ndim[-2:]: if len(ndim) == 2: image = np.expand_dims(image, 0) + cube = np.vstack((cube, image)) + elif len(ndim) == 3: + cube = np.vstack((cube, image[skipped_initial_ints:, :, :])) + num_ints -= skipped_initial_ints elif len(ndim) > 3: raise ValueError("4-dimensional input slope images not supported.") - cube = np.vstack((cube, image)) else: raise ValueError("Input images are of inconsistent size in x/y dimension.") exptimes.append([exptime] * num_ints) diff --git a/jwql/pull_jwql_branch.sh b/jwql/pull_jwql_branch.sh index 73c7fb8b4..95a1c94b4 100644 --- a/jwql/pull_jwql_branch.sh +++ b/jwql/pull_jwql_branch.sh @@ -1,8 +1,22 @@ #!/bin/bash function echo_format { - echo "WARNING! the optional parameters should only be used during a JWQL release in production" + echo "" echo "Usage: $0 [-r|--reset_service] [-n|--notify ]" + echo "" + echo "WARNING! the optional parameters should only be used during a JWQL release in production" + echo "branch: the git branch to pull from" + echo "[-r|--reset_service]: Reset the jwql service" + echo "[-n|--notify ]: Notify via provided email" + echo "" + echo "Local:" + echo "$ bash pull_jwql_branch.sh develop" + echo "" + echo "Test:" + echo "$ bash pull_jwql_branch.sh v1.2 -r" + echo "" + echo "Production:" + echo "$ bash pull_jwql_branch.sh v1.2 -r -n group_email_address@stsci.edu" } # Check if the required number of arguments are provided @@ -54,7 +68,7 @@ git fetch origin $branch_name git pull origin $branch_name git fetch origin --tags -# 2. Bring the server down and back up +# 2. Bring the service down if [ "$reset" = true ]; then sudo /bin/systemctl stop jwql.service fi @@ -65,7 +79,7 @@ pip install -e .. # 4. Merge Any Migrations python ./website/manage.py migrate -# 5. Bring the server back up +# 5. Bring the service back up if [ "$reset" = true ]; then sudo /bin/systemctl start jwql.service fi @@ -76,8 +90,7 @@ python ./database/database_interface.py # 7. Send out notification email if [ "$notify" = true ] && [ -n "$recipient" ]; then subject="JWQL $branch_name Released" - message_content="Hello, A new version of JWQL ($branch_name) has just been deployed to jwql.stsci.edu. Visit https://github.com/spacetelescope/jwql/releases for more information." + message_content="Hello, A new version of JWQL ($branch_name) has just been released. Visit https://github.com/spacetelescope/jwql/releases for more information." echo "$message_content" | mail -s "$subject" "$recipient" echo "Notification Email Sent" - echo "Deployment Complete!" fi \ No newline at end of file diff --git a/jwql/tests/test_dark_monitor.py b/jwql/tests/test_dark_monitor.py index a9fe14547..ab869cc06 100644 --- a/jwql/tests/test_dark_monitor.py +++ b/jwql/tests/test_dark_monitor.py @@ -28,10 +28,150 @@ from jwql.instrument_monitors.common_monitors import dark_monitor from jwql.tests.resources import has_test_db from jwql.utils.monitor_utils import mast_query_darks +from jwql.utils.constants import DARK_MONITOR_BETWEEN_EPOCH_THRESHOLD_TIME from jwql.utils.utils import get_config from jwql.utils.constants import ON_GITHUB_ACTIONS +def generate_data_for_file_splitting_test(): + # Define data for parameterized test_split_files_into_sub_lists calls + files = [f'file_{idx}.fits' for idx in range(10)] + now = Time.now().mjd + deltat = [26., 25., 24., 23., 22., 4., 3., 2., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 5. # integrations + integration_list = [3, 3, 2, 2, 2, 1, 1, 1, 1, 1] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits', 'file_4.fits'], + ['file_5.fits', 'file_6.fits', 'file_7.fits', 'file_8.fits', 'file_9.fits'] + ] + test1 = (files, start_times, end_times, integration_list, threshold, expected) + + # Final epoch may not be over. Not enough ints in final epoch + deltat = [26., 25., 24., 23., 22., 4., 3., 2., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 6. # integrations + integration_list = [3, 3, 2, 2, 2, 1, 1, 1, 1, 1] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits', 'file_4.fits'] + ] + test2 = (files, start_times, end_times, integration_list, threshold, expected) + + # Final epoch may not be over. Not enough ints in final subgroup of final epoch + deltat = [26., 25., 24., 23., 22., 4., 3., 2., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 6. # integrations + integration_list = [3, 3, 2, 2, 2, 1, 3, 3, 2, 2] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits', 'file_4.fits'], + ['file_5.fits', 'file_6.fits', 'file_7.fits'] + ] + test3 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [40., 39., 38., 37., 36., 18., 17., 16., 15., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 5. # integrations + integration_list = [3, 3, 2, 2, 2, 1, 1, 1, 1, 1] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits', 'file_4.fits'], + ['file_5.fits', 'file_6.fits', 'file_7.fits', 'file_8.fits'] + ] + test4 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [40., 39., 38., 37., 36., 18., 17., 16., 15., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 6. # integrations + integration_list = [3, 3, 2, 2, 2, 1, 1, 1, 1, 1] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits', 'file_4.fits'], + ['file_5.fits', 'file_6.fits', 'file_7.fits', 'file_8.fits'] + ] + test5 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [9., 8., 7., 6., 5., 4., 3., 2., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + integration_list = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + threshold = 6 + expected = [['file_0.fits', 'file_1.fits', 'file_2.fits', 'file_3.fits', 'file_4.fits', 'file_5.fits']] + test6 = (files, start_times, end_times, integration_list, threshold, expected) + + threshold = 9 + expected = [['file_0.fits', 'file_1.fits', 'file_2.fits', 'file_3.fits', 'file_4.fits', 'file_5.fits', + 'file_6.fits', 'file_7.fits', 'file_8.fits']] + test7 = (files, start_times, end_times, integration_list, threshold, expected) + + integration_list = [1] * len(start_times) + threshold = 10 + expected = [['file_0.fits', 'file_1.fits', 'file_2.fits', 'file_3.fits', 'file_4.fits', 'file_5.fits', + 'file_6.fits', 'file_7.fits', 'file_8.fits', 'file_9.fits'] + ] + test8 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [23., 22., 21., 20., 19., 18., 17., 16., 15., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + integration_list = [1] * len(start_times) + threshold = 10 + expected = [['file_0.fits', 'file_1.fits', 'file_2.fits', 'file_3.fits', 'file_4.fits', 'file_5.fits', + 'file_6.fits', 'file_7.fits', 'file_8.fits'] + ] + test9 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [9., 8., 7., 6., 5., 4., 3., 2., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + integration_list = [1] * len(start_times) + threshold = 10 + expected = [['file_0.fits', 'file_1.fits', 'file_2.fits', 'file_3.fits', 'file_4.fits', 'file_5.fits', + 'file_6.fits', 'file_7.fits', 'file_8.fits', 'file_9.fits'] + ] + test10 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [9., 8., 7., 6., 5., 4., 3., 2., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + integration_list = [1] * len(start_times) + threshold = 11 + expected = [] + test11 = (files, start_times, end_times, integration_list, threshold, expected) + + deltat = [40., 39., 38., 37., 24., 23., 22., 21., 1., 0.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 6 # integrations + integration_list = [3, 3, 2, 2, 2, 1, 1, 1, 1, 1] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits'], + ['file_4.fits', 'file_5.fits', 'file_6.fits', 'file_7.fits'] + ] + test12 = (files, start_times, end_times, integration_list, threshold, expected) + + # In this case, the final 2 files are grouped together due to being taken close + # in time to one another. However, they do not contain enough integrations to + # reach the threshold. Since these are the final two files, we have no way of + # knowing if they are just the first two observations of a larger set that should + # be grouped. Therefore, the dark monitor ignores these final two files, under + # the assumption that they will be used the next time the monitor is run. + deltat = [50., 49., 48., 47., 34., 33., 32., 31., 20., 19.] + start_times = [now - dt for dt in deltat] + end_times = [s + 0.1 for s in start_times] + threshold = 6 # integrations + integration_list = [3, 3, 2, 2, 2, 1, 1, 1, 1, 1] + expected = [['file_0.fits', 'file_1.fits'], + ['file_2.fits', 'file_3.fits'], + ['file_4.fits', 'file_5.fits', 'file_6.fits', 'file_7.fits'] + ] + test13 = (files, start_times, end_times, integration_list, threshold, expected) + + return [test1, test2, test3, test4, test5, test6, test7, test8, test9, test10, test11, test12, test13] + + def test_find_hot_dead_pixels(): """Test hot and dead pixel searches""" monitor = dark_monitor.Dark() @@ -137,6 +277,16 @@ def test_shift_to_full_frame(): assert np.all(new_coords[1] == np.array([518, 515])) +@pytest.mark.parametrize("files,start_times,end_times,integration_list,threshold,expected", generate_data_for_file_splitting_test()) +def test_split_files_into_sub_lists(files, start_times, end_times, integration_list, threshold, expected): + """Test that file lists are appropriately split into subgroups for separate monitor runs""" + d = dark_monitor.Dark() + d.instrument = 'nircam' + d.split_files_into_sub_lists(files, start_times, end_times, integration_list, threshold) + + assert d.file_batches == expected + + @pytest.mark.skipif(not has_test_db(), reason='Modifies test database.') def test_add_bad_pix(): coord = ([1, 2, 3], [4, 5, 6]) diff --git a/jwql/utils/calculations.py b/jwql/utils/calculations.py index 34e866c2e..a2a44ac3c 100644 --- a/jwql/utils/calculations.py +++ b/jwql/utils/calculations.py @@ -17,6 +17,7 @@ """ import numpy as np +import warnings from astropy.modeling import fitting, models from astropy.stats import sigma_clip @@ -169,8 +170,9 @@ def mean_stdev(image, sigma_threshold=3): stdev_value : float Sigma-clipped standard deviation of image """ - - clipped, lower, upper = sigmaclip(image, low=sigma_threshold, high=sigma_threshold) + # Ignore the warning about NaNs being clipped. + warnings.filterwarnings('ignore', message='Input data contains invalid values (NaNs or infs), which were automatically clipped.*') + clipped = sigma_clip(image, sigma=sigma_threshold, masked=False) mean_value = np.mean(clipped) stdev_value = np.std(clipped) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index d6791f489..5a588f439 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -207,6 +207,18 @@ # Types of potential bad pixels identified by the dark current monitor DARK_MONITOR_BADPIX_TYPES = ["hot", "dead", "noisy"] +# Minimum amount of time, in days, between epochs of dark current observations. If the +# dark monitor sees this much time, or longer, between two dark current files, it assumes +# that the two files are part of separate epochs. This means the monitor will run separately +# on these files, rather than bundling them together into a batch, where they would have +# been combined into a mean dark rate +DARK_MONITOR_BETWEEN_EPOCH_THRESHOLD_TIME = {'nircam': 10., + 'niriss': 10., + 'miri': 0.00001, # Treat each MIRI exposure separately + 'nirspec': 10., + 'fgs': 10. + } + # Maximum number of potential new bad pixels to overplot on the dark monitor # mean dark image plot. Too many overplotted points starts to obscure the image # itself, and are most likely not really new bad pixels @@ -616,6 +628,10 @@ # Maximum number of records returned by MAST for a single query MAST_QUERY_LIMIT = 550000 +# Minimum number of groups per integration required to include data +# in the dark current monitor +MINIMUM_DARK_CURRENT_GROUPS = 10 + # Expected position sensor values for MIRI. Used by the EDB monitor # to filter out bad values. Tuple values are the expected value and # the standard deviation associated with the value @@ -653,6 +669,52 @@ }, } +# Names of all of the monitor database tables +MONITOR_TABLE_NAMES = [ + "fgs_bad_pixel_query_history", "fgs_bad_pixel_stats", + "miri_bad_pixel_query_history", "miri_bad_pixel_stats", + "nircam_bad_pixel_query_history", "nircam_bad_pixel_stats", + "niriss_bad_pixel_query_history", "niriss_bad_pixel_stats", + "nirspec_bad_pixel_query_history", "nirspec_bad_pixel_stats", + "nircam_bias_query_history", "nircam_bias_stats", + "niriss_bias_query_history", "niriss_bias_stats", + "nirspec_bias_query_history", "nirspec_bias_stats", + "nircam_claw_query_history", "nircam_claw_stats", + "monitor", + "central_storage", + "filesystem_characteristics", + "filesystem_general", + "filesystem_instrument", + "fgs_anomaly", + "miri_anomaly", + "nircam_anomaly", + "niriss_anomaly", + "nirspec_anomaly", + "fgs_cosmic_ray_query_history", "fgs_cosmic_ray_stats", + "miri_cosmic_ray_query_history", "miri_cosmic_ray_stats", + "nircam_cosmic_ray_query_history", "nircam_cosmic_ray_stats", + "niriss_cosmic_ray_query_history", "niriss_cosmic_ray_stats", + "nirspec_cosmic_ray_query_history", "nirspec_cosmic_ray_stats", + "fgs_dark_dark_current", "fgs_dark_pixel_stats", "fgs_dark_query_history", + "miri_dark_dark_current", "miri_dark_pixel_stats", "miri_dark_query_history", + "nircam_dark_dark_current", "nircam_dark_pixel_stats", "nircam_dark_query_history", + "niriss_dark_dark_current", "niriss_dark_pixel_stats", "niriss_dark_query_history", + "nirspec_dark_dark_current", "nirspec_dark_pixel_stats", "nirspec_dark_query_history", + "fgs_edb_blocks_stats", "fgs_edb_daily_stats", "fgs_edb_every_change_stats", "fgs_edb_time_interval_stats", "fgs_edb_time_stats", + "miri_edb_blocks_stats", "miri_edb_daily_stats", "miri_edb_every_change_stats", "miri_edb_time_interval_stats", "miri_edb_time_stats", + "nircam_edb_blocks_stats", "nircam_edb_daily_stats", "nircam_edb_every_change_stats", "nircam_edb_time_interval_stats", "nircam_edb_time_stats", + "niriss_edb_blocks_stats", "niriss_edb_daily_stats", "niriss_edb_every_change_stats", "niriss_edb_time_interval_stats", "niriss_edb_time_stats", + "nirspec_edb_blocks_stats", "nirspec_edb_daily_stats", "nirspec_edb_every_change_stats", "nirspec_edb_time_interval_stats", "nirspec_edb_time_stats", + "nirspec_grating_stats", + "fgs_readnoise_query_history", "fgs_readnoise_stats", + "miri_readnoise_query_history", "miri_readnoise_stats", + "nircam_readnoise_query_history", "nircam_readnoise_stats", + "niriss_readnoise_query_history", "niriss_readnoise_stats", + "nirspec_readnoise_query_history", "nirspec_readnoise_stats", + "miri_ta_query_history", "miri_ta_stats", + "nirspec_ta_query_history", "nirspec_ta_stats" +] + # Suffix for msa files MSA_SUFFIX = ["msa"] diff --git a/jwql/utils/instrument_properties.py b/jwql/utils/instrument_properties.py index 33d900b4d..88acb5465 100644 --- a/jwql/utils/instrument_properties.py +++ b/jwql/utils/instrument_properties.py @@ -129,6 +129,11 @@ def amplifier_info(filename, omit_reference_pixels=True): except KeyError: raise KeyError('DQ extension not found.') + # If the file contains multiple frames (e.g. rateints file) + # keep just the first + if len(data_quality.shape) == 3: + data_quality = data_quality[0, :, :] + # Reference pixels should be flagged in the DQ array with the # REFERENCE_PIXEL flag. Find the science pixels by looping for # pixels that don't have that bit set. diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py index 49218c6fa..536ac1ad4 100644 --- a/jwql/utils/monitor_utils.py +++ b/jwql/utils/monitor_utils.py @@ -19,9 +19,9 @@ import datetime import os from astroquery.mast import Mast, Observations +import numpy as np from django import setup - from jwql.database.database_interface import Monitor, engine from jwql.utils.constants import ASIC_TEMPLATES, JWST_DATAPRODUCTS, MAST_QUERY_LIMIT from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS @@ -154,6 +154,11 @@ def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None): if len(query['data']) > 0: query_results.extend(query['data']) + # Put the file entries in chronological order + expstarts = [e['expstart'] for e in query_results] + idx = np.argsort(expstarts) + query_results = list(np.array(query_results)[idx]) + return query_results diff --git a/jwql/website/apps/jwql/migrations/0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more.py b/jwql/website/apps/jwql/migrations/0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more.py new file mode 100644 index 000000000..b963ad7ad --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more.py @@ -0,0 +1,384 @@ +# Generated by Django 4.2.5 on 2024-02-23 16:50 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0016_anomalies_bright_object_not_a_short_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='NIRSpecReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'nirspec_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRSpecReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'nirspec_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRISSReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'niriss_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRISSReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'niriss_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'nircam_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'nircam_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamClawStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('filename', models.CharField(blank=True, null=True)), + ('proposal', models.CharField(blank=True, null=True)), + ('obs', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('filter', models.CharField(blank=True, null=True)), + ('pupil', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('expstart_mjd', models.FloatField(blank=True, null=True)), + ('effexptm', models.FloatField(blank=True, null=True)), + ('ra', models.FloatField(blank=True, null=True)), + ('dec', models.FloatField(blank=True, null=True)), + ('pa_v3', models.FloatField(blank=True, null=True)), + ('mean', models.FloatField(blank=True, null=True)), + ('median', models.FloatField(blank=True, null=True)), + ('stddev', models.FloatField(blank=True, null=True)), + ('frac_masked', models.FloatField(blank=True, null=True)), + ('skyflat_filename', models.CharField(blank=True, null=True)), + ], + options={ + 'db_table': 'nircam_claw_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamClawQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('instrument', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ], + options={ + 'db_table': 'nircam_claw_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='MIRIReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'miri_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='MIRIReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'miri_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='FGSReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'fgs_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='FGSReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'fgs_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0018_nircamclawstats_doy_nircamclawstats_total_bkg.py b/jwql/website/apps/jwql/migrations/0018_nircamclawstats_doy_nircamclawstats_total_bkg.py new file mode 100644 index 000000000..c5efd9125 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0018_nircamclawstats_doy_nircamclawstats_total_bkg.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.5 on 2024-02-23 16:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='nircamclawstats', + name='doy', + field=models.FloatField(blank=True, null=True), + ), + migrations.AddField( + model_name='nircamclawstats', + name='total_bkg', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/monitor_models/bad_pixel.py b/jwql/website/apps/jwql/monitor_models/bad_pixel.py index 151aa39ff..463331deb 100644 --- a/jwql/website/apps/jwql/monitor_models/bad_pixel.py +++ b/jwql/website/apps/jwql/monitor_models/bad_pixel.py @@ -48,7 +48,6 @@ class Meta: managed = True db_table = 'fgs_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSBadPixelStats(models.Model): @@ -67,7 +66,6 @@ class Meta: managed = True db_table = 'fgs_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIBadPixelQueryHistory(models.Model): @@ -88,7 +86,6 @@ class Meta: managed = True db_table = 'miri_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIBadPixelStats(models.Model): @@ -107,7 +104,6 @@ class Meta: managed = True db_table = 'miri_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamBadPixelQueryHistory(models.Model): @@ -128,7 +124,6 @@ class Meta: managed = True db_table = 'nircam_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamBadPixelStats(models.Model): @@ -147,7 +142,6 @@ class Meta: managed = True db_table = 'nircam_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBadPixelQueryHistory(models.Model): @@ -168,7 +162,6 @@ class Meta: managed = True db_table = 'niriss_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBadPixelStats(models.Model): @@ -187,7 +180,6 @@ class Meta: managed = True db_table = 'niriss_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBadPixelQueryHistory(models.Model): @@ -208,7 +200,6 @@ class Meta: managed = True db_table = 'nirspec_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBadPixelStats(models.Model): @@ -227,4 +218,3 @@ class Meta: managed = True db_table = 'nirspec_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/bias.py b/jwql/website/apps/jwql/monitor_models/bias.py index 306ef3b29..c8245b9fe 100644 --- a/jwql/website/apps/jwql/monitor_models/bias.py +++ b/jwql/website/apps/jwql/monitor_models/bias.py @@ -44,7 +44,6 @@ class Meta: managed = True db_table = 'nircam_bias_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamBiasStats(models.Model): @@ -74,7 +73,6 @@ class Meta: managed = True db_table = 'nircam_bias_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBiasQueryHistory(models.Model): @@ -91,7 +89,6 @@ class Meta: managed = True db_table = 'niriss_bias_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBiasStats(models.Model): @@ -121,7 +118,6 @@ class Meta: managed = True db_table = 'niriss_bias_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBiasQueryHistory(models.Model): @@ -138,7 +134,6 @@ class Meta: managed = True db_table = 'nirspec_bias_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBiasStats(models.Model): @@ -168,4 +163,3 @@ class Meta: managed = True db_table = 'nirspec_bias_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/claw.py b/jwql/website/apps/jwql/monitor_models/claw.py index 4e92ea3d4..a1c6c93e3 100644 --- a/jwql/website/apps/jwql/monitor_models/claw.py +++ b/jwql/website/apps/jwql/monitor_models/claw.py @@ -40,7 +40,6 @@ class Meta: managed = True db_table = 'nircam_claw_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamClawStats(models.Model): @@ -62,9 +61,10 @@ class NIRCamClawStats(models.Model): stddev = models.FloatField(blank=True, null=True) frac_masked = models.FloatField(blank=True, null=True) skyflat_filename = models.CharField(blank=True, null=True) + doy = models.FloatField(blank=True, null=True) + total_bkg = models.FloatField(blank=True, null=True) class Meta: managed = True db_table = 'nircam_claw_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 9809e843c..d0bed2afc 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -8,154 +8,153 @@ a data table. This module defines models that are used to store data related to the JWQL monitors. -Usage ------ + Usage + ----- -JWQL uses the django database models for creating tables, updating table fields, adding -new data to tables, and retrieving data from tables. For instrument monitors, in particular, -there are a number of issues that may be relevant. + JWQL uses the django database models for creating tables, updating table fields, adding + new data to tables, and retrieving data from tables. For instrument monitors, in particular, + there are a number of issues that may be relevant. -In general, django model documentation can be found -`on the django website `_. -Unfortunately, finding a particular bit of documentation in django can be a challenge, so -a few quick-reference notes are provided below. + In general, django model documentation can be found + `on the django website `_. + Unfortunately, finding a particular bit of documentation in django can be a challenge, so + a few quick-reference notes are provided below. -Retrieving Data ---------------- + Retrieving Data + --------------- -Django retrieves data directly from its model tables. So, for example, if you want to -select data from the `MIRIMyMonitorStats` table, you must first import the relevant -object: + Django retrieves data directly from its model tables. So, for example, if you want to + select data from the `MIRIMyMonitorStats` table, you must first import the relevant + object: -.. code-block:: python + .. code-block:: python - from jwql.website.apps.jwql.monitor_models.my_monitor import MIRIMyMonitorStats + from jwql.website.apps.jwql.monitor_models.my_monitor import MIRIMyMonitorStats -Then, you would access the database contents via the `objects` member of the class. For -example, to search the `MIRIMyMonitorStats` table for all entries matching a given -aperture, and to sort them with the most recent date at the top, you might do a query like -the following: + Then, you would access the database contents via the `objects` member of the class. For + example, to search the `MIRIMyMonitorStats` table for all entries matching a given + aperture, and to sort them with the most recent date at the top, you might do a query like + the following: -.. code-block:: python + .. code-block:: python - aperture = "my_miri_aperture" + aperture = "my_miri_aperture" - records = MIRIMyMonitorStats.objects.filter(aperture__iexact=aperture).order_by("-mjd_end").all() + records = MIRIMyMonitorStats.objects.filter(aperture__iexact=aperture).order_by("-mjd_end").all() -In the above code, + In the above code, -* The `filter()` function selects matching records from the full table. You can use - multiple filter statements, or a single filter function with multiple filters. `filter()` - statements are always combined with an implicit AND. -* If you have a long filter statement and want to separate it from the query statement, - you can create a dictionary and add it in with the `**` prepended. The dictionary - equivalent to the above would be `{'aperture__iexact': aperture}` -* The text before the double underscore is a field name, and the text afterwards describes - the type of comparison. `iexact` indicates "case-insensitive exact match". You can also - use a variety of standard SQL comparisons (`like`, `startswith`, `gte`, etc.) -* If you want to get only records that *don't* match a pattern, then you can use the - `exclude()` function, which otherwise operates exactly the same as `filter()`. -* In the `order_by()` function, the `-` at the start is used to reverse the sort order, - and the `mjd_end` is the name of the field to be sorted by. -* The `all()` statement indicates that you want all the values returned. `get()` returns - a single value and can be iterated on, `first()` returns only the first value, etc. + * The `filter()` function selects matching records from the full table. You can use + multiple filter statements, or a single filter function with multiple filters. `filter()` + statements are always combined with an implicit AND. + * If you have a long filter statement and want to separate it from the query statement, + you can create a dictionary and add it in with the `**` prepended. The dictionary + equivalent to the above would be `{'aperture__iexact': aperture}` + * The text before the double underscore is a field name, and the text afterwards describes + the type of comparison. `iexact` indicates "case-insensitive exact match". You can also + use a variety of standard SQL comparisons (`like`, `startswith`, `gte`, etc.) + * If you want to get only records that *don't* match a pattern, then you can use the + `exclude()` function, which otherwise operates exactly the same as `filter()`. + * In the `order_by()` function, the `-` at the start is used to reverse the sort order, + and the `mjd_end` is the name of the field to be sorted by. + * The `all()` statement indicates that you want all the values returned. `get()` returns + a single value and can be iterated on, `first()` returns only the first value, etc. -As an example of multiple filters, the code below: + As an example of multiple filters, the code below: -.. code-block:: python + .. code-block:: python - records = MIRIMyMonitorStats.objects.filter(aperture__iexact=ap, mjd_end__gte=60000) + records = MIRIMyMonitorStats.objects.filter(aperture__iexact=ap, mjd_end__gte=60000) - filters = { - "aperture__iexact": ap, - "mjd_end__gte": 60000 - } - records = MIRIMyMonitorStats.objects.filter(**filters) + filters = { + "aperture__iexact": ap, + "mjd_end__gte": 60000 + } + records = MIRIMyMonitorStats.objects.filter(**filters) -show two different ways of combining a search for a particular aperture *and* only data -taken more recently than MJD=60000. + show two different ways of combining a search for a particular aperture *and* only data + taken more recently than MJD=60000. -Note that django executes queries lazily, meaning that it will only actually *do* the -query when it needs the results. The above statement, for example, will not actually -run the query. Instead, it will be run when you operate on it, such as + Note that django executes queries lazily, meaning that it will only actually *do* the + query when it needs the results. The above statement, for example, will not actually + run the query. Instead, it will be run when you operate on it, such as -* Getting the length of the result with e.g. `len(records)` -* Printing out any of the results -* Asking for the value of one of the fields (e.g. `records[3].aperture`) + * Getting the length of the result with e.g. `len(records)` + * Printing out any of the results + * Asking for the value of one of the fields (e.g. `records[3].aperture`) -Retrieving Specific Columns -=========================== + Retrieving Specific Columns + =========================== -Django offers two ways of doing this. The first one is to use the `only()` function, which -immediately loads only the relevant columns. For example, + Django offers two ways of doing this. The first one is to use the `only()` function, which + immediately loads only the relevant columns. For example, -.. code-block:: python + .. code-block:: python - records = MIRIMyMonitorStats.objects.only("aperture", "mjd_start", "relevant_item") + records = MIRIMyMonitorStats.objects.only("aperture", "mjd_start", "relevant_item") -will immediately load only the three columns selected (although the rest will be retrieved -from the database, and can still be accessed, for no immediately understandable reason). -The other method is the `defer()` method, which loads every column *except* the ones listed. + will immediately load only the three columns selected (although the rest will be retrieved + from the database, and can still be accessed, for no immediately understandable reason). + The other method is the `defer()` method, which loads every column *except* the ones listed. -Q Objects -========= + Q Objects + ========= -In order to make more complex queries, Django supplies "Q Objects", which are essentially -encapsulated filters which can be combined using logical operators. For more on this, see -`the django Q object documentation `_. + In order to make more complex queries, Django supplies "Q Objects", which are essentially + encapsulated filters which can be combined using logical operators. For more on this, see + `the django Q object documentation `_. -Storing Data ------------- + Storing Data + ------------ -Django also uses the model tables (and objects) directly for storing new data. For example, -if you have a monitor table defined as below: + Django also uses the model tables (and objects) directly for storing new data. For example, + if you have a monitor table defined as below: -.. code-block:: python + .. code-block:: python - from django.db import models - from django.contrib.postgres.fields import ArrayField + from django.db import models + from django.contrib.postgres.fields import ArrayField - class NIRISSMyMonitorStats(models.Model): - aperture = models.CharField(blank=True, null=True) - mean = models.FloatField(blank=True, null=True) - median = models.FloatField(blank=True, null=True) - stddev = models.FloatField(blank=True, null=True) - counts = ArrayField(models.FloatField()) - entry_date = models.DateTimeField(blank=True, null=True) + class NIRISSMyMonitorStats(models.Model): + aperture = models.CharField(blank=True, null=True) + mean = models.FloatField(blank=True, null=True) + median = models.FloatField(blank=True, null=True) + stddev = models.FloatField(blank=True, null=True) + counts = ArrayField(models.FloatField()) + entry_date = models.DateTimeField(blank=True, null=True) - class Meta: - managed = True - db_table = 'niriss_my_monitor_stats' - unique_together = (('id', 'entry_date'),) - app_label = 'monitors' + class Meta: + managed = True + db_table = 'niriss_my_monitor_stats' + unique_together = (('id', 'entry_date'),) -then you would create a new entry as follows: + then you would create a new entry as follows: -.. code-block:: python + .. code-block:: python - values = { - "aperture": "my_aperture", - "mean": float(mean), - "median": float(median), - "stddev": float(stddev), - "counts": list(counts.astype(float)), - "entry_date": datetime.datetime.now() - } + values = { + "aperture": "my_aperture", + "mean": float(mean), + "median": float(median), + "stddev": float(stddev), + "counts": list(counts.astype(float)), + "entry_date": datetime.datetime.now() + } - entry = NIRISSMyMonitorStats(**values) - entry.save() - -There are (as usual) a few things to note above: + entry = NIRISSMyMonitorStats(**values) + entry.save() -* Django doesn't have a built-in array data type, so you need to import it from the - database-compatibility layers. The ArrayField takes, as a required argument, the type - of data that makes up the array. -* In the Meta sub-class of the monitor class, the `app_label = 'monitors'` statement is - required so that django knows that the model should be stored in the monitors table. -* The `float()` casts are required because the database interface doesn't understand - numpy data types. -* The `list()` cast is required because the database interface doesn't understand the - numpy `ndarray` data type + There are (as usual) a few things to note above: + + * Django doesn't have a built-in array data type, so you need to import it from the + database-compatibility layers. The ArrayField takes, as a required argument, the type + of data that makes up the array. + * In the Meta sub-class of the monitor class, the `db_table_comment = 'monitors'` statement is + required so that django knows that the model should be stored in the monitors table. + * The `float()` casts are required because the database interface doesn't understand + numpy data types. + * The `list()` cast is required because the database interface doesn't understand the + numpy `ndarray` data type Authors ------- @@ -179,50 +178,6 @@ class Meta: from django.contrib.postgres.fields import ArrayField -class MonitorRouter: - """ - A router to control all database operations on models in the - JWQLDB (monitors) database. - """ - - route_app_labels = {"monitors"} - - def db_for_read(self, model, **hints): - """ - Attempts to read monitor models go to monitors db. - """ - if model._meta.app_label in self.route_app_labels: - return "monitors" - return None - - def db_for_write(self, model, **hints): - """ - Attempts to write monitor models go to monitors db. - """ - if model._meta.app_label in self.route_app_labels: - return "monitors" - return None - - def allow_relation(self, obj1, obj2, **hints): - """ - Allow relations between tables in the monitors DB. - """ - if ( - obj1._meta.app_label in self.route_app_labels - or obj2._meta.app_label in self.route_app_labels - ): - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): - """ - Make sure the monitors apps only appear in the 'monitors' database. - """ - if app_label in self.route_app_labels: - return db == "monitors" - return None - - class Monitor(models.Model): monitor_name = models.CharField() start_time = models.DateTimeField() @@ -233,7 +188,6 @@ class Monitor(models.Model): class Meta: managed = True db_table = 'monitor' - app_label = 'monitors' class CentralStorage(models.Model): @@ -246,7 +200,6 @@ class CentralStorage(models.Model): class Meta: managed = True db_table = 'central_storage' - app_label = 'monitors' class FilesystemCharacteristics(models.Model): @@ -258,7 +211,6 @@ class FilesystemCharacteristics(models.Model): class Meta: managed = True db_table = 'filesystem_characteristics' - app_label = 'monitors' class FilesystemGeneral(models.Model): @@ -273,7 +225,6 @@ class FilesystemGeneral(models.Model): class Meta: managed = True db_table = 'filesystem_general' - app_label = 'monitors' class FilesystemInstrument(models.Model): @@ -287,7 +238,6 @@ class Meta: managed = True db_table = 'filesystem_instrument' unique_together = (('date', 'instrument', 'filetype'),) - app_label = 'monitors' class FgsAnomaly(models.Model): @@ -308,7 +258,6 @@ class FgsAnomaly(models.Model): class Meta: managed = True db_table = 'fgs_anomaly' - app_label = 'monitors' class MiriAnomaly(models.Model): @@ -334,7 +283,6 @@ class MiriAnomaly(models.Model): class Meta: managed = True db_table = 'miri_anomaly' - app_label = 'monitors' class NircamAnomaly(models.Model): @@ -360,7 +308,6 @@ class NircamAnomaly(models.Model): class Meta: managed = True db_table = 'nircam_anomaly' - app_label = 'monitors' class NirissAnomaly(models.Model): @@ -383,7 +330,6 @@ class NirissAnomaly(models.Model): class Meta: managed = True db_table = 'niriss_anomaly' - app_label = 'monitors' class NirspecAnomaly(models.Model): @@ -406,4 +352,3 @@ class NirspecAnomaly(models.Model): class Meta: managed = True db_table = 'nirspec_anomaly' - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/cosmic_ray.py b/jwql/website/apps/jwql/monitor_models/cosmic_ray.py index f9084aa06..cdff2eb22 100644 --- a/jwql/website/apps/jwql/monitor_models/cosmic_ray.py +++ b/jwql/website/apps/jwql/monitor_models/cosmic_ray.py @@ -43,7 +43,6 @@ class Meta: managed = True db_table = 'fgs_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSCosmicRayStats(models.Model): @@ -61,7 +60,6 @@ class Meta: managed = True db_table = 'fgs_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRICosmicRayQueryHistory(models.Model): @@ -77,7 +75,6 @@ class Meta: managed = True db_table = 'miri_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRICosmicRayStats(models.Model): @@ -95,7 +92,6 @@ class Meta: managed = True db_table = 'miri_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamCosmicRayQueryHistory(models.Model): @@ -111,7 +107,6 @@ class Meta: managed = True db_table = 'nircam_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamCosmicRayStats(models.Model): @@ -129,7 +124,6 @@ class Meta: managed = True db_table = 'nircam_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSCosmicRayQueryHistory(models.Model): @@ -145,7 +139,6 @@ class Meta: managed = True db_table = 'niriss_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSCosmicRayStats(models.Model): @@ -163,7 +156,6 @@ class Meta: managed = True db_table = 'niriss_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecCosmicRayQueryHistory(models.Model): @@ -179,7 +171,6 @@ class Meta: managed = True db_table = 'nirspec_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecCosmicRayStats(models.Model): @@ -197,4 +188,3 @@ class Meta: managed = True db_table = 'nirspec_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/dark_current.py b/jwql/website/apps/jwql/monitor_models/dark_current.py index 29d9a9523..41ae1ccac 100644 --- a/jwql/website/apps/jwql/monitor_models/dark_current.py +++ b/jwql/website/apps/jwql/monitor_models/dark_current.py @@ -60,7 +60,6 @@ class Meta: managed = True db_table = 'fgs_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSDarkPixelStats(models.Model): @@ -80,7 +79,6 @@ class Meta: managed = True db_table = 'fgs_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSDarkQueryHistory(models.Model): @@ -97,7 +95,6 @@ class Meta: managed = True db_table = 'fgs_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIDarkDarkCurrent(models.Model): @@ -130,7 +127,6 @@ class Meta: managed = True db_table = 'miri_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIDarkPixelStats(models.Model): @@ -150,7 +146,6 @@ class Meta: managed = True db_table = 'miri_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIDarkQueryHistory(models.Model): @@ -167,7 +162,6 @@ class Meta: managed = True db_table = 'miri_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamDarkDarkCurrent(models.Model): @@ -200,7 +194,6 @@ class Meta: managed = True db_table = 'nircam_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamDarkPixelStats(models.Model): @@ -220,7 +213,6 @@ class Meta: managed = True db_table = 'nircam_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamDarkQueryHistory(models.Model): @@ -237,7 +229,6 @@ class Meta: managed = True db_table = 'nircam_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSDarkDarkCurrent(models.Model): @@ -270,7 +261,6 @@ class Meta: managed = True db_table = 'niriss_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSDarkPixelStats(models.Model): @@ -290,7 +280,6 @@ class Meta: managed = True db_table = 'niriss_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSDarkQueryHistory(models.Model): @@ -307,7 +296,6 @@ class Meta: managed = True db_table = 'niriss_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecDarkDarkCurrent(models.Model): @@ -340,7 +328,6 @@ class Meta: managed = True db_table = 'nirspec_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecDarkPixelStats(models.Model): @@ -360,7 +347,6 @@ class Meta: managed = True db_table = 'nirspec_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecDarkQueryHistory(models.Model): @@ -377,4 +363,3 @@ class Meta: managed = True db_table = 'nirspec_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/edb.py b/jwql/website/apps/jwql/monitor_models/edb.py index cc9d8ec82..2cad15418 100644 --- a/jwql/website/apps/jwql/monitor_models/edb.py +++ b/jwql/website/apps/jwql/monitor_models/edb.py @@ -45,7 +45,6 @@ class Meta: managed = True db_table = 'fgs_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbDailyStats(models.Model): @@ -63,7 +62,6 @@ class Meta: managed = True db_table = 'fgs_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbEveryChangeStats(models.Model): @@ -81,7 +79,6 @@ class Meta: managed = True db_table = 'fgs_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbTimeIntervalStats(models.Model): @@ -99,7 +96,6 @@ class Meta: managed = True db_table = 'fgs_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbTimeStats(models.Model): @@ -114,7 +110,6 @@ class Meta: managed = True db_table = 'fgs_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbBlocksStats(models.Model): @@ -132,7 +127,6 @@ class Meta: managed = True db_table = 'miri_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbDailyStats(models.Model): @@ -150,7 +144,6 @@ class Meta: managed = True db_table = 'miri_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbEveryChangeStats(models.Model): @@ -168,7 +161,6 @@ class Meta: managed = True db_table = 'miri_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbTimeIntervalStats(models.Model): @@ -186,7 +178,6 @@ class Meta: managed = True db_table = 'miri_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbTimeStats(models.Model): @@ -201,7 +192,6 @@ class Meta: managed = True db_table = 'miri_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbBlocksStats(models.Model): @@ -219,7 +209,6 @@ class Meta: managed = True db_table = 'nircam_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbDailyStats(models.Model): @@ -237,7 +226,6 @@ class Meta: managed = True db_table = 'nircam_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbEveryChangeStats(models.Model): @@ -255,7 +243,6 @@ class Meta: managed = True db_table = 'nircam_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbTimeIntervalStats(models.Model): @@ -273,7 +260,6 @@ class Meta: managed = True db_table = 'nircam_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbTimeStats(models.Model): @@ -288,7 +274,6 @@ class Meta: managed = True db_table = 'nircam_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbBlocksStats(models.Model): @@ -306,7 +291,6 @@ class Meta: managed = True db_table = 'niriss_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbDailyStats(models.Model): @@ -324,7 +308,6 @@ class Meta: managed = True db_table = 'niriss_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbEveryChangeStats(models.Model): @@ -342,7 +325,6 @@ class Meta: managed = True db_table = 'niriss_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbTimeIntervalStats(models.Model): @@ -360,7 +342,6 @@ class Meta: managed = True db_table = 'niriss_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbTimeStats(models.Model): @@ -375,7 +356,6 @@ class Meta: managed = True db_table = 'niriss_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbBlocksStats(models.Model): @@ -393,7 +373,6 @@ class Meta: managed = True db_table = 'nirspec_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbDailyStats(models.Model): @@ -411,7 +390,6 @@ class Meta: managed = True db_table = 'nirspec_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbEveryChangeStats(models.Model): @@ -429,7 +407,6 @@ class Meta: managed = True db_table = 'nirspec_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbTimeIntervalStats(models.Model): @@ -447,7 +424,6 @@ class Meta: managed = True db_table = 'nirspec_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbTimeStats(models.Model): @@ -462,4 +438,3 @@ class Meta: managed = True db_table = 'nirspec_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/grating.py b/jwql/website/apps/jwql/monitor_models/grating.py index 9e4268879..1c2029049 100644 --- a/jwql/website/apps/jwql/monitor_models/grating.py +++ b/jwql/website/apps/jwql/monitor_models/grating.py @@ -40,7 +40,6 @@ class Meta: managed = True db_table = 'nirspec_grating_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecGratingStats(models.Model): @@ -70,4 +69,3 @@ class Meta: managed = True db_table = 'nirspec_grating_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/readnoise.py b/jwql/website/apps/jwql/monitor_models/readnoise.py index 5a500c709..5e616aa71 100644 --- a/jwql/website/apps/jwql/monitor_models/readnoise.py +++ b/jwql/website/apps/jwql/monitor_models/readnoise.py @@ -44,7 +44,6 @@ class Meta: managed = True db_table = 'fgs_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSReadnoiseStats(models.Model): @@ -88,7 +87,6 @@ class Meta: managed = True db_table = 'fgs_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIReadnoiseQueryHistory(models.Model): @@ -105,7 +103,6 @@ class Meta: managed = True db_table = 'miri_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIReadnoiseStats(models.Model): @@ -149,7 +146,6 @@ class Meta: managed = True db_table = 'miri_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamReadnoiseQueryHistory(models.Model): @@ -166,7 +162,6 @@ class Meta: managed = True db_table = 'nircam_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamReadnoiseStats(models.Model): @@ -210,7 +205,6 @@ class Meta: managed = True db_table = 'nircam_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSReadnoiseQueryHistory(models.Model): @@ -227,7 +221,6 @@ class Meta: managed = True db_table = 'niriss_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSReadnoiseStats(models.Model): @@ -271,7 +264,6 @@ class Meta: managed = True db_table = 'niriss_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecReadnoiseQueryHistory(models.Model): @@ -288,7 +280,6 @@ class Meta: managed = True db_table = 'nirspec_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecReadnoiseStats(models.Model): @@ -332,4 +323,3 @@ class Meta: managed = True db_table = 'nirspec_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/ta.py b/jwql/website/apps/jwql/monitor_models/ta.py index e48433dee..93a8b269b 100644 --- a/jwql/website/apps/jwql/monitor_models/ta.py +++ b/jwql/website/apps/jwql/monitor_models/ta.py @@ -44,7 +44,6 @@ class Meta: managed = True db_table = 'miri_ta_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRITaStats(models.Model): @@ -63,7 +62,6 @@ class Meta: managed = True db_table = 'miri_ta_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecTaQueryHistory(models.Model): @@ -80,7 +78,6 @@ class Meta: managed = True db_table = 'nirspec_ta_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecTaStats(models.Model): @@ -122,4 +119,3 @@ class Meta: managed = True db_table = 'nirspec_ta_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py index 1d9788314..1a4e7a670 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py @@ -60,7 +60,7 @@ class DarkHistPlot(): plot : bokeh.figure Figure containing the histogram plot """ - def __init__(self, aperture, data): + def __init__(self, aperture, data, obsdate): """Create the plot Parameters @@ -74,6 +74,7 @@ def __init__(self, aperture, data): """ self.data = data self.aperture = aperture + self.obsdate = obsdate self.create_plot() def calc_bin_edges(self, centers): @@ -95,6 +96,9 @@ def calc_bin_edges(self, centers): def create_plot(self): """Place the data in a CoumnDataSource and create the plot """ + title_str = f'{self.aperture}: Dark Rate Histogram. {self.obsdate.strftime("%d %b %Y")}' + x_label = 'Dark Rate (DN/sec)' + y_label = 'Number of Pixels' if len(self.data) > 0: # Specify which key ("amplifier") to show. If there is data for amp='5', # show that, as it will be the data for the entire detector. If not then @@ -109,10 +113,6 @@ def create_plot(self): else: use_amp = '1' - title_str = f'{self.aperture}: Dark Rate Histogram' - x_label = 'Dark Rate (DN/sec)' - y_label = 'Number of Pixels' - # If there are histogram data for multiple amps, then we can plot each histogram. if len(self.data) > 1: # Looks like the histogram data for the individual amps is not being saved @@ -256,7 +256,7 @@ def create_plot(self): class DarkMonitorData(): - """Retrive dark monitor data from the database tables + """Retrieve dark monitor data from the database tables Attributes ---------- @@ -465,7 +465,7 @@ def __init__(self, instrument): # Retrieve data from database. Since the mean dark image plots are # produced by the dark monitor itself, all we need for that is the - # name of the file. then we need the histogram and trending data. All + # name of the file. Then we need the histogram and trending data. All # of this is in the dark monitor stats table. No need to query the # dark monitor pixel table. self.db.retrieve_data(self.aperture, get_pixtable_for_detector=False) @@ -482,7 +482,7 @@ def __init__(self, instrument): self.get_trending_data() # Now that we have all the data, create the acutal plots - self.hist_plots[aperture] = DarkHistPlot(self.aperture, self.hist_data).plot + self.hist_plots[aperture] = DarkHistPlot(self.aperture, self.hist_data, self.hist_date).plot self.trending_plots[aperture] = DarkTrendPlot(self.aperture, self.mean_dark, self.stdev_dark, self.obstime).plot def ensure_all_full_frame_apertures(self): @@ -541,8 +541,7 @@ def get_latest_histogram_data(self): self.hist_data = {} if len(self._entry_dates) > 0: # Find the index of the most recent entry - # self._aperture_entries = np.where((self._apertures == aperture))[0] - latest_date = np.max(self._entry_dates) # [self._aperture_entries]) + latest_date = np.max(self._entry_dates) # Get indexes of entries for all amps that were added in the # most recent run of the monitor for the aperture. All entries @@ -552,12 +551,15 @@ def get_latest_histogram_data(self): most_recent_idx = np.where(self._entry_dates > (latest_date - delta_time))[0] # Store the histogram data in a dictionary where the keys are the - # amplifier values (note that these are strings e.g. '1''), and the + # amplifier values (note that these are strings e.g. '1'), and the # values are tuples of (x, y) lists for idx in most_recent_idx: self.hist_data[self.db.stats_data[idx].amplifier] = (self.db.stats_data[idx].hist_dark_values, self.db.stats_data[idx].hist_amplitudes) + # Keep track of the observation date of the most recent entry + self.hist_date = self.db.stats_data[most_recent_idx[0]].obs_mid_time + def get_trending_data(self): """Organize data for the trending plot. Here we need all the data for the aperture. Keep amplifier-specific data separated. @@ -579,10 +581,10 @@ def stats_data_to_lists(self): """Create arrays from some of the stats database columns that are used by multiple plot types """ - # apertures = np.array([e.aperture for e in self.db.stats_data]) self._amplifiers = np.array([e.amplifier for e in self.db.stats_data]) self._entry_dates = np.array([e.entry_date for e in self.db.stats_data]) self._mean = np.array([e.mean for e in self.db.stats_data]) + self._readpatt = np.array([e.readpattern for e in self.db.stats_data]) self._stdev = np.array([e.stdev for e in self.db.stats_data]) self._obs_mid_time = np.array([e.obs_mid_time for e in self.db.stats_data]) self._stats_mean_dark_image_files = np.array([e.mean_dark_image_file for e in self.db.stats_data]) @@ -682,7 +684,8 @@ def create_plot(self): error_upper=error_upper, time=self.obstime[use_amp] ) - ) + ) + self.plot = figure(title=f'{self.aperture}: Mean +/- 1-sigma Dark Rate', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 79ac6f127..fa6f48204 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -157,10 +157,9 @@ def claw_monitor(request): template = "claw_monitor.html" # Get all recent claw stack images from the last 10 days - query = NIRCamClawStats.objects # .filter(expstart_mjd__gte=(Time.now().mjd - 10)) - query = query.order_by('-expstart_mjd').all().values('expstart_mjd', 'skyflat_filename') - df = pd.DataFrame.from_records(query) - recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10])) + query = NIRCamClawStats.objects.filter(expstart_mjd__gte=(Time.now().mjd - 10)) + query = query.order_by('-expstart_mjd').all().values('skyflat_filename') + recent_files = list(pd.unique(pd.DataFrame.from_records(query)['skyflat_filename'])) output_dir_claws = static(os.path.join("outputs", "claw_monitor", "claw_stacks")) claw_stacks = [os.path.join(output_dir_claws, filename) for filename in recent_files] diff --git a/jwql/website/apps/jwql/router.py b/jwql/website/apps/jwql/router.py new file mode 100644 index 000000000..dbfd9f3a2 --- /dev/null +++ b/jwql/website/apps/jwql/router.py @@ -0,0 +1,69 @@ +"""Defines the query routing for the monitor database tables. + +In Django, database queries are assumed to go to the default database unless either the +`using` field/keyword is defined or a routing table sends it to a different database. In +this case, all monitor tables should be routed to the monitors database, and the router +should otherwise express no opinion (by returning None). + +Authors +------- + - Brian York + +Use +--- + This module is not intended to be used outside of Django asking about it. + +References +---------- + For more information please see: + ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` +""" +from jwql.utils.constants import MONITOR_TABLE_NAMES + + +class MonitorRouter: + """ + A router to control all database operations on models in the + JWQLDB (monitors) database. + """ + + def db_for_read(self, model, **hints): + """ + Attempts to read monitor models go to monitors db. + """ + if model._meta.db_table in MONITOR_TABLE_NAMES: + return "monitors" + return None + + def db_for_write(self, model, **hints): + """ + Attempts to write monitor models go to monitors db. + """ + if model._meta.db_table in MONITOR_TABLE_NAMES: + return "monitors" + return None + + def allow_relation(self, obj1, obj2, **hints): + """ + Allow relations between tables in the monitors DB. + """ + if ( + obj1._meta.db_table in MONITOR_TABLE_NAMES + and obj2._meta.db_table in MONITOR_TABLE_NAMES + ): + return True + return None + + def allow_migrate(self, db, app_label, model_name=None, **hints): + """ + Make sure the monitors apps only appear in the 'monitors' database. + """ + model_names = [name.replace("_", "") for name in MONITOR_TABLE_NAMES] + if app_label == 'jwql': + if model_name in model_names: + if db == "monitors": + return True + return False + elif db == "monitors": + return False + return None diff --git a/jwql/website/jwql_proj/settings.py b/jwql/website/jwql_proj/settings.py index c0e72240f..a8adf9fa3 100644 --- a/jwql/website/jwql_proj/settings.py +++ b/jwql/website/jwql_proj/settings.py @@ -111,7 +111,7 @@ 'default': get_config()['django_databases']['default'], 'monitors': get_config()['django_databases']['monitors'] } -DATABASE_ROUTERS = ["jwql.website.apps.jwql.monitor_models.common.MonitorRouter"] +DATABASE_ROUTERS = ["jwql.website.apps.jwql.router.MonitorRouter"] # Password validation # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators