diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py index b5388309d..b73d916c0 100644 --- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py +++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py @@ -138,7 +138,7 @@ def make_background_plots(self, plot_type='bkg'): """ columns = ['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd', 'entry_date', 'mean', 'median', - 'stddev', 'frac_masked'] # , 'total_bkg'] + 'stddev', 'frac_masked', 'total_bkg'] # Get all of the background data. background_data = NIRCamClawStats.objects.all().values(*columns) @@ -192,8 +192,7 @@ def make_background_plots(self, plot_type='bkg'): df = df[df['stddev'] != 0] # older data has no accurate stddev measures plot_data = df['stddev'].values if plot_type == 'model': - total_bkg = [1. for x in df['median'].values] - plot_data = df['median'].values # / df['total_bkg'].values + plot_data = df['median'].values / df['total_bkg'].values plot_expstarts = df['expstart_mjd'].values # Plot the background data over time @@ -331,8 +330,8 @@ def process(self): 'stddev': float(stddev), 'frac_masked': len(segmap_orig[(segmap_orig != 0) | (dq & 1 != 0)]) / (segmap_orig.shape[0] * segmap_orig.shape[1]), 'skyflat_filename': os.path.basename(self.outfile), - # 'doy': float(doy), - # 'total_bkg': float(total_bkg), + 'doy': float(doy), + 'total_bkg': float(total_bkg), 'entry_date': datetime.datetime.now() } entry = self.stats_table(**claw_db_entry) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index a797fe6a5..5a588f439 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -669,6 +669,52 @@ }, } +# Names of all of the monitor database tables +MONITOR_TABLE_NAMES = [ + "fgs_bad_pixel_query_history", "fgs_bad_pixel_stats", + "miri_bad_pixel_query_history", "miri_bad_pixel_stats", + "nircam_bad_pixel_query_history", "nircam_bad_pixel_stats", + "niriss_bad_pixel_query_history", "niriss_bad_pixel_stats", + "nirspec_bad_pixel_query_history", "nirspec_bad_pixel_stats", + "nircam_bias_query_history", "nircam_bias_stats", + "niriss_bias_query_history", "niriss_bias_stats", + "nirspec_bias_query_history", "nirspec_bias_stats", + "nircam_claw_query_history", "nircam_claw_stats", + "monitor", + "central_storage", + "filesystem_characteristics", + "filesystem_general", + "filesystem_instrument", + "fgs_anomaly", + "miri_anomaly", + "nircam_anomaly", + "niriss_anomaly", + "nirspec_anomaly", + "fgs_cosmic_ray_query_history", "fgs_cosmic_ray_stats", + "miri_cosmic_ray_query_history", "miri_cosmic_ray_stats", + "nircam_cosmic_ray_query_history", "nircam_cosmic_ray_stats", + "niriss_cosmic_ray_query_history", "niriss_cosmic_ray_stats", + "nirspec_cosmic_ray_query_history", "nirspec_cosmic_ray_stats", + "fgs_dark_dark_current", "fgs_dark_pixel_stats", "fgs_dark_query_history", + "miri_dark_dark_current", "miri_dark_pixel_stats", "miri_dark_query_history", + "nircam_dark_dark_current", "nircam_dark_pixel_stats", "nircam_dark_query_history", + "niriss_dark_dark_current", "niriss_dark_pixel_stats", "niriss_dark_query_history", + "nirspec_dark_dark_current", "nirspec_dark_pixel_stats", "nirspec_dark_query_history", + "fgs_edb_blocks_stats", "fgs_edb_daily_stats", "fgs_edb_every_change_stats", "fgs_edb_time_interval_stats", "fgs_edb_time_stats", + "miri_edb_blocks_stats", "miri_edb_daily_stats", "miri_edb_every_change_stats", "miri_edb_time_interval_stats", "miri_edb_time_stats", + "nircam_edb_blocks_stats", "nircam_edb_daily_stats", "nircam_edb_every_change_stats", "nircam_edb_time_interval_stats", "nircam_edb_time_stats", + "niriss_edb_blocks_stats", "niriss_edb_daily_stats", "niriss_edb_every_change_stats", "niriss_edb_time_interval_stats", "niriss_edb_time_stats", + "nirspec_edb_blocks_stats", "nirspec_edb_daily_stats", "nirspec_edb_every_change_stats", "nirspec_edb_time_interval_stats", "nirspec_edb_time_stats", + "nirspec_grating_stats", + "fgs_readnoise_query_history", "fgs_readnoise_stats", + "miri_readnoise_query_history", "miri_readnoise_stats", + "nircam_readnoise_query_history", "nircam_readnoise_stats", + "niriss_readnoise_query_history", "niriss_readnoise_stats", + "nirspec_readnoise_query_history", "nirspec_readnoise_stats", + "miri_ta_query_history", "miri_ta_stats", + "nirspec_ta_query_history", "nirspec_ta_stats" +] + # Suffix for msa files MSA_SUFFIX = ["msa"] diff --git a/jwql/website/apps/jwql/migrations/0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more.py b/jwql/website/apps/jwql/migrations/0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more.py new file mode 100644 index 000000000..b963ad7ad --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more.py @@ -0,0 +1,384 @@ +# Generated by Django 4.2.5 on 2024-02-23 16:50 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0016_anomalies_bright_object_not_a_short_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='NIRSpecReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'nirspec_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRSpecReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'nirspec_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRISSReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'niriss_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRISSReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'niriss_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'nircam_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'nircam_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamClawStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('filename', models.CharField(blank=True, null=True)), + ('proposal', models.CharField(blank=True, null=True)), + ('obs', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('filter', models.CharField(blank=True, null=True)), + ('pupil', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('expstart_mjd', models.FloatField(blank=True, null=True)), + ('effexptm', models.FloatField(blank=True, null=True)), + ('ra', models.FloatField(blank=True, null=True)), + ('dec', models.FloatField(blank=True, null=True)), + ('pa_v3', models.FloatField(blank=True, null=True)), + ('mean', models.FloatField(blank=True, null=True)), + ('median', models.FloatField(blank=True, null=True)), + ('stddev', models.FloatField(blank=True, null=True)), + ('frac_masked', models.FloatField(blank=True, null=True)), + ('skyflat_filename', models.CharField(blank=True, null=True)), + ], + options={ + 'db_table': 'nircam_claw_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='NIRCamClawQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('instrument', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ], + options={ + 'db_table': 'nircam_claw_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='MIRIReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'miri_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='MIRIReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'miri_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='FGSReadnoiseStats', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uncal_filename', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('detector', models.CharField(blank=True, null=True)), + ('subarray', models.CharField(blank=True, null=True)), + ('read_pattern', models.CharField(blank=True, null=True)), + ('nints', models.CharField(blank=True, null=True)), + ('ngroups', models.CharField(blank=True, null=True)), + ('expstart', models.CharField(blank=True, null=True)), + ('readnoise_filename', models.CharField(blank=True, null=True)), + ('full_image_mean', models.FloatField(blank=True, null=True)), + ('full_image_stddev', models.FloatField(blank=True, null=True)), + ('full_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('full_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('readnoise_diff_image', models.CharField(blank=True, null=True)), + ('diff_image_mean', models.FloatField(blank=True, null=True)), + ('diff_image_stddev', models.FloatField(blank=True, null=True)), + ('diff_image_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('diff_image_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ('amp1_mean', models.FloatField(blank=True, null=True)), + ('amp1_stddev', models.FloatField(blank=True, null=True)), + ('amp1_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp1_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_mean', models.FloatField(blank=True, null=True)), + ('amp2_stddev', models.FloatField(blank=True, null=True)), + ('amp2_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp2_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_mean', models.FloatField(blank=True, null=True)), + ('amp3_stddev', models.FloatField(blank=True, null=True)), + ('amp3_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp3_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_mean', models.FloatField(blank=True, null=True)), + ('amp4_stddev', models.FloatField(blank=True, null=True)), + ('amp4_n', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('amp4_bin_centers', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ], + options={ + 'db_table': 'fgs_readnoise_stats', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + migrations.CreateModel( + name='FGSReadnoiseQueryHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('instrument', models.CharField(blank=True, null=True)), + ('aperture', models.CharField(blank=True, null=True)), + ('start_time_mjd', models.FloatField(blank=True, null=True)), + ('end_time_mjd', models.FloatField(blank=True, null=True)), + ('entries_found', models.IntegerField(blank=True, null=True)), + ('files_found', models.IntegerField(blank=True, null=True)), + ('run_monitor', models.BooleanField(blank=True, null=True)), + ('entry_date', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'db_table': 'fgs_readnoise_query_history', + 'managed': True, + 'unique_together': {('id', 'entry_date')}, + }, + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0018_nircamclawstats_doy_nircamclawstats_total_bkg.py b/jwql/website/apps/jwql/migrations/0018_nircamclawstats_doy_nircamclawstats_total_bkg.py new file mode 100644 index 000000000..c5efd9125 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0018_nircamclawstats_doy_nircamclawstats_total_bkg.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.5 on 2024-02-23 16:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0017_nirspecreadnoisestats_nirspecreadnoisequeryhistory_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='nircamclawstats', + name='doy', + field=models.FloatField(blank=True, null=True), + ), + migrations.AddField( + model_name='nircamclawstats', + name='total_bkg', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/monitor_models/bad_pixel.py b/jwql/website/apps/jwql/monitor_models/bad_pixel.py index 151aa39ff..463331deb 100644 --- a/jwql/website/apps/jwql/monitor_models/bad_pixel.py +++ b/jwql/website/apps/jwql/monitor_models/bad_pixel.py @@ -48,7 +48,6 @@ class Meta: managed = True db_table = 'fgs_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSBadPixelStats(models.Model): @@ -67,7 +66,6 @@ class Meta: managed = True db_table = 'fgs_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIBadPixelQueryHistory(models.Model): @@ -88,7 +86,6 @@ class Meta: managed = True db_table = 'miri_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIBadPixelStats(models.Model): @@ -107,7 +104,6 @@ class Meta: managed = True db_table = 'miri_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamBadPixelQueryHistory(models.Model): @@ -128,7 +124,6 @@ class Meta: managed = True db_table = 'nircam_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamBadPixelStats(models.Model): @@ -147,7 +142,6 @@ class Meta: managed = True db_table = 'nircam_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBadPixelQueryHistory(models.Model): @@ -168,7 +162,6 @@ class Meta: managed = True db_table = 'niriss_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBadPixelStats(models.Model): @@ -187,7 +180,6 @@ class Meta: managed = True db_table = 'niriss_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBadPixelQueryHistory(models.Model): @@ -208,7 +200,6 @@ class Meta: managed = True db_table = 'nirspec_bad_pixel_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBadPixelStats(models.Model): @@ -227,4 +218,3 @@ class Meta: managed = True db_table = 'nirspec_bad_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/bias.py b/jwql/website/apps/jwql/monitor_models/bias.py index 306ef3b29..c8245b9fe 100644 --- a/jwql/website/apps/jwql/monitor_models/bias.py +++ b/jwql/website/apps/jwql/monitor_models/bias.py @@ -44,7 +44,6 @@ class Meta: managed = True db_table = 'nircam_bias_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamBiasStats(models.Model): @@ -74,7 +73,6 @@ class Meta: managed = True db_table = 'nircam_bias_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBiasQueryHistory(models.Model): @@ -91,7 +89,6 @@ class Meta: managed = True db_table = 'niriss_bias_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSBiasStats(models.Model): @@ -121,7 +118,6 @@ class Meta: managed = True db_table = 'niriss_bias_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBiasQueryHistory(models.Model): @@ -138,7 +134,6 @@ class Meta: managed = True db_table = 'nirspec_bias_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecBiasStats(models.Model): @@ -168,4 +163,3 @@ class Meta: managed = True db_table = 'nirspec_bias_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/claw.py b/jwql/website/apps/jwql/monitor_models/claw.py index 4e92ea3d4..a1c6c93e3 100644 --- a/jwql/website/apps/jwql/monitor_models/claw.py +++ b/jwql/website/apps/jwql/monitor_models/claw.py @@ -40,7 +40,6 @@ class Meta: managed = True db_table = 'nircam_claw_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamClawStats(models.Model): @@ -62,9 +61,10 @@ class NIRCamClawStats(models.Model): stddev = models.FloatField(blank=True, null=True) frac_masked = models.FloatField(blank=True, null=True) skyflat_filename = models.CharField(blank=True, null=True) + doy = models.FloatField(blank=True, null=True) + total_bkg = models.FloatField(blank=True, null=True) class Meta: managed = True db_table = 'nircam_claw_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 9809e843c..d0bed2afc 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -8,154 +8,153 @@ a data table. This module defines models that are used to store data related to the JWQL monitors. -Usage ------ + Usage + ----- -JWQL uses the django database models for creating tables, updating table fields, adding -new data to tables, and retrieving data from tables. For instrument monitors, in particular, -there are a number of issues that may be relevant. + JWQL uses the django database models for creating tables, updating table fields, adding + new data to tables, and retrieving data from tables. For instrument monitors, in particular, + there are a number of issues that may be relevant. -In general, django model documentation can be found -`on the django website `_. -Unfortunately, finding a particular bit of documentation in django can be a challenge, so -a few quick-reference notes are provided below. + In general, django model documentation can be found + `on the django website `_. + Unfortunately, finding a particular bit of documentation in django can be a challenge, so + a few quick-reference notes are provided below. -Retrieving Data ---------------- + Retrieving Data + --------------- -Django retrieves data directly from its model tables. So, for example, if you want to -select data from the `MIRIMyMonitorStats` table, you must first import the relevant -object: + Django retrieves data directly from its model tables. So, for example, if you want to + select data from the `MIRIMyMonitorStats` table, you must first import the relevant + object: -.. code-block:: python + .. code-block:: python - from jwql.website.apps.jwql.monitor_models.my_monitor import MIRIMyMonitorStats + from jwql.website.apps.jwql.monitor_models.my_monitor import MIRIMyMonitorStats -Then, you would access the database contents via the `objects` member of the class. For -example, to search the `MIRIMyMonitorStats` table for all entries matching a given -aperture, and to sort them with the most recent date at the top, you might do a query like -the following: + Then, you would access the database contents via the `objects` member of the class. For + example, to search the `MIRIMyMonitorStats` table for all entries matching a given + aperture, and to sort them with the most recent date at the top, you might do a query like + the following: -.. code-block:: python + .. code-block:: python - aperture = "my_miri_aperture" + aperture = "my_miri_aperture" - records = MIRIMyMonitorStats.objects.filter(aperture__iexact=aperture).order_by("-mjd_end").all() + records = MIRIMyMonitorStats.objects.filter(aperture__iexact=aperture).order_by("-mjd_end").all() -In the above code, + In the above code, -* The `filter()` function selects matching records from the full table. You can use - multiple filter statements, or a single filter function with multiple filters. `filter()` - statements are always combined with an implicit AND. -* If you have a long filter statement and want to separate it from the query statement, - you can create a dictionary and add it in with the `**` prepended. The dictionary - equivalent to the above would be `{'aperture__iexact': aperture}` -* The text before the double underscore is a field name, and the text afterwards describes - the type of comparison. `iexact` indicates "case-insensitive exact match". You can also - use a variety of standard SQL comparisons (`like`, `startswith`, `gte`, etc.) -* If you want to get only records that *don't* match a pattern, then you can use the - `exclude()` function, which otherwise operates exactly the same as `filter()`. -* In the `order_by()` function, the `-` at the start is used to reverse the sort order, - and the `mjd_end` is the name of the field to be sorted by. -* The `all()` statement indicates that you want all the values returned. `get()` returns - a single value and can be iterated on, `first()` returns only the first value, etc. + * The `filter()` function selects matching records from the full table. You can use + multiple filter statements, or a single filter function with multiple filters. `filter()` + statements are always combined with an implicit AND. + * If you have a long filter statement and want to separate it from the query statement, + you can create a dictionary and add it in with the `**` prepended. The dictionary + equivalent to the above would be `{'aperture__iexact': aperture}` + * The text before the double underscore is a field name, and the text afterwards describes + the type of comparison. `iexact` indicates "case-insensitive exact match". You can also + use a variety of standard SQL comparisons (`like`, `startswith`, `gte`, etc.) + * If you want to get only records that *don't* match a pattern, then you can use the + `exclude()` function, which otherwise operates exactly the same as `filter()`. + * In the `order_by()` function, the `-` at the start is used to reverse the sort order, + and the `mjd_end` is the name of the field to be sorted by. + * The `all()` statement indicates that you want all the values returned. `get()` returns + a single value and can be iterated on, `first()` returns only the first value, etc. -As an example of multiple filters, the code below: + As an example of multiple filters, the code below: -.. code-block:: python + .. code-block:: python - records = MIRIMyMonitorStats.objects.filter(aperture__iexact=ap, mjd_end__gte=60000) + records = MIRIMyMonitorStats.objects.filter(aperture__iexact=ap, mjd_end__gte=60000) - filters = { - "aperture__iexact": ap, - "mjd_end__gte": 60000 - } - records = MIRIMyMonitorStats.objects.filter(**filters) + filters = { + "aperture__iexact": ap, + "mjd_end__gte": 60000 + } + records = MIRIMyMonitorStats.objects.filter(**filters) -show two different ways of combining a search for a particular aperture *and* only data -taken more recently than MJD=60000. + show two different ways of combining a search for a particular aperture *and* only data + taken more recently than MJD=60000. -Note that django executes queries lazily, meaning that it will only actually *do* the -query when it needs the results. The above statement, for example, will not actually -run the query. Instead, it will be run when you operate on it, such as + Note that django executes queries lazily, meaning that it will only actually *do* the + query when it needs the results. The above statement, for example, will not actually + run the query. Instead, it will be run when you operate on it, such as -* Getting the length of the result with e.g. `len(records)` -* Printing out any of the results -* Asking for the value of one of the fields (e.g. `records[3].aperture`) + * Getting the length of the result with e.g. `len(records)` + * Printing out any of the results + * Asking for the value of one of the fields (e.g. `records[3].aperture`) -Retrieving Specific Columns -=========================== + Retrieving Specific Columns + =========================== -Django offers two ways of doing this. The first one is to use the `only()` function, which -immediately loads only the relevant columns. For example, + Django offers two ways of doing this. The first one is to use the `only()` function, which + immediately loads only the relevant columns. For example, -.. code-block:: python + .. code-block:: python - records = MIRIMyMonitorStats.objects.only("aperture", "mjd_start", "relevant_item") + records = MIRIMyMonitorStats.objects.only("aperture", "mjd_start", "relevant_item") -will immediately load only the three columns selected (although the rest will be retrieved -from the database, and can still be accessed, for no immediately understandable reason). -The other method is the `defer()` method, which loads every column *except* the ones listed. + will immediately load only the three columns selected (although the rest will be retrieved + from the database, and can still be accessed, for no immediately understandable reason). + The other method is the `defer()` method, which loads every column *except* the ones listed. -Q Objects -========= + Q Objects + ========= -In order to make more complex queries, Django supplies "Q Objects", which are essentially -encapsulated filters which can be combined using logical operators. For more on this, see -`the django Q object documentation `_. + In order to make more complex queries, Django supplies "Q Objects", which are essentially + encapsulated filters which can be combined using logical operators. For more on this, see + `the django Q object documentation `_. -Storing Data ------------- + Storing Data + ------------ -Django also uses the model tables (and objects) directly for storing new data. For example, -if you have a monitor table defined as below: + Django also uses the model tables (and objects) directly for storing new data. For example, + if you have a monitor table defined as below: -.. code-block:: python + .. code-block:: python - from django.db import models - from django.contrib.postgres.fields import ArrayField + from django.db import models + from django.contrib.postgres.fields import ArrayField - class NIRISSMyMonitorStats(models.Model): - aperture = models.CharField(blank=True, null=True) - mean = models.FloatField(blank=True, null=True) - median = models.FloatField(blank=True, null=True) - stddev = models.FloatField(blank=True, null=True) - counts = ArrayField(models.FloatField()) - entry_date = models.DateTimeField(blank=True, null=True) + class NIRISSMyMonitorStats(models.Model): + aperture = models.CharField(blank=True, null=True) + mean = models.FloatField(blank=True, null=True) + median = models.FloatField(blank=True, null=True) + stddev = models.FloatField(blank=True, null=True) + counts = ArrayField(models.FloatField()) + entry_date = models.DateTimeField(blank=True, null=True) - class Meta: - managed = True - db_table = 'niriss_my_monitor_stats' - unique_together = (('id', 'entry_date'),) - app_label = 'monitors' + class Meta: + managed = True + db_table = 'niriss_my_monitor_stats' + unique_together = (('id', 'entry_date'),) -then you would create a new entry as follows: + then you would create a new entry as follows: -.. code-block:: python + .. code-block:: python - values = { - "aperture": "my_aperture", - "mean": float(mean), - "median": float(median), - "stddev": float(stddev), - "counts": list(counts.astype(float)), - "entry_date": datetime.datetime.now() - } + values = { + "aperture": "my_aperture", + "mean": float(mean), + "median": float(median), + "stddev": float(stddev), + "counts": list(counts.astype(float)), + "entry_date": datetime.datetime.now() + } - entry = NIRISSMyMonitorStats(**values) - entry.save() - -There are (as usual) a few things to note above: + entry = NIRISSMyMonitorStats(**values) + entry.save() -* Django doesn't have a built-in array data type, so you need to import it from the - database-compatibility layers. The ArrayField takes, as a required argument, the type - of data that makes up the array. -* In the Meta sub-class of the monitor class, the `app_label = 'monitors'` statement is - required so that django knows that the model should be stored in the monitors table. -* The `float()` casts are required because the database interface doesn't understand - numpy data types. -* The `list()` cast is required because the database interface doesn't understand the - numpy `ndarray` data type + There are (as usual) a few things to note above: + + * Django doesn't have a built-in array data type, so you need to import it from the + database-compatibility layers. The ArrayField takes, as a required argument, the type + of data that makes up the array. + * In the Meta sub-class of the monitor class, the `db_table_comment = 'monitors'` statement is + required so that django knows that the model should be stored in the monitors table. + * The `float()` casts are required because the database interface doesn't understand + numpy data types. + * The `list()` cast is required because the database interface doesn't understand the + numpy `ndarray` data type Authors ------- @@ -179,50 +178,6 @@ class Meta: from django.contrib.postgres.fields import ArrayField -class MonitorRouter: - """ - A router to control all database operations on models in the - JWQLDB (monitors) database. - """ - - route_app_labels = {"monitors"} - - def db_for_read(self, model, **hints): - """ - Attempts to read monitor models go to monitors db. - """ - if model._meta.app_label in self.route_app_labels: - return "monitors" - return None - - def db_for_write(self, model, **hints): - """ - Attempts to write monitor models go to monitors db. - """ - if model._meta.app_label in self.route_app_labels: - return "monitors" - return None - - def allow_relation(self, obj1, obj2, **hints): - """ - Allow relations between tables in the monitors DB. - """ - if ( - obj1._meta.app_label in self.route_app_labels - or obj2._meta.app_label in self.route_app_labels - ): - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): - """ - Make sure the monitors apps only appear in the 'monitors' database. - """ - if app_label in self.route_app_labels: - return db == "monitors" - return None - - class Monitor(models.Model): monitor_name = models.CharField() start_time = models.DateTimeField() @@ -233,7 +188,6 @@ class Monitor(models.Model): class Meta: managed = True db_table = 'monitor' - app_label = 'monitors' class CentralStorage(models.Model): @@ -246,7 +200,6 @@ class CentralStorage(models.Model): class Meta: managed = True db_table = 'central_storage' - app_label = 'monitors' class FilesystemCharacteristics(models.Model): @@ -258,7 +211,6 @@ class FilesystemCharacteristics(models.Model): class Meta: managed = True db_table = 'filesystem_characteristics' - app_label = 'monitors' class FilesystemGeneral(models.Model): @@ -273,7 +225,6 @@ class FilesystemGeneral(models.Model): class Meta: managed = True db_table = 'filesystem_general' - app_label = 'monitors' class FilesystemInstrument(models.Model): @@ -287,7 +238,6 @@ class Meta: managed = True db_table = 'filesystem_instrument' unique_together = (('date', 'instrument', 'filetype'),) - app_label = 'monitors' class FgsAnomaly(models.Model): @@ -308,7 +258,6 @@ class FgsAnomaly(models.Model): class Meta: managed = True db_table = 'fgs_anomaly' - app_label = 'monitors' class MiriAnomaly(models.Model): @@ -334,7 +283,6 @@ class MiriAnomaly(models.Model): class Meta: managed = True db_table = 'miri_anomaly' - app_label = 'monitors' class NircamAnomaly(models.Model): @@ -360,7 +308,6 @@ class NircamAnomaly(models.Model): class Meta: managed = True db_table = 'nircam_anomaly' - app_label = 'monitors' class NirissAnomaly(models.Model): @@ -383,7 +330,6 @@ class NirissAnomaly(models.Model): class Meta: managed = True db_table = 'niriss_anomaly' - app_label = 'monitors' class NirspecAnomaly(models.Model): @@ -406,4 +352,3 @@ class NirspecAnomaly(models.Model): class Meta: managed = True db_table = 'nirspec_anomaly' - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/cosmic_ray.py b/jwql/website/apps/jwql/monitor_models/cosmic_ray.py index f9084aa06..cdff2eb22 100644 --- a/jwql/website/apps/jwql/monitor_models/cosmic_ray.py +++ b/jwql/website/apps/jwql/monitor_models/cosmic_ray.py @@ -43,7 +43,6 @@ class Meta: managed = True db_table = 'fgs_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSCosmicRayStats(models.Model): @@ -61,7 +60,6 @@ class Meta: managed = True db_table = 'fgs_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRICosmicRayQueryHistory(models.Model): @@ -77,7 +75,6 @@ class Meta: managed = True db_table = 'miri_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRICosmicRayStats(models.Model): @@ -95,7 +92,6 @@ class Meta: managed = True db_table = 'miri_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamCosmicRayQueryHistory(models.Model): @@ -111,7 +107,6 @@ class Meta: managed = True db_table = 'nircam_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamCosmicRayStats(models.Model): @@ -129,7 +124,6 @@ class Meta: managed = True db_table = 'nircam_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSCosmicRayQueryHistory(models.Model): @@ -145,7 +139,6 @@ class Meta: managed = True db_table = 'niriss_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSCosmicRayStats(models.Model): @@ -163,7 +156,6 @@ class Meta: managed = True db_table = 'niriss_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecCosmicRayQueryHistory(models.Model): @@ -179,7 +171,6 @@ class Meta: managed = True db_table = 'nirspec_cosmic_ray_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecCosmicRayStats(models.Model): @@ -197,4 +188,3 @@ class Meta: managed = True db_table = 'nirspec_cosmic_ray_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/dark_current.py b/jwql/website/apps/jwql/monitor_models/dark_current.py index 29d9a9523..41ae1ccac 100644 --- a/jwql/website/apps/jwql/monitor_models/dark_current.py +++ b/jwql/website/apps/jwql/monitor_models/dark_current.py @@ -60,7 +60,6 @@ class Meta: managed = True db_table = 'fgs_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSDarkPixelStats(models.Model): @@ -80,7 +79,6 @@ class Meta: managed = True db_table = 'fgs_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSDarkQueryHistory(models.Model): @@ -97,7 +95,6 @@ class Meta: managed = True db_table = 'fgs_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIDarkDarkCurrent(models.Model): @@ -130,7 +127,6 @@ class Meta: managed = True db_table = 'miri_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIDarkPixelStats(models.Model): @@ -150,7 +146,6 @@ class Meta: managed = True db_table = 'miri_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIDarkQueryHistory(models.Model): @@ -167,7 +162,6 @@ class Meta: managed = True db_table = 'miri_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamDarkDarkCurrent(models.Model): @@ -200,7 +194,6 @@ class Meta: managed = True db_table = 'nircam_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamDarkPixelStats(models.Model): @@ -220,7 +213,6 @@ class Meta: managed = True db_table = 'nircam_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamDarkQueryHistory(models.Model): @@ -237,7 +229,6 @@ class Meta: managed = True db_table = 'nircam_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSDarkDarkCurrent(models.Model): @@ -270,7 +261,6 @@ class Meta: managed = True db_table = 'niriss_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSDarkPixelStats(models.Model): @@ -290,7 +280,6 @@ class Meta: managed = True db_table = 'niriss_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSDarkQueryHistory(models.Model): @@ -307,7 +296,6 @@ class Meta: managed = True db_table = 'niriss_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecDarkDarkCurrent(models.Model): @@ -340,7 +328,6 @@ class Meta: managed = True db_table = 'nirspec_dark_dark_current' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecDarkPixelStats(models.Model): @@ -360,7 +347,6 @@ class Meta: managed = True db_table = 'nirspec_dark_pixel_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecDarkQueryHistory(models.Model): @@ -377,4 +363,3 @@ class Meta: managed = True db_table = 'nirspec_dark_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/edb.py b/jwql/website/apps/jwql/monitor_models/edb.py index cc9d8ec82..2cad15418 100644 --- a/jwql/website/apps/jwql/monitor_models/edb.py +++ b/jwql/website/apps/jwql/monitor_models/edb.py @@ -45,7 +45,6 @@ class Meta: managed = True db_table = 'fgs_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbDailyStats(models.Model): @@ -63,7 +62,6 @@ class Meta: managed = True db_table = 'fgs_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbEveryChangeStats(models.Model): @@ -81,7 +79,6 @@ class Meta: managed = True db_table = 'fgs_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbTimeIntervalStats(models.Model): @@ -99,7 +96,6 @@ class Meta: managed = True db_table = 'fgs_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSEdbTimeStats(models.Model): @@ -114,7 +110,6 @@ class Meta: managed = True db_table = 'fgs_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbBlocksStats(models.Model): @@ -132,7 +127,6 @@ class Meta: managed = True db_table = 'miri_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbDailyStats(models.Model): @@ -150,7 +144,6 @@ class Meta: managed = True db_table = 'miri_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbEveryChangeStats(models.Model): @@ -168,7 +161,6 @@ class Meta: managed = True db_table = 'miri_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbTimeIntervalStats(models.Model): @@ -186,7 +178,6 @@ class Meta: managed = True db_table = 'miri_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIEdbTimeStats(models.Model): @@ -201,7 +192,6 @@ class Meta: managed = True db_table = 'miri_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbBlocksStats(models.Model): @@ -219,7 +209,6 @@ class Meta: managed = True db_table = 'nircam_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbDailyStats(models.Model): @@ -237,7 +226,6 @@ class Meta: managed = True db_table = 'nircam_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbEveryChangeStats(models.Model): @@ -255,7 +243,6 @@ class Meta: managed = True db_table = 'nircam_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbTimeIntervalStats(models.Model): @@ -273,7 +260,6 @@ class Meta: managed = True db_table = 'nircam_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamEdbTimeStats(models.Model): @@ -288,7 +274,6 @@ class Meta: managed = True db_table = 'nircam_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbBlocksStats(models.Model): @@ -306,7 +291,6 @@ class Meta: managed = True db_table = 'niriss_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbDailyStats(models.Model): @@ -324,7 +308,6 @@ class Meta: managed = True db_table = 'niriss_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbEveryChangeStats(models.Model): @@ -342,7 +325,6 @@ class Meta: managed = True db_table = 'niriss_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbTimeIntervalStats(models.Model): @@ -360,7 +342,6 @@ class Meta: managed = True db_table = 'niriss_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSEdbTimeStats(models.Model): @@ -375,7 +356,6 @@ class Meta: managed = True db_table = 'niriss_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbBlocksStats(models.Model): @@ -393,7 +373,6 @@ class Meta: managed = True db_table = 'nirspec_edb_blocks_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbDailyStats(models.Model): @@ -411,7 +390,6 @@ class Meta: managed = True db_table = 'nirspec_edb_daily_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbEveryChangeStats(models.Model): @@ -429,7 +407,6 @@ class Meta: managed = True db_table = 'nirspec_edb_every_change_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbTimeIntervalStats(models.Model): @@ -447,7 +424,6 @@ class Meta: managed = True db_table = 'nirspec_edb_time_interval_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecEdbTimeStats(models.Model): @@ -462,4 +438,3 @@ class Meta: managed = True db_table = 'nirspec_edb_time_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/grating.py b/jwql/website/apps/jwql/monitor_models/grating.py index 9e4268879..1c2029049 100644 --- a/jwql/website/apps/jwql/monitor_models/grating.py +++ b/jwql/website/apps/jwql/monitor_models/grating.py @@ -40,7 +40,6 @@ class Meta: managed = True db_table = 'nirspec_grating_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecGratingStats(models.Model): @@ -70,4 +69,3 @@ class Meta: managed = True db_table = 'nirspec_grating_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/readnoise.py b/jwql/website/apps/jwql/monitor_models/readnoise.py index 5a500c709..5e616aa71 100644 --- a/jwql/website/apps/jwql/monitor_models/readnoise.py +++ b/jwql/website/apps/jwql/monitor_models/readnoise.py @@ -44,7 +44,6 @@ class Meta: managed = True db_table = 'fgs_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class FGSReadnoiseStats(models.Model): @@ -88,7 +87,6 @@ class Meta: managed = True db_table = 'fgs_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIReadnoiseQueryHistory(models.Model): @@ -105,7 +103,6 @@ class Meta: managed = True db_table = 'miri_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRIReadnoiseStats(models.Model): @@ -149,7 +146,6 @@ class Meta: managed = True db_table = 'miri_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamReadnoiseQueryHistory(models.Model): @@ -166,7 +162,6 @@ class Meta: managed = True db_table = 'nircam_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRCamReadnoiseStats(models.Model): @@ -210,7 +205,6 @@ class Meta: managed = True db_table = 'nircam_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSReadnoiseQueryHistory(models.Model): @@ -227,7 +221,6 @@ class Meta: managed = True db_table = 'niriss_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRISSReadnoiseStats(models.Model): @@ -271,7 +264,6 @@ class Meta: managed = True db_table = 'niriss_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecReadnoiseQueryHistory(models.Model): @@ -288,7 +280,6 @@ class Meta: managed = True db_table = 'nirspec_readnoise_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecReadnoiseStats(models.Model): @@ -332,4 +323,3 @@ class Meta: managed = True db_table = 'nirspec_readnoise_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_models/ta.py b/jwql/website/apps/jwql/monitor_models/ta.py index e48433dee..93a8b269b 100644 --- a/jwql/website/apps/jwql/monitor_models/ta.py +++ b/jwql/website/apps/jwql/monitor_models/ta.py @@ -44,7 +44,6 @@ class Meta: managed = True db_table = 'miri_ta_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class MIRITaStats(models.Model): @@ -63,7 +62,6 @@ class Meta: managed = True db_table = 'miri_ta_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecTaQueryHistory(models.Model): @@ -80,7 +78,6 @@ class Meta: managed = True db_table = 'nirspec_ta_query_history' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' class NIRSpecTaStats(models.Model): @@ -122,4 +119,3 @@ class Meta: managed = True db_table = 'nirspec_ta_stats' unique_together = (('id', 'entry_date'),) - app_label = 'monitors' diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 79ac6f127..fa6f48204 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -157,10 +157,9 @@ def claw_monitor(request): template = "claw_monitor.html" # Get all recent claw stack images from the last 10 days - query = NIRCamClawStats.objects # .filter(expstart_mjd__gte=(Time.now().mjd - 10)) - query = query.order_by('-expstart_mjd').all().values('expstart_mjd', 'skyflat_filename') - df = pd.DataFrame.from_records(query) - recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10])) + query = NIRCamClawStats.objects.filter(expstart_mjd__gte=(Time.now().mjd - 10)) + query = query.order_by('-expstart_mjd').all().values('skyflat_filename') + recent_files = list(pd.unique(pd.DataFrame.from_records(query)['skyflat_filename'])) output_dir_claws = static(os.path.join("outputs", "claw_monitor", "claw_stacks")) claw_stacks = [os.path.join(output_dir_claws, filename) for filename in recent_files] diff --git a/jwql/website/apps/jwql/router.py b/jwql/website/apps/jwql/router.py new file mode 100644 index 000000000..5b08fa440 --- /dev/null +++ b/jwql/website/apps/jwql/router.py @@ -0,0 +1,64 @@ +"""Defines the query routing for the monitor database tables. + +In Django, database queries are assumed to go to the default database unless either the +`using` field/keyword is defined or a routing table sends it to a different database. In +this case, all monitor tables should be routed to the monitors database, and the router +should otherwise express no opinion (by returning None). + +Authors +------- + - Brian York + +Use +--- + This module is not intended to be used outside of Django asking about it. + +References +---------- + For more information please see: + ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` +""" +from jwql.utils.constants import MONITOR_TABLE_NAMES + + +class MonitorRouter: + """ + A router to control all database operations on models in the + JWQLDB (monitors) database. + """ + + def db_for_read(self, model, **hints): + """ + Attempts to read monitor models go to monitors db. + """ + if model._meta.db_table in MONITOR_TABLE_NAMES: + return "monitors" + return None + + def db_for_write(self, model, **hints): + """ + Attempts to write monitor models go to monitors db. + """ + if model._meta.db_table in MONITOR_TABLE_NAMES: + return "monitors" + return None + + def allow_relation(self, obj1, obj2, **hints): + """ + Allow relations between tables in the monitors DB. + """ + if ( + obj1._meta.db_table in MONITOR_TABLE_NAMES + and obj2._meta.db_table in MONITOR_TABLE_NAMES + ): + return True + return None + + def allow_migrate(self, db, app_label, model_name=None, **hints): + """ + Make sure the monitors apps only appear in the 'monitors' database. + """ + model_names = [name.replace("_", "") for name in MONITOR_TABLE_NAMES] + if app_label == 'jwql' and model_name in model_names: + return db == "monitors" + return None diff --git a/jwql/website/jwql_proj/settings.py b/jwql/website/jwql_proj/settings.py index c0e72240f..a8adf9fa3 100644 --- a/jwql/website/jwql_proj/settings.py +++ b/jwql/website/jwql_proj/settings.py @@ -111,7 +111,7 @@ 'default': get_config()['django_databases']['default'], 'monitors': get_config()['django_databases']['monitors'] } -DATABASE_ROUTERS = ["jwql.website.apps.jwql.monitor_models.common.MonitorRouter"] +DATABASE_ROUTERS = ["jwql.website.apps.jwql.router.MonitorRouter"] # Password validation # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators