From 477072f37d93b36ed907d77d43204d74f1f600f8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:22:52 -0400 Subject: [PATCH 01/23] create wrapper func for saving png files --- .../common_monitors/dark_monitor.py | 5 ++-- jwql/utils/utils.py | 23 ++++++++++++++++++- .../monitor_pages/monitor_bad_pixel_bokeh.py | 6 ++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c1c2ecc4c..467a8bd1c 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -84,7 +84,6 @@ from astropy.modeling import models from astropy.stats import sigma_clipped_stats from astropy.time import Time -from bokeh.io import export_png from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend from bokeh.models import LinearColorMapper from bokeh.plotting import figure @@ -106,7 +105,7 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt') @@ -350,7 +349,7 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no self.plot.add_layout(legend, 'below') # Save the plot in a png - export_png(self.plot, filename=output_filename) + save_png(self.plot, filename=output_filename) set_permissions(output_filename) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 33d0c272c..75cf49ca6 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -45,6 +45,7 @@ from bokeh.plotting import figure import numpy as np from PIL import Image +from selenium import webdriver from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ @@ -170,7 +171,7 @@ def create_png_from_fits(filename, outdir): # Save the plot in a png output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - export_png(plot, filename=output_filename) + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: @@ -756,6 +757,26 @@ def read_png(filename): return img +def save_png(fig, filename=''): + """Starting with selenium version 4.10.0, our testing has shown that on the JWQL + servers, we need to specify an instance of a web driver when exporting a Bokeh + figure as a png. This is a wrapper function that creates the web driver instance + and calls Bokeh's export_png function. + + Parameters + ---------- + fig : bokeh.plotting.figure + Bokeh figure to be saved as a png + + filename : str + Filename to use for the png file + """ + options = webdriver.FirefoxOptions() + options.add_argument('-headless') + driver = webdriver.Firefox(options=options) + export_png(fig, filename=filename, webdriver=driver) + + def grouper(iterable, chunksize): """ Take a list of items (iterable), and group it into chunks of chunksize, with the diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 48c47e216..b172df2d4 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -22,7 +22,7 @@ from astropy.stats import sigma_clipped_stats from astropy.time import Time from bokeh.embed import components, file_html -from bokeh.io import export_png, show +from bokeh.io import show from bokeh.layouts import layout from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title from bokeh.plotting import figure @@ -40,7 +40,7 @@ from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.permissions import set_permissions -from jwql.utils.utils import filesystem_path, get_config, read_png +from jwql.utils.utils import filesystem_path, get_config, read_png, save_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) OUTPUT_DIR = get_config()['outputs'] @@ -593,7 +593,7 @@ def switch_to_png(self, filename, title): Title to add to the Figure """ # Save the figure as a png - export_png(self.plot, filename=filename) + save_png(self.plot, filename=filename) set_permissions(filename) # Read in the png and insert into a replacement figure From 03da9a8916d3dc438a7caaae96c0710a2708092b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:38:38 -0400 Subject: [PATCH 02/23] Quit driver when done. Add test --- jwql/tests/test_utils.py | 19 ++++++++++++++++++- jwql/utils/utils.py | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index c2f6795c1..27bee1b5e 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -22,7 +22,11 @@ from pathlib import Path import pytest -from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config +from bokeh.models import LinearColorMapper +from bokeh.plotting import figure +import numpy as np + +from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config # Determine if tests are being run on Github Actions @@ -479,6 +483,19 @@ def test_filesystem_path(): assert check == location +def test_save_png(): + """Test that we can create a png file""" + plot = figure(title='test',tools='') + image = np.zeros((200,200)) + image[100:105, 100:105] = 1 + ny, nx = image.shape + mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + #save_png(plot, filename='test.png') + from bokeh.io import export_png + export_png(plot, filename='test.png') + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_validate_config(): """Test that the config validator works.""" diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 75cf49ca6..182c18191 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -775,6 +775,7 @@ def save_png(fig, filename=''): options.add_argument('-headless') driver = webdriver.Firefox(options=options) export_png(fig, filename=filename, webdriver=driver) + driver.quit() def grouper(iterable, chunksize): From f239ef2ccc290c3f05c657611e0d7903044fcf7a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:54:07 -0400 Subject: [PATCH 03/23] RTD needs selenium now --- rtd_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index aa1047b5e..d4b88e1fd 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -8,6 +8,7 @@ jwst==1.11.4 pygments==2.16.1 pytest==7.4.0 redis==5.0.0 +selenium==4.11.3 sphinx==6.2.1 sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 From 23d02e79a7a3dd9752c4505fdccc2a36f791059c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 16:05:08 -0400 Subject: [PATCH 04/23] Fix test. Adjust rtd version --- jwql/tests/test_utils.py | 4 +--- rtd_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 27bee1b5e..813bb0432 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -491,9 +491,7 @@ def test_save_png(): ny, nx = image.shape mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - #save_png(plot, filename='test.png') - from bokeh.io import export_png - export_png(plot, filename='test.png') + save_png(plot, filename='test.png') @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') diff --git a/rtd_requirements.txt b/rtd_requirements.txt index d4b88e1fd..39500c2b6 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -8,7 +8,7 @@ jwst==1.11.4 pygments==2.16.1 pytest==7.4.0 redis==5.0.0 -selenium==4.11.3 +selenium==4.11.2 sphinx==6.2.1 sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 From b85a1b0f7080afc1a58a1e8070b22377cf878b58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:21:42 +0000 Subject: [PATCH 05/23] Bump cryptography from 41.0.3 to 41.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4391be8b3..d4f3a5616 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ bandit==1.7.5 beautifulsoup4==4.12.2 bokeh==2.4.3 celery==5.3.1 -cryptography==41.0.3 +cryptography==41.0.4 django==4.2.3 inflection==0.5.1 ipython==8.14.0 From 7ce0d9c89e9ace520c98f2487762e5cb208156e4 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 25 Sep 2023 13:19:07 -0400 Subject: [PATCH 06/23] Fix typo in env file names in readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 70b4e173b..377005a11 100644 --- a/README.md +++ b/README.md @@ -83,16 +83,16 @@ source activate base/root **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step. -Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): +Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): ``` -conda env create -f environment_python_3_9.yml +conda env create -f environment_python_3.9.yml ``` or ``` -conda env create -f environment_python_3_10.yml +conda env create -f environment_python_3.10.yml ``` ### Configuration File From ebf05a9e22ca555ee3f7d9b48fcd48dd0068de5c Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Thu, 28 Sep 2023 10:59:37 -0400 Subject: [PATCH 07/23] Include migrations in source control --- .../apps/jwql/migrations/0001_initial.py | 45 +++++++++ .../migrations/0002_auto_20220913_1525.py | 27 ++++++ .../migrations/0003_auto_20220921_0955.py | 58 +++++++++++ .../migrations/0004_auto_20220922_0911.py | 33 +++++++ .../migrations/0005_auto_20220922_1422.py | 23 +++++ .../migrations/0006_auto_20230214_1624.py | 95 +++++++++++++++++++ .../migrations/0007_auto_20230222_1157.py | 22 +++++ .../migrations/0008_rootfileinfo_exp_type.py | 18 ++++ .../migrations/0009_auto_20230303_0930.py | 63 ++++++++++++ .../migrations/0010_auto_20230313_1053.py | 23 +++++ jwql/website/apps/jwql/migrations/__init__.py | 0 11 files changed, 407 insertions(+) create mode 100644 jwql/website/apps/jwql/migrations/0001_initial.py create mode 100644 jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py create mode 100644 jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py create mode 100644 jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py create mode 100644 jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py create mode 100644 jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py create mode 100644 jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py create mode 100644 jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py create mode 100644 jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py create mode 100644 jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py create mode 100644 jwql/website/apps/jwql/migrations/__init__.py diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py new file mode 100644 index 000000000..9de5ca6a9 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 3.1.7 on 2022-09-09 17:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ImageData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')), + ('pub_date', models.DateTimeField(verbose_name='date published')), + ('filepath', models.FilePathField(path='/user/lchambers/jwql/')), + ], + options={ + 'verbose_name_plural': 'image data', + 'db_table': 'imagedata', + }, + ), + migrations.CreateModel( + name='InstrumentFilterHandler', + fields=[ + ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)), + ], + ), + migrations.CreateModel( + name='ThumbnailFilterInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('observation', models.PositiveIntegerField()), + ('proposal', models.PositiveIntegerField()), + ('root_name', models.CharField(max_length=300)), + ('marked_viewed', models.BooleanField(default=False)), + ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')), + ], + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py new file mode 100644 index 000000000..049901811 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py @@ -0,0 +1,27 @@ +# Generated by Django 3.1.7 on 2022-09-13 20:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='id', + ), + migrations.AlterField( + model_name='instrumentfilterhandler', + name='instrument', + field=models.TextField(max_length=10, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='thumbnailfilterinfo', + name='root_name', + field=models.TextField(max_length=300, primary_key=True, serialize=False), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py new file mode 100644 index 000000000..df7bc6965 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py @@ -0,0 +1,58 @@ +# Generated by Django 3.1.7 on 2022-09-21 14:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0002_auto_20220913_1525'), + ] + + operations = [ + migrations.CreateModel( + name='Archive', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)), + ], + options={ + 'ordering': ['instrument'], + }, + ), + migrations.CreateModel( + name='Observation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)), + ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')), + ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')), + ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')), + ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)), + ], + options={ + 'ordering': ['-obsnum'], + }, + ), + migrations.CreateModel( + name='Proposal', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)), + ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')), + ], + options={ + 'ordering': ['-prop_id'], + 'unique_together': {('prop_id', 'archive')}, + }, + ), + migrations.DeleteModel( + name='ImageData', + ), + migrations.AddField( + model_name='observation', + name='proposal', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py new file mode 100644 index 000000000..998dd2ce4 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.7 on 2022-09-22 14:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0003_auto_20220921_0955'), + ] + + operations = [ + migrations.AddField( + model_name='thumbnailfilterinfo', + name='obsnum', + field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3), + preserve_default=False, + ), + migrations.CreateModel( + name='RootFileInfo', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7)), + ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)), + ('viewed', models.BooleanField(default=False)), + ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')), + ], + options={ + 'ordering': ['-root_name'], + }, + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py new file mode 100644 index 000000000..3c51cbe23 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2022-09-22 19:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0004_auto_20220922_0911'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='inst_handler', + ), + migrations.DeleteModel( + name='InstrumentFilterHandler', + ), + migrations.DeleteModel( + name='ThumbnailFilterInfo', + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py new file mode 100644 index 000000000..493608df3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py @@ -0,0 +1,95 @@ +# Generated by Django 3.1.7 on 2023-02-14 21:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0005_auto_20220922_1422'), + ] + + operations = [ + migrations.CreateModel( + name='Anomalies', + fields=[ + ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')), + ('cosmic_ray_shower', models.BooleanField(default=False)), + ('diffraction_spike', models.BooleanField(default=False)), + ('excessive_saturation', models.BooleanField(default=False)), + ('guidestar_failure', models.BooleanField(default=False)), + ('persistence', models.BooleanField(default=False)), + ('crosstalk', models.BooleanField(default=False)), + ('data_transfer_error', models.BooleanField(default=False)), + ('ghost', models.BooleanField(default=False)), + ('snowball', models.BooleanField(default=False)), + ('column_pull_up', models.BooleanField(default=False)), + ('column_pull_down', models.BooleanField(default=False)), + ('dominant_msa_leakage', models.BooleanField(default=False)), + ('dragons_breath', models.BooleanField(default=False)), + ('mrs_glow', models.BooleanField(default=False)), + ('mrs_zipper', models.BooleanField(default=False)), + ('internal_reflection', models.BooleanField(default=False)), + ('optical_short', models.BooleanField(default=False)), + ('row_pull_up', models.BooleanField(default=False)), + ('row_pull_down', models.BooleanField(default=False)), + ('lrs_contamination', models.BooleanField(default=False)), + ('tree_rings', models.BooleanField(default=False)), + ('scattered_light', models.BooleanField(default=False)), + ('claws', models.BooleanField(default=False)), + ('wisps', models.BooleanField(default=False)), + ('tilt_event', models.BooleanField(default=False)), + ('light_saber', models.BooleanField(default=False)), + ('other', models.BooleanField(default=False)), + ], + options={ + 'ordering': ['-root_file_info'], + }, + ), + migrations.AddField( + model_name='proposal', + name='cat_type', + field=models.CharField(default='', help_text='Category Type', max_length=10), + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperature', + field=models.CharField(default='', help_text='Aperature', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(default='', help_text='Detector', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(default='', help_text='Instrument name', max_length=7), + ), + migrations.AddField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(default='', help_text='Grating', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(default='', help_text='Pupil', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(default='', help_text='Read Pattern', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt_num', + field=models.IntegerField(default=0, help_text='Read Pattern Number'), + ), + migrations.AddField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(default='', help_text='Subarray', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py new file mode 100644 index 000000000..2d4ffdba3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.7 on 2023-02-22 16:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0006_auto_20230214_1624'), + ] + + operations = [ + migrations.RemoveField( + model_name='rootfileinfo', + name='aperature', + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(default='', help_text='Aperture', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py new file mode 100644 index 000000000..2c67cd955 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.7 on 2023-02-22 17:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0007_auto_20230222_1157'), + ] + + operations = [ + migrations.AddField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(default='', help_text='Exposure Type', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py new file mode 100644 index 000000000..d0dbafc76 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py @@ -0,0 +1,63 @@ +# Generated by Django 3.1.7 on 2023-03-03 14:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0008_rootfileinfo_exp_type'), + ] + + operations = [ + migrations.RenameField( + model_name='proposal', + old_name='cat_type', + new_name='category', + ), + migrations.AddField( + model_name='rootfileinfo', + name='expstart', + field=models.FloatField(default=0.0, help_text='Exposure Start Time'), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py new file mode 100644 index 000000000..3d80e95fe --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2023-03-13 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0009_auto_20230303_0930'), + ] + + operations = [ + migrations.AddField( + model_name='anomalies', + name='flag_date', + field=models.DateTimeField(blank=True, help_text='flag date', null=True), + ), + migrations.AddField( + model_name='anomalies', + name='user', + field=models.CharField(blank=True, default='', help_text='user', max_length=50, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From a22b9ccc95958ff5a29f6ddbdc35978cbdd64682 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 14:51:41 -0400 Subject: [PATCH 08/23] pep8 fixes --- .../common_monitors/dark_monitor.py | 20 ++++--------- jwql/utils/utils.py | 10 +++++-- .../monitor_pages/monitor_bad_pixel_bokeh.py | 28 +++++-------------- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 467a8bd1c..c8d30c92a 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -275,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no # Create figure start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y") - end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") + end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='') # tools='pan,box_zoom,reset,wheel_zoom,save') self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 # Create the color mapper that will be used to scale the image - mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image and add color bar imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -291,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec') self.plot.add_layout(color_bar, 'right') - # Add hover tool for all pixel values - #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'), - # ('value:', '@image') - # ], - # renderers=[imgplot]) - #self.plot.tools.append(hover_tool) - if (('FULL' in self.aperture) or ('_CEN' in self.aperture)): if hotxy is not None: @@ -337,14 +330,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no base_start = Time(float(base_parts[3]), format='mjd').tt.datetime base_end = Time(float(base_parts[5]), format='mjd').tt.datetime base_start_time = base_start.strftime("%m/%d/%Y") - base_end_time = base_end.strftime("%m/%d/%Y") + base_end_time = base_end.strftime("%m/%d/%Y") legend_title = f'Compared to dark from {base_start_time} to {base_end_time}' else: legend_title = 'Compared to previous mean dark' legend = Legend(items=[hot_legend, dead_legend, noisy_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') @@ -352,7 +345,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no save_png(self.plot, filename=output_filename) set_permissions(output_filename) - def get_metadata(self, filename): """Collect basic metadata from a fits file @@ -650,14 +642,14 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] ) - ) + ) # Overplot the bad pixel locations badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', source=sources[pix_type], color=colors[pix_type]) # Add to the legend - if numpix > 0: + if numpix > 0: if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT: text = f"{numpix} pix {adjective[pix_type]} than baseline" else: diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 182c18191..4b8951717 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -158,8 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -170,8 +169,13 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png +<<<<<<< Updated upstream output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) save_png(plot, filename=output_filename) +======= + output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) + export_png(plot, filename=output_filename) +>>>>>>> Stashed changes permissions.set_permissions(output_filename) return output_filename else: @@ -750,7 +754,7 @@ def read_png(filename): # Copy the RGBA image into view, flipping it so it comes right-side up # with a lower-left origin - view[:,:,:] = np.flipud(np.asarray(rgba_img)) + view[:, :, :] = np.flipud(np.asarray(rgba_img)) else: view = None # Return the 2D version diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index b172df2d4..385efaa1f 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -74,8 +74,6 @@ def __init__(self, instrument): # Get the relevant database tables self.identify_tables() - #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') - self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument]) if self.instrument == 'miri': self.detectors = ['MIRIMAGE'] @@ -128,7 +126,6 @@ def modify_bokeh_saved_html(self): self._html = "".join(newlines) - def run(self): # Right now, the aperture name in the query history table is used as the title of the @@ -163,7 +160,7 @@ def run(self): # Insert into our html template and save template_dir = os.path.join(os.path.dirname(__file__), '../templates') template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html') - temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} + temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div': div} self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) # Modify the html such that our Django-related lines are kept in place, @@ -264,7 +261,6 @@ def __init__(self, pixel_table, instrument, detector): for badtype in self.badtypes: self.get_trending_data(badtype) - def get_most_recent_entry(self): """Get all nedded data from the database tables. """ @@ -312,7 +308,7 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ .all() # Organize the results @@ -463,7 +459,7 @@ def create_plot(self): self.plot = figure(title=title_text, tools=tools, x_axis_label="Pixel Number", y_axis_label="Pixel Number") else: - self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot = figure(tools='') self.plot.toolbar.logo = None self.plot.toolbar_location = None self.plot.min_border = 0 @@ -475,18 +471,9 @@ def create_plot(self): # Plot image if image is not None: ny, nx = image.shape - #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - - - # Shift the figure title slightly right in this case to get it # to align with the axes - #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*, - # tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5) - - - else: # If the background image is not present, manually set the x and y range self.plot.x_range.start = 0 @@ -511,11 +498,10 @@ def create_plot(self): legend = Legend(items=[plot_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') - def overplot_bad_pix(self): """Add a scatter plot of potential new bad pixels to the plot @@ -537,7 +523,7 @@ def overplot_bad_pix(self): # If there are no new bad pixels, write text within the figure mentioning that txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2], text=[f'No new {self.badpix_type} pixels found'])) - glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value': '20px'}) self.plot.add_glyph(txt_source, glyph) # Insert a fake one, in order to get the plot to be made @@ -668,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") @@ -696,7 +682,7 @@ def create_plot(self): time_pad = datetime.timedelta(days=1) self.plot.x_range.start = min(self.time) - time_pad self.plot.x_range.end = max(self.time) + time_pad - self.plot.grid.grid_line_color="white" + self.plot.grid.grid_line_color = "white" self.plot.xaxis.axis_label = 'Date' self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels' From 7c2cd6618ee81e39cb7b9b09723f96f4beca4c51 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 15:03:00 -0400 Subject: [PATCH 09/23] more pep8 --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 4 ++-- jwql/tests/test_utils.py | 6 +++--- jwql/utils/utils.py | 7 +------ .../apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py | 6 +++--- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c8d30c92a..c73dd3543 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -641,8 +641,8 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] - ) - ) + ) + ) # Overplot the bad pixel locations badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 813bb0432..cd50c6a01 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -485,11 +485,11 @@ def test_filesystem_path(): def test_save_png(): """Test that we can create a png file""" - plot = figure(title='test',tools='') - image = np.zeros((200,200)) + plot = figure(title='test', tools='') + image = np.zeros((200, 200)) image[100:105, 100:105] = 1 ny, nx = image.shape - mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) + mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1) imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") save_png(plot, filename='test.png') diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 4b8951717..ca4df70a3 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -169,13 +169,8 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png -<<<<<<< Updated upstream - output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - save_png(plot, filename=output_filename) -======= output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) - export_png(plot, filename=output_filename) ->>>>>>> Stashed changes + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 385efaa1f..5e522f7cd 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -308,8 +308,8 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ - .all() + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .all() # Organize the results num_pix = [] @@ -654,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") From 6fc7ad384071ed1bdd884c63a6c7ed22aa33174f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 15:04:49 -0400 Subject: [PATCH 10/23] pep8, make up your mind --- jwql/utils/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index ca4df70a3..113dd66c9 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -49,11 +49,11 @@ from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ - FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ - FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ - FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ - FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ - JWST_INSTRUMENT_NAMES_SHORTHAND + FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ + FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ + FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ + FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ + JWST_INSTRUMENT_NAMES_SHORTHAND __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -158,7 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev))) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, From f81de385fa3105aa2470b774014bddcb243d00d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:50:14 +0000 Subject: [PATCH 11/23] Bump scipy from 1.9.3 to 1.10.0 Bumps [scipy](https://github.com/scipy/scipy) from 1.9.3 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.9.3...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f3a5616..4270e9d45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ pyvo==1.4.2 pyyaml==6.0 redis==5.0.0 ruff==0.0.285 -scipy==1.9.3 +scipy==1.10.0 selenium==4.11.2 setuptools==68.0.0 sphinx==6.2.1 From 8c2fd616cdaf51ca280ff0dd77cf0a2dbf922e82 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:07:04 -0400 Subject: [PATCH 12/23] update deps in environment files --- environment_python_3.10.yml | 50 ++++++++++++------------ environment_python_3.9.yml | 46 +++++++++++----------- requirements.txt | 78 ++++++++++++++++++------------------- 3 files changed, 87 insertions(+), 87 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f61973680..7856cf201 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs>=18.15.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - postgresql>=15.2 + - psycopg2>=2.9.3 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - - python=3.10.12 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - python=3.10.13 + - pyyaml=6.0.1 + - redis=5.0.1 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.2 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - redis==5.0.1 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index d3724b1d8..554355df3 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - psycopg2=2.9.7 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.9.17 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - pyyaml=6.0.1 + - redis=5.0.1 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.2 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - redis==5.0.1 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/requirements.txt b/requirements.txt index 4270e9d45..6b8d88d41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,40 @@ -astropy==5.3.2 -astroquery==0.4.6 -bandit==1.7.5 -beautifulsoup4==4.12.2 -bokeh==2.4.3 -celery==5.3.1 -cryptography==41.0.4 -django==4.2.3 -inflection==0.5.1 -ipython==8.14.0 -jinja2==3.1.2 -jsonschema==4.19.0 -jwst==1.11.4 -matplotlib==3.7.2 -nodejs==20.5.1 -numpy==1.25.2 -numpydoc==1.5.0 -pandas==2.0.3 -psycopg2-binary==2.9.7 -pysiaf==0.20.0 -pysqlite3==0.5.1 -pytest==7.4.0 -pytest-cov==4.1.0 -pytest-mock==3.11.1 -pyvo==1.4.2 -pyyaml==6.0 -redis==5.0.0 -ruff==0.0.285 -scipy==1.10.0 -selenium==4.11.2 -setuptools==68.0.0 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 -sqlalchemy==2.0.20 -stdatamodels==1.7.2 -stsci_rtd_theme==1.0.0 -twine==4.0.2 -vine==5.0.0 -wtforms==3.0.1 +astropy +astroquery +bandit +beautifulsoup4 +bokeh +celery +cryptography +django +inflection +ipython +jinja2 +jsonschema +jwst +matplotlib +nodejs +numpy +numpydoc +pandas +psycopg2-binary +pysiaf +pysqlite3 +pytest +pytest-cov +pytest-mock +pyvo +pyyaml +redis +ruff +scipy +selenium +setuptools +sphinx +sphinx_rtd_theme +sqlalchemy +stdatamodels +stsci_rtd_theme +twine +vine +wtforms git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles From c0de0112325fa00644caa9cbe619d25bb63aa670 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:18:34 -0400 Subject: [PATCH 13/23] update versions in requirements.txt --- requirements.txt | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6b8d88d41..bb289de5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,40 @@ -astropy -astroquery -bandit -beautifulsoup4 -bokeh -celery -cryptography -django -inflection -ipython -jinja2 -jsonschema -jwst -matplotlib -nodejs -numpy -numpydoc -pandas -psycopg2-binary -pysiaf -pysqlite3 -pytest -pytest-cov -pytest-mock -pyvo -pyyaml -redis -ruff -scipy -selenium -setuptools -sphinx -sphinx_rtd_theme -sqlalchemy -stdatamodels -stsci_rtd_theme -twine -vine -wtforms +astropy==5.3.3 +astroquery==0.4.6 +bandit==1.7.5 +beautifulsoup4==4.12.2 +bokeh==2.4.3 +celery==5.3.4 +cryptography==41.0.4 +django==4.2.5 +inflection==0.5.1 +ipython==8.16.1 +jinja2==3.1.2 +jsonschema==4.19.1 +jwst==1.12.2 +matplotlib==3.8.0 +nodejs==20.8.0 +numpy==1.25.2 +numpydoc==1.5.0 +pandas==2.1.1 +psycopg2-binary==2.9.7 +pysiaf==0.20.0 +pysqlite3==0.5.2 +pytest==7.4.2 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +pyvo==1.4.2 +pyyaml==6.0.1 +redis==5.0.1 +ruff==0.0.292 +scipy==1.9.3 +selenium==4.13.0 +setuptools==68.2.2 +sphinx==7.2.6 +sphinx_rtd_theme==1.3.0 +sqlalchemy==2.0.21 +stdatamodels==1.8.3 +stsci_rtd_theme==1.0.0 +twine==4.0.2 +vine==5.0.0 +wtforms==3.0.1 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles From 8ae4da424b1cf99c8a6e20cabe504c938285c228 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:21:14 -0400 Subject: [PATCH 14/23] put redis back to 5.0.0 --- environment_python_3.10.yml | 2 +- jwql/utils/logging_functions.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 7856cf201..505bdb7c0 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -45,7 +45,7 @@ dependencies: - pytest-mock=3.11.1 - python=3.10.13 - pyyaml=6.0.1 - - redis=5.0.1 + - redis=5.0.0 - ruff=0.0.292 - scipy=1.9.3 - setuptools=68.2.2 diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index abd82a45e..3ef464a40 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -106,7 +106,7 @@ def configure_logging(module): logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S %p', - level=logging.INFO) + level=logging.DEBUG) print('Log file initialized to {}'.format(log_file)) set_permissions(log_file) diff --git a/requirements.txt b/requirements.txt index bb289de5a..d82a3409f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0.1 -redis==5.0.1 +redis==5.0.0. ruff==0.0.292 scipy==1.9.3 selenium==4.13.0 From 9ef1cd9356335913eab0562282cad68e16f1f579 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:23:10 -0400 Subject: [PATCH 15/23] whoops, accidental logging change --- jwql/utils/logging_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index 3ef464a40..abd82a45e 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -106,7 +106,7 @@ def configure_logging(module): logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S %p', - level=logging.DEBUG) + level=logging.INFO) print('Log file initialized to {}'.format(log_file)) set_permissions(log_file) From b4e4c38e856adfa88e82483dcfc4f81b6e5bbb52 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 11:04:47 -0400 Subject: [PATCH 16/23] revert all redis references to 5.0.0 --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 505bdb7c0..518fbc902 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -62,7 +62,7 @@ dependencies: - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.1 + - redis==5.0.0 - selenium==4.13.0 - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 554355df3..5b70010a8 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -45,7 +45,7 @@ dependencies: - pytest-mock=3.11.1 - python=3.9.17 - pyyaml=6.0.1 - - redis=5.0.1 + - redis=5.0.0 - ruff=0.0.292 - scipy=1.9.3 - setuptools=68.2.2 @@ -62,7 +62,7 @@ dependencies: - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.1 + - redis==5.0.0 - selenium==4.13.0 - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 diff --git a/requirements.txt b/requirements.txt index d82a3409f..01aeb4f7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0.1 -redis==5.0.0. +redis==5.0.0 ruff==0.0.292 scipy==1.9.3 selenium==4.13.0 From b874639b0e45e0a67b8813a9b36a4d12f02af7a1 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 12:51:46 -0400 Subject: [PATCH 17/23] mamba didn't like 3.10.13 --- environment_python_3.10.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 518fbc902..f60c31d27 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -43,7 +43,7 @@ dependencies: - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - - python=3.10.13 + - python=3.10.12 - pyyaml=6.0.1 - redis=5.0.0 - ruff=0.0.292 From 6edf1262793467580a131f5c745682e980d8cecb Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 15:32:45 -0400 Subject: [PATCH 18/23] update for 3.10.12 --- environment_python_3.10.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f60c31d27..fd927ce30 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,24 +22,24 @@ channels: - defaults dependencies: - - astropy=5.3.3 + - astropy=5.3.4 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - celery=5.3.4 - cryptography=41.0.4 - - django=4.2.5 + - django=4.2.6 - inflection=0.5.1 - ipython=8.16.1 - jinja2=3.1.2 - jsonschema=4.19.1 - matplotlib=3.8.0 - - nodejs>=18.15.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.1.1 - pip=23.2.1 - - postgresql>=15.2 - - psycopg2>=2.9.3 + - postgresql=15.4 + - psycopg2=2.9.7 - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 From 91feef45fdbf47a9e994c28ef8b3921ecae7d68f Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:29:43 -0400 Subject: [PATCH 19/23] update rtd_requirements.txt --- rtd_requirements.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 39500c2b6..f4f42830e 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,16 +1,16 @@ -sphinx_automodapi==0.15.0 +sphinx_automodapi>=0.15.0 bokeh==2.4.3 -celery==5.3.1 -cython==3.0.0 -django==4.2.3 -docutils==0.18.1 -jwst==1.11.4 +celery==5.3.4 +cython>=3.0.0 +django==4.2.5 +docutils>=0.18.1 +jwst==1.12.3 pygments==2.16.1 -pytest==7.4.0 +pytest==7.4.2 redis==5.0.0 -selenium==4.11.2 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 +selenium==4.13.0 +sphinx==7.2.6 +sphinx_rtd_theme==1.30 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From 059310ac90171af382c1c91b130861e20cdb0c6d Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:35:59 -0400 Subject: [PATCH 20/23] jwst changed out from under me --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index fd927ce30..9f2ed409b 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -58,7 +58,7 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.12.2 + - jwst==1.12.3 - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 5b70010a8..4f4159244 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -58,7 +58,7 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.12.2 + - jwst==1.12.3 - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 diff --git a/requirements.txt b/requirements.txt index 01aeb4f7f..6a7f670cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ inflection==0.5.1 ipython==8.16.1 jinja2==3.1.2 jsonschema==4.19.1 -jwst==1.12.2 +jwst==1.12.3 matplotlib==3.8.0 nodejs==20.8.0 numpy==1.25.2 From fdfd4b674a9a6db8468fe16e72a235e537894079 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:40:32 -0400 Subject: [PATCH 21/23] update setuptools version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c17d3dc28..133af8c79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] From f365329f7023f0d18596e59ee6f1979d76d51f31 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:51:14 -0400 Subject: [PATCH 22/23] setuptools greater than --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 133af8c79..24f701717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools>=68.0.0", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] From f7ecd691e9034e3828a21406ebe00372721ec09a Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:54:54 -0400 Subject: [PATCH 23/23] no thank you sphinx_theme. stsci_theme 4ever --- rtd_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index f4f42830e..abac6b128 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -10,7 +10,6 @@ pytest==7.4.2 redis==5.0.0 selenium==4.13.0 sphinx==7.2.6 -sphinx_rtd_theme==1.30 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles