From 53dd4705c71b1a6d6d37a405797b1d1960aa6e61 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Wed, 3 Oct 2018 16:27:09 -0400
Subject: [PATCH 01/33] Started adding anomaly form to image view.
---
jwql/jwql_monitors/generate_preview_images.py | 12 ++++----
jwql/utils/preview_image.py | 10 +++----
jwql/website/apps/jwql/static/css/jwql.css | 6 ++++
.../apps/jwql/templates/view_image.html | 30 +++++++++++++------
jwql/website/apps/jwql/views.py | 3 +-
5 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py
index be78072fa..60c7ee207 100755
--- a/jwql/jwql_monitors/generate_preview_images.py
+++ b/jwql/jwql_monitors/generate_preview_images.py
@@ -322,7 +322,7 @@ def create_mosaic(filenames):
elif datadim == 3:
full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan
else:
- raise ValueError((f'Difference image for {filenames[0]} must be either 2D or 3D.'))
+ raise ValueError('Difference image for {} must be either 2D or 3D.'.format(filenames[0]))
# Place the data from the individual detectors in the appropriate
# places in the final image
@@ -538,11 +538,11 @@ def generate_preview_images():
if not os.path.exists(preview_output_directory):
os.makedirs(preview_output_directory)
permissions.set_permissions(preview_output_directory)
- logging.info(f'Created directory {preview_output_directory}')
+ logging.info('Created directory {}'.format(preview_output_directory))
if not os.path.exists(thumbnail_output_directory):
os.makedirs(thumbnail_output_directory)
permissions.set_permissions(thumbnail_output_directory)
- logging.info(f'Created directory {thumbnail_output_directory}')
+ logging.info('Created directory {}'.format(thumbnail_output_directory))
# If the exposure contains more than one file (because more
# than one detector was used), then create a mosaic
@@ -553,7 +553,7 @@ def generate_preview_images():
mosaic_image, mosaic_dq = create_mosaic(file_list)
logging.info('Created mosiac for:')
for item in file_list:
- logging.info(f'\t{item}')
+ logging.info('\t{}'.format(item))
except (ValueError, FileNotFoundError) as error:
logging.error(error)
dummy_file = create_dummy_filename(file_list)
@@ -636,7 +636,7 @@ def group_filenames(input_files):
detector = filename_parts['detector'].upper()
suffix = filename_parts['suffix']
- observation_base = f'jw{program}{observation}{visit}_{visit_group}{parallel}{activity}_{exposure}_'
+ observation_base = 'jw{}{}{}_{}{}{}_{}_'.format(program, observation, visit, visit_group, parallel, activity, exposure)
if detector in NIRCAM_SHORTWAVE_DETECTORS:
detector_str = 'NRC[AB][1234]'
@@ -644,7 +644,7 @@ def group_filenames(input_files):
detector_str = 'NRC[AB]5'
else: # non-NIRCam detectors - should never be used I think??
detector_str = detector
- match_str = f'{observation_base}{detector_str}_{suffix}.fits'
+ match_str = '{}{}_{}.fits'.format(observation_base, detector_str, suffix)
match_str = os.path.join(file_directory, match_str)
pattern = re.compile(match_str, re.IGNORECASE)
diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py
index f51a5bc8e..39a640d50 100755
--- a/jwql/utils/preview_image.py
+++ b/jwql/utils/preview_image.py
@@ -200,7 +200,7 @@ def get_data(self, filename, ext):
else:
data = hdulist[ext].data.astype(np.float)
else:
- raise ValueError((f'WARNING: no {ext} extension in {filename}!'))
+ raise ValueError('WARNING: no {} extension in {}!'.format(ext, filename))
if 'PIXELDQ' in extnames:
dq = hdulist['PIXELDQ'].data
dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0)
@@ -216,7 +216,7 @@ def get_data(self, filename, ext):
self.xlen = hdulist[0].header['SUBSIZE1']
self.ylen = hdulist[0].header['SUBSIZE2']
else:
- raise FileNotFoundError((f'WARNING: {filename} does not exist!'))
+ raise FileNotFoundError('WARNING: {} does not exist!'.format(filename))
return data, dq
@@ -257,7 +257,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
# Check the input scaling
if scale not in ['linear', 'log']:
- raise ValueError((f'WARNING: scaling option {scale} not supported.'))
+ raise ValueError('WARNING: scaling option {} not supported.'.format(scale))
# Set the figure size
yd, xd = image.shape
@@ -417,6 +417,6 @@ def save_image(self, fname, thumbnail=False):
if thumbnail:
thumb_fname = fname.replace('.jpg', '.thumb')
os.rename(fname, thumb_fname)
- logging.info(f'Saved image to {thumb_fname}')
+ logging.info('Saved image to {}'.format(thumb_fname))
else:
- logging.info(f'Saved image to {fname}')
+ logging.info('Saved image to {}'.format(fname))
diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css
index b2f109bb9..9c8748877 100644
--- a/jwql/website/apps/jwql/static/css/jwql.css
+++ b/jwql/website/apps/jwql/static/css/jwql.css
@@ -373,3 +373,9 @@ li:hover .nav-link, .navbar-brand:hover {
li.dropdown:hover .dropdown-menu {
display: block;
}
+
+ul.no-bullets {
+ list-style: none;
+ padding-left:10px;
+ line-height:25px;
+}
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 2359726c2..9e6266e8a 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -137,20 +137,32 @@ {{ file_root }}
-
+
Download FITS
Download JPEG
- Submit Anomaly
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index b778658b2..fc0964a54 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -280,6 +280,7 @@ def view_image(request, inst, file_root, rewrite=False):
'jpg_files': image_info['all_jpegs'],
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
- 'num_ints': image_info['num_ints']}
+ 'num_ints': image_info['num_ints'],
+ 'anomalies': ['foo','bar','baz']}
return render(request, template, context)
From f24bbb68e39c30ed0cab6d446005d88f64d90e33 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Wed, 3 Oct 2018 16:27:09 -0400
Subject: [PATCH 02/33] Started adding anomaly form to image view.
---
jwql/jwql_monitors/generate_preview_images.py | 12 ++++----
jwql/utils/preview_image.py | 10 +++----
jwql/website/apps/jwql/static/css/jwql.css | 6 ++++
.../apps/jwql/templates/view_image.html | 30 +++++++++++++------
jwql/website/apps/jwql/views.py | 3 +-
5 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py
index be78072fa..60c7ee207 100755
--- a/jwql/jwql_monitors/generate_preview_images.py
+++ b/jwql/jwql_monitors/generate_preview_images.py
@@ -322,7 +322,7 @@ def create_mosaic(filenames):
elif datadim == 3:
full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan
else:
- raise ValueError((f'Difference image for {filenames[0]} must be either 2D or 3D.'))
+ raise ValueError('Difference image for {} must be either 2D or 3D.'.format(filenames[0]))
# Place the data from the individual detectors in the appropriate
# places in the final image
@@ -538,11 +538,11 @@ def generate_preview_images():
if not os.path.exists(preview_output_directory):
os.makedirs(preview_output_directory)
permissions.set_permissions(preview_output_directory)
- logging.info(f'Created directory {preview_output_directory}')
+ logging.info('Created directory {}'.format(preview_output_directory))
if not os.path.exists(thumbnail_output_directory):
os.makedirs(thumbnail_output_directory)
permissions.set_permissions(thumbnail_output_directory)
- logging.info(f'Created directory {thumbnail_output_directory}')
+ logging.info('Created directory {}'.format(thumbnail_output_directory))
# If the exposure contains more than one file (because more
# than one detector was used), then create a mosaic
@@ -553,7 +553,7 @@ def generate_preview_images():
mosaic_image, mosaic_dq = create_mosaic(file_list)
logging.info('Created mosiac for:')
for item in file_list:
- logging.info(f'\t{item}')
+ logging.info('\t{}'.format(item))
except (ValueError, FileNotFoundError) as error:
logging.error(error)
dummy_file = create_dummy_filename(file_list)
@@ -636,7 +636,7 @@ def group_filenames(input_files):
detector = filename_parts['detector'].upper()
suffix = filename_parts['suffix']
- observation_base = f'jw{program}{observation}{visit}_{visit_group}{parallel}{activity}_{exposure}_'
+ observation_base = 'jw{}{}{}_{}{}{}_{}_'.format(program, observation, visit, visit_group, parallel, activity, exposure)
if detector in NIRCAM_SHORTWAVE_DETECTORS:
detector_str = 'NRC[AB][1234]'
@@ -644,7 +644,7 @@ def group_filenames(input_files):
detector_str = 'NRC[AB]5'
else: # non-NIRCam detectors - should never be used I think??
detector_str = detector
- match_str = f'{observation_base}{detector_str}_{suffix}.fits'
+ match_str = '{}{}_{}.fits'.format(observation_base, detector_str, suffix)
match_str = os.path.join(file_directory, match_str)
pattern = re.compile(match_str, re.IGNORECASE)
diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py
index f51a5bc8e..39a640d50 100755
--- a/jwql/utils/preview_image.py
+++ b/jwql/utils/preview_image.py
@@ -200,7 +200,7 @@ def get_data(self, filename, ext):
else:
data = hdulist[ext].data.astype(np.float)
else:
- raise ValueError((f'WARNING: no {ext} extension in {filename}!'))
+ raise ValueError('WARNING: no {} extension in {}!'.format(ext, filename))
if 'PIXELDQ' in extnames:
dq = hdulist['PIXELDQ'].data
dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0)
@@ -216,7 +216,7 @@ def get_data(self, filename, ext):
self.xlen = hdulist[0].header['SUBSIZE1']
self.ylen = hdulist[0].header['SUBSIZE2']
else:
- raise FileNotFoundError((f'WARNING: {filename} does not exist!'))
+ raise FileNotFoundError('WARNING: {} does not exist!'.format(filename))
return data, dq
@@ -257,7 +257,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
# Check the input scaling
if scale not in ['linear', 'log']:
- raise ValueError((f'WARNING: scaling option {scale} not supported.'))
+ raise ValueError('WARNING: scaling option {} not supported.'.format(scale))
# Set the figure size
yd, xd = image.shape
@@ -417,6 +417,6 @@ def save_image(self, fname, thumbnail=False):
if thumbnail:
thumb_fname = fname.replace('.jpg', '.thumb')
os.rename(fname, thumb_fname)
- logging.info(f'Saved image to {thumb_fname}')
+ logging.info('Saved image to {}'.format(thumb_fname))
else:
- logging.info(f'Saved image to {fname}')
+ logging.info('Saved image to {}'.format(fname))
diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css
index b2f109bb9..9c8748877 100644
--- a/jwql/website/apps/jwql/static/css/jwql.css
+++ b/jwql/website/apps/jwql/static/css/jwql.css
@@ -373,3 +373,9 @@ li:hover .nav-link, .navbar-brand:hover {
li.dropdown:hover .dropdown-menu {
display: block;
}
+
+ul.no-bullets {
+ list-style: none;
+ padding-left:10px;
+ line-height:25px;
+}
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 2359726c2..9e6266e8a 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -137,20 +137,32 @@ {{ file_root }}
-
+
Download FITS
Download JPEG
- Submit Anomaly
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index b778658b2..fc0964a54 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -280,6 +280,7 @@ def view_image(request, inst, file_root, rewrite=False):
'jpg_files': image_info['all_jpegs'],
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
- 'num_ints': image_info['num_ints']}
+ 'num_ints': image_info['num_ints'],
+ 'anomalies': ['foo','bar','baz']}
return render(request, template, context)
From 675bf9c21329fde2f9ed7c017fa4a1b06449d982 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Fri, 5 Oct 2018 11:15:27 -0400
Subject: [PATCH 03/33] Added list of anomalies to image view.
---
jwql/database/database_interface.py | 4 ++--
jwql/website/apps/jwql/templates/view_image.html | 2 +-
jwql/website/apps/jwql/views.py | 3 ++-
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index dc80873ed..18e2532eb 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -147,8 +147,8 @@ def __repr__(self):
def colnames(self):
"""A list of all the column names in this table"""
# Get the columns
- a_list = [col for col, val in self.__dict__.items()
- if isinstance(val, bool)]
+ a_list = [col for col, val in self._sa_instance_state.attrs.items()
+ if col not in ['id', 'filename', 'flag_date']]
return a_list
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 9e6266e8a..2201df8ea 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -155,7 +155,7 @@ Anomalies in this image:
{{ anom }}
{% endfor %}
- Submit Anomalies
+ Submit Anomalies
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index fc0964a54..466883112 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -47,6 +47,7 @@
from .data_containers import get_proposal_info
from .data_containers import thumbnails
from jwql.utils.utils import get_config, JWST_INSTRUMENTS, MONITORS
+from jwql.database import database_interface as di
FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')
@@ -281,6 +282,6 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': ['foo','bar','baz']}
+ 'anomalies': di.Anomaly().colnames}
return render(request, template, context)
From 0203ba055f4592e04f3ea0639293c691dfa808fd Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Fri, 5 Oct 2018 11:15:27 -0400
Subject: [PATCH 04/33] Added list of anomalies to image view.
---
jwql/database/database_interface.py | 4 ++--
jwql/website/apps/jwql/templates/view_image.html | 2 +-
jwql/website/apps/jwql/views.py | 3 ++-
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index dc80873ed..18e2532eb 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -147,8 +147,8 @@ def __repr__(self):
def colnames(self):
"""A list of all the column names in this table"""
# Get the columns
- a_list = [col for col, val in self.__dict__.items()
- if isinstance(val, bool)]
+ a_list = [col for col, val in self._sa_instance_state.attrs.items()
+ if col not in ['id', 'filename', 'flag_date']]
return a_list
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 9e6266e8a..2201df8ea 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -155,7 +155,7 @@ Anomalies in this image:
{{ anom }}
{% endfor %}
- Submit Anomalies
+ Submit Anomalies
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index fc0964a54..466883112 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -47,6 +47,7 @@
from .data_containers import get_proposal_info
from .data_containers import thumbnails
from jwql.utils.utils import get_config, JWST_INSTRUMENTS, MONITORS
+from jwql.database import database_interface as di
FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')
@@ -281,6 +282,6 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': ['foo','bar','baz']}
+ 'anomalies': di.Anomaly().colnames}
return render(request, template, context)
From 3866b896b0076d69965621c6a477d7eb198f6373 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Fri, 5 Oct 2018 15:27:25 -0400
Subject: [PATCH 05/33] Added previous anomaly tags to image preview.
---
jwql/website/apps/jwql/templates/view_image.html | 1 +
jwql/website/apps/jwql/views.py | 11 ++++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 2201df8ea..16d4fb0f5 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -150,6 +150,7 @@ {{ file_root }}
Anomalies in this image:
+
Previously flagged: {{ prev_anom }}
{% for anom in anomalies %}
- {{ anom }}
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 466883112..726341093 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -275,6 +275,14 @@ def view_image(request, inst, file_root, rewrite=False):
"""
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
+
+ # Get a list of previously flagged anomalies
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
+ all_records = query.data_frame()
+ prev_anom = ', '.join([col for col, val in
+ np.sum(all_records, axis=0).items() if val])
+
+ # Build the context
context = {'inst': inst,
'file_root': file_root,
'tools': MONITORS,
@@ -282,6 +290,7 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': di.Anomaly().colnames}
+ 'anomalies': di.Anomaly().colnames,
+ 'prev_anom': prev_anom}
return render(request, template, context)
From db78ae406b6d9e73a2420bda8ddb64e1e3cebde8 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Fri, 5 Oct 2018 15:27:25 -0400
Subject: [PATCH 06/33] Added previous anomaly tags to image preview.
---
jwql/website/apps/jwql/templates/view_image.html | 1 +
jwql/website/apps/jwql/views.py | 11 ++++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 2201df8ea..16d4fb0f5 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -150,6 +150,7 @@ {{ file_root }}
Anomalies in this image:
+
Previously flagged: {{ prev_anom }}
{% for anom in anomalies %}
- {{ anom }}
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 466883112..726341093 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -275,6 +275,14 @@ def view_image(request, inst, file_root, rewrite=False):
"""
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
+
+ # Get a list of previously flagged anomalies
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
+ all_records = query.data_frame()
+ prev_anom = ', '.join([col for col, val in
+ np.sum(all_records, axis=0).items() if val])
+
+ # Build the context
context = {'inst': inst,
'file_root': file_root,
'tools': MONITORS,
@@ -282,6 +290,7 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': di.Anomaly().colnames}
+ 'anomalies': di.Anomaly().colnames,
+ 'prev_anom': prev_anom}
return render(request, template, context)
From d25450d04446edd69ea56db569519625b256b248 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Wed, 14 Nov 2018 11:55:19 -0500
Subject: [PATCH 07/33] Testing anomaly view with preloaded checkboxes.
---
jwql/website/apps/jwql/templates/view_image.html | 7 +++++--
jwql/website/apps/jwql/views.py | 10 ++++++----
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 16d4fb0f5..86bb6a532 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -150,10 +150,13 @@ {{ file_root }}
Anomalies in this image:
-
Previously flagged: {{ prev_anom }}
Submit Anomalies
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 726341093..ae1a07e1a 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -277,10 +277,12 @@ def view_image(request, inst, file_root, rewrite=False):
image_info = get_image_info(file_root, rewrite)
# Get a list of previously flagged anomalies
- query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
- all_records = query.data_frame()
- prev_anom = ', '.join([col for col, val in
- np.sum(all_records, axis=0).items() if val])
+ prev_anom = None
+ # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
+ # all_records = query.data_frame()
+ # if not all_records.empty:
+ # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
context = {'inst': inst,
From 1130a8d2897e5debf68a0d939da0a3453638e570 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Wed, 14 Nov 2018 11:55:19 -0500
Subject: [PATCH 08/33] Testing anomaly view with preloaded checkboxes.
---
jwql/website/apps/jwql/templates/view_image.html | 7 +++++--
jwql/website/apps/jwql/views.py | 10 ++++++----
2 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 16d4fb0f5..86bb6a532 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -150,10 +150,13 @@ {{ file_root }}
Anomalies in this image:
-
Previously flagged: {{ prev_anom }}
Submit Anomalies
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 726341093..ae1a07e1a 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -277,10 +277,12 @@ def view_image(request, inst, file_root, rewrite=False):
image_info = get_image_info(file_root, rewrite)
# Get a list of previously flagged anomalies
- query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
- all_records = query.data_frame()
- prev_anom = ', '.join([col for col, val in
- np.sum(all_records, axis=0).items() if val])
+ prev_anom = None
+ # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
+ # all_records = query.data_frame()
+ # if not all_records.empty:
+ # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
context = {'inst': inst,
From 65d9c9b77cbf599747c3ae23b6b05b9241afab90 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Tue, 18 Dec 2018 14:59:55 -0500
Subject: [PATCH 09/33] Fixed anomaly ORM to load dynamically, use memory db
for testing, and write tests.
---
jwql/database/database_interface.py | 110 +++++++++---------
jwql/tests/test_database_interface.py | 49 ++++++++
.../apps/jwql/templates/view_image.html | 6 +-
jwql/website/apps/jwql/views.py | 14 ++-
4 files changed, 112 insertions(+), 67 deletions(-)
create mode 100755 jwql/tests/test_database_interface.py
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 18e2532eb..751aace93 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -27,14 +27,10 @@
from datetime import datetime
import pandas as pd
-from sqlalchemy import Boolean
-from sqlalchemy import Column
+from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, String, Table
from sqlalchemy import create_engine
-from sqlalchemy import DateTime
-from sqlalchemy import Integer
-from sqlalchemy import MetaData
-from sqlalchemy import String
from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.query import Query
@@ -92,65 +88,63 @@ def load_connection(connection_string):
base = declarative_base(engine)
Session = sessionmaker(bind=engine)
session = Session()
- meta = MetaData()
+ meta = MetaData(engine)
+
+ # Make sure it has an anomalies table
+ if not engine.has_table('anomalies'):
+ print("No 'anomalies' table. Generating one now...")
+
+ # Define anomaly table column names
+ columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
+ 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
+ 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
+ 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
+ 'fringing', 'guidestar_failure', 'banding', 'persistence',
+ 'prominent_blobs', 'trail', 'scattered_light', 'other']
+
+ # Create a table with the appropriate Columns
+ anomalies = Table('anomalies', meta,
+ Column('id', Integer, primary_key=True, nullable=False),
+ Column('filename', String, nullable=False),
+ Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())),
+ *[Column(name, String, nullable=False, server_default="False") for name in columns])
+
+ # Implement it
+ meta.create_all()
return session, base, engine, meta
+# Load the objects for the database
session, base, engine, meta = load_connection(SETTINGS['connection_string'])
-class Anomaly(base):
- """ORM for the anomalies table"""
- # Name the table
- __tablename__ = 'anomalies'
-
- # Define the columns
- id = Column(Integer, primary_key=True, nullable=False)
- filename = Column(String, nullable=False)
- flag_date = Column(DateTime, nullable=False, default=datetime.now())
- bowtie = Column(Boolean, nullable=False, default=False)
- snowball = Column(Boolean, nullable=False, default=False)
- cosmic_ray_shower = Column(Boolean, nullable=False, default=False)
- crosstalk = Column(Boolean, nullable=False, default=False)
- cte_correction_error = Column(Boolean, nullable=False, default=False)
- data_transfer_error = Column(Boolean, nullable=False, default=False)
- detector_ghost = Column(Boolean, nullable=False, default=False)
- diamond = Column(Boolean, nullable=False, default=False)
- diffraction_spike = Column(Boolean, nullable=False, default=False)
- dragon_breath = Column(Boolean, nullable=False, default=False)
- earth_limb = Column(Boolean, nullable=False, default=False)
- excessive_saturation = Column(Boolean, nullable=False, default=False)
- figure8_ghost = Column(Boolean, nullable=False, default=False)
- filter_ghost = Column(Boolean, nullable=False, default=False)
- fringing = Column(Boolean, nullable=False, default=False)
- guidestar_failure = Column(Boolean, nullable=False, default=False)
- banding = Column(Boolean, nullable=False, default=False)
- persistence = Column(Boolean, nullable=False, default=False)
- prominent_blobs = Column(Boolean, nullable=False, default=False)
- trail = Column(Boolean, nullable=False, default=False)
- scattered_light = Column(Boolean, nullable=False, default=False)
- other = Column(Boolean, nullable=False, default=False)
-
- def __repr__(self):
- """Return the canonical string representation of the object"""
- # Get the columns that are True
- a_list = [col for col, val in self.__dict__.items()
- if val is True and isinstance(val, bool)]
-
- txt = ('Anomaly {0.id}: {0.filename} flagged at '
- '{0.flag_date} for {1}').format(self, a_list)
-
- return txt
-
- @property
- def colnames(self):
- """A list of all the column names in this table"""
- # Get the columns
- a_list = [col for col, val in self._sa_instance_state.attrs.items()
- if col not in ['id', 'filename', 'flag_date']]
-
- return a_list
+# Make convenience methods for Base class
+@property
+def colnames(self):
+ """A list of all the column names in this table"""
+ # Get the columns
+ a_list = sorted([col for col, val in self._sa_instance_state.attrs.items()
+ if col not in ['id', 'filename', 'flag_date']])
+
+ return a_list
+
+
+@property
+def names(self):
+ """A list of human readable names for all the columns in this table"""
+ return [name.replace('_', ' ') for name in self.colnames]
+
+
+# Generate Base class and add methods
+Base = automap_base()
+Base.colnames = colnames
+Base.names = names
+Base.prepare(engine, reflect=True)
+
+
+# Automap Anomaly class from Base class for anomalies table
+Anomaly = Base.classes.anomalies
if __name__ == '__main__':
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
new file mode 100755
index 000000000..a1ea4a1b8
--- /dev/null
+++ b/jwql/tests/test_database_interface.py
@@ -0,0 +1,49 @@
+#! /usr/bin/env python
+
+"""Tests for the ``database_interface.py`` module.
+
+Authors
+-------
+
+ Joe Filippazzo
+
+Use
+---
+
+ These tests can be run via the command line (omit the ``-s`` to
+ suppress verbose output to stdout):
+ ::
+
+ pytest -s database_interface.py
+"""
+
+from jwql.database import database_interface as di
+
+
+def test_anomaly_table():
+ """Test to see that the database has an anomalies table"""
+ assert 'anomalies' in di.engine.table_names()
+
+
+def test_anomaly_records():
+ """Test to see that new records can be entered"""
+ # Add some data
+ di.session.add(di.Anomaly(filename='foo1', bowtie="True"))
+ di.session.commit()
+
+ # Test the bowties column
+ bowties = di.session.query(di.Anomaly).filter(di.Anomaly.bowtie == "True")
+ assert bowties.data_frame.iloc[0]['bowtie'] == "True"
+
+ # Test the other columns
+ non_bowties = [col for col in di.Anomaly().colnames if col != 'bowtie']
+ assert all([i == "False" for i in bowties.data_frame.iloc[0][non_bowties]])
+
+
+def test_names_colnames():
+ """Test that the column names are correct"""
+ # Make sure we get non-empty lists
+ anom = di.Anomaly()
+ assert isinstance(anom.colnames, list) and len(anom.colnames) > 0
+ assert isinstance(anom.names, list) and len(anom.names) == len(anom.colnames)
+ assert all([i == j.replace('_', ' ') for i, j in zip(anom.names, anom.colnames)])
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 86bb6a532..cdde71f6e 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -151,11 +151,11 @@ {{ file_root }}
Anomalies in this image:
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index ae1a07e1a..38c1f81bf 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -278,13 +278,15 @@ def view_image(request, inst, file_root, rewrite=False):
# Get a list of previously flagged anomalies
prev_anom = None
- # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
- # all_records = query.data_frame()
- # if not all_records.empty:
- # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- prev_anom = ', '.join(['bowtie', 'snowball'])
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ all_records = query.data_frame()
+ print(all_records)
+ if not all_records.empty:
+ prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ # prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
+ anom = di.Anomaly()
context = {'inst': inst,
'file_root': file_root,
'tools': MONITORS,
@@ -292,7 +294,7 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': di.Anomaly().colnames,
+ 'anomalies': zip(anom.colnames, anom.names),
'prev_anom': prev_anom}
return render(request, template, context)
From 4676f9ec44adffba307e08756ea612213c255cfa Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Tue, 18 Dec 2018 14:59:55 -0500
Subject: [PATCH 10/33] Fixed anomaly ORM to load dynamically, use memory db
for testing, and write tests.
---
jwql/database/database_interface.py | 110 +++++++++---------
jwql/tests/test_database_interface.py | 49 ++++++++
.../apps/jwql/templates/view_image.html | 6 +-
jwql/website/apps/jwql/views.py | 14 ++-
4 files changed, 112 insertions(+), 67 deletions(-)
create mode 100755 jwql/tests/test_database_interface.py
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 18e2532eb..751aace93 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -27,14 +27,10 @@
from datetime import datetime
import pandas as pd
-from sqlalchemy import Boolean
-from sqlalchemy import Column
+from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, String, Table
from sqlalchemy import create_engine
-from sqlalchemy import DateTime
-from sqlalchemy import Integer
-from sqlalchemy import MetaData
-from sqlalchemy import String
from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.query import Query
@@ -92,65 +88,63 @@ def load_connection(connection_string):
base = declarative_base(engine)
Session = sessionmaker(bind=engine)
session = Session()
- meta = MetaData()
+ meta = MetaData(engine)
+
+ # Make sure it has an anomalies table
+ if not engine.has_table('anomalies'):
+ print("No 'anomalies' table. Generating one now...")
+
+ # Define anomaly table column names
+ columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
+ 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
+ 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
+ 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
+ 'fringing', 'guidestar_failure', 'banding', 'persistence',
+ 'prominent_blobs', 'trail', 'scattered_light', 'other']
+
+ # Create a table with the appropriate Columns
+ anomalies = Table('anomalies', meta,
+ Column('id', Integer, primary_key=True, nullable=False),
+ Column('filename', String, nullable=False),
+ Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())),
+ *[Column(name, String, nullable=False, server_default="False") for name in columns])
+
+ # Implement it
+ meta.create_all()
return session, base, engine, meta
+# Load the objects for the database
session, base, engine, meta = load_connection(SETTINGS['connection_string'])
-class Anomaly(base):
- """ORM for the anomalies table"""
- # Name the table
- __tablename__ = 'anomalies'
-
- # Define the columns
- id = Column(Integer, primary_key=True, nullable=False)
- filename = Column(String, nullable=False)
- flag_date = Column(DateTime, nullable=False, default=datetime.now())
- bowtie = Column(Boolean, nullable=False, default=False)
- snowball = Column(Boolean, nullable=False, default=False)
- cosmic_ray_shower = Column(Boolean, nullable=False, default=False)
- crosstalk = Column(Boolean, nullable=False, default=False)
- cte_correction_error = Column(Boolean, nullable=False, default=False)
- data_transfer_error = Column(Boolean, nullable=False, default=False)
- detector_ghost = Column(Boolean, nullable=False, default=False)
- diamond = Column(Boolean, nullable=False, default=False)
- diffraction_spike = Column(Boolean, nullable=False, default=False)
- dragon_breath = Column(Boolean, nullable=False, default=False)
- earth_limb = Column(Boolean, nullable=False, default=False)
- excessive_saturation = Column(Boolean, nullable=False, default=False)
- figure8_ghost = Column(Boolean, nullable=False, default=False)
- filter_ghost = Column(Boolean, nullable=False, default=False)
- fringing = Column(Boolean, nullable=False, default=False)
- guidestar_failure = Column(Boolean, nullable=False, default=False)
- banding = Column(Boolean, nullable=False, default=False)
- persistence = Column(Boolean, nullable=False, default=False)
- prominent_blobs = Column(Boolean, nullable=False, default=False)
- trail = Column(Boolean, nullable=False, default=False)
- scattered_light = Column(Boolean, nullable=False, default=False)
- other = Column(Boolean, nullable=False, default=False)
-
- def __repr__(self):
- """Return the canonical string representation of the object"""
- # Get the columns that are True
- a_list = [col for col, val in self.__dict__.items()
- if val is True and isinstance(val, bool)]
-
- txt = ('Anomaly {0.id}: {0.filename} flagged at '
- '{0.flag_date} for {1}').format(self, a_list)
-
- return txt
-
- @property
- def colnames(self):
- """A list of all the column names in this table"""
- # Get the columns
- a_list = [col for col, val in self._sa_instance_state.attrs.items()
- if col not in ['id', 'filename', 'flag_date']]
-
- return a_list
+# Make convenience methods for Base class
+@property
+def colnames(self):
+ """A list of all the column names in this table"""
+ # Get the columns
+ a_list = sorted([col for col, val in self._sa_instance_state.attrs.items()
+ if col not in ['id', 'filename', 'flag_date']])
+
+ return a_list
+
+
+@property
+def names(self):
+ """A list of human readable names for all the columns in this table"""
+ return [name.replace('_', ' ') for name in self.colnames]
+
+
+# Generate Base class and add methods
+Base = automap_base()
+Base.colnames = colnames
+Base.names = names
+Base.prepare(engine, reflect=True)
+
+
+# Automap Anomaly class from Base class for anomalies table
+Anomaly = Base.classes.anomalies
if __name__ == '__main__':
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
new file mode 100755
index 000000000..a1ea4a1b8
--- /dev/null
+++ b/jwql/tests/test_database_interface.py
@@ -0,0 +1,49 @@
+#! /usr/bin/env python
+
+"""Tests for the ``database_interface.py`` module.
+
+Authors
+-------
+
+ Joe Filippazzo
+
+Use
+---
+
+ These tests can be run via the command line (omit the ``-s`` to
+ suppress verbose output to stdout):
+ ::
+
+ pytest -s database_interface.py
+"""
+
+from jwql.database import database_interface as di
+
+
+def test_anomaly_table():
+ """Test to see that the database has an anomalies table"""
+ assert 'anomalies' in di.engine.table_names()
+
+
+def test_anomaly_records():
+ """Test to see that new records can be entered"""
+ # Add some data
+ di.session.add(di.Anomaly(filename='foo1', bowtie="True"))
+ di.session.commit()
+
+ # Test the bowties column
+ bowties = di.session.query(di.Anomaly).filter(di.Anomaly.bowtie == "True")
+ assert bowties.data_frame.iloc[0]['bowtie'] == "True"
+
+ # Test the other columns
+ non_bowties = [col for col in di.Anomaly().colnames if col != 'bowtie']
+ assert all([i == "False" for i in bowties.data_frame.iloc[0][non_bowties]])
+
+
+def test_names_colnames():
+ """Test that the column names are correct"""
+ # Make sure we get non-empty lists
+ anom = di.Anomaly()
+ assert isinstance(anom.colnames, list) and len(anom.colnames) > 0
+ assert isinstance(anom.names, list) and len(anom.names) == len(anom.colnames)
+ assert all([i == j.replace('_', ' ') for i, j in zip(anom.names, anom.colnames)])
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 86bb6a532..cdde71f6e 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -151,11 +151,11 @@ {{ file_root }}
Anomalies in this image:
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index ae1a07e1a..38c1f81bf 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -278,13 +278,15 @@ def view_image(request, inst, file_root, rewrite=False):
# Get a list of previously flagged anomalies
prev_anom = None
- # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root)
- # all_records = query.data_frame()
- # if not all_records.empty:
- # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- prev_anom = ', '.join(['bowtie', 'snowball'])
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ all_records = query.data_frame()
+ print(all_records)
+ if not all_records.empty:
+ prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ # prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
+ anom = di.Anomaly()
context = {'inst': inst,
'file_root': file_root,
'tools': MONITORS,
@@ -292,7 +294,7 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': di.Anomaly().colnames,
+ 'anomalies': zip(anom.colnames, anom.names),
'prev_anom': prev_anom}
return render(request, template, context)
From a6d64688b805806f4c291853998e5fe802821d6f Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Wed, 30 Jan 2019 11:29:12 -0500
Subject: [PATCH 11/33] Paceholder until test db is up.
---
jwql/website/apps/jwql/views.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 38c1f81bf..2d22ceda3 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -277,13 +277,13 @@ def view_image(request, inst, file_root, rewrite=False):
image_info = get_image_info(file_root, rewrite)
# Get a list of previously flagged anomalies
- prev_anom = None
- query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
- all_records = query.data_frame()
- print(all_records)
- if not all_records.empty:
- prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- # prev_anom = ', '.join(['bowtie', 'snowball'])
+ # prev_anom = None
+ # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ # all_records = query.data_frame()
+ # print(all_records)
+ # if not all_records.empty:
+ # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
anom = di.Anomaly()
From d220b94684fa5fdfb2fafd2ca6f9edf33c5974f5 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Wed, 30 Jan 2019 11:29:12 -0500
Subject: [PATCH 12/33] Paceholder until test db is up.
---
jwql/website/apps/jwql/views.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 38c1f81bf..2d22ceda3 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -277,13 +277,13 @@ def view_image(request, inst, file_root, rewrite=False):
image_info = get_image_info(file_root, rewrite)
# Get a list of previously flagged anomalies
- prev_anom = None
- query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
- all_records = query.data_frame()
- print(all_records)
- if not all_records.empty:
- prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- # prev_anom = ', '.join(['bowtie', 'snowball'])
+ # prev_anom = None
+ # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ # all_records = query.data_frame()
+ # print(all_records)
+ # if not all_records.empty:
+ # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
anom = di.Anomaly()
From 30310cada53ffe0ab992e8769a6514fd6f1292ec Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Mon, 25 Mar 2019 14:19:28 -0400
Subject: [PATCH 13/33] Enabled db lookup for anomaly table
---
jwql/website/apps/jwql/views.py | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 2d22ceda3..0fa6cc7db 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -277,13 +277,12 @@ def view_image(request, inst, file_root, rewrite=False):
image_info = get_image_info(file_root, rewrite)
# Get a list of previously flagged anomalies
- # prev_anom = None
- # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
- # all_records = query.data_frame()
- # print(all_records)
- # if not all_records.empty:
- # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- prev_anom = ', '.join(['bowtie', 'snowball'])
+ prev_anom = None
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ all_records = query.data_frame()
+ if not all_records.empty:
+ prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ # prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
anom = di.Anomaly()
From c3caed3e3d1337875a77714308f59370abab24d2 Mon Sep 17 00:00:00 2001
From: Joe Filippazzo
Date: Mon, 25 Mar 2019 14:19:28 -0400
Subject: [PATCH 14/33] Enabled db lookup for anomaly table
---
jwql/website/apps/jwql/views.py | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 2d22ceda3..0fa6cc7db 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -277,13 +277,12 @@ def view_image(request, inst, file_root, rewrite=False):
image_info = get_image_info(file_root, rewrite)
# Get a list of previously flagged anomalies
- # prev_anom = None
- # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
- # all_records = query.data_frame()
- # print(all_records)
- # if not all_records.empty:
- # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- prev_anom = ', '.join(['bowtie', 'snowball'])
+ prev_anom = None
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ all_records = query.data_frame()
+ if not all_records.empty:
+ prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ # prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
anom = di.Anomaly()
From 0ac57b91de69f813f4bbc46cd6a9ce3994080c9e Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 25 Mar 2019 14:54:48 -0400
Subject: [PATCH 15/33] Attempting to insert a record into database upon page
load
---
jwql/database/database_interface.py | 2 +-
jwql/website/apps/jwql/views.py | 12 +++++++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 1cf2200ff..b5645f847 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -147,7 +147,7 @@ def load_connection(connection_string):
Column('id', Integer, primary_key=True, nullable=False),
Column('filename', String, nullable=False),
Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())),
- *[Column(name, String, nullable=False, server_default="False") for name in columns])
+ *[Column(name, String, nullable=True, server_default=False) for name in columns])
# Implement it
meta.create_all()
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index ab0fa90a6..727d06c7e 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -35,6 +35,7 @@
placed in the ``jwql/utils/`` directory.
"""
+import datetime
import os
from django.http import JsonResponse
@@ -50,7 +51,7 @@
from .data_containers import thumbnails_ajax
from .forms import FileSearchForm
from jwql.database import database_interface as di
-from jwql.utils.constants import JWST_INSTRUMENTS, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS
from jwql.utils.utils import get_base_url, get_config
FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')
@@ -389,9 +390,18 @@ def view_image(request, inst, file_root, rewrite=False):
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
+
+ # Fake insert a record
+ data_dict = {}
+ data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
+ data_dict['flag_date'] = datetime.datetime.now()
+ data_dict['bowtie'] = True
+ di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
+
# Get a list of previously flagged anomalies
prev_anom = None
query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ print(query)
all_records = query.data_frame()
if not all_records.empty:
prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
From 86ab670737d3e99fee4740abe8238895643de2dc Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 25 Mar 2019 14:54:48 -0400
Subject: [PATCH 16/33] Attempting to insert a record into database upon page
load
---
jwql/database/database_interface.py | 2 +-
jwql/website/apps/jwql/views.py | 12 +++++++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 1cf2200ff..b5645f847 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -147,7 +147,7 @@ def load_connection(connection_string):
Column('id', Integer, primary_key=True, nullable=False),
Column('filename', String, nullable=False),
Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())),
- *[Column(name, String, nullable=False, server_default="False") for name in columns])
+ *[Column(name, String, nullable=True, server_default=False) for name in columns])
# Implement it
meta.create_all()
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index ab0fa90a6..727d06c7e 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -35,6 +35,7 @@
placed in the ``jwql/utils/`` directory.
"""
+import datetime
import os
from django.http import JsonResponse
@@ -50,7 +51,7 @@
from .data_containers import thumbnails_ajax
from .forms import FileSearchForm
from jwql.database import database_interface as di
-from jwql.utils.constants import JWST_INSTRUMENTS, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS
from jwql.utils.utils import get_base_url, get_config
FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')
@@ -389,9 +390,18 @@ def view_image(request, inst, file_root, rewrite=False):
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
+
+ # Fake insert a record
+ data_dict = {}
+ data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
+ data_dict['flag_date'] = datetime.datetime.now()
+ data_dict['bowtie'] = True
+ di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
+
# Get a list of previously flagged anomalies
prev_anom = None
query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
+ print(query)
all_records = query.data_frame()
if not all_records.empty:
prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
From cfbd10e3b6ffc62c931bae507aa8d487ba55f189 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 26 Mar 2019 16:44:39 -0400
Subject: [PATCH 17/33] Moved anomaly table creation to its own ORM factory (to
help with future instrument-specific anomaly tables).
---
jwql/database/database_interface.py | 88 +++++++++++++----------------
jwql/website/apps/jwql/views.py | 16 +++---
2 files changed, 46 insertions(+), 58 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index b5645f847..64c7e81c0 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -85,6 +85,7 @@
@property
def data_frame(self):
"""Method to return a ``pandas.DataFrame`` of the results"""
+ print(pd.read_sql(self.statement, self.session.bind))
return pd.read_sql(self.statement, self.session.bind)
Query.data_frame = data_frame
@@ -130,28 +131,6 @@ def load_connection(connection_string):
session = Session()
meta = MetaData(engine)
- # Make sure it has an anomalies table
- if not engine.has_table('anomalies'):
- print("No 'anomalies' table. Generating one now...")
-
- # Define anomaly table column names
- columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
- 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
- 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
- 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
- 'fringing', 'guidestar_failure', 'banding', 'persistence',
- 'prominent_blobs', 'trail', 'scattered_light', 'other']
-
- # Create a table with the appropriate Columns
- anomalies = Table('anomalies', meta,
- Column('id', Integer, primary_key=True, nullable=False),
- Column('filename', String, nullable=False),
- Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())),
- *[Column(name, String, nullable=True, server_default=False) for name in columns])
-
- # Implement it
- meta.create_all()
-
return session, base, engine, meta
# Import a global session. If running from readthedocs, pass a dummy connection string
@@ -162,33 +141,6 @@ def load_connection(connection_string):
SETTINGS = utils.get_config()
session, base, engine, meta = load_connection(SETTINGS['connection_string'])
-# Make convenience methods for Base class
-@property
-def colnames(self):
- """A list of all the column names in this table"""
- # Get the columns
- a_list = sorted([col for col, val in self._sa_instance_state.attrs.items()
- if col not in ['id', 'filename', 'flag_date']])
-
- return a_list
-
-
-@property
-def names(self):
- """A list of human readable names for all the columns in this table"""
- return [name.replace('_', ' ') for name in self.colnames]
-
-
-# Generate Base class and add methods
-Base = automap_base()
-Base.colnames = colnames
-Base.names = names
-Base.prepare(engine, reflect=True)
-
-
-# Automap Anomaly class from Base class for anomalies table
-Anomaly = Base.classes.anomalies
-
class Monitor(base):
"""ORM for the ``monitor`` table"""
@@ -205,6 +157,43 @@ class Monitor(base):
log_file = Column(String(), nullable=False)
+def anomaly_orm_factory(class_name):
+ """Create a ``SQLAlchemy`` ORM Class for an anomaly table.
+
+ Parameters
+ ----------
+ class_name : str
+ The name of the class to be created
+
+ Returns
+ -------
+ class : obj
+ The ``SQLAlchemy`` ORM
+ """
+
+ # Initialize a dictionary to hold the column metadata
+ data_dict = {}
+ data_dict['__tablename__'] = class_name.lower()
+
+ # Define anomaly table column names
+ data_dict['columns'] = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
+ 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
+ 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
+ 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
+ 'fringing', 'guidestar_failure', 'banding', 'persistence',
+ 'prominent_blobs', 'trail', 'scattered_light', 'other']
+ data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']]
+
+ # Create a table with the appropriate Columns
+ data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
+ data_dict['filename'] = Column(String(), nullable=False)
+ data_dict['flag_date'] = Column(DateTime, nullable=False)
+ for column in data_dict['columns']:
+ data_dict[column] = Column(Boolean, nullable=False, default=False)
+
+ return type(class_name, (base,), data_dict)
+
+
def get_monitor_columns(data_dict, table_name):
"""Read in the corresponding table definition text file to
generate ``SQLAlchemy`` columns for the table.
@@ -311,6 +300,7 @@ class : obj
return type(class_name, (base,), data_dict)
# Create tables from ORM factory
+Anomaly = anomaly_orm_factory('anomaly')
# NIRCamDarkQueries = monitor_orm_factory('nircam_dark_queries')
# NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
# NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 727d06c7e..bfbb48c40 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -40,6 +40,7 @@
from django.http import JsonResponse
from django.shortcuts import render
+import numpy as np
from .data_containers import get_acknowledgements, get_edb_components
from .data_containers import get_dashboard_components
@@ -384,28 +385,25 @@ def view_image(request, inst, file_root, rewrite=False):
HttpResponse object
Outgoing response sent to the webpage
"""
+
# Ensure the instrument is correctly capitalized
inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
-
# Fake insert a record
data_dict = {}
data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
data_dict['flag_date'] = datetime.datetime.now()
- data_dict['bowtie'] = True
+ data_dict['fringing'] = True
di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
- # Get a list of previously flagged anomalies
- prev_anom = None
- query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
- print(query)
- all_records = query.data_frame()
+ # Get most previously flagged anomalies
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root).order_by(di.Anomaly.flag_date.desc()).limit(1)
+ all_records = query.data_frame
if not all_records.empty:
prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- # prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
anom = di.Anomaly()
@@ -415,7 +413,7 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': zip(anom.colnames, anom.names),
+ 'anomalies': zip(anom.columns, anom.names),
'prev_anom': prev_anom}
return render(request, template, context)
From 6557bada2df44e0ce696f99fa6a6e16c6b50c8c5 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 26 Mar 2019 16:44:39 -0400
Subject: [PATCH 18/33] Moved anomaly table creation to its own ORM factory (to
help with future instrument-specific anomaly tables).
---
jwql/database/database_interface.py | 88 +++++++++++++----------------
jwql/website/apps/jwql/views.py | 16 +++---
2 files changed, 46 insertions(+), 58 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index b5645f847..64c7e81c0 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -85,6 +85,7 @@
@property
def data_frame(self):
"""Method to return a ``pandas.DataFrame`` of the results"""
+ print(pd.read_sql(self.statement, self.session.bind))
return pd.read_sql(self.statement, self.session.bind)
Query.data_frame = data_frame
@@ -130,28 +131,6 @@ def load_connection(connection_string):
session = Session()
meta = MetaData(engine)
- # Make sure it has an anomalies table
- if not engine.has_table('anomalies'):
- print("No 'anomalies' table. Generating one now...")
-
- # Define anomaly table column names
- columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
- 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
- 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
- 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
- 'fringing', 'guidestar_failure', 'banding', 'persistence',
- 'prominent_blobs', 'trail', 'scattered_light', 'other']
-
- # Create a table with the appropriate Columns
- anomalies = Table('anomalies', meta,
- Column('id', Integer, primary_key=True, nullable=False),
- Column('filename', String, nullable=False),
- Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())),
- *[Column(name, String, nullable=True, server_default=False) for name in columns])
-
- # Implement it
- meta.create_all()
-
return session, base, engine, meta
# Import a global session. If running from readthedocs, pass a dummy connection string
@@ -162,33 +141,6 @@ def load_connection(connection_string):
SETTINGS = utils.get_config()
session, base, engine, meta = load_connection(SETTINGS['connection_string'])
-# Make convenience methods for Base class
-@property
-def colnames(self):
- """A list of all the column names in this table"""
- # Get the columns
- a_list = sorted([col for col, val in self._sa_instance_state.attrs.items()
- if col not in ['id', 'filename', 'flag_date']])
-
- return a_list
-
-
-@property
-def names(self):
- """A list of human readable names for all the columns in this table"""
- return [name.replace('_', ' ') for name in self.colnames]
-
-
-# Generate Base class and add methods
-Base = automap_base()
-Base.colnames = colnames
-Base.names = names
-Base.prepare(engine, reflect=True)
-
-
-# Automap Anomaly class from Base class for anomalies table
-Anomaly = Base.classes.anomalies
-
class Monitor(base):
"""ORM for the ``monitor`` table"""
@@ -205,6 +157,43 @@ class Monitor(base):
log_file = Column(String(), nullable=False)
+def anomaly_orm_factory(class_name):
+ """Create a ``SQLAlchemy`` ORM Class for an anomaly table.
+
+ Parameters
+ ----------
+ class_name : str
+ The name of the class to be created
+
+ Returns
+ -------
+ class : obj
+ The ``SQLAlchemy`` ORM
+ """
+
+ # Initialize a dictionary to hold the column metadata
+ data_dict = {}
+ data_dict['__tablename__'] = class_name.lower()
+
+ # Define anomaly table column names
+ data_dict['columns'] = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
+ 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
+ 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
+ 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
+ 'fringing', 'guidestar_failure', 'banding', 'persistence',
+ 'prominent_blobs', 'trail', 'scattered_light', 'other']
+ data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']]
+
+ # Create a table with the appropriate Columns
+ data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
+ data_dict['filename'] = Column(String(), nullable=False)
+ data_dict['flag_date'] = Column(DateTime, nullable=False)
+ for column in data_dict['columns']:
+ data_dict[column] = Column(Boolean, nullable=False, default=False)
+
+ return type(class_name, (base,), data_dict)
+
+
def get_monitor_columns(data_dict, table_name):
"""Read in the corresponding table definition text file to
generate ``SQLAlchemy`` columns for the table.
@@ -311,6 +300,7 @@ class : obj
return type(class_name, (base,), data_dict)
# Create tables from ORM factory
+Anomaly = anomaly_orm_factory('anomaly')
# NIRCamDarkQueries = monitor_orm_factory('nircam_dark_queries')
# NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
# NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 727d06c7e..bfbb48c40 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -40,6 +40,7 @@
from django.http import JsonResponse
from django.shortcuts import render
+import numpy as np
from .data_containers import get_acknowledgements, get_edb_components
from .data_containers import get_dashboard_components
@@ -384,28 +385,25 @@ def view_image(request, inst, file_root, rewrite=False):
HttpResponse object
Outgoing response sent to the webpage
"""
+
# Ensure the instrument is correctly capitalized
inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
-
# Fake insert a record
data_dict = {}
data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
data_dict['flag_date'] = datetime.datetime.now()
- data_dict['bowtie'] = True
+ data_dict['fringing'] = True
di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
- # Get a list of previously flagged anomalies
- prev_anom = None
- query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root)
- print(query)
- all_records = query.data_frame()
+ # Get most previously flagged anomalies
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root).order_by(di.Anomaly.flag_date.desc()).limit(1)
+ all_records = query.data_frame
if not all_records.empty:
prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- # prev_anom = ', '.join(['bowtie', 'snowball'])
# Build the context
anom = di.Anomaly()
@@ -415,7 +413,7 @@ def view_image(request, inst, file_root, rewrite=False):
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': zip(anom.colnames, anom.names),
+ 'anomalies': zip(anom.columns, anom.names),
'prev_anom': prev_anom}
return render(request, template, context)
From f63fa4cb128ba27fa1acd839a1bb4eac2b7eda4f Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 2 Apr 2019 11:23:20 -0400
Subject: [PATCH 19/33] Added user column to anomaly table
---
jwql/database/database_interface.py | 2 ++
jwql/website/apps/jwql/views.py | 10 +++++++---
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 64c7e81c0..6034d6d9f 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -188,6 +188,8 @@ class : obj
data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
data_dict['filename'] = Column(String(), nullable=False)
data_dict['flag_date'] = Column(DateTime, nullable=False)
+ data_dict['user'] = Column(String(), nullable=False)
+
for column in data_dict['columns']:
data_dict[column] = Column(Boolean, nullable=False, default=False)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index bfbb48c40..fd913e069 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -51,6 +51,7 @@
from .data_containers import thumbnails
from .data_containers import thumbnails_ajax
from .forms import FileSearchForm
+from .oauth import auth_info
from jwql.database import database_interface as di
from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS
from jwql.utils.utils import get_base_url, get_config
@@ -365,8 +366,8 @@ def view_header(request, inst, file):
return render(request, template, context)
-
-def view_image(request, inst, file_root, rewrite=False):
+@auth_info
+def view_image(request, user, inst, file_root, rewrite=False):
"""Generate the image view page
Parameters
@@ -392,11 +393,14 @@ def view_image(request, inst, file_root, rewrite=False):
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
+ print(user)
+
# Fake insert a record
data_dict = {}
data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
data_dict['flag_date'] = datetime.datetime.now()
- data_dict['fringing'] = True
+ data_dict['bowtie'] = False
+ data_dict['user'] = user['ezid']
di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
# Get most previously flagged anomalies
From fe1f272338b22d590ce39efd9fb7363020523080 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 2 Apr 2019 11:23:20 -0400
Subject: [PATCH 20/33] Added user column to anomaly table
---
jwql/database/database_interface.py | 2 ++
jwql/website/apps/jwql/views.py | 10 +++++++---
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 64c7e81c0..6034d6d9f 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -188,6 +188,8 @@ class : obj
data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
data_dict['filename'] = Column(String(), nullable=False)
data_dict['flag_date'] = Column(DateTime, nullable=False)
+ data_dict['user'] = Column(String(), nullable=False)
+
for column in data_dict['columns']:
data_dict[column] = Column(Boolean, nullable=False, default=False)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index bfbb48c40..fd913e069 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -51,6 +51,7 @@
from .data_containers import thumbnails
from .data_containers import thumbnails_ajax
from .forms import FileSearchForm
+from .oauth import auth_info
from jwql.database import database_interface as di
from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS
from jwql.utils.utils import get_base_url, get_config
@@ -365,8 +366,8 @@ def view_header(request, inst, file):
return render(request, template, context)
-
-def view_image(request, inst, file_root, rewrite=False):
+@auth_info
+def view_image(request, user, inst, file_root, rewrite=False):
"""Generate the image view page
Parameters
@@ -392,11 +393,14 @@ def view_image(request, inst, file_root, rewrite=False):
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
+ print(user)
+
# Fake insert a record
data_dict = {}
data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
data_dict['flag_date'] = datetime.datetime.now()
- data_dict['fringing'] = True
+ data_dict['bowtie'] = False
+ data_dict['user'] = user['ezid']
di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
# Get most previously flagged anomalies
From 53ad89fee89c773b222932113cf4c846615b0882 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 21 May 2019 12:19:22 -0400
Subject: [PATCH 21/33] Moved anomaly definitions to constants.py
---
jwql/database/database_interface.py | 65 +----------------------------
jwql/utils/constants.py | 20 +++++++++
2 files changed, 22 insertions(+), 63 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index dc1367caa..477ed9ab1 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -79,7 +79,7 @@
from sqlalchemy.orm.query import Query
from sqlalchemy.types import ARRAY
-from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
+from jwql.utils.constants import ANOMALIES, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
from jwql.utils.utils import get_config
@@ -144,62 +144,6 @@ def load_connection(connection_string):
session, base, engine, meta = load_connection(SETTINGS['connection_string'])
-class Anomaly(base):
- """ORM for the ``anomalies`` table"""
-
- # Name the table
- __tablename__ = 'anomalies'
-
- # Define the columns
- id = Column(Integer, primary_key=True, nullable=False)
- filename = Column(String, nullable=False)
- flag_date = Column(DateTime, nullable=False, default=datetime.now())
- bowtie = Column(Boolean, nullable=False, default=False)
- snowball = Column(Boolean, nullable=False, default=False)
- cosmic_ray_shower = Column(Boolean, nullable=False, default=False)
- crosstalk = Column(Boolean, nullable=False, default=False)
- cte_correction_error = Column(Boolean, nullable=False, default=False)
- data_transfer_error = Column(Boolean, nullable=False, default=False)
- detector_ghost = Column(Boolean, nullable=False, default=False)
- diamond = Column(Boolean, nullable=False, default=False)
- diffraction_spike = Column(Boolean, nullable=False, default=False)
- dragon_breath = Column(Boolean, nullable=False, default=False)
- earth_limb = Column(Boolean, nullable=False, default=False)
- excessive_saturation = Column(Boolean, nullable=False, default=False)
- figure8_ghost = Column(Boolean, nullable=False, default=False)
- filter_ghost = Column(Boolean, nullable=False, default=False)
- fringing = Column(Boolean, nullable=False, default=False)
- guidestar_failure = Column(Boolean, nullable=False, default=False)
- banding = Column(Boolean, nullable=False, default=False)
- persistence = Column(Boolean, nullable=False, default=False)
- prominent_blobs = Column(Boolean, nullable=False, default=False)
- trail = Column(Boolean, nullable=False, default=False)
- scattered_light = Column(Boolean, nullable=False, default=False)
- other = Column(Boolean, nullable=False, default=False)
-
- def __repr__(self):
- """Return the canonical string representation of the object"""
-
- # Get the columns that are True
- a_list = [col for col, val in self.__dict__.items()
- if val is True and isinstance(val, bool)]
-
- txt = ('Anomaly {0.id}: {0.filename} flagged at '
- '{0.flag_date} for {1}').format(self, a_list)
-
- return txt
-
- @property
- def colnames(self):
- """A list of all the column names in this table"""
-
- # Get the columns
- a_list = [col for col, val in self.__dict__.items()
- if isinstance(val, bool)]
-
- return a_list
-
-
class FilesystemGeneral(base):
"""ORM for the general (non instrument specific) filesystem monitor
table"""
@@ -277,12 +221,7 @@ class : obj
data_dict['__tablename__'] = class_name.lower()
# Define anomaly table column names
- data_dict['columns'] = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
- 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
- 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
- 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
- 'fringing', 'guidestar_failure', 'banding', 'persistence',
- 'prominent_blobs', 'trail', 'scattered_light', 'other']
+ data_dict['columns'] = ANOMALIES
data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']]
# Create a table with the appropriate Columns
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 3cb8fa7b8..562c413fb 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -25,6 +25,26 @@
'3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]}
}
+# Defines the possible anomalies to flag through the web app
+ANOMALIES = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
+ 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
+ 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
+ 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
+ 'fringing', 'guidestar_failure', 'banding', 'persistence',
+ 'prominent_blobs', 'trail', 'scattered_light', 'other']
+
+# Defines the possible anomalies (with rendered name) to flag through the web app
+ANOMALY_CHOICES = [
+ ('bowtie', 'Bowtie'), ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'),
+ ('crosstalk', 'Crosstalk'), ('cte_correction_error', 'CTE Correction Error'),
+ ('data_transfer_error', 'Data Transfer Error'), ('detector_ghost', 'Detector Ghost'),
+ ('diamond', 'Diamond Feature'), ('diffraction_spike', 'Diffraction Spike'),
+ ('dragon_breath', 'Dragons Breath'), ('earth_limb', 'Earth Limb'),
+ ('excessive_saturation', 'Excessive Saturation'), ('figure8_ghost', 'Figure 8 Ghost'),
+ ('filter_ghost', 'Filter Ghost'), ('fringing', 'Fringing'), ('guidestar_failure', 'Guidestar Failure'),
+ ('banding', 'Banding'), ('persistence', 'Persistence'), ('prominent_blobs', 'Prominent Blobs'),
+ ('trail', 'Trail'), ('scattered_light', 'Scattered Light'), ('other', 'Other')]
+
# Possible suffix types for nominal files
GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints',
From d2e638a42ef7767e20fb96c5c665802ef1a6c0e9 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 21 May 2019 12:19:47 -0400
Subject: [PATCH 22/33] Making use of django forms to define anomaly options
---
jwql/website/apps/jwql/forms.py | 10 ++-
.../apps/jwql/templates/view_image.html | 75 +++++++------------
jwql/website/apps/jwql/views.py | 11 ++-
3 files changed, 44 insertions(+), 52 deletions(-)
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index 67d172d17..0f65864d3 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -48,12 +48,20 @@ def view_function(request):
from django.shortcuts import redirect
from jwedb.edb_interface import is_valid_mnemonic
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND
+from jwql.utils.constants import ANOMALY_CHOICES, JWST_INSTRUMENT_NAMES_SHORTHAND
from jwql.utils.utils import get_config, filename_parser
FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem')
+class AnomalySubmitForm(forms.Form):
+ """
+ """
+
+ # Define search field
+ anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, initial='', widget=forms.CheckboxSelectMultiple(), required=True)
+
+
class FileSearchForm(forms.Form):
"""Single-field form to search for a proposal or fileroot."""
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 4c32f8c97..16445ad31 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -50,20 +50,33 @@ {{ file_root }}
-
-
-
Anomalies in this image:
-
+ {% endif %}
+
+
+ {{ csrf_input }}
+
+
+ {% for field in form %}
+
+ {% endfor %}
+
+
@@ -84,42 +97,6 @@
Anomalies in this image:
Lauren needs to figure out what to do with these: {{suffixes}}
{% endif %}
-
-
-
-
{% endblock %}
\ No newline at end of file
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 17fafc590..3b3012417 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -53,7 +53,7 @@
from .data_containers import thumbnails_ajax
from .data_containers import data_trending
from .data_containers import nirspec_trending
-from .forms import FileSearchForm
+from .forms import AnomalySubmitForm, FileSearchForm
from .oauth import auth_info, auth_required
from jwql.database import database_interface as di
from jwql.utils.constants import JWST_INSTRUMENT_NAMES, MONITORS, JWST_INSTRUMENT_NAMES_MIXEDCASE
@@ -497,6 +497,12 @@ def view_image(request, user, inst, file_root, rewrite=False):
all_records = query.data_frame
if not all_records.empty:
prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ else:
+ prev_anom = ''
+
+
+ # Create a form instance and populate it with data from the request
+ form = AnomalySubmitForm(request.POST or None)
# Build the context
anom = di.Anomaly()
@@ -507,6 +513,7 @@ def view_image(request, user, inst, file_root, rewrite=False):
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
'anomalies': zip(anom.columns, anom.names),
- 'prev_anom': prev_anom}
+ 'prev_anom': prev_anom,
+ 'form': form}
return render(request, template, context)
From 3f4aa32d26af2946b61d8f01285c3265c3786346 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 21 May 2019 14:42:13 -0400
Subject: [PATCH 23/33] Implemented updating anomaly database table upon
anomaly form submit
---
jwql/database/database_interface.py | 2 +-
jwql/website/apps/jwql/forms.py | 25 +++++++++++++++++++++-
jwql/website/apps/jwql/views.py | 32 ++++++++---------------------
3 files changed, 34 insertions(+), 25 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 477ed9ab1..6af3d405e 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -226,7 +226,7 @@ class : obj
# Create a table with the appropriate Columns
data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
- data_dict['filename'] = Column(String(), nullable=False)
+ data_dict['rootname'] = Column(String(), nullable=False)
data_dict['flag_date'] = Column(DateTime, nullable=False)
data_dict['user'] = Column(String(), nullable=False)
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index 0f65864d3..dbbb959de 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -40,14 +40,17 @@ def view_function(request):
placed in the ``jwql/utils/`` directory.
"""
+
+import datetime
import glob
import os
from astropy.time import Time, TimeDelta
from django import forms
from django.shortcuts import redirect
-
from jwedb.edb_interface import is_valid_mnemonic
+
+from jwql.database import database_interface as di
from jwql.utils.constants import ANOMALY_CHOICES, JWST_INSTRUMENT_NAMES_SHORTHAND
from jwql.utils.utils import get_config, filename_parser
@@ -61,6 +64,26 @@ class AnomalySubmitForm(forms.Form):
# Define search field
anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, initial='', widget=forms.CheckboxSelectMultiple(), required=True)
+ def update_anomaly_table(self, rootname, user, anomaly_choices):
+ """
+ """
+
+ data_dict = {}
+ data_dict['rootname'] = rootname
+ data_dict['flag_date'] = datetime.datetime.now()
+ data_dict['user'] = user
+ for choice in anomaly_choices:
+ data_dict[choice] = True
+ di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
+
+ # # Get most previously flagged anomalies
+ # query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1)
+ # all_records = query.data_frame
+ # if not all_records.empty:
+ # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
+ # else:
+ # prev_anom = ''
+
class FileSearchForm(forms.Form):
"""Single-field form to search for a proposal or fileroot."""
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 3b3012417..33d308554 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -463,9 +463,11 @@ def view_image(request, user, inst, file_root, rewrite=False):
----------
request : HttpRequest object
Incoming request from the webpage
+ user : dict
+ A dictionary of user credentials.
inst : str
Name of JWST instrument
- file : str
+ file_root : str
FITS filename of selected image in filesystem
rewrite : bool, optional
Regenerate the jpg preview of `file` if it already exists?
@@ -482,38 +484,22 @@ def view_image(request, user, inst, file_root, rewrite=False):
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
- print(user)
-
- # Fake insert a record
- data_dict = {}
- data_dict['filename'] = 'jw00327001001_06101_00001_guider1'
- data_dict['flag_date'] = datetime.datetime.now()
- data_dict['bowtie'] = False
- data_dict['user'] = user['ezid']
- di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
-
- # Get most previously flagged anomalies
- query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root).order_by(di.Anomaly.flag_date.desc()).limit(1)
- all_records = query.data_frame
- if not all_records.empty:
- prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- else:
- prev_anom = ''
-
-
# Create a form instance and populate it with data from the request
form = AnomalySubmitForm(request.POST or None)
+ # If this is a POST request, we need to process the form data
+ if request.method == 'POST':
+ anomaly_choices = dict(request.POST)['anomaly_choices']
+ if form.is_valid():
+ form.update_anomaly_table(file_root, user['ezid'], anomaly_choices)
+
# Build the context
- anom = di.Anomaly()
context = {'inst': inst,
'file_root': file_root,
'jpg_files': image_info['all_jpegs'],
'fits_files': image_info['all_files'],
'suffixes': image_info['suffixes'],
'num_ints': image_info['num_ints'],
- 'anomalies': zip(anom.columns, anom.names),
- 'prev_anom': prev_anom,
'form': form}
return render(request, template, context)
From 02dd41c75b22fd126644504e08138b91c5029656 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 21 May 2019 16:36:27 -0400
Subject: [PATCH 24/33] Added docstrings and implemented automatic selection of
currently flagged anomalies
---
jwql/database/database_interface.py | 2 +-
jwql/website/apps/jwql/data_containers.py | 28 +++++++++++++++++++++
jwql/website/apps/jwql/forms.py | 30 +++++++++++++----------
jwql/website/apps/jwql/views.py | 12 +++++----
4 files changed, 53 insertions(+), 19 deletions(-)
diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 6af3d405e..b76a2c3b6 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -87,7 +87,7 @@
@property
def data_frame(self):
"""Method to return a ``pandas.DataFrame`` of the results"""
- print(pd.read_sql(self.statement, self.session.bind))
+
return pd.read_sql(self.statement, self.session.bind)
Query.data_frame = data_frame
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 7018b2b47..d5fd631ce 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -42,6 +42,7 @@
from astroquery.mast import Mast
from jwedb.edb_interface import mnemonic_inventory
+from jwql.database import database_interface as di
from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info
from jwql.instrument_monitors.miri_monitors.data_trending import dashboard as miri_dash
from jwql.instrument_monitors.nirspec_monitors.data_trending import dashboard as nirspec_dash
@@ -142,6 +143,33 @@ def get_all_proposals():
return proposals
+def get_current_flagged_anomalies(rootname):
+ """Return a list of currently flagged anomalies for the given
+ ``rootname``
+
+ Parameters
+ ----------
+ rootname : str
+ The rootname of interest (e.g.
+ ``jw86600008001_02101_00001_guider2/``)
+
+ Returns
+ -------
+ current_anomalies : list
+ A list of currently flagged anomalies for the given ``rootname``
+ (e.g. ``['snowball', 'crosstalk']``)
+ """
+
+ query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1)
+ all_records = query.data_frame
+ if not all_records.empty:
+ current_anomalies = [col for col, val in np.sum(all_records, axis=0).items() if val]
+ else:
+ current_anomalies = []
+
+ return current_anomalies
+
+
def get_dashboard_components():
"""Build and return dictionaries containing components and html
needed for the dashboard.
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index dbbb959de..5f97a6d73 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -58,14 +58,26 @@ def view_function(request):
class AnomalySubmitForm(forms.Form):
- """
- """
+ """A multiple choice field for specifying flagged anomalies."""
- # Define search field
- anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, initial='', widget=forms.CheckboxSelectMultiple(), required=True)
+ # Define anomaly choice field
+ anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, widget=forms.CheckboxSelectMultiple())
def update_anomaly_table(self, rootname, user, anomaly_choices):
- """
+ """Updated the ``anomaly`` table of the database with flagged
+ anomaly information
+
+ Parameters
+ ----------
+ rootname : str
+ The rootname of the image to flag (e.g.
+ ``jw86600008001_02101_00001_guider2``)
+ user : str
+ The ``ezid`` of the authenticated user that is flagging the
+ anomaly
+ anomaly_choices : list
+ A list of anomalies that are to be flagged (e.g.
+ ``['snowball', 'crosstalk']``)
"""
data_dict = {}
@@ -76,14 +88,6 @@ def update_anomaly_table(self, rootname, user, anomaly_choices):
data_dict[choice] = True
di.engine.execute(di.Anomaly.__table__.insert(), data_dict)
- # # Get most previously flagged anomalies
- # query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1)
- # all_records = query.data_frame
- # if not all_records.empty:
- # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val])
- # else:
- # prev_anom = ''
-
class FileSearchForm(forms.Form):
"""Single-field form to search for a proposal or fileroot."""
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 33d308554..4d743db7e 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -40,13 +40,13 @@
from django.http import JsonResponse
from django.shortcuts import render
-import numpy as np
from .data_containers import get_acknowledgements, get_edb_components
from .data_containers import get_dashboard_components
from .data_containers import get_filenames_by_instrument
from .data_containers import get_header_info
from .data_containers import get_image_info
+from .data_containers import get_current_flagged_anomalies
from .data_containers import get_proposal_info
from .data_containers import random_404_page
from .data_containers import thumbnails
@@ -55,7 +55,6 @@
from .data_containers import nirspec_trending
from .forms import AnomalySubmitForm, FileSearchForm
from .oauth import auth_info, auth_required
-from jwql.database import database_interface as di
from jwql.utils.constants import JWST_INSTRUMENT_NAMES, MONITORS, JWST_INSTRUMENT_NAMES_MIXEDCASE
from jwql.utils.utils import get_base_url, get_config
@@ -484,10 +483,13 @@ def view_image(request, user, inst, file_root, rewrite=False):
template = 'view_image.html'
image_info = get_image_info(file_root, rewrite)
- # Create a form instance and populate it with data from the request
- form = AnomalySubmitForm(request.POST or None)
+ # Determine current flagged anomalies
+ current_anomalies = get_current_flagged_anomalies(file_root)
- # If this is a POST request, we need to process the form data
+ # Create a form instance
+ form = AnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies})
+
+ # If this is a POST request, process the form data
if request.method == 'POST':
anomaly_choices = dict(request.POST)['anomaly_choices']
if form.is_valid():
From 01328c011cbd5dbd8ac9389260167af43094fec5 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 21 May 2019 16:43:24 -0400
Subject: [PATCH 25/33] Removed a bunch of anomaly categories that
probably/hopefully wont exist for JWST instruments
---
jwql/utils/constants.py | 21 ++++++---------------
1 file changed, 6 insertions(+), 15 deletions(-)
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 562c413fb..d74f2bb65 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -26,24 +26,15 @@
}
# Defines the possible anomalies to flag through the web app
-ANOMALIES = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk',
- 'cte_correction_error', 'data_transfer_error', 'detector_ghost',
- 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb',
- 'excessive_saturation', 'figure8_ghost', 'filter_ghost',
- 'fringing', 'guidestar_failure', 'banding', 'persistence',
- 'prominent_blobs', 'trail', 'scattered_light', 'other']
+ANOMALIES = ['snowball', 'cosmic_ray_shower', 'crosstalk', 'data_transfer_error', 'diffraction_spike',
+ 'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other']
# Defines the possible anomalies (with rendered name) to flag through the web app
ANOMALY_CHOICES = [
- ('bowtie', 'Bowtie'), ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'),
- ('crosstalk', 'Crosstalk'), ('cte_correction_error', 'CTE Correction Error'),
- ('data_transfer_error', 'Data Transfer Error'), ('detector_ghost', 'Detector Ghost'),
- ('diamond', 'Diamond Feature'), ('diffraction_spike', 'Diffraction Spike'),
- ('dragon_breath', 'Dragons Breath'), ('earth_limb', 'Earth Limb'),
- ('excessive_saturation', 'Excessive Saturation'), ('figure8_ghost', 'Figure 8 Ghost'),
- ('filter_ghost', 'Filter Ghost'), ('fringing', 'Fringing'), ('guidestar_failure', 'Guidestar Failure'),
- ('banding', 'Banding'), ('persistence', 'Persistence'), ('prominent_blobs', 'Prominent Blobs'),
- ('trail', 'Trail'), ('scattered_light', 'Scattered Light'), ('other', 'Other')]
+ ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'), ('crosstalk', 'Crosstalk'),
+ ('data_transfer_error', 'Data Transfer Error'), ('diffraction_spike', 'Diffraction Spike'),
+ ('excessive_saturation', 'Excessive Saturation'), ('ghost', 'Ghost'), ('guidestar_failure', 'Guidestar Failure'),
+ ('persistence', 'Persistence'), ('satellite_trail', 'Satellite Trail'), ('other', 'Other')]
# Possible suffix types for nominal files
GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
From e3795fbe92a7725c2025820b24809e327c649d2a Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 21 May 2019 17:34:19 -0400
Subject: [PATCH 26/33] Fixed HTML and CSS to get anomaly form to render where
it should.
---
jwql/website/apps/jwql/static/css/jwql.css | 13 ++-
.../apps/jwql/templates/view_image.html | 80 ++++++++++---------
2 files changed, 56 insertions(+), 37 deletions(-)
diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css
index f7e3340f6..d6cbc7f9b 100644
--- a/jwql/website/apps/jwql/static/css/jwql.css
+++ b/jwql/website/apps/jwql/static/css/jwql.css
@@ -1,5 +1,11 @@
+.anomaly_form {
+ position: absolute;
+ top: 50%;
+ transform: translateY(-50%);
+}
+
.APT_parameters {
- width: 20%
+ width: 20%
}
.banner {
@@ -182,6 +188,11 @@ display : inline;
display: inline;
}
+#id_anomaly_choices {
+ list-style: none;
+ padding-left: 0;
+}
+
/*Don't let the search bar be super long*/
.input-group {
width: 250px;
diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html
index 16445ad31..e15feb5be 100644
--- a/jwql/website/apps/jwql/templates/view_image.html
+++ b/jwql/website/apps/jwql/templates/view_image.html
@@ -40,43 +40,51 @@ {{ file_root }}
View Proposal {{ file_root[2:7] }}
-
-
-
-
-
From d97601f97edee7a54bb2042820cd60bdeb0a4bde Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 3 Jun 2019 13:39:54 -0400
Subject: [PATCH 28/33] Minor edit to appease @pep8speaks
---
jwql/utils/preview_image.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py
index 9f47e84de..0369dec36 100755
--- a/jwql/utils/preview_image.py
+++ b/jwql/utils/preview_image.py
@@ -213,7 +213,6 @@ def get_data(self, filename, ext):
yd, xd = data.shape[-2:]
dq = np.ones((yd, xd), dtype="bool")
-
# Collect information on aperture location within the
# full detector. This is needed for mosaicking NIRCam
# detectors later.
From 5fb0797cc4847cd0f4c4eb4adadd270a593498a3 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 3 Jun 2019 13:50:39 -0400
Subject: [PATCH 29/33] updated anomaly table tests based on recent changes
---
jwql/tests/test_database_interface.py | 29 +++++++++------------------
1 file changed, 10 insertions(+), 19 deletions(-)
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
index a1ea4a1b8..da13bb333 100755
--- a/jwql/tests/test_database_interface.py
+++ b/jwql/tests/test_database_interface.py
@@ -5,7 +5,7 @@
Authors
-------
- Joe Filippazzo
+ - Joe Filippazzo
Use
---
@@ -17,33 +17,24 @@
pytest -s database_interface.py
"""
+import datetime
+
from jwql.database import database_interface as di
def test_anomaly_table():
"""Test to see that the database has an anomalies table"""
- assert 'anomalies' in di.engine.table_names()
+
+ assert 'anomaly' in di.engine.table_names()
def test_anomaly_records():
"""Test to see that new records can be entered"""
+
# Add some data
- di.session.add(di.Anomaly(filename='foo1', bowtie="True"))
+ di.session.add(di.Anomaly(rootname='foo1', flag_date=datetime.datetime.today(), user='test', ghost=True))
di.session.commit()
- # Test the bowties column
- bowties = di.session.query(di.Anomaly).filter(di.Anomaly.bowtie == "True")
- assert bowties.data_frame.iloc[0]['bowtie'] == "True"
-
- # Test the other columns
- non_bowties = [col for col in di.Anomaly().colnames if col != 'bowtie']
- assert all([i == "False" for i in bowties.data_frame.iloc[0][non_bowties]])
-
-
-def test_names_colnames():
- """Test that the column names are correct"""
- # Make sure we get non-empty lists
- anom = di.Anomaly()
- assert isinstance(anom.colnames, list) and len(anom.colnames) > 0
- assert isinstance(anom.names, list) and len(anom.names) == len(anom.colnames)
- assert all([i == j.replace('_', ' ') for i, j in zip(anom.names, anom.colnames)])
+ # Test the ghosts column
+ ghosts = di.session.query(di.Anomaly).filter(di.Anomaly.ghost == "True")
+ assert ghosts.data_frame.iloc[0]['ghost'] == True
From f93437953750876eddd9e7936338460c2e06bd08 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 3 Jun 2019 14:30:54 -0400
Subject: [PATCH 30/33] Marking database interface tests with skipif since
jenkins server doesnt have access to dev database server
---
jwql/tests/test_database_interface.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
index da13bb333..4c078b8b3 100755
--- a/jwql/tests/test_database_interface.py
+++ b/jwql/tests/test_database_interface.py
@@ -21,13 +21,18 @@
from jwql.database import database_interface as di
+# Determine if tests are being run on jenkins
+ON_JENKINS = os.path.expanduser('~') == '/home/jenkins'
+
+@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.')
def test_anomaly_table():
"""Test to see that the database has an anomalies table"""
assert 'anomaly' in di.engine.table_names()
+@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.')
def test_anomaly_records():
"""Test to see that new records can be entered"""
From 0471dc64dabb8dcfa568ea36ffa8b9fa1fce252e Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 3 Jun 2019 14:45:16 -0400
Subject: [PATCH 31/33] Forgot os import
---
jwql/tests/test_database_interface.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
index 4c078b8b3..77daaa44f 100755
--- a/jwql/tests/test_database_interface.py
+++ b/jwql/tests/test_database_interface.py
@@ -18,6 +18,7 @@
"""
import datetime
+import os
from jwql.database import database_interface as di
From d51c6d580d415bea04169f8269aba6bf3415a733 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Mon, 3 Jun 2019 14:51:53 -0400
Subject: [PATCH 32/33] Forgot pytest import
---
jwql/tests/test_database_interface.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
index 77daaa44f..ab6a3d5b1 100755
--- a/jwql/tests/test_database_interface.py
+++ b/jwql/tests/test_database_interface.py
@@ -19,6 +19,7 @@
import datetime
import os
+import pytest
from jwql.database import database_interface as di
From d769a4b8a33eea824b345fc3fc28d575422ad034 Mon Sep 17 00:00:00 2001
From: Matthew Bourque
Date: Tue, 4 Jun 2019 17:04:36 -0400
Subject: [PATCH 33/33] Small changes based on @laurenmarietta review comments
---
environment_python_3_5.yml | 1 +
environment_python_3_6.yml | 1 +
jwql/tests/test_database_interface.py | 6 +++++-
jwql/utils/constants.py | 9 ++++-----
jwql/website/apps/jwql/forms.py | 1 +
requirements.txt | 1 +
6 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/environment_python_3_5.yml b/environment_python_3_5.yml
index 0ab098dff..43b44b4cc 100644
--- a/environment_python_3_5.yml
+++ b/environment_python_3_5.yml
@@ -8,6 +8,7 @@ dependencies:
- bokeh=0.13.0
- crds>=7.2.7
- django=2.1.1
+- inflection=0.3.1
- ipython=6.5.0
- jinja2=2.10
- jwst=0.13.0
diff --git a/environment_python_3_6.yml b/environment_python_3_6.yml
index 68a0fcced..7e5951ab1 100644
--- a/environment_python_3_6.yml
+++ b/environment_python_3_6.yml
@@ -8,6 +8,7 @@ dependencies:
- bokeh=1.1.0
- crds>=7.2.7
- django=2.1.7
+- inflection=0.3.1
- ipython=7.5.0
- jinja2=2.10
- jwst=0.13.1
diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py
index ab6a3d5b1..96bb884d0 100755
--- a/jwql/tests/test_database_interface.py
+++ b/jwql/tests/test_database_interface.py
@@ -6,6 +6,7 @@
-------
- Joe Filippazzo
+ - Matthew Bourque
Use
---
@@ -20,6 +21,8 @@
import datetime
import os
import pytest
+import random
+import string
from jwql.database import database_interface as di
@@ -39,7 +42,8 @@ def test_anomaly_records():
"""Test to see that new records can be entered"""
# Add some data
- di.session.add(di.Anomaly(rootname='foo1', flag_date=datetime.datetime.today(), user='test', ghost=True))
+ random_string = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for _ in range(10))
+ di.session.add(di.Anomaly(rootname=random_string, flag_date=datetime.datetime.today(), user='test', ghost=True))
di.session.commit()
# Test the ghosts column
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index d74f2bb65..1db166b1a 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -20,6 +20,9 @@
``utils.py``
"""
+import inflection
+
+
# Defines the x and y coordinates of amplifier boundaries
AMPLIFIER_BOUNDARIES = {'nircam': {'1': [(0, 0), (512, 2048)], '2': [(512, 0), (1024, 2048)],
'3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]}
@@ -30,11 +33,7 @@
'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other']
# Defines the possible anomalies (with rendered name) to flag through the web app
-ANOMALY_CHOICES = [
- ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'), ('crosstalk', 'Crosstalk'),
- ('data_transfer_error', 'Data Transfer Error'), ('diffraction_spike', 'Diffraction Spike'),
- ('excessive_saturation', 'Excessive Saturation'), ('ghost', 'Ghost'), ('guidestar_failure', 'Guidestar Failure'),
- ('persistence', 'Persistence'), ('satellite_trail', 'Satellite Trail'), ('other', 'Other')]
+ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES]
# Possible suffix types for nominal files
GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index 080a86ace..06e8aed35 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -10,6 +10,7 @@
- Lauren Chambers
- Johannes Sahlmann
+ - Matthew Bourque
Use
---
diff --git a/requirements.txt b/requirements.txt
index a23571589..f6d8241b8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,6 +4,7 @@ astroquery==0.3.9
authlib==0.11
bokeh==1.1.0
django==2.2.1
+inflection==0.3.1
ipython==7.5.0
jinja2==2.10.1
jwedb>=0.0.3