From 53dd4705c71b1a6d6d37a405797b1d1960aa6e61 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 3 Oct 2018 16:27:09 -0400 Subject: [PATCH 01/33] Started adding anomaly form to image view. --- jwql/jwql_monitors/generate_preview_images.py | 12 ++++---- jwql/utils/preview_image.py | 10 +++---- jwql/website/apps/jwql/static/css/jwql.css | 6 ++++ .../apps/jwql/templates/view_image.html | 30 +++++++++++++------ jwql/website/apps/jwql/views.py | 3 +- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py index be78072fa..60c7ee207 100755 --- a/jwql/jwql_monitors/generate_preview_images.py +++ b/jwql/jwql_monitors/generate_preview_images.py @@ -322,7 +322,7 @@ def create_mosaic(filenames): elif datadim == 3: full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan else: - raise ValueError((f'Difference image for {filenames[0]} must be either 2D or 3D.')) + raise ValueError('Difference image for {} must be either 2D or 3D.'.format(filenames[0])) # Place the data from the individual detectors in the appropriate # places in the final image @@ -538,11 +538,11 @@ def generate_preview_images(): if not os.path.exists(preview_output_directory): os.makedirs(preview_output_directory) permissions.set_permissions(preview_output_directory) - logging.info(f'Created directory {preview_output_directory}') + logging.info('Created directory {}'.format(preview_output_directory)) if not os.path.exists(thumbnail_output_directory): os.makedirs(thumbnail_output_directory) permissions.set_permissions(thumbnail_output_directory) - logging.info(f'Created directory {thumbnail_output_directory}') + logging.info('Created directory {}'.format(thumbnail_output_directory)) # If the exposure contains more than one file (because more # than one detector was used), then create a mosaic @@ -553,7 +553,7 @@ def generate_preview_images(): mosaic_image, mosaic_dq = create_mosaic(file_list) logging.info('Created mosiac for:') for item in file_list: - logging.info(f'\t{item}') + logging.info('\t{}'.format(item)) except (ValueError, FileNotFoundError) as error: logging.error(error) dummy_file = create_dummy_filename(file_list) @@ -636,7 +636,7 @@ def group_filenames(input_files): detector = filename_parts['detector'].upper() suffix = filename_parts['suffix'] - observation_base = f'jw{program}{observation}{visit}_{visit_group}{parallel}{activity}_{exposure}_' + observation_base = 'jw{}{}{}_{}{}{}_{}_'.format(program, observation, visit, visit_group, parallel, activity, exposure) if detector in NIRCAM_SHORTWAVE_DETECTORS: detector_str = 'NRC[AB][1234]' @@ -644,7 +644,7 @@ def group_filenames(input_files): detector_str = 'NRC[AB]5' else: # non-NIRCam detectors - should never be used I think?? detector_str = detector - match_str = f'{observation_base}{detector_str}_{suffix}.fits' + match_str = '{}{}_{}.fits'.format(observation_base, detector_str, suffix) match_str = os.path.join(file_directory, match_str) pattern = re.compile(match_str, re.IGNORECASE) diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py index f51a5bc8e..39a640d50 100755 --- a/jwql/utils/preview_image.py +++ b/jwql/utils/preview_image.py @@ -200,7 +200,7 @@ def get_data(self, filename, ext): else: data = hdulist[ext].data.astype(np.float) else: - raise ValueError((f'WARNING: no {ext} extension in {filename}!')) + raise ValueError('WARNING: no {} extension in {}!'.format(ext, filename)) if 'PIXELDQ' in extnames: dq = hdulist['PIXELDQ'].data dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0) @@ -216,7 +216,7 @@ def get_data(self, filename, ext): self.xlen = hdulist[0].header['SUBSIZE1'] self.ylen = hdulist[0].header['SUBSIZE2'] else: - raise FileNotFoundError((f'WARNING: {filename} does not exist!')) + raise FileNotFoundError('WARNING: {} does not exist!'.format(filename)) return data, dq @@ -257,7 +257,7 @@ def make_figure(self, image, integration_number, min_value, max_value, # Check the input scaling if scale not in ['linear', 'log']: - raise ValueError((f'WARNING: scaling option {scale} not supported.')) + raise ValueError('WARNING: scaling option {} not supported.'.format(scale)) # Set the figure size yd, xd = image.shape @@ -417,6 +417,6 @@ def save_image(self, fname, thumbnail=False): if thumbnail: thumb_fname = fname.replace('.jpg', '.thumb') os.rename(fname, thumb_fname) - logging.info(f'Saved image to {thumb_fname}') + logging.info('Saved image to {}'.format(thumb_fname)) else: - logging.info(f'Saved image to {fname}') + logging.info('Saved image to {}'.format(fname)) diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index b2f109bb9..9c8748877 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -373,3 +373,9 @@ li:hover .nav-link, .navbar-brand:hover { li.dropdown:hover .dropdown-menu { display: block; } + +ul.no-bullets { + list-style: none; + padding-left:10px; + line-height:25px; +} diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 2359726c2..9e6266e8a 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -137,20 +137,32 @@

{{ file_root }}

-
- - - Displaying integration 1/1
- {{ file_root }}_cal_integ0.jpg -
- -
+
+
+ + + Displaying integration 1/1
+ {{ file_root }}_cal_integ0.jpg +
+ +
+ + +
+
Anomalies in this image:
+
    + {% for anom in anomalies %} +
  •   {{ anom }}
  • + {% endfor %} +
+ Submit Anomalies +
+

Download FITS Download JPEG - Submit Anomaly

diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index b778658b2..fc0964a54 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -280,6 +280,7 @@ def view_image(request, inst, file_root, rewrite=False): 'jpg_files': image_info['all_jpegs'], 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], - 'num_ints': image_info['num_ints']} + 'num_ints': image_info['num_ints'], + 'anomalies': ['foo','bar','baz']} return render(request, template, context) From f24bbb68e39c30ed0cab6d446005d88f64d90e33 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 3 Oct 2018 16:27:09 -0400 Subject: [PATCH 02/33] Started adding anomaly form to image view. --- jwql/jwql_monitors/generate_preview_images.py | 12 ++++---- jwql/utils/preview_image.py | 10 +++---- jwql/website/apps/jwql/static/css/jwql.css | 6 ++++ .../apps/jwql/templates/view_image.html | 30 +++++++++++++------ jwql/website/apps/jwql/views.py | 3 +- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py index be78072fa..60c7ee207 100755 --- a/jwql/jwql_monitors/generate_preview_images.py +++ b/jwql/jwql_monitors/generate_preview_images.py @@ -322,7 +322,7 @@ def create_mosaic(filenames): elif datadim == 3: full_array = np.zeros((datashape[0], full_ydim, full_xdim)) * np.nan else: - raise ValueError((f'Difference image for {filenames[0]} must be either 2D or 3D.')) + raise ValueError('Difference image for {} must be either 2D or 3D.'.format(filenames[0])) # Place the data from the individual detectors in the appropriate # places in the final image @@ -538,11 +538,11 @@ def generate_preview_images(): if not os.path.exists(preview_output_directory): os.makedirs(preview_output_directory) permissions.set_permissions(preview_output_directory) - logging.info(f'Created directory {preview_output_directory}') + logging.info('Created directory {}'.format(preview_output_directory)) if not os.path.exists(thumbnail_output_directory): os.makedirs(thumbnail_output_directory) permissions.set_permissions(thumbnail_output_directory) - logging.info(f'Created directory {thumbnail_output_directory}') + logging.info('Created directory {}'.format(thumbnail_output_directory)) # If the exposure contains more than one file (because more # than one detector was used), then create a mosaic @@ -553,7 +553,7 @@ def generate_preview_images(): mosaic_image, mosaic_dq = create_mosaic(file_list) logging.info('Created mosiac for:') for item in file_list: - logging.info(f'\t{item}') + logging.info('\t{}'.format(item)) except (ValueError, FileNotFoundError) as error: logging.error(error) dummy_file = create_dummy_filename(file_list) @@ -636,7 +636,7 @@ def group_filenames(input_files): detector = filename_parts['detector'].upper() suffix = filename_parts['suffix'] - observation_base = f'jw{program}{observation}{visit}_{visit_group}{parallel}{activity}_{exposure}_' + observation_base = 'jw{}{}{}_{}{}{}_{}_'.format(program, observation, visit, visit_group, parallel, activity, exposure) if detector in NIRCAM_SHORTWAVE_DETECTORS: detector_str = 'NRC[AB][1234]' @@ -644,7 +644,7 @@ def group_filenames(input_files): detector_str = 'NRC[AB]5' else: # non-NIRCam detectors - should never be used I think?? detector_str = detector - match_str = f'{observation_base}{detector_str}_{suffix}.fits' + match_str = '{}{}_{}.fits'.format(observation_base, detector_str, suffix) match_str = os.path.join(file_directory, match_str) pattern = re.compile(match_str, re.IGNORECASE) diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py index f51a5bc8e..39a640d50 100755 --- a/jwql/utils/preview_image.py +++ b/jwql/utils/preview_image.py @@ -200,7 +200,7 @@ def get_data(self, filename, ext): else: data = hdulist[ext].data.astype(np.float) else: - raise ValueError((f'WARNING: no {ext} extension in {filename}!')) + raise ValueError('WARNING: no {} extension in {}!'.format(ext, filename)) if 'PIXELDQ' in extnames: dq = hdulist['PIXELDQ'].data dq = (dq & dqflags.pixel['NON_SCIENCE'] == 0) @@ -216,7 +216,7 @@ def get_data(self, filename, ext): self.xlen = hdulist[0].header['SUBSIZE1'] self.ylen = hdulist[0].header['SUBSIZE2'] else: - raise FileNotFoundError((f'WARNING: {filename} does not exist!')) + raise FileNotFoundError('WARNING: {} does not exist!'.format(filename)) return data, dq @@ -257,7 +257,7 @@ def make_figure(self, image, integration_number, min_value, max_value, # Check the input scaling if scale not in ['linear', 'log']: - raise ValueError((f'WARNING: scaling option {scale} not supported.')) + raise ValueError('WARNING: scaling option {} not supported.'.format(scale)) # Set the figure size yd, xd = image.shape @@ -417,6 +417,6 @@ def save_image(self, fname, thumbnail=False): if thumbnail: thumb_fname = fname.replace('.jpg', '.thumb') os.rename(fname, thumb_fname) - logging.info(f'Saved image to {thumb_fname}') + logging.info('Saved image to {}'.format(thumb_fname)) else: - logging.info(f'Saved image to {fname}') + logging.info('Saved image to {}'.format(fname)) diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index b2f109bb9..9c8748877 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -373,3 +373,9 @@ li:hover .nav-link, .navbar-brand:hover { li.dropdown:hover .dropdown-menu { display: block; } + +ul.no-bullets { + list-style: none; + padding-left:10px; + line-height:25px; +} diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 2359726c2..9e6266e8a 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -137,20 +137,32 @@

{{ file_root }}

-
- - - Displaying integration 1/1
- {{ file_root }}_cal_integ0.jpg -
- -
+
+
+ + + Displaying integration 1/1
+ {{ file_root }}_cal_integ0.jpg +
+ +
+ + +
+
Anomalies in this image:
+
    + {% for anom in anomalies %} +
  •   {{ anom }}
  • + {% endfor %} +
+ Submit Anomalies +
+

Download FITS Download JPEG - Submit Anomaly

diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index b778658b2..fc0964a54 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -280,6 +280,7 @@ def view_image(request, inst, file_root, rewrite=False): 'jpg_files': image_info['all_jpegs'], 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], - 'num_ints': image_info['num_ints']} + 'num_ints': image_info['num_ints'], + 'anomalies': ['foo','bar','baz']} return render(request, template, context) From 675bf9c21329fde2f9ed7c017fa4a1b06449d982 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Fri, 5 Oct 2018 11:15:27 -0400 Subject: [PATCH 03/33] Added list of anomalies to image view. --- jwql/database/database_interface.py | 4 ++-- jwql/website/apps/jwql/templates/view_image.html | 2 +- jwql/website/apps/jwql/views.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index dc80873ed..18e2532eb 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -147,8 +147,8 @@ def __repr__(self): def colnames(self): """A list of all the column names in this table""" # Get the columns - a_list = [col for col, val in self.__dict__.items() - if isinstance(val, bool)] + a_list = [col for col, val in self._sa_instance_state.attrs.items() + if col not in ['id', 'filename', 'flag_date']] return a_list diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 9e6266e8a..2201df8ea 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -155,7 +155,7 @@
Anomalies in this image:
  •   {{ anom }}
  • {% endfor %} - Submit Anomalies + Submit Anomalies diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index fc0964a54..466883112 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -47,6 +47,7 @@ from .data_containers import get_proposal_info from .data_containers import thumbnails from jwql.utils.utils import get_config, JWST_INSTRUMENTS, MONITORS +from jwql.database import database_interface as di FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') @@ -281,6 +282,6 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': ['foo','bar','baz']} + 'anomalies': di.Anomaly().colnames} return render(request, template, context) From 0203ba055f4592e04f3ea0639293c691dfa808fd Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Fri, 5 Oct 2018 11:15:27 -0400 Subject: [PATCH 04/33] Added list of anomalies to image view. --- jwql/database/database_interface.py | 4 ++-- jwql/website/apps/jwql/templates/view_image.html | 2 +- jwql/website/apps/jwql/views.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index dc80873ed..18e2532eb 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -147,8 +147,8 @@ def __repr__(self): def colnames(self): """A list of all the column names in this table""" # Get the columns - a_list = [col for col, val in self.__dict__.items() - if isinstance(val, bool)] + a_list = [col for col, val in self._sa_instance_state.attrs.items() + if col not in ['id', 'filename', 'flag_date']] return a_list diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 9e6266e8a..2201df8ea 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -155,7 +155,7 @@
    Anomalies in this image:
  •   {{ anom }}
  • {% endfor %} - Submit Anomalies + Submit Anomalies diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index fc0964a54..466883112 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -47,6 +47,7 @@ from .data_containers import get_proposal_info from .data_containers import thumbnails from jwql.utils.utils import get_config, JWST_INSTRUMENTS, MONITORS +from jwql.database import database_interface as di FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') @@ -281,6 +282,6 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': ['foo','bar','baz']} + 'anomalies': di.Anomaly().colnames} return render(request, template, context) From 3866b896b0076d69965621c6a477d7eb198f6373 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Fri, 5 Oct 2018 15:27:25 -0400 Subject: [PATCH 05/33] Added previous anomaly tags to image preview. --- jwql/website/apps/jwql/templates/view_image.html | 1 + jwql/website/apps/jwql/views.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 2201df8ea..16d4fb0f5 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -150,6 +150,7 @@

    {{ file_root }}

    Anomalies in this image:
    +

    Previously flagged: {{ prev_anom }}

      {% for anom in anomalies %}
    •   {{ anom }}
    • diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 466883112..726341093 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -275,6 +275,14 @@ def view_image(request, inst, file_root, rewrite=False): """ template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + + # Get a list of previously flagged anomalies + query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) + all_records = query.data_frame() + prev_anom = ', '.join([col for col, val in + np.sum(all_records, axis=0).items() if val]) + + # Build the context context = {'inst': inst, 'file_root': file_root, 'tools': MONITORS, @@ -282,6 +290,7 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': di.Anomaly().colnames} + 'anomalies': di.Anomaly().colnames, + 'prev_anom': prev_anom} return render(request, template, context) From db78ae406b6d9e73a2420bda8ddb64e1e3cebde8 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Fri, 5 Oct 2018 15:27:25 -0400 Subject: [PATCH 06/33] Added previous anomaly tags to image preview. --- jwql/website/apps/jwql/templates/view_image.html | 1 + jwql/website/apps/jwql/views.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 2201df8ea..16d4fb0f5 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -150,6 +150,7 @@

      {{ file_root }}

      Anomalies in this image:
      +

      Previously flagged: {{ prev_anom }}

        {% for anom in anomalies %}
      •   {{ anom }}
      • diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 466883112..726341093 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -275,6 +275,14 @@ def view_image(request, inst, file_root, rewrite=False): """ template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + + # Get a list of previously flagged anomalies + query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) + all_records = query.data_frame() + prev_anom = ', '.join([col for col, val in + np.sum(all_records, axis=0).items() if val]) + + # Build the context context = {'inst': inst, 'file_root': file_root, 'tools': MONITORS, @@ -282,6 +290,7 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': di.Anomaly().colnames} + 'anomalies': di.Anomaly().colnames, + 'prev_anom': prev_anom} return render(request, template, context) From d25450d04446edd69ea56db569519625b256b248 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 14 Nov 2018 11:55:19 -0500 Subject: [PATCH 07/33] Testing anomaly view with preloaded checkboxes. --- jwql/website/apps/jwql/templates/view_image.html | 7 +++++-- jwql/website/apps/jwql/views.py | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 16d4fb0f5..86bb6a532 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -150,10 +150,13 @@

        {{ file_root }}

        Anomalies in this image:
        -

        Previously flagged: {{ prev_anom }}

          {% for anom in anomalies %} -
        •   {{ anom }}
        • + {% if anom in prev_anom %} +
        •   {{ anom }}
        • + {% else %} +
        •   {{ anom }}
        • + {% endif %} {% endfor %}
        Submit Anomalies diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 726341093..ae1a07e1a 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -277,10 +277,12 @@ def view_image(request, inst, file_root, rewrite=False): image_info = get_image_info(file_root, rewrite) # Get a list of previously flagged anomalies - query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) - all_records = query.data_frame() - prev_anom = ', '.join([col for col, val in - np.sum(all_records, axis=0).items() if val]) + prev_anom = None + # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) + # all_records = query.data_frame() + # if not all_records.empty: + # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context context = {'inst': inst, From 1130a8d2897e5debf68a0d939da0a3453638e570 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 14 Nov 2018 11:55:19 -0500 Subject: [PATCH 08/33] Testing anomaly view with preloaded checkboxes. --- jwql/website/apps/jwql/templates/view_image.html | 7 +++++-- jwql/website/apps/jwql/views.py | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 16d4fb0f5..86bb6a532 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -150,10 +150,13 @@

        {{ file_root }}

        Anomalies in this image:
        -

        Previously flagged: {{ prev_anom }}

          {% for anom in anomalies %} -
        •   {{ anom }}
        • + {% if anom in prev_anom %} +
        •   {{ anom }}
        • + {% else %} +
        •   {{ anom }}
        • + {% endif %} {% endfor %}
        Submit Anomalies diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 726341093..ae1a07e1a 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -277,10 +277,12 @@ def view_image(request, inst, file_root, rewrite=False): image_info = get_image_info(file_root, rewrite) # Get a list of previously flagged anomalies - query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) - all_records = query.data_frame() - prev_anom = ', '.join([col for col, val in - np.sum(all_records, axis=0).items() if val]) + prev_anom = None + # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) + # all_records = query.data_frame() + # if not all_records.empty: + # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context context = {'inst': inst, From 65d9c9b77cbf599747c3ae23b6b05b9241afab90 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 18 Dec 2018 14:59:55 -0500 Subject: [PATCH 09/33] Fixed anomaly ORM to load dynamically, use memory db for testing, and write tests. --- jwql/database/database_interface.py | 110 +++++++++--------- jwql/tests/test_database_interface.py | 49 ++++++++ .../apps/jwql/templates/view_image.html | 6 +- jwql/website/apps/jwql/views.py | 14 ++- 4 files changed, 112 insertions(+), 67 deletions(-) create mode 100755 jwql/tests/test_database_interface.py diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 18e2532eb..751aace93 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -27,14 +27,10 @@ from datetime import datetime import pandas as pd -from sqlalchemy import Boolean -from sqlalchemy import Column +from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, String, Table from sqlalchemy import create_engine -from sqlalchemy import DateTime -from sqlalchemy import Integer -from sqlalchemy import MetaData -from sqlalchemy import String from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.query import Query @@ -92,65 +88,63 @@ def load_connection(connection_string): base = declarative_base(engine) Session = sessionmaker(bind=engine) session = Session() - meta = MetaData() + meta = MetaData(engine) + + # Make sure it has an anomalies table + if not engine.has_table('anomalies'): + print("No 'anomalies' table. Generating one now...") + + # Define anomaly table column names + columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', + 'cte_correction_error', 'data_transfer_error', 'detector_ghost', + 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', + 'excessive_saturation', 'figure8_ghost', 'filter_ghost', + 'fringing', 'guidestar_failure', 'banding', 'persistence', + 'prominent_blobs', 'trail', 'scattered_light', 'other'] + + # Create a table with the appropriate Columns + anomalies = Table('anomalies', meta, + Column('id', Integer, primary_key=True, nullable=False), + Column('filename', String, nullable=False), + Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())), + *[Column(name, String, nullable=False, server_default="False") for name in columns]) + + # Implement it + meta.create_all() return session, base, engine, meta +# Load the objects for the database session, base, engine, meta = load_connection(SETTINGS['connection_string']) -class Anomaly(base): - """ORM for the anomalies table""" - # Name the table - __tablename__ = 'anomalies' - - # Define the columns - id = Column(Integer, primary_key=True, nullable=False) - filename = Column(String, nullable=False) - flag_date = Column(DateTime, nullable=False, default=datetime.now()) - bowtie = Column(Boolean, nullable=False, default=False) - snowball = Column(Boolean, nullable=False, default=False) - cosmic_ray_shower = Column(Boolean, nullable=False, default=False) - crosstalk = Column(Boolean, nullable=False, default=False) - cte_correction_error = Column(Boolean, nullable=False, default=False) - data_transfer_error = Column(Boolean, nullable=False, default=False) - detector_ghost = Column(Boolean, nullable=False, default=False) - diamond = Column(Boolean, nullable=False, default=False) - diffraction_spike = Column(Boolean, nullable=False, default=False) - dragon_breath = Column(Boolean, nullable=False, default=False) - earth_limb = Column(Boolean, nullable=False, default=False) - excessive_saturation = Column(Boolean, nullable=False, default=False) - figure8_ghost = Column(Boolean, nullable=False, default=False) - filter_ghost = Column(Boolean, nullable=False, default=False) - fringing = Column(Boolean, nullable=False, default=False) - guidestar_failure = Column(Boolean, nullable=False, default=False) - banding = Column(Boolean, nullable=False, default=False) - persistence = Column(Boolean, nullable=False, default=False) - prominent_blobs = Column(Boolean, nullable=False, default=False) - trail = Column(Boolean, nullable=False, default=False) - scattered_light = Column(Boolean, nullable=False, default=False) - other = Column(Boolean, nullable=False, default=False) - - def __repr__(self): - """Return the canonical string representation of the object""" - # Get the columns that are True - a_list = [col for col, val in self.__dict__.items() - if val is True and isinstance(val, bool)] - - txt = ('Anomaly {0.id}: {0.filename} flagged at ' - '{0.flag_date} for {1}').format(self, a_list) - - return txt - - @property - def colnames(self): - """A list of all the column names in this table""" - # Get the columns - a_list = [col for col, val in self._sa_instance_state.attrs.items() - if col not in ['id', 'filename', 'flag_date']] - - return a_list +# Make convenience methods for Base class +@property +def colnames(self): + """A list of all the column names in this table""" + # Get the columns + a_list = sorted([col for col, val in self._sa_instance_state.attrs.items() + if col not in ['id', 'filename', 'flag_date']]) + + return a_list + + +@property +def names(self): + """A list of human readable names for all the columns in this table""" + return [name.replace('_', ' ') for name in self.colnames] + + +# Generate Base class and add methods +Base = automap_base() +Base.colnames = colnames +Base.names = names +Base.prepare(engine, reflect=True) + + +# Automap Anomaly class from Base class for anomalies table +Anomaly = Base.classes.anomalies if __name__ == '__main__': diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py new file mode 100755 index 000000000..a1ea4a1b8 --- /dev/null +++ b/jwql/tests/test_database_interface.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python + +"""Tests for the ``database_interface.py`` module. + +Authors +------- + + Joe Filippazzo + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s database_interface.py +""" + +from jwql.database import database_interface as di + + +def test_anomaly_table(): + """Test to see that the database has an anomalies table""" + assert 'anomalies' in di.engine.table_names() + + +def test_anomaly_records(): + """Test to see that new records can be entered""" + # Add some data + di.session.add(di.Anomaly(filename='foo1', bowtie="True")) + di.session.commit() + + # Test the bowties column + bowties = di.session.query(di.Anomaly).filter(di.Anomaly.bowtie == "True") + assert bowties.data_frame.iloc[0]['bowtie'] == "True" + + # Test the other columns + non_bowties = [col for col in di.Anomaly().colnames if col != 'bowtie'] + assert all([i == "False" for i in bowties.data_frame.iloc[0][non_bowties]]) + + +def test_names_colnames(): + """Test that the column names are correct""" + # Make sure we get non-empty lists + anom = di.Anomaly() + assert isinstance(anom.colnames, list) and len(anom.colnames) > 0 + assert isinstance(anom.names, list) and len(anom.names) == len(anom.colnames) + assert all([i == j.replace('_', ' ') for i, j in zip(anom.names, anom.colnames)]) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 86bb6a532..cdde71f6e 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -151,11 +151,11 @@

        {{ file_root }}

        Anomalies in this image:
          - {% for anom in anomalies %} + {% for anom, name in anomalies %} {% if anom in prev_anom %} -
        •   {{ anom }}
        • +
        •   {{ name }}
        • {% else %} -
        •   {{ anom }}
        • +
        •   {{ name }}
        • {% endif %} {% endfor %}
        diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index ae1a07e1a..38c1f81bf 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -278,13 +278,15 @@ def view_image(request, inst, file_root, rewrite=False): # Get a list of previously flagged anomalies prev_anom = None - # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) - # all_records = query.data_frame() - # if not all_records.empty: - # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - prev_anom = ', '.join(['bowtie', 'snowball']) + query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + all_records = query.data_frame() + print(all_records) + if not all_records.empty: + prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + # prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context + anom = di.Anomaly() context = {'inst': inst, 'file_root': file_root, 'tools': MONITORS, @@ -292,7 +294,7 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': di.Anomaly().colnames, + 'anomalies': zip(anom.colnames, anom.names), 'prev_anom': prev_anom} return render(request, template, context) From 4676f9ec44adffba307e08756ea612213c255cfa Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Tue, 18 Dec 2018 14:59:55 -0500 Subject: [PATCH 10/33] Fixed anomaly ORM to load dynamically, use memory db for testing, and write tests. --- jwql/database/database_interface.py | 110 +++++++++--------- jwql/tests/test_database_interface.py | 49 ++++++++ .../apps/jwql/templates/view_image.html | 6 +- jwql/website/apps/jwql/views.py | 14 ++- 4 files changed, 112 insertions(+), 67 deletions(-) create mode 100755 jwql/tests/test_database_interface.py diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 18e2532eb..751aace93 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -27,14 +27,10 @@ from datetime import datetime import pandas as pd -from sqlalchemy import Boolean -from sqlalchemy import Column +from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, String, Table from sqlalchemy import create_engine -from sqlalchemy import DateTime -from sqlalchemy import Integer -from sqlalchemy import MetaData -from sqlalchemy import String from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import sessionmaker from sqlalchemy.orm.query import Query @@ -92,65 +88,63 @@ def load_connection(connection_string): base = declarative_base(engine) Session = sessionmaker(bind=engine) session = Session() - meta = MetaData() + meta = MetaData(engine) + + # Make sure it has an anomalies table + if not engine.has_table('anomalies'): + print("No 'anomalies' table. Generating one now...") + + # Define anomaly table column names + columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', + 'cte_correction_error', 'data_transfer_error', 'detector_ghost', + 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', + 'excessive_saturation', 'figure8_ghost', 'filter_ghost', + 'fringing', 'guidestar_failure', 'banding', 'persistence', + 'prominent_blobs', 'trail', 'scattered_light', 'other'] + + # Create a table with the appropriate Columns + anomalies = Table('anomalies', meta, + Column('id', Integer, primary_key=True, nullable=False), + Column('filename', String, nullable=False), + Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())), + *[Column(name, String, nullable=False, server_default="False") for name in columns]) + + # Implement it + meta.create_all() return session, base, engine, meta +# Load the objects for the database session, base, engine, meta = load_connection(SETTINGS['connection_string']) -class Anomaly(base): - """ORM for the anomalies table""" - # Name the table - __tablename__ = 'anomalies' - - # Define the columns - id = Column(Integer, primary_key=True, nullable=False) - filename = Column(String, nullable=False) - flag_date = Column(DateTime, nullable=False, default=datetime.now()) - bowtie = Column(Boolean, nullable=False, default=False) - snowball = Column(Boolean, nullable=False, default=False) - cosmic_ray_shower = Column(Boolean, nullable=False, default=False) - crosstalk = Column(Boolean, nullable=False, default=False) - cte_correction_error = Column(Boolean, nullable=False, default=False) - data_transfer_error = Column(Boolean, nullable=False, default=False) - detector_ghost = Column(Boolean, nullable=False, default=False) - diamond = Column(Boolean, nullable=False, default=False) - diffraction_spike = Column(Boolean, nullable=False, default=False) - dragon_breath = Column(Boolean, nullable=False, default=False) - earth_limb = Column(Boolean, nullable=False, default=False) - excessive_saturation = Column(Boolean, nullable=False, default=False) - figure8_ghost = Column(Boolean, nullable=False, default=False) - filter_ghost = Column(Boolean, nullable=False, default=False) - fringing = Column(Boolean, nullable=False, default=False) - guidestar_failure = Column(Boolean, nullable=False, default=False) - banding = Column(Boolean, nullable=False, default=False) - persistence = Column(Boolean, nullable=False, default=False) - prominent_blobs = Column(Boolean, nullable=False, default=False) - trail = Column(Boolean, nullable=False, default=False) - scattered_light = Column(Boolean, nullable=False, default=False) - other = Column(Boolean, nullable=False, default=False) - - def __repr__(self): - """Return the canonical string representation of the object""" - # Get the columns that are True - a_list = [col for col, val in self.__dict__.items() - if val is True and isinstance(val, bool)] - - txt = ('Anomaly {0.id}: {0.filename} flagged at ' - '{0.flag_date} for {1}').format(self, a_list) - - return txt - - @property - def colnames(self): - """A list of all the column names in this table""" - # Get the columns - a_list = [col for col, val in self._sa_instance_state.attrs.items() - if col not in ['id', 'filename', 'flag_date']] - - return a_list +# Make convenience methods for Base class +@property +def colnames(self): + """A list of all the column names in this table""" + # Get the columns + a_list = sorted([col for col, val in self._sa_instance_state.attrs.items() + if col not in ['id', 'filename', 'flag_date']]) + + return a_list + + +@property +def names(self): + """A list of human readable names for all the columns in this table""" + return [name.replace('_', ' ') for name in self.colnames] + + +# Generate Base class and add methods +Base = automap_base() +Base.colnames = colnames +Base.names = names +Base.prepare(engine, reflect=True) + + +# Automap Anomaly class from Base class for anomalies table +Anomaly = Base.classes.anomalies if __name__ == '__main__': diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py new file mode 100755 index 000000000..a1ea4a1b8 --- /dev/null +++ b/jwql/tests/test_database_interface.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python + +"""Tests for the ``database_interface.py`` module. + +Authors +------- + + Joe Filippazzo + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s database_interface.py +""" + +from jwql.database import database_interface as di + + +def test_anomaly_table(): + """Test to see that the database has an anomalies table""" + assert 'anomalies' in di.engine.table_names() + + +def test_anomaly_records(): + """Test to see that new records can be entered""" + # Add some data + di.session.add(di.Anomaly(filename='foo1', bowtie="True")) + di.session.commit() + + # Test the bowties column + bowties = di.session.query(di.Anomaly).filter(di.Anomaly.bowtie == "True") + assert bowties.data_frame.iloc[0]['bowtie'] == "True" + + # Test the other columns + non_bowties = [col for col in di.Anomaly().colnames if col != 'bowtie'] + assert all([i == "False" for i in bowties.data_frame.iloc[0][non_bowties]]) + + +def test_names_colnames(): + """Test that the column names are correct""" + # Make sure we get non-empty lists + anom = di.Anomaly() + assert isinstance(anom.colnames, list) and len(anom.colnames) > 0 + assert isinstance(anom.names, list) and len(anom.names) == len(anom.colnames) + assert all([i == j.replace('_', ' ') for i, j in zip(anom.names, anom.colnames)]) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 86bb6a532..cdde71f6e 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -151,11 +151,11 @@

        {{ file_root }}

        Anomalies in this image:
          - {% for anom in anomalies %} + {% for anom, name in anomalies %} {% if anom in prev_anom %} -
        •   {{ anom }}
        • +
        •   {{ name }}
        • {% else %} -
        •   {{ anom }}
        • +
        •   {{ name }}
        • {% endif %} {% endfor %}
        diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index ae1a07e1a..38c1f81bf 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -278,13 +278,15 @@ def view_image(request, inst, file_root, rewrite=False): # Get a list of previously flagged anomalies prev_anom = None - # query = di.session.query(di.Anomaly).filter(di.Anomaly.file_root == file_root) - # all_records = query.data_frame() - # if not all_records.empty: - # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - prev_anom = ', '.join(['bowtie', 'snowball']) + query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + all_records = query.data_frame() + print(all_records) + if not all_records.empty: + prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + # prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context + anom = di.Anomaly() context = {'inst': inst, 'file_root': file_root, 'tools': MONITORS, @@ -292,7 +294,7 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': di.Anomaly().colnames, + 'anomalies': zip(anom.colnames, anom.names), 'prev_anom': prev_anom} return render(request, template, context) From a6d64688b805806f4c291853998e5fe802821d6f Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 30 Jan 2019 11:29:12 -0500 Subject: [PATCH 11/33] Paceholder until test db is up. --- jwql/website/apps/jwql/views.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 38c1f81bf..2d22ceda3 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -277,13 +277,13 @@ def view_image(request, inst, file_root, rewrite=False): image_info = get_image_info(file_root, rewrite) # Get a list of previously flagged anomalies - prev_anom = None - query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) - all_records = query.data_frame() - print(all_records) - if not all_records.empty: - prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - # prev_anom = ', '.join(['bowtie', 'snowball']) + # prev_anom = None + # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + # all_records = query.data_frame() + # print(all_records) + # if not all_records.empty: + # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context anom = di.Anomaly() From d220b94684fa5fdfb2fafd2ca6f9edf33c5974f5 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Wed, 30 Jan 2019 11:29:12 -0500 Subject: [PATCH 12/33] Paceholder until test db is up. --- jwql/website/apps/jwql/views.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 38c1f81bf..2d22ceda3 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -277,13 +277,13 @@ def view_image(request, inst, file_root, rewrite=False): image_info = get_image_info(file_root, rewrite) # Get a list of previously flagged anomalies - prev_anom = None - query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) - all_records = query.data_frame() - print(all_records) - if not all_records.empty: - prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - # prev_anom = ', '.join(['bowtie', 'snowball']) + # prev_anom = None + # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + # all_records = query.data_frame() + # print(all_records) + # if not all_records.empty: + # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context anom = di.Anomaly() From 30310cada53ffe0ab992e8769a6514fd6f1292ec Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 25 Mar 2019 14:19:28 -0400 Subject: [PATCH 13/33] Enabled db lookup for anomaly table --- jwql/website/apps/jwql/views.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 2d22ceda3..0fa6cc7db 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -277,13 +277,12 @@ def view_image(request, inst, file_root, rewrite=False): image_info = get_image_info(file_root, rewrite) # Get a list of previously flagged anomalies - # prev_anom = None - # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) - # all_records = query.data_frame() - # print(all_records) - # if not all_records.empty: - # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - prev_anom = ', '.join(['bowtie', 'snowball']) + prev_anom = None + query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + all_records = query.data_frame() + if not all_records.empty: + prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + # prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context anom = di.Anomaly() From c3caed3e3d1337875a77714308f59370abab24d2 Mon Sep 17 00:00:00 2001 From: Joe Filippazzo Date: Mon, 25 Mar 2019 14:19:28 -0400 Subject: [PATCH 14/33] Enabled db lookup for anomaly table --- jwql/website/apps/jwql/views.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 2d22ceda3..0fa6cc7db 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -277,13 +277,12 @@ def view_image(request, inst, file_root, rewrite=False): image_info = get_image_info(file_root, rewrite) # Get a list of previously flagged anomalies - # prev_anom = None - # query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) - # all_records = query.data_frame() - # print(all_records) - # if not all_records.empty: - # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - prev_anom = ', '.join(['bowtie', 'snowball']) + prev_anom = None + query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + all_records = query.data_frame() + if not all_records.empty: + prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + # prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context anom = di.Anomaly() From 0ac57b91de69f813f4bbc46cd6a9ce3994080c9e Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 25 Mar 2019 14:54:48 -0400 Subject: [PATCH 15/33] Attempting to insert a record into database upon page load --- jwql/database/database_interface.py | 2 +- jwql/website/apps/jwql/views.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 1cf2200ff..b5645f847 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -147,7 +147,7 @@ def load_connection(connection_string): Column('id', Integer, primary_key=True, nullable=False), Column('filename', String, nullable=False), Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())), - *[Column(name, String, nullable=False, server_default="False") for name in columns]) + *[Column(name, String, nullable=True, server_default=False) for name in columns]) # Implement it meta.create_all() diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index ab0fa90a6..727d06c7e 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -35,6 +35,7 @@ placed in the ``jwql/utils/`` directory. """ +import datetime import os from django.http import JsonResponse @@ -50,7 +51,7 @@ from .data_containers import thumbnails_ajax from .forms import FileSearchForm from jwql.database import database_interface as di -from jwql.utils.constants import JWST_INSTRUMENTS, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS from jwql.utils.utils import get_base_url, get_config FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') @@ -389,9 +390,18 @@ def view_image(request, inst, file_root, rewrite=False): template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + + # Fake insert a record + data_dict = {} + data_dict['filename'] = 'jw00327001001_06101_00001_guider1' + data_dict['flag_date'] = datetime.datetime.now() + data_dict['bowtie'] = True + di.engine.execute(di.Anomaly.__table__.insert(), data_dict) + # Get a list of previously flagged anomalies prev_anom = None query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + print(query) all_records = query.data_frame() if not all_records.empty: prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) From 86ab670737d3e99fee4740abe8238895643de2dc Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 25 Mar 2019 14:54:48 -0400 Subject: [PATCH 16/33] Attempting to insert a record into database upon page load --- jwql/database/database_interface.py | 2 +- jwql/website/apps/jwql/views.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 1cf2200ff..b5645f847 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -147,7 +147,7 @@ def load_connection(connection_string): Column('id', Integer, primary_key=True, nullable=False), Column('filename', String, nullable=False), Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())), - *[Column(name, String, nullable=False, server_default="False") for name in columns]) + *[Column(name, String, nullable=True, server_default=False) for name in columns]) # Implement it meta.create_all() diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index ab0fa90a6..727d06c7e 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -35,6 +35,7 @@ placed in the ``jwql/utils/`` directory. """ +import datetime import os from django.http import JsonResponse @@ -50,7 +51,7 @@ from .data_containers import thumbnails_ajax from .forms import FileSearchForm from jwql.database import database_interface as di -from jwql.utils.constants import JWST_INSTRUMENTS, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS from jwql.utils.utils import get_base_url, get_config FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') @@ -389,9 +390,18 @@ def view_image(request, inst, file_root, rewrite=False): template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + + # Fake insert a record + data_dict = {} + data_dict['filename'] = 'jw00327001001_06101_00001_guider1' + data_dict['flag_date'] = datetime.datetime.now() + data_dict['bowtie'] = True + di.engine.execute(di.Anomaly.__table__.insert(), data_dict) + # Get a list of previously flagged anomalies prev_anom = None query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) + print(query) all_records = query.data_frame() if not all_records.empty: prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) From cfbd10e3b6ffc62c931bae507aa8d487ba55f189 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 26 Mar 2019 16:44:39 -0400 Subject: [PATCH 17/33] Moved anomaly table creation to its own ORM factory (to help with future instrument-specific anomaly tables). --- jwql/database/database_interface.py | 88 +++++++++++++---------------- jwql/website/apps/jwql/views.py | 16 +++--- 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index b5645f847..64c7e81c0 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -85,6 +85,7 @@ @property def data_frame(self): """Method to return a ``pandas.DataFrame`` of the results""" + print(pd.read_sql(self.statement, self.session.bind)) return pd.read_sql(self.statement, self.session.bind) Query.data_frame = data_frame @@ -130,28 +131,6 @@ def load_connection(connection_string): session = Session() meta = MetaData(engine) - # Make sure it has an anomalies table - if not engine.has_table('anomalies'): - print("No 'anomalies' table. Generating one now...") - - # Define anomaly table column names - columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', - 'cte_correction_error', 'data_transfer_error', 'detector_ghost', - 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', - 'excessive_saturation', 'figure8_ghost', 'filter_ghost', - 'fringing', 'guidestar_failure', 'banding', 'persistence', - 'prominent_blobs', 'trail', 'scattered_light', 'other'] - - # Create a table with the appropriate Columns - anomalies = Table('anomalies', meta, - Column('id', Integer, primary_key=True, nullable=False), - Column('filename', String, nullable=False), - Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())), - *[Column(name, String, nullable=True, server_default=False) for name in columns]) - - # Implement it - meta.create_all() - return session, base, engine, meta # Import a global session. If running from readthedocs, pass a dummy connection string @@ -162,33 +141,6 @@ def load_connection(connection_string): SETTINGS = utils.get_config() session, base, engine, meta = load_connection(SETTINGS['connection_string']) -# Make convenience methods for Base class -@property -def colnames(self): - """A list of all the column names in this table""" - # Get the columns - a_list = sorted([col for col, val in self._sa_instance_state.attrs.items() - if col not in ['id', 'filename', 'flag_date']]) - - return a_list - - -@property -def names(self): - """A list of human readable names for all the columns in this table""" - return [name.replace('_', ' ') for name in self.colnames] - - -# Generate Base class and add methods -Base = automap_base() -Base.colnames = colnames -Base.names = names -Base.prepare(engine, reflect=True) - - -# Automap Anomaly class from Base class for anomalies table -Anomaly = Base.classes.anomalies - class Monitor(base): """ORM for the ``monitor`` table""" @@ -205,6 +157,43 @@ class Monitor(base): log_file = Column(String(), nullable=False) +def anomaly_orm_factory(class_name): + """Create a ``SQLAlchemy`` ORM Class for an anomaly table. + + Parameters + ---------- + class_name : str + The name of the class to be created + + Returns + ------- + class : obj + The ``SQLAlchemy`` ORM + """ + + # Initialize a dictionary to hold the column metadata + data_dict = {} + data_dict['__tablename__'] = class_name.lower() + + # Define anomaly table column names + data_dict['columns'] = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', + 'cte_correction_error', 'data_transfer_error', 'detector_ghost', + 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', + 'excessive_saturation', 'figure8_ghost', 'filter_ghost', + 'fringing', 'guidestar_failure', 'banding', 'persistence', + 'prominent_blobs', 'trail', 'scattered_light', 'other'] + data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']] + + # Create a table with the appropriate Columns + data_dict['id'] = Column(Integer, primary_key=True, nullable=False) + data_dict['filename'] = Column(String(), nullable=False) + data_dict['flag_date'] = Column(DateTime, nullable=False) + for column in data_dict['columns']: + data_dict[column] = Column(Boolean, nullable=False, default=False) + + return type(class_name, (base,), data_dict) + + def get_monitor_columns(data_dict, table_name): """Read in the corresponding table definition text file to generate ``SQLAlchemy`` columns for the table. @@ -311,6 +300,7 @@ class : obj return type(class_name, (base,), data_dict) # Create tables from ORM factory +Anomaly = anomaly_orm_factory('anomaly') # NIRCamDarkQueries = monitor_orm_factory('nircam_dark_queries') # NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats') # NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current') diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 727d06c7e..bfbb48c40 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -40,6 +40,7 @@ from django.http import JsonResponse from django.shortcuts import render +import numpy as np from .data_containers import get_acknowledgements, get_edb_components from .data_containers import get_dashboard_components @@ -384,28 +385,25 @@ def view_image(request, inst, file_root, rewrite=False): HttpResponse object Outgoing response sent to the webpage """ + # Ensure the instrument is correctly capitalized inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] template = 'view_image.html' image_info = get_image_info(file_root, rewrite) - # Fake insert a record data_dict = {} data_dict['filename'] = 'jw00327001001_06101_00001_guider1' data_dict['flag_date'] = datetime.datetime.now() - data_dict['bowtie'] = True + data_dict['fringing'] = True di.engine.execute(di.Anomaly.__table__.insert(), data_dict) - # Get a list of previously flagged anomalies - prev_anom = None - query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) - print(query) - all_records = query.data_frame() + # Get most previously flagged anomalies + query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root).order_by(di.Anomaly.flag_date.desc()).limit(1) + all_records = query.data_frame if not all_records.empty: prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - # prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context anom = di.Anomaly() @@ -415,7 +413,7 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': zip(anom.colnames, anom.names), + 'anomalies': zip(anom.columns, anom.names), 'prev_anom': prev_anom} return render(request, template, context) From 6557bada2df44e0ce696f99fa6a6e16c6b50c8c5 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 26 Mar 2019 16:44:39 -0400 Subject: [PATCH 18/33] Moved anomaly table creation to its own ORM factory (to help with future instrument-specific anomaly tables). --- jwql/database/database_interface.py | 88 +++++++++++++---------------- jwql/website/apps/jwql/views.py | 16 +++--- 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index b5645f847..64c7e81c0 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -85,6 +85,7 @@ @property def data_frame(self): """Method to return a ``pandas.DataFrame`` of the results""" + print(pd.read_sql(self.statement, self.session.bind)) return pd.read_sql(self.statement, self.session.bind) Query.data_frame = data_frame @@ -130,28 +131,6 @@ def load_connection(connection_string): session = Session() meta = MetaData(engine) - # Make sure it has an anomalies table - if not engine.has_table('anomalies'): - print("No 'anomalies' table. Generating one now...") - - # Define anomaly table column names - columns = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', - 'cte_correction_error', 'data_transfer_error', 'detector_ghost', - 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', - 'excessive_saturation', 'figure8_ghost', 'filter_ghost', - 'fringing', 'guidestar_failure', 'banding', 'persistence', - 'prominent_blobs', 'trail', 'scattered_light', 'other'] - - # Create a table with the appropriate Columns - anomalies = Table('anomalies', meta, - Column('id', Integer, primary_key=True, nullable=False), - Column('filename', String, nullable=False), - Column('flag_date', DateTime, nullable=False, server_default=str(datetime.now())), - *[Column(name, String, nullable=True, server_default=False) for name in columns]) - - # Implement it - meta.create_all() - return session, base, engine, meta # Import a global session. If running from readthedocs, pass a dummy connection string @@ -162,33 +141,6 @@ def load_connection(connection_string): SETTINGS = utils.get_config() session, base, engine, meta = load_connection(SETTINGS['connection_string']) -# Make convenience methods for Base class -@property -def colnames(self): - """A list of all the column names in this table""" - # Get the columns - a_list = sorted([col for col, val in self._sa_instance_state.attrs.items() - if col not in ['id', 'filename', 'flag_date']]) - - return a_list - - -@property -def names(self): - """A list of human readable names for all the columns in this table""" - return [name.replace('_', ' ') for name in self.colnames] - - -# Generate Base class and add methods -Base = automap_base() -Base.colnames = colnames -Base.names = names -Base.prepare(engine, reflect=True) - - -# Automap Anomaly class from Base class for anomalies table -Anomaly = Base.classes.anomalies - class Monitor(base): """ORM for the ``monitor`` table""" @@ -205,6 +157,43 @@ class Monitor(base): log_file = Column(String(), nullable=False) +def anomaly_orm_factory(class_name): + """Create a ``SQLAlchemy`` ORM Class for an anomaly table. + + Parameters + ---------- + class_name : str + The name of the class to be created + + Returns + ------- + class : obj + The ``SQLAlchemy`` ORM + """ + + # Initialize a dictionary to hold the column metadata + data_dict = {} + data_dict['__tablename__'] = class_name.lower() + + # Define anomaly table column names + data_dict['columns'] = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', + 'cte_correction_error', 'data_transfer_error', 'detector_ghost', + 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', + 'excessive_saturation', 'figure8_ghost', 'filter_ghost', + 'fringing', 'guidestar_failure', 'banding', 'persistence', + 'prominent_blobs', 'trail', 'scattered_light', 'other'] + data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']] + + # Create a table with the appropriate Columns + data_dict['id'] = Column(Integer, primary_key=True, nullable=False) + data_dict['filename'] = Column(String(), nullable=False) + data_dict['flag_date'] = Column(DateTime, nullable=False) + for column in data_dict['columns']: + data_dict[column] = Column(Boolean, nullable=False, default=False) + + return type(class_name, (base,), data_dict) + + def get_monitor_columns(data_dict, table_name): """Read in the corresponding table definition text file to generate ``SQLAlchemy`` columns for the table. @@ -311,6 +300,7 @@ class : obj return type(class_name, (base,), data_dict) # Create tables from ORM factory +Anomaly = anomaly_orm_factory('anomaly') # NIRCamDarkQueries = monitor_orm_factory('nircam_dark_queries') # NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats') # NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current') diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 727d06c7e..bfbb48c40 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -40,6 +40,7 @@ from django.http import JsonResponse from django.shortcuts import render +import numpy as np from .data_containers import get_acknowledgements, get_edb_components from .data_containers import get_dashboard_components @@ -384,28 +385,25 @@ def view_image(request, inst, file_root, rewrite=False): HttpResponse object Outgoing response sent to the webpage """ + # Ensure the instrument is correctly capitalized inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] template = 'view_image.html' image_info = get_image_info(file_root, rewrite) - # Fake insert a record data_dict = {} data_dict['filename'] = 'jw00327001001_06101_00001_guider1' data_dict['flag_date'] = datetime.datetime.now() - data_dict['bowtie'] = True + data_dict['fringing'] = True di.engine.execute(di.Anomaly.__table__.insert(), data_dict) - # Get a list of previously flagged anomalies - prev_anom = None - query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root) - print(query) - all_records = query.data_frame() + # Get most previously flagged anomalies + query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root).order_by(di.Anomaly.flag_date.desc()).limit(1) + all_records = query.data_frame if not all_records.empty: prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - # prev_anom = ', '.join(['bowtie', 'snowball']) # Build the context anom = di.Anomaly() @@ -415,7 +413,7 @@ def view_image(request, inst, file_root, rewrite=False): 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': zip(anom.colnames, anom.names), + 'anomalies': zip(anom.columns, anom.names), 'prev_anom': prev_anom} return render(request, template, context) From f63fa4cb128ba27fa1acd839a1bb4eac2b7eda4f Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 2 Apr 2019 11:23:20 -0400 Subject: [PATCH 19/33] Added user column to anomaly table --- jwql/database/database_interface.py | 2 ++ jwql/website/apps/jwql/views.py | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 64c7e81c0..6034d6d9f 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -188,6 +188,8 @@ class : obj data_dict['id'] = Column(Integer, primary_key=True, nullable=False) data_dict['filename'] = Column(String(), nullable=False) data_dict['flag_date'] = Column(DateTime, nullable=False) + data_dict['user'] = Column(String(), nullable=False) + for column in data_dict['columns']: data_dict[column] = Column(Boolean, nullable=False, default=False) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index bfbb48c40..fd913e069 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -51,6 +51,7 @@ from .data_containers import thumbnails from .data_containers import thumbnails_ajax from .forms import FileSearchForm +from .oauth import auth_info from jwql.database import database_interface as di from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS from jwql.utils.utils import get_base_url, get_config @@ -365,8 +366,8 @@ def view_header(request, inst, file): return render(request, template, context) - -def view_image(request, inst, file_root, rewrite=False): +@auth_info +def view_image(request, user, inst, file_root, rewrite=False): """Generate the image view page Parameters @@ -392,11 +393,14 @@ def view_image(request, inst, file_root, rewrite=False): template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + print(user) + # Fake insert a record data_dict = {} data_dict['filename'] = 'jw00327001001_06101_00001_guider1' data_dict['flag_date'] = datetime.datetime.now() - data_dict['fringing'] = True + data_dict['bowtie'] = False + data_dict['user'] = user['ezid'] di.engine.execute(di.Anomaly.__table__.insert(), data_dict) # Get most previously flagged anomalies From fe1f272338b22d590ce39efd9fb7363020523080 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 2 Apr 2019 11:23:20 -0400 Subject: [PATCH 20/33] Added user column to anomaly table --- jwql/database/database_interface.py | 2 ++ jwql/website/apps/jwql/views.py | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 64c7e81c0..6034d6d9f 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -188,6 +188,8 @@ class : obj data_dict['id'] = Column(Integer, primary_key=True, nullable=False) data_dict['filename'] = Column(String(), nullable=False) data_dict['flag_date'] = Column(DateTime, nullable=False) + data_dict['user'] = Column(String(), nullable=False) + for column in data_dict['columns']: data_dict[column] = Column(Boolean, nullable=False, default=False) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index bfbb48c40..fd913e069 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -51,6 +51,7 @@ from .data_containers import thumbnails from .data_containers import thumbnails_ajax from .forms import FileSearchForm +from .oauth import auth_info from jwql.database import database_interface as di from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MONITORS from jwql.utils.utils import get_base_url, get_config @@ -365,8 +366,8 @@ def view_header(request, inst, file): return render(request, template, context) - -def view_image(request, inst, file_root, rewrite=False): +@auth_info +def view_image(request, user, inst, file_root, rewrite=False): """Generate the image view page Parameters @@ -392,11 +393,14 @@ def view_image(request, inst, file_root, rewrite=False): template = 'view_image.html' image_info = get_image_info(file_root, rewrite) + print(user) + # Fake insert a record data_dict = {} data_dict['filename'] = 'jw00327001001_06101_00001_guider1' data_dict['flag_date'] = datetime.datetime.now() - data_dict['fringing'] = True + data_dict['bowtie'] = False + data_dict['user'] = user['ezid'] di.engine.execute(di.Anomaly.__table__.insert(), data_dict) # Get most previously flagged anomalies From 53ad89fee89c773b222932113cf4c846615b0882 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 21 May 2019 12:19:22 -0400 Subject: [PATCH 21/33] Moved anomaly definitions to constants.py --- jwql/database/database_interface.py | 65 +---------------------------- jwql/utils/constants.py | 20 +++++++++ 2 files changed, 22 insertions(+), 63 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index dc1367caa..477ed9ab1 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -79,7 +79,7 @@ from sqlalchemy.orm.query import Query from sqlalchemy.types import ARRAY -from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES +from jwql.utils.constants import ANOMALIES, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES from jwql.utils.utils import get_config @@ -144,62 +144,6 @@ def load_connection(connection_string): session, base, engine, meta = load_connection(SETTINGS['connection_string']) -class Anomaly(base): - """ORM for the ``anomalies`` table""" - - # Name the table - __tablename__ = 'anomalies' - - # Define the columns - id = Column(Integer, primary_key=True, nullable=False) - filename = Column(String, nullable=False) - flag_date = Column(DateTime, nullable=False, default=datetime.now()) - bowtie = Column(Boolean, nullable=False, default=False) - snowball = Column(Boolean, nullable=False, default=False) - cosmic_ray_shower = Column(Boolean, nullable=False, default=False) - crosstalk = Column(Boolean, nullable=False, default=False) - cte_correction_error = Column(Boolean, nullable=False, default=False) - data_transfer_error = Column(Boolean, nullable=False, default=False) - detector_ghost = Column(Boolean, nullable=False, default=False) - diamond = Column(Boolean, nullable=False, default=False) - diffraction_spike = Column(Boolean, nullable=False, default=False) - dragon_breath = Column(Boolean, nullable=False, default=False) - earth_limb = Column(Boolean, nullable=False, default=False) - excessive_saturation = Column(Boolean, nullable=False, default=False) - figure8_ghost = Column(Boolean, nullable=False, default=False) - filter_ghost = Column(Boolean, nullable=False, default=False) - fringing = Column(Boolean, nullable=False, default=False) - guidestar_failure = Column(Boolean, nullable=False, default=False) - banding = Column(Boolean, nullable=False, default=False) - persistence = Column(Boolean, nullable=False, default=False) - prominent_blobs = Column(Boolean, nullable=False, default=False) - trail = Column(Boolean, nullable=False, default=False) - scattered_light = Column(Boolean, nullable=False, default=False) - other = Column(Boolean, nullable=False, default=False) - - def __repr__(self): - """Return the canonical string representation of the object""" - - # Get the columns that are True - a_list = [col for col, val in self.__dict__.items() - if val is True and isinstance(val, bool)] - - txt = ('Anomaly {0.id}: {0.filename} flagged at ' - '{0.flag_date} for {1}').format(self, a_list) - - return txt - - @property - def colnames(self): - """A list of all the column names in this table""" - - # Get the columns - a_list = [col for col, val in self.__dict__.items() - if isinstance(val, bool)] - - return a_list - - class FilesystemGeneral(base): """ORM for the general (non instrument specific) filesystem monitor table""" @@ -277,12 +221,7 @@ class : obj data_dict['__tablename__'] = class_name.lower() # Define anomaly table column names - data_dict['columns'] = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', - 'cte_correction_error', 'data_transfer_error', 'detector_ghost', - 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', - 'excessive_saturation', 'figure8_ghost', 'filter_ghost', - 'fringing', 'guidestar_failure', 'banding', 'persistence', - 'prominent_blobs', 'trail', 'scattered_light', 'other'] + data_dict['columns'] = ANOMALIES data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']] # Create a table with the appropriate Columns diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 3cb8fa7b8..562c413fb 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -25,6 +25,26 @@ '3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]} } +# Defines the possible anomalies to flag through the web app +ANOMALIES = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', + 'cte_correction_error', 'data_transfer_error', 'detector_ghost', + 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', + 'excessive_saturation', 'figure8_ghost', 'filter_ghost', + 'fringing', 'guidestar_failure', 'banding', 'persistence', + 'prominent_blobs', 'trail', 'scattered_light', 'other'] + +# Defines the possible anomalies (with rendered name) to flag through the web app +ANOMALY_CHOICES = [ + ('bowtie', 'Bowtie'), ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'), + ('crosstalk', 'Crosstalk'), ('cte_correction_error', 'CTE Correction Error'), + ('data_transfer_error', 'Data Transfer Error'), ('detector_ghost', 'Detector Ghost'), + ('diamond', 'Diamond Feature'), ('diffraction_spike', 'Diffraction Spike'), + ('dragon_breath', 'Dragons Breath'), ('earth_limb', 'Earth Limb'), + ('excessive_saturation', 'Excessive Saturation'), ('figure8_ghost', 'Figure 8 Ghost'), + ('filter_ghost', 'Filter Ghost'), ('fringing', 'Fringing'), ('guidestar_failure', 'Guidestar Failure'), + ('banding', 'Banding'), ('persistence', 'Persistence'), ('prominent_blobs', 'Prominent Blobs'), + ('trail', 'Trail'), ('scattered_light', 'Scattered Light'), ('other', 'Other')] + # Possible suffix types for nominal files GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d', 'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints', From d2e638a42ef7767e20fb96c5c665802ef1a6c0e9 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 21 May 2019 12:19:47 -0400 Subject: [PATCH 22/33] Making use of django forms to define anomaly options --- jwql/website/apps/jwql/forms.py | 10 ++- .../apps/jwql/templates/view_image.html | 75 +++++++------------ jwql/website/apps/jwql/views.py | 11 ++- 3 files changed, 44 insertions(+), 52 deletions(-) diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index 67d172d17..0f65864d3 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -48,12 +48,20 @@ def view_function(request): from django.shortcuts import redirect from jwedb.edb_interface import is_valid_mnemonic -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_SHORTHAND +from jwql.utils.constants import ANOMALY_CHOICES, JWST_INSTRUMENT_NAMES_SHORTHAND from jwql.utils.utils import get_config, filename_parser FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') +class AnomalySubmitForm(forms.Form): + """ + """ + + # Define search field + anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, initial='', widget=forms.CheckboxSelectMultiple(), required=True) + + class FileSearchForm(forms.Form): """Single-field form to search for a proposal or fileroot.""" diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 4c32f8c97..16445ad31 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -50,20 +50,33 @@

        {{ file_root }}

        - -
        -
        Anomalies in this image:
        -
          - {% for anom, name in anomalies %} - {% if anom in prev_anom %} -
        •   {{ name }}
        • - {% else %} -
        •   {{ name }}
        • - {% endif %} + +
          + + {% if form.errors %} + {% for field in form %} + {% for error in field.errors %} +
          + {{ error|escape }} +
          + {% endfor %} {% endfor %} -
        - Submit Anomalies -
        + {% endif %} + + + {{ csrf_input }} + + + {% for field in form %} +
        + {{ field }} + {% if field.help_text %} +

        {{ field.help_text|safe }}

        + {% endif %} +
        + {% endfor %} + +

        @@ -84,42 +97,6 @@

        Anomalies in this image:
        Lauren needs to figure out what to do with these: {{suffixes}} {% endif %} - - - - {% endblock %} \ No newline at end of file diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 17fafc590..3b3012417 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -53,7 +53,7 @@ from .data_containers import thumbnails_ajax from .data_containers import data_trending from .data_containers import nirspec_trending -from .forms import FileSearchForm +from .forms import AnomalySubmitForm, FileSearchForm from .oauth import auth_info, auth_required from jwql.database import database_interface as di from jwql.utils.constants import JWST_INSTRUMENT_NAMES, MONITORS, JWST_INSTRUMENT_NAMES_MIXEDCASE @@ -497,6 +497,12 @@ def view_image(request, user, inst, file_root, rewrite=False): all_records = query.data_frame if not all_records.empty: prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + else: + prev_anom = '' + + + # Create a form instance and populate it with data from the request + form = AnomalySubmitForm(request.POST or None) # Build the context anom = di.Anomaly() @@ -507,6 +513,7 @@ def view_image(request, user, inst, file_root, rewrite=False): 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], 'anomalies': zip(anom.columns, anom.names), - 'prev_anom': prev_anom} + 'prev_anom': prev_anom, + 'form': form} return render(request, template, context) From 3f4aa32d26af2946b61d8f01285c3265c3786346 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 21 May 2019 14:42:13 -0400 Subject: [PATCH 23/33] Implemented updating anomaly database table upon anomaly form submit --- jwql/database/database_interface.py | 2 +- jwql/website/apps/jwql/forms.py | 25 +++++++++++++++++++++- jwql/website/apps/jwql/views.py | 32 ++++++++--------------------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 477ed9ab1..6af3d405e 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -226,7 +226,7 @@ class : obj # Create a table with the appropriate Columns data_dict['id'] = Column(Integer, primary_key=True, nullable=False) - data_dict['filename'] = Column(String(), nullable=False) + data_dict['rootname'] = Column(String(), nullable=False) data_dict['flag_date'] = Column(DateTime, nullable=False) data_dict['user'] = Column(String(), nullable=False) diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index 0f65864d3..dbbb959de 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -40,14 +40,17 @@ def view_function(request): placed in the ``jwql/utils/`` directory. """ + +import datetime import glob import os from astropy.time import Time, TimeDelta from django import forms from django.shortcuts import redirect - from jwedb.edb_interface import is_valid_mnemonic + +from jwql.database import database_interface as di from jwql.utils.constants import ANOMALY_CHOICES, JWST_INSTRUMENT_NAMES_SHORTHAND from jwql.utils.utils import get_config, filename_parser @@ -61,6 +64,26 @@ class AnomalySubmitForm(forms.Form): # Define search field anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, initial='', widget=forms.CheckboxSelectMultiple(), required=True) + def update_anomaly_table(self, rootname, user, anomaly_choices): + """ + """ + + data_dict = {} + data_dict['rootname'] = rootname + data_dict['flag_date'] = datetime.datetime.now() + data_dict['user'] = user + for choice in anomaly_choices: + data_dict[choice] = True + di.engine.execute(di.Anomaly.__table__.insert(), data_dict) + + # # Get most previously flagged anomalies + # query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1) + # all_records = query.data_frame + # if not all_records.empty: + # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) + # else: + # prev_anom = '' + class FileSearchForm(forms.Form): """Single-field form to search for a proposal or fileroot.""" diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 3b3012417..33d308554 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -463,9 +463,11 @@ def view_image(request, user, inst, file_root, rewrite=False): ---------- request : HttpRequest object Incoming request from the webpage + user : dict + A dictionary of user credentials. inst : str Name of JWST instrument - file : str + file_root : str FITS filename of selected image in filesystem rewrite : bool, optional Regenerate the jpg preview of `file` if it already exists? @@ -482,38 +484,22 @@ def view_image(request, user, inst, file_root, rewrite=False): template = 'view_image.html' image_info = get_image_info(file_root, rewrite) - print(user) - - # Fake insert a record - data_dict = {} - data_dict['filename'] = 'jw00327001001_06101_00001_guider1' - data_dict['flag_date'] = datetime.datetime.now() - data_dict['bowtie'] = False - data_dict['user'] = user['ezid'] - di.engine.execute(di.Anomaly.__table__.insert(), data_dict) - - # Get most previously flagged anomalies - query = di.session.query(di.Anomaly).filter(di.Anomaly.filename == file_root).order_by(di.Anomaly.flag_date.desc()).limit(1) - all_records = query.data_frame - if not all_records.empty: - prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - else: - prev_anom = '' - - # Create a form instance and populate it with data from the request form = AnomalySubmitForm(request.POST or None) + # If this is a POST request, we need to process the form data + if request.method == 'POST': + anomaly_choices = dict(request.POST)['anomaly_choices'] + if form.is_valid(): + form.update_anomaly_table(file_root, user['ezid'], anomaly_choices) + # Build the context - anom = di.Anomaly() context = {'inst': inst, 'file_root': file_root, 'jpg_files': image_info['all_jpegs'], 'fits_files': image_info['all_files'], 'suffixes': image_info['suffixes'], 'num_ints': image_info['num_ints'], - 'anomalies': zip(anom.columns, anom.names), - 'prev_anom': prev_anom, 'form': form} return render(request, template, context) From 02dd41c75b22fd126644504e08138b91c5029656 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 21 May 2019 16:36:27 -0400 Subject: [PATCH 24/33] Added docstrings and implemented automatic selection of currently flagged anomalies --- jwql/database/database_interface.py | 2 +- jwql/website/apps/jwql/data_containers.py | 28 +++++++++++++++++++++ jwql/website/apps/jwql/forms.py | 30 +++++++++++++---------- jwql/website/apps/jwql/views.py | 12 +++++---- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 6af3d405e..b76a2c3b6 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -87,7 +87,7 @@ @property def data_frame(self): """Method to return a ``pandas.DataFrame`` of the results""" - print(pd.read_sql(self.statement, self.session.bind)) + return pd.read_sql(self.statement, self.session.bind) Query.data_frame = data_frame diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 7018b2b47..d5fd631ce 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -42,6 +42,7 @@ from astroquery.mast import Mast from jwedb.edb_interface import mnemonic_inventory +from jwql.database import database_interface as di from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info from jwql.instrument_monitors.miri_monitors.data_trending import dashboard as miri_dash from jwql.instrument_monitors.nirspec_monitors.data_trending import dashboard as nirspec_dash @@ -142,6 +143,33 @@ def get_all_proposals(): return proposals +def get_current_flagged_anomalies(rootname): + """Return a list of currently flagged anomalies for the given + ``rootname`` + + Parameters + ---------- + rootname : str + The rootname of interest (e.g. + ``jw86600008001_02101_00001_guider2/``) + + Returns + ------- + current_anomalies : list + A list of currently flagged anomalies for the given ``rootname`` + (e.g. ``['snowball', 'crosstalk']``) + """ + + query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1) + all_records = query.data_frame + if not all_records.empty: + current_anomalies = [col for col, val in np.sum(all_records, axis=0).items() if val] + else: + current_anomalies = [] + + return current_anomalies + + def get_dashboard_components(): """Build and return dictionaries containing components and html needed for the dashboard. diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index dbbb959de..5f97a6d73 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -58,14 +58,26 @@ def view_function(request): class AnomalySubmitForm(forms.Form): - """ - """ + """A multiple choice field for specifying flagged anomalies.""" - # Define search field - anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, initial='', widget=forms.CheckboxSelectMultiple(), required=True) + # Define anomaly choice field + anomaly_choices = forms.MultipleChoiceField(choices=ANOMALY_CHOICES, widget=forms.CheckboxSelectMultiple()) def update_anomaly_table(self, rootname, user, anomaly_choices): - """ + """Updated the ``anomaly`` table of the database with flagged + anomaly information + + Parameters + ---------- + rootname : str + The rootname of the image to flag (e.g. + ``jw86600008001_02101_00001_guider2``) + user : str + The ``ezid`` of the authenticated user that is flagging the + anomaly + anomaly_choices : list + A list of anomalies that are to be flagged (e.g. + ``['snowball', 'crosstalk']``) """ data_dict = {} @@ -76,14 +88,6 @@ def update_anomaly_table(self, rootname, user, anomaly_choices): data_dict[choice] = True di.engine.execute(di.Anomaly.__table__.insert(), data_dict) - # # Get most previously flagged anomalies - # query = di.session.query(di.Anomaly).filter(di.Anomaly.rootname == rootname).order_by(di.Anomaly.flag_date.desc()).limit(1) - # all_records = query.data_frame - # if not all_records.empty: - # prev_anom = ', '.join([col for col, val in np.sum(all_records, axis=0).items() if val]) - # else: - # prev_anom = '' - class FileSearchForm(forms.Form): """Single-field form to search for a proposal or fileroot.""" diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 33d308554..4d743db7e 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -40,13 +40,13 @@ from django.http import JsonResponse from django.shortcuts import render -import numpy as np from .data_containers import get_acknowledgements, get_edb_components from .data_containers import get_dashboard_components from .data_containers import get_filenames_by_instrument from .data_containers import get_header_info from .data_containers import get_image_info +from .data_containers import get_current_flagged_anomalies from .data_containers import get_proposal_info from .data_containers import random_404_page from .data_containers import thumbnails @@ -55,7 +55,6 @@ from .data_containers import nirspec_trending from .forms import AnomalySubmitForm, FileSearchForm from .oauth import auth_info, auth_required -from jwql.database import database_interface as di from jwql.utils.constants import JWST_INSTRUMENT_NAMES, MONITORS, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_base_url, get_config @@ -484,10 +483,13 @@ def view_image(request, user, inst, file_root, rewrite=False): template = 'view_image.html' image_info = get_image_info(file_root, rewrite) - # Create a form instance and populate it with data from the request - form = AnomalySubmitForm(request.POST or None) + # Determine current flagged anomalies + current_anomalies = get_current_flagged_anomalies(file_root) - # If this is a POST request, we need to process the form data + # Create a form instance + form = AnomalySubmitForm(request.POST or None, initial={'anomaly_choices': current_anomalies}) + + # If this is a POST request, process the form data if request.method == 'POST': anomaly_choices = dict(request.POST)['anomaly_choices'] if form.is_valid(): From 01328c011cbd5dbd8ac9389260167af43094fec5 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 21 May 2019 16:43:24 -0400 Subject: [PATCH 25/33] Removed a bunch of anomaly categories that probably/hopefully wont exist for JWST instruments --- jwql/utils/constants.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 562c413fb..d74f2bb65 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -26,24 +26,15 @@ } # Defines the possible anomalies to flag through the web app -ANOMALIES = ['bowtie', 'snowball', 'cosmic_ray_shower', 'crosstalk', - 'cte_correction_error', 'data_transfer_error', 'detector_ghost', - 'diamond', 'diffraction_spike', 'dragon_breath', 'earth_limb', - 'excessive_saturation', 'figure8_ghost', 'filter_ghost', - 'fringing', 'guidestar_failure', 'banding', 'persistence', - 'prominent_blobs', 'trail', 'scattered_light', 'other'] +ANOMALIES = ['snowball', 'cosmic_ray_shower', 'crosstalk', 'data_transfer_error', 'diffraction_spike', + 'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other'] # Defines the possible anomalies (with rendered name) to flag through the web app ANOMALY_CHOICES = [ - ('bowtie', 'Bowtie'), ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'), - ('crosstalk', 'Crosstalk'), ('cte_correction_error', 'CTE Correction Error'), - ('data_transfer_error', 'Data Transfer Error'), ('detector_ghost', 'Detector Ghost'), - ('diamond', 'Diamond Feature'), ('diffraction_spike', 'Diffraction Spike'), - ('dragon_breath', 'Dragons Breath'), ('earth_limb', 'Earth Limb'), - ('excessive_saturation', 'Excessive Saturation'), ('figure8_ghost', 'Figure 8 Ghost'), - ('filter_ghost', 'Filter Ghost'), ('fringing', 'Fringing'), ('guidestar_failure', 'Guidestar Failure'), - ('banding', 'Banding'), ('persistence', 'Persistence'), ('prominent_blobs', 'Prominent Blobs'), - ('trail', 'Trail'), ('scattered_light', 'Scattered Light'), ('other', 'Other')] + ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'), ('crosstalk', 'Crosstalk'), + ('data_transfer_error', 'Data Transfer Error'), ('diffraction_spike', 'Diffraction Spike'), + ('excessive_saturation', 'Excessive Saturation'), ('ghost', 'Ghost'), ('guidestar_failure', 'Guidestar Failure'), + ('persistence', 'Persistence'), ('satellite_trail', 'Satellite Trail'), ('other', 'Other')] # Possible suffix types for nominal files GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d', From e3795fbe92a7725c2025820b24809e327c649d2a Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 21 May 2019 17:34:19 -0400 Subject: [PATCH 26/33] Fixed HTML and CSS to get anomaly form to render where it should. --- jwql/website/apps/jwql/static/css/jwql.css | 13 ++- .../apps/jwql/templates/view_image.html | 80 ++++++++++--------- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index f7e3340f6..d6cbc7f9b 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -1,5 +1,11 @@ +.anomaly_form { + position: absolute; + top: 50%; + transform: translateY(-50%); +} + .APT_parameters { - width: 20% + width: 20% } .banner { @@ -182,6 +188,11 @@ display : inline; display: inline; } +#id_anomaly_choices { + list-style: none; + padding-left: 0; +} + /*Don't let the search bar be super long*/ .input-group { width: 250px; diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 16445ad31..e15feb5be 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -40,43 +40,51 @@

        {{ file_root }}

        View Proposal {{ file_root[2:7] }}

        - -
        - - - Displaying integration 1/1
        - {{ file_root }}_cal_integ0.jpg -
        - -
        - - -
        - - {% if form.errors %} - {% for field in form %} - {% for error in field.errors %} -
        - {{ error|escape }} -
        - {% endfor %} - {% endfor %} - {% endif %} - - - {{ csrf_input }} - - - {% for field in form %} -
        - {{ field }} - {% if field.help_text %} -

        {{ field.help_text|safe }}

        +
        + + +
        + + + Displaying integration 1/1
        + {{ file_root }}_cal_integ0.jpg +
        + +
        + + +
        + +
        +
        Submit Anomaly
        + + + {% if form.errors %} + {% for field in form %} + {% for error in field.errors %} +
        + {{ error|escape }} +
        + {% endfor %} + {% endfor %} {% endif %} -
        - {% endfor %} - - + + + {{ csrf_input }} + + + {% for field in form %} + {{ field }} + {% if field.help_text %} +

        {{ field.help_text|safe }}

        + {% endif %} + {% endfor %} + + +
        +
        + +

        From 92258c13c40fe0675c6cd50869c1b2253cae6229 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 3 Jun 2019 13:34:57 -0400 Subject: [PATCH 27/33] Putting widget elements in a list to allow for easier use of css --- jwql/website/apps/jwql/static/css/jwql.css | 4 ++++ jwql/website/apps/jwql/templates/view_image.html | 7 +++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index d6cbc7f9b..b801c8358 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -1,3 +1,7 @@ +.anomaly_choice { + list-style: none; +} + .anomaly_form { position: absolute; top: 50%; diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index e15feb5be..8362ab77a 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -74,10 +74,9 @@

        Submit Anomaly
        {% for field in form %} - {{ field }} - {% if field.help_text %} -

        {{ field.help_text|safe }}

        - {% endif %} + {% for subwidget in field.subwidgets %} +
      • {{subwidget}}
      • + {% endfor %} {% endfor %} From d97601f97edee7a54bb2042820cd60bdeb0a4bde Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 3 Jun 2019 13:39:54 -0400 Subject: [PATCH 28/33] Minor edit to appease @pep8speaks --- jwql/utils/preview_image.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py index 9f47e84de..0369dec36 100755 --- a/jwql/utils/preview_image.py +++ b/jwql/utils/preview_image.py @@ -213,7 +213,6 @@ def get_data(self, filename, ext): yd, xd = data.shape[-2:] dq = np.ones((yd, xd), dtype="bool") - # Collect information on aperture location within the # full detector. This is needed for mosaicking NIRCam # detectors later. From 5fb0797cc4847cd0f4c4eb4adadd270a593498a3 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 3 Jun 2019 13:50:39 -0400 Subject: [PATCH 29/33] updated anomaly table tests based on recent changes --- jwql/tests/test_database_interface.py | 29 +++++++++------------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py index a1ea4a1b8..da13bb333 100755 --- a/jwql/tests/test_database_interface.py +++ b/jwql/tests/test_database_interface.py @@ -5,7 +5,7 @@ Authors ------- - Joe Filippazzo + - Joe Filippazzo Use --- @@ -17,33 +17,24 @@ pytest -s database_interface.py """ +import datetime + from jwql.database import database_interface as di def test_anomaly_table(): """Test to see that the database has an anomalies table""" - assert 'anomalies' in di.engine.table_names() + + assert 'anomaly' in di.engine.table_names() def test_anomaly_records(): """Test to see that new records can be entered""" + # Add some data - di.session.add(di.Anomaly(filename='foo1', bowtie="True")) + di.session.add(di.Anomaly(rootname='foo1', flag_date=datetime.datetime.today(), user='test', ghost=True)) di.session.commit() - # Test the bowties column - bowties = di.session.query(di.Anomaly).filter(di.Anomaly.bowtie == "True") - assert bowties.data_frame.iloc[0]['bowtie'] == "True" - - # Test the other columns - non_bowties = [col for col in di.Anomaly().colnames if col != 'bowtie'] - assert all([i == "False" for i in bowties.data_frame.iloc[0][non_bowties]]) - - -def test_names_colnames(): - """Test that the column names are correct""" - # Make sure we get non-empty lists - anom = di.Anomaly() - assert isinstance(anom.colnames, list) and len(anom.colnames) > 0 - assert isinstance(anom.names, list) and len(anom.names) == len(anom.colnames) - assert all([i == j.replace('_', ' ') for i, j in zip(anom.names, anom.colnames)]) + # Test the ghosts column + ghosts = di.session.query(di.Anomaly).filter(di.Anomaly.ghost == "True") + assert ghosts.data_frame.iloc[0]['ghost'] == True From f93437953750876eddd9e7936338460c2e06bd08 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 3 Jun 2019 14:30:54 -0400 Subject: [PATCH 30/33] Marking database interface tests with skipif since jenkins server doesnt have access to dev database server --- jwql/tests/test_database_interface.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py index da13bb333..4c078b8b3 100755 --- a/jwql/tests/test_database_interface.py +++ b/jwql/tests/test_database_interface.py @@ -21,13 +21,18 @@ from jwql.database import database_interface as di +# Determine if tests are being run on jenkins +ON_JENKINS = os.path.expanduser('~') == '/home/jenkins' + +@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.') def test_anomaly_table(): """Test to see that the database has an anomalies table""" assert 'anomaly' in di.engine.table_names() +@pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.') def test_anomaly_records(): """Test to see that new records can be entered""" From 0471dc64dabb8dcfa568ea36ffa8b9fa1fce252e Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 3 Jun 2019 14:45:16 -0400 Subject: [PATCH 31/33] Forgot os import --- jwql/tests/test_database_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py index 4c078b8b3..77daaa44f 100755 --- a/jwql/tests/test_database_interface.py +++ b/jwql/tests/test_database_interface.py @@ -18,6 +18,7 @@ """ import datetime +import os from jwql.database import database_interface as di From d51c6d580d415bea04169f8269aba6bf3415a733 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 3 Jun 2019 14:51:53 -0400 Subject: [PATCH 32/33] Forgot pytest import --- jwql/tests/test_database_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py index 77daaa44f..ab6a3d5b1 100755 --- a/jwql/tests/test_database_interface.py +++ b/jwql/tests/test_database_interface.py @@ -19,6 +19,7 @@ import datetime import os +import pytest from jwql.database import database_interface as di From d769a4b8a33eea824b345fc3fc28d575422ad034 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 4 Jun 2019 17:04:36 -0400 Subject: [PATCH 33/33] Small changes based on @laurenmarietta review comments --- environment_python_3_5.yml | 1 + environment_python_3_6.yml | 1 + jwql/tests/test_database_interface.py | 6 +++++- jwql/utils/constants.py | 9 ++++----- jwql/website/apps/jwql/forms.py | 1 + requirements.txt | 1 + 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/environment_python_3_5.yml b/environment_python_3_5.yml index 0ab098dff..43b44b4cc 100644 --- a/environment_python_3_5.yml +++ b/environment_python_3_5.yml @@ -8,6 +8,7 @@ dependencies: - bokeh=0.13.0 - crds>=7.2.7 - django=2.1.1 +- inflection=0.3.1 - ipython=6.5.0 - jinja2=2.10 - jwst=0.13.0 diff --git a/environment_python_3_6.yml b/environment_python_3_6.yml index 68a0fcced..7e5951ab1 100644 --- a/environment_python_3_6.yml +++ b/environment_python_3_6.yml @@ -8,6 +8,7 @@ dependencies: - bokeh=1.1.0 - crds>=7.2.7 - django=2.1.7 +- inflection=0.3.1 - ipython=7.5.0 - jinja2=2.10 - jwst=0.13.1 diff --git a/jwql/tests/test_database_interface.py b/jwql/tests/test_database_interface.py index ab6a3d5b1..96bb884d0 100755 --- a/jwql/tests/test_database_interface.py +++ b/jwql/tests/test_database_interface.py @@ -6,6 +6,7 @@ ------- - Joe Filippazzo + - Matthew Bourque Use --- @@ -20,6 +21,8 @@ import datetime import os import pytest +import random +import string from jwql.database import database_interface as di @@ -39,7 +42,8 @@ def test_anomaly_records(): """Test to see that new records can be entered""" # Add some data - di.session.add(di.Anomaly(rootname='foo1', flag_date=datetime.datetime.today(), user='test', ghost=True)) + random_string = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for _ in range(10)) + di.session.add(di.Anomaly(rootname=random_string, flag_date=datetime.datetime.today(), user='test', ghost=True)) di.session.commit() # Test the ghosts column diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index d74f2bb65..1db166b1a 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -20,6 +20,9 @@ ``utils.py`` """ +import inflection + + # Defines the x and y coordinates of amplifier boundaries AMPLIFIER_BOUNDARIES = {'nircam': {'1': [(0, 0), (512, 2048)], '2': [(512, 0), (1024, 2048)], '3': [(1024, 0), (1536, 2048)], '4': [(1536, 0), (2048, 2048)]} @@ -30,11 +33,7 @@ 'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other'] # Defines the possible anomalies (with rendered name) to flag through the web app -ANOMALY_CHOICES = [ - ('snowball', 'Snowball'), ('cosmic_ray_shower', 'Cosmic Ray Shower'), ('crosstalk', 'Crosstalk'), - ('data_transfer_error', 'Data Transfer Error'), ('diffraction_spike', 'Diffraction Spike'), - ('excessive_saturation', 'Excessive Saturation'), ('ghost', 'Ghost'), ('guidestar_failure', 'Guidestar Failure'), - ('persistence', 'Persistence'), ('satellite_trail', 'Satellite Trail'), ('other', 'Other')] +ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES] # Possible suffix types for nominal files GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d', diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index 080a86ace..06e8aed35 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -10,6 +10,7 @@ - Lauren Chambers - Johannes Sahlmann + - Matthew Bourque Use --- diff --git a/requirements.txt b/requirements.txt index a23571589..f6d8241b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ astroquery==0.3.9 authlib==0.11 bokeh==1.1.0 django==2.2.1 +inflection==0.3.1 ipython==7.5.0 jinja2==2.10.1 jwedb>=0.0.3