Merge pull request #1234 from melanieclarke/dependency_updates

Updates for sqlalchemy>=2, django>=3.2

environment_python_3_10.yml

@@ -28,7 +28,7 @@ dependencies:
   - celery=5.2.7
   - codecov=2.1.12
   - cryptography=39.0.1
-  - django=3.1.8
+  - django=4.1.7
   - flake8=6.0.0
   - inflection=0.5.1
   - ipython=8.10.0
@@ -38,7 +38,7 @@ dependencies:
   - nodejs=18.12.1
   - numpy=1.24.2
   - numpydoc=1.5.0
-  - pandas=1.5.3
+  - pandas=2.0.0
   - pip=23.0
   - postgresql=15.2
   - psycopg2=2.9.3
@@ -52,7 +52,7 @@ dependencies:
   - setuptools=67.3.1
   - sphinx=6.1.3
   - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=1.4.46
+  - sqlalchemy=2.0.8
   - twine=4.0.2
   - wtforms=3.0.1
 

environment_python_3_8.yml

@@ -28,7 +28,7 @@ dependencies:
   - celery=5.2.7
   - codecov=2.1.12
   - cryptography=39.0.1
-  - django=3.1.8
+  - django=4.1.7
   - flake8=6.0.0
   - inflection=0.5.1
   - ipython=8.10.0
@@ -38,7 +38,7 @@ dependencies:
   - nodejs=18.12.1
   - numpy=1.24.2
   - numpydoc=1.5.0
-  - pandas=1.5.3
+  - pandas=2.0.0
   - pip=23.0
   - postgresql=15.2
   - psycopg2=2.9.3
@@ -52,7 +52,7 @@ dependencies:
   - setuptools=67.3.1
   - sphinx=6.1.3
   - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=1.4.46
+  - sqlalchemy=2.0.8
   - twine=4.0.2
   - wtforms=3.0.1
 

environment_python_3_9.yml

@@ -28,7 +28,7 @@ dependencies:
   - celery=5.2.7
   - codecov=2.1.12
   - cryptography=39.0.1
-  - django=3.1.8
+  - django=4.1.7
   - flake8=6.0.0
   - inflection=0.5.1
   - ipython=8.10.0
@@ -38,7 +38,7 @@ dependencies:
   - nodejs=18.12.1
   - numpy=1.24.2
   - numpydoc=1.5.0
-  - pandas=1.5.3
+  - pandas=2.0.0
   - pip=23.0
   - postgresql=15.2
   - psycopg2=2.9.3
@@ -52,7 +52,7 @@ dependencies:
   - setuptools=67.3.1
   - sphinx=6.1.3
   - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=1.4.46
+  - sqlalchemy=2.0.8
   - twine=4.0.2
   - wtforms=3.0.1
 

jwql/__init__.py

@@ -1,22 +1,13 @@
-import os
-import pkg_resources
+from importlib.metadata import version
 from jwql.utils import utils
 
-module_path = pkg_resources.resource_filename('jwql', '')
-setup_path = os.path.normpath(os.path.join(module_path, '../setup.py'))
-
+__version__ = version('jwql')
 try:
-    with open(setup_path) as f:
-        data = f.readlines()
-
-    for line in data:
-        if 'VERSION =' in line:
-            __version__ = line.split(' ')[-1].replace("'", "").strip()
 
     config_version = utils.get_config()['jwql_version']
     if __version__ != config_version:
-        print("Warning: config file JWQL version is {}, while JWQL is using {}".format(config_version, __version__))
+        print(f"Warning: config file JWQL version is {config_version}, "
+              f"while JWQL is using {__version__}")
 
 except FileNotFoundError:
-    print('Could not determine jwql version')
-    __version__ = '0.0.0'
+    print('Could not determine jwql config version')

jwql/database/clone_tables.py

@@ -50,13 +50,9 @@
 import sys
 
 import pandas
-from psycopg2.errors import UndefinedTable
-from sqlalchemy.exc import ProgrammingError
 
 from jwql.database import database_interface
-from jwql.database.database_interface import base, set_read_permissions
 from jwql.database.database_interface import INSTRUMENT_TABLES, MONITOR_TABLES
-from jwql.utils.utils import get_config
 
 
 if __name__ == '__main__':

jwql/database/database_interface.py

@@ -75,11 +75,10 @@
 from sqlalchemy import MetaData
 from sqlalchemy import String
 from sqlalchemy import Time
+from sqlalchemy import text
 from sqlalchemy import UniqueConstraint
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import declarative_base, sessionmaker
 from sqlalchemy.orm.query import Query
-from sqlalchemy.sql import text
 from sqlalchemy.types import ARRAY
 
 from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT
@@ -94,7 +93,7 @@
 @property
 def data_frame(self):
     """Method to return a ``pandas.DataFrame`` of the results"""
-
+    # NOTE: this requires pandas>=2 if sqlalchemy>=2
     return pd.read_sql(self.statement, self.session.bind)
 
 
@@ -135,11 +134,11 @@ def load_connection(connection_string):
     ``ascql``:
         https://github.com/spacetelescope/acsql/blob/master/acsql/database/database_interface.py
     """
-    engine = create_engine(connection_string, echo=False, client_encoding='utf8', encoding='utf8')
-    base = declarative_base(engine)
+    engine = create_engine(connection_string, echo=False)
+    base = declarative_base()
     Session = sessionmaker(bind=engine)
     session = Session()
-    meta = MetaData(engine)
+    meta = MetaData()
 
     return session, base, engine, meta
 
@@ -431,16 +430,6 @@ class : obj
     return type(class_name, (base,), data_dict)
 
 
-def set_read_permissions():
-    """Set read permissions for db tables"""
-
-    db_username = SETTINGS['database']['user']
-    db_username = '_'.join(db_username.split('_')[:-1])
-    db_account = '{}_read'.format(db_username)
-    command = 'GRANT SELECT ON ALL TABLES IN SCHEMA public TO {};'.format(db_account)
-    engine.execute(command)
-
-
 # Create tables from ORM factory
 NIRCamAnomaly = anomaly_orm_factory('nircam_anomaly')
 NIRISSAnomaly = anomaly_orm_factory('niriss_anomaly')

jwql/database/reset_database.py

@@ -29,7 +29,7 @@
 from sqlalchemy.exc import ProgrammingError
 import sys
 
-from jwql.database.database_interface import base, set_read_permissions
+from jwql.database.database_interface import base
 from jwql.database.database_interface import INSTRUMENT_TABLES, MONITOR_TABLES
 from jwql.utils.utils import get_config
 

jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py

@@ -95,7 +95,7 @@
 from jwst_reffiles.bad_pixel_mask import bad_pixel_mask
 import numpy as np
 
-from jwql.database.database_interface import session
+from jwql.database.database_interface import engine, session
 from jwql.database.database_interface import NIRCamBadPixelQueryHistory, NIRCamBadPixelStats
 from jwql.database.database_interface import NIRISSBadPixelQueryHistory, NIRISSBadPixelStats
 from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats
@@ -441,7 +441,8 @@ def add_bad_pix(self, coordinates, pixel_type, files, obs_start_time, obs_mid_ti
                  'obs_end_time': obs_end_time,
                  'baseline_file': baseline_file,
                  'entry_date': datetime.datetime.now()}
-        self.pixel_table.__table__.insert().execute(entry)
+        with engine.begin() as connection:
+            connection.execute(self.pixel_table.__table__.insert(), entry)
 
     def filter_query_results(self, results, datatype):
         """Filter MAST query results. For input flats, keep only those
@@ -1287,7 +1288,8 @@ def run(self):
                              'run_bpix_from_flats': run_flats,
                              'run_monitor': run_flats or run_darks,
                              'entry_date': datetime.datetime.now()}
-                self.query_table.__table__.insert().execute(new_entry)
+                with engine.begin() as connection:
+                    connection.execute(self.query_table.__table__.insert(), new_entry)
                 logging.info('\tUpdated the query history table')
 
         # Update the figures to be shown in the web app. Only update figures

jwql/instrument_monitors/common_monitors/bias_monitor.py

@@ -49,7 +49,7 @@
 from pysiaf import Siaf  # noqa: E402 (module import not at top)
 from sqlalchemy.sql.expression import and_  # noqa: E402 (module import not at top)
 
-from jwql.database.database_interface import session  # noqa: E402 (module import not at top)
+from jwql.database.database_interface import session, engine  # noqa: E402 (module import not at top)
 from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory  # noqa: E402 (module import not at top)
 from jwql.database.database_interface import NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats  # noqa: E402 (module import not at top)
 from jwql.instrument_monitors import pipeline_tools  # noqa: E402 (module import not at top)
@@ -430,7 +430,8 @@ def process(self, file_list):
                 bias_db_entry[key] = float(amp_medians[key])
 
             # Add this new entry to the bias database table
-            self.stats_table.__table__.insert().execute(bias_db_entry)
+            with engine.begin() as connection:
+                connection.execute(self.stats_table.__table__.insert(), bias_db_entry)
             logging.info('\tNew entry added to bias database table: {}'.format(bias_db_entry))
 
             # Remove the raw and calibrated files to save memory space
@@ -533,7 +534,8 @@ def run(self):
                              'files_found': len(new_files),
                              'run_monitor': monitor_run,
                              'entry_date': datetime.datetime.now()}
-                self.query_table.__table__.insert().execute(new_entry)
+                with engine.begin() as connection:
+                    connection.execute(self.query_table.__table__.insert(), new_entry)
                 logging.info('\tUpdated the query history table')
 
             # Update the bias monitor plots

jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py

@@ -60,7 +60,7 @@
 from jwql.database.database_interface import NIRSpecCosmicRayStats
 from jwql.database.database_interface import FGSCosmicRayQueryHistory
 from jwql.database.database_interface import FGSCosmicRayStats
-from jwql.database.database_interface import session
+from jwql.database.database_interface import session, engine
 from jwql.jwql_monitors import monitor_mast
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS
@@ -702,7 +702,8 @@ def process(self, file_list):
                                            'magnitude': cosmic_ray_mags,
                                            'outliers': outlier_mags
                                            }
-                    self.stats_table.__table__.insert().execute(cosmic_ray_db_entry)
+                    with engine.begin() as connection:
+                        connection.execute(self.stats_table.__table__.insert(), cosmic_ray_db_entry)
 
                     logging.info("Successfully inserted into database. \n")
 
@@ -820,7 +821,8 @@ def run(self):
                              'files_found': len(new_entries),
                              'run_monitor': monitor_run,
                              'entry_date': datetime.datetime.now()}
-                self.query_table.__table__.insert().execute(new_entry)
+                with engine.begin() as connection:
+                    connection.execute(self.query_table.__table__.insert(), new_entry)
                 logging.info('\tUpdated the query history table')
 
     def query_mast(self):

jwql/instrument_monitors/common_monitors/dark_monitor.py

@@ -93,7 +93,7 @@
 from sqlalchemy import func
 from sqlalchemy.sql.expression import and_
 
-from jwql.database.database_interface import session
+from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRCamDarkQueryHistory, NIRCamDarkPixelStats, NIRCamDarkDarkCurrent
 from jwql.database.database_interface import NIRISSDarkQueryHistory, NIRISSDarkPixelStats, NIRISSDarkDarkCurrent
 from jwql.database.database_interface import MIRIDarkQueryHistory, MIRIDarkPixelStats, MIRIDarkDarkCurrent
@@ -232,7 +232,8 @@ def add_bad_pix(self, coordinates, pixel_type, files, mean_filename, baseline_fi
                  'mean_dark_image_file': os.path.basename(mean_filename),
                  'baseline_file': os.path.basename(baseline_filename),
                  'entry_date': datetime.datetime.now()}
-        self.pixel_table.__table__.insert().execute(entry)
+        with engine.begin() as connection:
+            connection.execute(self.pixel_table.__table__.insert(), entry)
 
     def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, noisyxy=None, baseline_file=None):
         """Create and save a png containing the mean dark slope image,
@@ -861,7 +862,8 @@ def process(self, file_list):
                              'hist_amplitudes': histogram[key],
                              'entry_date': datetime.datetime.now()
                              }
-            self.stats_table.__table__.insert().execute(dark_db_entry)
+            with engine.begin() as connection:
+                connection.execute(self.stats_table.__table__.insert(), dark_db_entry)
 
     def read_baseline_slope_image(self, filename):
         """Read in a baseline mean slope image and associated standard
@@ -1049,7 +1051,9 @@ def run(self):
                                  'files_found': len(new_entries),
                                  'run_monitor': monitor_run,
                                  'entry_date': datetime.datetime.now()}
-                    self.query_table.__table__.insert().execute(new_entry)
+                    with engine.begin() as connection:
+                        connection.execute(
+                            self.query_table.__table__.insert(), new_entry)
                     logging.info('\tUpdated the query history table')
 
         logging.info('Dark Monitor completed successfully.')

jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py

@@ -390,7 +390,7 @@
     NIRISSEDBTimeIntervalStats, NIRISSEDBEveryChangeStats, MIRIEDBDailyStats, MIRIEDBBlockStats, \
     MIRIEDBTimeIntervalStats, MIRIEDBEveryChangeStats, FGSEDBDailyStats, FGSEDBBlockStats, \
     FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats, NIRSpecEDBDailyStats, NIRSpecEDBBlockStats, \
-    NIRSpecEDBTimeIntervalStats, NIRSpecEDBEveryChangeStats, session
+    NIRSpecEDBTimeIntervalStats, NIRSpecEDBEveryChangeStats, session, engine
 from jwql.edb import engineering_database as ed
 from jwql.instrument_monitors.common_monitors.edb_telemetry_monitor_utils import condition
 from jwql.instrument_monitors.common_monitors.edb_telemetry_monitor_utils import utils
@@ -527,7 +527,8 @@ def add_new_block_db_entry(self, mnem, query_time):
                     'min': mins,
                     'entry_date': datetime.datetime.now()
                     }
-        self.history_table.__table__.insert().execute(db_entry)
+        with engine.begin() as connection:
+            connection.execute(self.history_table.__table__.insert(), db_entry)
 
     def add_new_every_change_db_entry(self, mnem, mnem_dict, dependency_name, query_time):
         """Add new entries to the database table for "every change"
@@ -565,7 +566,9 @@ def add_new_every_change_db_entry(self, mnem, mnem_dict, dependency_name, query_
                         'latest_query': query_time,
                         'entry_date': datetime.datetime.now()
                         }
-            self.history_table.__table__.insert().execute(db_entry)
+            with engine.begin() as connection:
+                connection.execute(
+                    self.history_table.__table__.insert(), db_entry)
 
     def calc_timed_stats(self, mnem_data, bintime, sigma=3):
         """Not currently used.
@@ -2192,4 +2195,4 @@ def plot_every_change_data(data, mnem_name, units, show_plot=False, savefig=True
 
     monitor = EdbMnemonicMonitor()
     monitor.execute(args.mnem_to_query, plot_start_dt, plot_end_dt)
-    monitor_utils.update_monitor_table(module, start_time, log_file)
+    monitor_utils.update_monitor_table(module, start_time, log_file)

jwql/instrument_monitors/common_monitors/readnoise_monitor.py

@@ -54,7 +54,7 @@
 from jwql.database.database_interface import NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats  # noqa: E348 (comparison to true)
 from jwql.database.database_interface import NIRISSReadnoiseQueryHistory, NIRISSReadnoiseStats  # noqa: E348 (comparison to true)
 from jwql.database.database_interface import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats  # noqa: E348 (comparison to true)
-from jwql.database.database_interface import session  # noqa: E348 (comparison to true)
+from jwql.database.database_interface import session, engine  # noqa: E348 (comparison to true)
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline  # noqa: E348 (comparison to true)
 from jwql.instrument_monitors import pipeline_tools  # noqa: E348 (comparison to true)
 from jwql.utils import instrument_properties, monitor_utils  # noqa: E348 (comparison to true)
@@ -525,7 +525,8 @@ def process(self, file_list):
                     readnoise_db_entry[key] = amp_stats[key].astype(float)
 
             # Add this new entry to the readnoise database table
-            self.stats_table.__table__.insert().execute(readnoise_db_entry)
+            with engine.begin() as connection:
+                connection.execute(self.stats_table.__table__.insert(), readnoise_db_entry)
             logging.info('\tNew entry added to readnoise database table')
 
             # Remove the raw and calibrated files to save memory space
@@ -649,7 +650,8 @@ def run(self):
                              'files_found': len(new_files),
                              'run_monitor': monitor_run,
                              'entry_date': datetime.datetime.now()}
-                self.query_table.__table__.insert().execute(new_entry)
+                with engine.begin() as connection:
+                    connection.execute(self.query_table.__table__.insert(), new_entry)
                 logging.info('\tUpdated the query history table')
 
         logging.info('Readnoise Monitor completed successfully.')