Merge branch 'develop' into release-0.19.0

docs/source/common_monitors.rst

@@ -0,0 +1,9 @@
+***************
+common_monitors
+***************
+
+dark_monitor.py
+---------------
+.. automodule:: jwql.instrument_monitors.common_monitors.dark_monitor
+    :members:
+    :undoc-members:

docs/source/index.rst

@@ -23,9 +23,11 @@ API documentation
    :maxdepth: 1
    :caption: Contents:
 
+   common_monitors.rst
    database.rst
    edb.rst
    jwql_monitors.rst
+   instrument_monitors.rst
    tests.rst
    utils.rst
    website.rst

docs/source/instrument_monitors.rst

@@ -0,0 +1,9 @@
+*******************
+instrument_monitors
+*******************
+
+pipeline_tools.py
+-----------------
+.. automodule:: jwql.instrument_monitors.pipeline_tools
+    :members:
+    :undoc-members:

docs/source/tests.rst

@@ -8,12 +8,30 @@ test_api_views.py
     :members:
     :undoc-members:
 
+test_calculations.py
+--------------------
+.. automodule:: jwql.tests.test_calculations
+    :members:
+    :undoc-members:
+
+test_dark_monitor.py
+--------------------
+.. automodule:: jwql.tests.test_dark_monitor
+    :members:
+    :undoc-members:
+
 test_edb_interface.py
 ---------------------
 .. automodule:: jwql.tests.test_edb_interface
     :members:
     :undoc-members:
 
+test_instrument_properties.py
+-----------------------------
+.. automodule:: jwql.tests.test_instrument_properties
+    :members:
+    :undoc-members:
+
 test_loading_times.py
 --------------------
 .. automodule:: jwql.tests.test_loading_times
@@ -32,6 +50,12 @@ test_permissions.py
     :members:
     :undoc-members:
 
+test_pipeline_tools.py
+----------------------
+.. automodule:: jwql.tests.test_pipeline_tools
+    :members:
+    :undoc-members:
+
 test_plotting.py
 ----------------
 .. automodule:: jwql.tests.test_plotting

docs/source/utils.rst

@@ -2,12 +2,24 @@
 utils
 *****
 
+calculations.py
+---------------
+.. automodule:: jwql.utils.calculations
+    :members:
+    :undoc-members:
+
 constants.py
 ------------
 .. automodule:: jwql.utils.constants
     :members:
     :undoc-members:
 
+instrument_properties.py
+------------------------
+.. automodule:: jwql.utils.instrument_properties
+    :members:
+    :undoc-members:
+
 logging_functions.py
 --------------------
 .. automodule:: jwql.utils.logging_functions

environment.yml

@@ -3,14 +3,15 @@ channels:
 - http://ssb.stsci.edu/astroconda-dev
 - defaults
 dependencies:
-- asdf>=2.3.0
-- astropy
+- asdf=2.3.1
+- astropy>=3.1.2
 - astroquery=0.3.9
 - bokeh=1.1.0
+- crds>=7.2.7
 - django=2.1.7
 - ipython=7.3.0
 - jinja2=2.10
-- jwst
+- jwst=0.13.1
 - matplotlib=3.0.2
 - numpy=1.16.2
 - numpydoc=0.8.0
@@ -29,4 +30,5 @@ dependencies:
 - pip:
   - authlib==0.10
   - codecov==2.0.15
+  - pysiaf==0.2.5
   - sphinx-automodapi==0.10

jwql/database/database_interface.py

@@ -70,12 +70,10 @@
 from sqlalchemy import Enum
 from sqlalchemy import Float
 from sqlalchemy import Integer
-from sqlalchemy import Float
 from sqlalchemy import MetaData
 from sqlalchemy import String
 from sqlalchemy import Time
 from sqlalchemy import UniqueConstraint
-from sqlalchemy.dialects import postgresql
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.query import Query
@@ -137,8 +135,8 @@ def load_connection(connection_string):
     return session, base, engine, meta
 
 
-# Import a global session.  If running from readthedocs, pass a dummy connection string
-if 'build' and 'project' in socket.gethostname():
+# Import a global session.  If running from readthedocs or Jenkins, pass a dummy connection string
+if 'build' and 'project' in socket.gethostname() or os.path.expanduser('~') == '/home/jenkins':
     dummy_connection_string = 'postgresql+psycopg2://account:password@hostname:0000/db_name'
     session, base, engine, meta = load_connection(dummy_connection_string)
 else:
@@ -256,7 +254,7 @@ class Monitor(base):
     start_time = Column(DateTime, nullable=False)
     end_time = Column(DateTime, nullable=True)
     status = Column(Enum('SUCESS', 'FAILURE', name='monitor_status'), nullable=True)
-    affected_tables = Column(postgresql.ARRAY(String, dimensions=1), nullable=True)
+    affected_tables = Column(ARRAY(String, dimensions=1), nullable=True)
     log_file = Column(String(), nullable=False)
 
 
@@ -287,11 +285,14 @@ def get_monitor_columns(data_dict, table_name):
                       'date': Date(),
                       'time': Time(),
                       'datetime': DateTime,
-                      'bool': Boolean}
+                      'bool': Boolean
+                      }
 
     # Get the data from the table definition file
+    instrument = table_name.split('_')[0]
     table_definition_file = os.path.join(os.path.split(__file__)[0],
                                          'monitor_table_definitions',
+                                         instrument.lower(),
                                          '{}.txt'.format(table_name))
     with open(table_definition_file, 'r') as f:
         data = f.readlines()
@@ -302,9 +303,20 @@ def get_monitor_columns(data_dict, table_name):
         column_name = column_definition[0]
         data_type = column_definition[1]
 
+        if 'array' in data_type:
+            dtype, _a, dimension = data_type.split('_')
+            dimension = int(dimension[0])
+            array = True
+        else:
+            dtype = data_type
+            array = False
+
         # Create a new column
-        if data_type in list(data_type_dict.keys()):
-            data_dict[column_name.lower()] = Column(data_type_dict[data_type])
+        if dtype in list(data_type_dict.keys()):
+            if array:
+                data_dict[column_name.lower()] = Column(ARRAY(data_type_dict[dtype], dimensions=dimension))
+            else:
+                data_dict[column_name.lower()] = Column(data_type_dict[dtype])
         else:
             raise ValueError('Unrecognized column type: {}:{}'.format(column_name, data_type))
 
@@ -355,7 +367,7 @@ class : obj
     # Columns specific to all monitor ORMs
     data_dict['id'] = Column(Integer, primary_key=True, nullable=False)
     data_dict['entry_date'] = Column(DateTime, unique=True, nullable=False, default=datetime.now())
-    data_dict['__table_args__'] = (UniqueConstraint('id', 'entry_date', name='monitor_uc'),)
+    data_dict['__table_args__'] = (UniqueConstraint('id', 'entry_date', name='{}_uc'.format(data_dict['__tablename__'])),)
 
     # Get monitor-specific columns
     data_dict = get_monitor_columns(data_dict, data_dict['__tablename__'])
@@ -365,10 +377,23 @@ class : obj
 
     return type(class_name, (base,), data_dict)
 
+
 # Create tables from ORM factory
-# NIRCamDarkQueries = monitor_orm_factory('nircam_dark_queries')
-# NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
-# NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')
+NIRCamDarkQueryHistory = monitor_orm_factory('nircam_dark_query_history')
+NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
+NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')
+NIRISSDarkQueryHistory = monitor_orm_factory('niriss_dark_query_history')
+NIRISSDarkPixelStats = monitor_orm_factory('niriss_dark_pixel_stats')
+NIRISSDarkDarkCurrent = monitor_orm_factory('niriss_dark_dark_current')
+NIRSpecDarkQueryHistory = monitor_orm_factory('nirspec_dark_query_history')
+NIRSpecDarkPixelStats = monitor_orm_factory('nirspec_dark_pixel_stats')
+NIRSpecDarkDarkCurrent = monitor_orm_factory('nirspec_dark_dark_current')
+MIRIDarkQueryHistory = monitor_orm_factory('miri_dark_query_history')
+MIRIDarkPixelStats = monitor_orm_factory('miri_dark_pixel_stats')
+MIRIDarkDarkCurrent = monitor_orm_factory('miri_dark_dark_current')
+FGSDarkQueryHistory = monitor_orm_factory('fgs_dark_query_history')
+FGSDarkPixelStats = monitor_orm_factory('fgs_dark_pixel_stats')
+FGSDarkDarkCurrent = monitor_orm_factory('fgs_dark_dark_current')
 
 
 if __name__ == '__main__':

jwql/database/monitor_table_definitions/fgs/fgs_dark_dark_current.txt

@@ -0,0 +1,19 @@
+APERTURE, string
+AMPLIFIER, string
+MEAN, float
+STDEV, float
+SOURCE_FILES, string_array_1d
+GAUSS_AMPLITUDE, float_array_1d
+GUASS_PEAK, float_array_1d
+GAUSS_WIDTH, float_array_1d
+GAUSS_CHISQ, float
+DOUBLE_GAUSS_AMPLITUDE1, float_array_1d
+DOUBLE_GAUSS_PEAK1, float_array_1d
+DOUBLE_GAUSS_WIDTH1, float_array_1d
+DOUBLE_GAUSS_AMPLITUDE2, float_array_1d
+DOUBLE_GAUSS_PEAK2, float_array_1d
+DOUBLE_GAUSS_WIDTH2, float_array_1d
+DOUBLE_GAUSS_CHISQ, float
+MEAN_DARK_IMAGE_FILE, string
+HIST_DARK_VALUES, float_array_1d
+HIST_AMPLITUDES, float_array_1d

jwql/database/monitor_table_definitions/fgs/fgs_dark_pixel_stats.txt

@@ -0,0 +1,7 @@
+DETECTOR, string
+X_COORD, integer_array_1d
+Y_COORD, integer_array_1d
+TYPE, string
+SOURCE_FILES, string_array_1d
+MEAN_DARK_IMAGE_FILE, string
+BASELINE_FILE, string

jwql/database/monitor_table_definitions/fgs/fgs_dark_query_history.txt

@@ -0,0 +1,6 @@
+INSTRUMENT, string
+APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+FILES_FOUND, integer
+RUN_MONITOR, bool

jwql/database/monitor_table_definitions/miri/miri_dark_dark_current.txt

@@ -0,0 +1,19 @@
+APERTURE, string
+AMPLIFIER, string
+MEAN, float
+STDEV, float
+SOURCE_FILES, string_array_1d
+GAUSS_AMPLITUDE, float_array_1d
+GUASS_PEAK, float_array_1d
+GAUSS_WIDTH, float_array_1d
+GAUSS_CHISQ, float
+DOUBLE_GAUSS_AMPLITUDE1, float_array_1d
+DOUBLE_GAUSS_PEAK1, float_array_1d
+DOUBLE_GAUSS_WIDTH1, float_array_1d
+DOUBLE_GAUSS_AMPLITUDE2, float_array_1d
+DOUBLE_GAUSS_PEAK2, float_array_1d
+DOUBLE_GAUSS_WIDTH2, float_array_1d
+DOUBLE_GAUSS_CHISQ, float
+MEAN_DARK_IMAGE_FILE, string
+HIST_DARK_VALUES, float_array_1d
+HIST_AMPLITUDES, float_array_1d

jwql/database/monitor_table_definitions/miri/miri_dark_pixel_stats.txt

@@ -0,0 +1,7 @@
+DETECTOR, string
+X_COORD, integer_array_1d
+Y_COORD, integer_array_1d
+TYPE, string
+SOURCE_FILES, string_array_1d
+MEAN_DARK_IMAGE_FILE, string
+BASELINE_FILE, string

jwql/database/monitor_table_definitions/miri/miri_dark_query_history.txt

@@ -0,0 +1,6 @@
+INSTRUMENT, string
+APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+FILES_FOUND, integer
+RUN_MONITOR, bool

jwql/database/monitor_table_definitions/nircam/nircam_dark_dark_current.txt

@@ -1,2 +1,19 @@
+APERTURE, string
+AMPLIFIER, string
 MEAN, float
-STDEV, float
+STDEV, float
+SOURCE_FILES, string_array_1d
+GAUSS_AMPLITUDE, float_array_1d
+GUASS_PEAK, float_array_1d
+GAUSS_WIDTH, float_array_1d
+GAUSS_CHISQ, float
+DOUBLE_GAUSS_AMPLITUDE1, float_array_1d
+DOUBLE_GAUSS_PEAK1, float_array_1d
+DOUBLE_GAUSS_WIDTH1, float_array_1d
+DOUBLE_GAUSS_AMPLITUDE2, float_array_1d
+DOUBLE_GAUSS_PEAK2, float_array_1d
+DOUBLE_GAUSS_WIDTH2, float_array_1d
+DOUBLE_GAUSS_CHISQ, float
+MEAN_DARK_IMAGE_FILE, string
+HIST_DARK_VALUES, float_array_1d
+HIST_AMPLITUDES, float_array_1d

jwql/database/monitor_table_definitions/nircam/nircam_dark_pixel_stats.txt

@@ -1,3 +1,7 @@
-X_COORD, integer
-Y_COORD, integer
-TYPE, string
+DETECTOR, string
+X_COORD, integer_array_1d
+Y_COORD, integer_array_1d
+TYPE, string
+SOURCE_FILES, string_array_1d
+MEAN_DARK_IMAGE_FILE, string
+BASELINE_FILE, string

jwql/database/monitor_table_definitions/nircam/nircam_dark_query_history.txt

@@ -1,3 +1,6 @@
-START_TIME, datetime
-DETECTOR, string
+INSTRUMENT, string
 APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+FILES_FOUND, integer
+RUN_MONITOR, bool

jwql/database/monitor_table_definitions/niriss/niriss_dark_dark_current.txt

@@ -0,0 +1,19 @@
+APERTURE, string
+AMPLIFIER, string
+MEAN, float
+STDEV, float
+SOURCE_FILES, string_array_1d
+GAUSS_AMPLITUDE, float_array_1d
+GUASS_PEAK, float_array_1d
+GAUSS_WIDTH, float_array_1d
+GAUSS_CHISQ, float
+DOUBLE_GAUSS_AMPLITUDE1, float_array_1d
+DOUBLE_GAUSS_PEAK1, float_array_1d
+DOUBLE_GAUSS_WIDTH1, float_array_1d
+DOUBLE_GAUSS_AMPLITUDE2, float_array_1d
+DOUBLE_GAUSS_PEAK2, float_array_1d
+DOUBLE_GAUSS_WIDTH2, float_array_1d
+DOUBLE_GAUSS_CHISQ, float
+MEAN_DARK_IMAGE_FILE, string
+HIST_DARK_VALUES, float_array_1d
+HIST_AMPLITUDES, float_array_1d

jwql/database/monitor_table_definitions/niriss/niriss_dark_pixel_stats.txt

@@ -0,0 +1,7 @@
+DETECTOR, string
+X_COORD, integer_array_1d
+Y_COORD, integer_array_1d
+TYPE, string
+SOURCE_FILES, string_array_1d
+MEAN_DARK_IMAGE_FILE, string
+BASELINE_FILE, string

jwql/database/monitor_table_definitions/niriss/niriss_dark_query_history.txt

@@ -0,0 +1,6 @@
+INSTRUMENT, string
+APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+FILES_FOUND, integer
+RUN_MONITOR, bool