Merge branch 'main' of https://github.com/spacetelescope/romancal

.github/workflows/contexts.yml

@@ -0,0 +1,26 @@
+name: contexts
+
+on:
+  workflow_call:
+    outputs:
+      roman:
+        value: ${{ jobs.contexts.outputs.roman }}
+  workflow_dispatch:
+
+jobs:
+  contexts:
+    name: retrieve latest CRDS contexts
+    runs-on: ubuntu-latest
+    outputs:
+      roman: ${{ steps.roman_crds_context.outputs.pmap }}
+    steps:
+      - id: roman_crds_context
+        env:
+          OBSERVATORY: roman
+          CRDS_SERVER_URL: https://roman-crds.stsci.edu
+        run: >
+          echo "pmap=$(
+          curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}", null], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ --retry 8 |
+          python -c "import sys, json; print(json.load(sys.stdin)['result'])"
+          )" >> $GITHUB_OUTPUT
+      - run: if [[ ! -z "${{ steps.roman_crds_context.outputs.pmap }}" ]]; then echo ${{ steps.roman_crds_context.outputs.pmap }}; else exit 1; fi

.github/workflows/roman_ci.yml

@@ -41,7 +41,7 @@ jobs:
     with:
       minimal: true
   latest_crds_contexts:
-    uses: spacetelescope/crds/.github/workflows/contexts.yml@master
+    uses: ./.github/workflows/contexts.yml
   crds_context:
     needs: [ latest_crds_contexts ]
     runs-on: ubuntu-latest

.github/workflows/roman_ci_cron.yaml

@@ -41,7 +41,7 @@ jobs:
       minimal: true
   latest_crds_contexts:
     if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests')))
-    uses: spacetelescope/crds/.github/workflows/contexts.yml@master
+    uses: ./.github/workflows/contexts.yml
   crds_context:
     needs: [ latest_crds_contexts ]
     runs-on: ubuntu-latest

.github/workflows/tests_devdeps.yml

@@ -42,7 +42,7 @@ jobs:
       minimal: true
   latest_crds_contexts:
     if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))
-    uses: spacetelescope/crds/.github/workflows/contexts.yml@master
+    uses: ./.github/workflows/contexts.yml
   crds_context:
     needs: [ latest_crds_contexts ]
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ exclude: ".*\\.asdf$"
 repos:
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.5.0
+  rev: v5.0.0
   hooks:
     - id: check-added-large-files
     - id: check-ast
@@ -28,13 +28,13 @@ repos:
     - id: text-unicode-replacement-char
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: 'v3.15.1'
+  rev: 'v3.18.0'
   hooks:
     - id: pyupgrade
       args: ["--py38-plus"]
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: 'v0.2.2'
+  rev: 'v0.6.9'
   hooks:
     - id: ruff
       args: ["--fix"]
@@ -45,12 +45,12 @@ repos:
     - id: isort
 
 - repo: https://github.com/psf/black
-  rev: 24.2.0
+  rev: 24.10.0
   hooks:
     - id: black
 
 - repo: https://github.com/PyCQA/bandit
-  rev: 1.7.7
+  rev: 1.7.10
   hooks:
     - id: bandit
       args: ["-c", "pyproject.toml"]

changes/1433.general.rst

@@ -0,0 +1 @@
+Removed now unused lib.dms.

changes/1471.associations.rst

@@ -0,0 +1 @@
+Update skycell_asn docs and add skycell_asn as a script at install time

docs/conf.py

@@ -274,7 +274,7 @@ def check_sphinx_version(expected_version):
 # further.  For a list of options available for each theme, see the
 # documentation.
 
-html_theme_options = {"collapse_navigation": True, "version_selector": True}
+html_theme_options = {"collapse_navigation": True}
 
 #        "nosidebar": "false",
 #        "sidebarbgcolor": "#4db8ff",

docs/roman/associations/commands.rst

@@ -7,3 +7,4 @@ Association Commands
    :maxdepth: 2
 
    asn_from_list.rst
+   skycell_asn.rst

docs/roman/associations/skycell_asn.rst

@@ -0,0 +1,91 @@
+.. _skycell_asn:
+
+skycell_asn
+===========
+
+Create an association using either the command line tool
+``skycell_asn`` or through the Python API using either
+:class:`romancal.associations.skycell_asn.Main` or
+:func:`romancal.associations.skycellasn.skycell_asn`
+
+
+Associations
+^^^^^^^^^^^^
+
+Refer to TBD for a full description of associations.
+
+To create an association, use the following command:
+
+.. code-block:: python
+
+		skycell_asn file_list -o my_program
+
+where the individual calibrated detector files are in the current directory and the -o is the root
+for the output associations. To get a complete list of options you can run the command with the
+\-h option
+
+.. code-block:: python
+
+		skycell_asn -h
+
+
+By knowing the structure of the conventional file name you can generate custom associations based
+on the information included in the file names.
+The current naming convention for the input files consists of a Visit_ID + Exposure_ID + detector + suffix
+Where the Visit_ID is a 19 digit string
+PPPPPCCAAASSSOOOVVV, with
+
+PPPPP = Program number
+CC = Execution plan number
+AAA = Pass number (within execution plan)
+SSS = Segment Number (within pass)
+OOO = Observation number (within the leg, not to be confused with the Observation ID)
+VVV = Visit Number (within the observation)
+
+and the Exposure_ID is a four digits designating the exposures
+
+eeee = Exposure number (within the visit)
+
+The detector is WFI01, WFI02, ... WFI18
+
+The suffix indicates the processing level of the file and to create products based on the
+skycells the suffix should generally be 'cal'.
+
+and the file name is constructed as,
+rPPPPPCCAAASSSOOOVVV_eeee_detector_suffix.asdf
+
+To give a more concrete example we'll use the APT example program for the
+High Latitude Wide Angle Survey (hlwas). The program is 00991, the execution plan number is 01,
+there are 12 passes, 169 segments, 5 observations and a various number of visits.
+For this example we want to select a single filter, say F158, and that is observation 003.
+So to generate the visit level associations for observation 001 we would select the files using the bash
+command line,
+
+.. code-block:: text
+
+		skycell_asn r0099101001001003001_*_cal.asdf  -o  r0099101001001003001 --product-type visit
+
+Where the wildcard selects all the exposures for visit 001 and generates associations based on the skycells
+the exposures will populate. This will generate associations based the skycells that the exposures can
+contribute data to. The association files will be json files with names based
+
+.. code-block:: text
+
+	r0099101001001003001_<skycell name>_<product_type>_<filter>_<release product name>_i2d_asn.json
+
+or for the selections above
+
+.. code-block:: text
+
+	r0099101001001003001_r257dp63x98y83_visit_F158_prompt_i2d_asn.json
+
+where the skycell name can vary based on the location on the celestial sphere and the i2d indicates
+that this is resampled 2-d imaging data. The release product name can be changed from the default
+by adding the optional argument --release-product <new name> to the command line.
+
+An analogous command to generate the pass level products, again setting observation to 003 to only select
+the F158 filter.
+
+.. code-block:: text
+
+		skycell_asn r0099101???003*_*_cal.asdf  -o  r0099101 --product-type pass

pyproject.toml

@@ -91,6 +91,7 @@ schema_editor = "romancal.scripts.schema_editor:main"
 schemadoc = "romancal.scripts.schemadoc:main"
 verify_install_requires = "romancal.scripts.verify_install_requires:main"
 asn_from_list = "romancal.associations.asn_from_list:Main"
+skycell_asn = "romancal.associations.skycell_asn:Main"
 
 [build-system]
 requires = [

romancal/dark_current/tests/test_dark.py

@@ -31,7 +31,7 @@ def test_dark_step_interface(instrument, exptype):
 
     # Test dark results
     assert (result.data == ramp_model.data).all()
-    assert type(result) == RampModel
+    assert isinstance(result, RampModel)
     assert result.validate() is None
     assert result.data.shape == shape
     assert result.groupdq.shape == shape
@@ -99,7 +99,7 @@ def test_dark_step_output_dark_file(tmp_path, instrument, exptype):
     dark_out_file_model = rdm.open(path)
 
     # Test dark file results
-    assert type(dark_out_file_model) == DarkRefModel
+    assert isinstance(dark_out_file_model, DarkRefModel)
     assert dark_out_file_model.validate() is None
     assert dark_out_file_model.data.shape == shape
     assert dark_out_file_model.dq.shape == shape[1:]
@@ -132,7 +132,7 @@ def test_dark_step_getbestrefs(tmp_path, instrument, exptype):
     dark_out_file_model = rdm.open(path)
 
     # Test dark file results
-    assert type(dark_out_file_model) == DarkRefModel
+    assert isinstance(dark_out_file_model, DarkRefModel)
     assert dark_out_file_model.validate() is None
     assert dark_out_file_model.data.shape == shape
     assert dark_out_file_model.dq.shape == shape[1:]

romancal/linearity/tests/test_linearity.py

@@ -6,11 +6,10 @@
 
 import numpy as np
 import pytest
-from roman_datamodels import maker_utils
+from roman_datamodels import dqflags, maker_utils
 from roman_datamodels.datamodels import LinearityRefModel, ScienceRawModel
 
 from romancal.dq_init import DQInitStep
-from romancal.lib import dqflags
 from romancal.linearity import LinearityStep
 
 

romancal/regtest/test_ramp_fitting.py

@@ -10,14 +10,34 @@
 
 import pytest
 import roman_datamodels as rdm
+from stpipe import log as stpipe_log
 
-from romancal.lib.dms import log_result
 from romancal.lib.suffix import replace_suffix
 from romancal.ramp_fitting import RampFitStep
 
 from .regtestdata import compare_asdf
 
 
+def log_result(requirement, message, result):
+    """Log individual test results that relate to a requirement
+
+    Parameters
+    ----------
+    requirement : str
+        The requirement being logged. I.e "DMS363"
+
+    message : str
+        Message describing what is being tested
+
+    result : bool
+        The result of the test
+    """
+    logger = stpipe_log.delegator.log
+    result_text = "PASS" if result else "FAIL"
+    log_msg = f"{requirement} MSG: {message}.......{result_text}"
+    logger.info(log_msg)
+
+
 # ##########
 # Conditions
 # ##########

romancal/resample/tests/test_resample.py

@@ -346,11 +346,9 @@ def test_resampledata_init_default(exposure_1):
 @pytest.mark.parametrize("input_models", [list()])
 def test_resampledata_init_invalid_input(input_models):
     """Test that ResampleData will raise an exception on invalid inputs."""
-    with pytest.raises(Exception) as exec_info:
+    with pytest.raises(ValueError):
         ResampleData(input_models)
 
-    assert type(exec_info.value) == ValueError
-
 
 def test_resampledata_do_drizzle_many_to_one_default_no_rotation_single_exposure(
     exposure_1,