From 8ee070ab6e7f8e4971548283f2d785eb152ce65d Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 10 Oct 2024 13:41:59 -0400 Subject: [PATCH 1/5] remove lib.dms --- romancal/lib/dms.py | 27 --------------------------- romancal/regtest/test_ramp_fitting.py | 22 +++++++++++++++++++++- 2 files changed, 21 insertions(+), 28 deletions(-) delete mode 100644 romancal/lib/dms.py diff --git a/romancal/lib/dms.py b/romancal/lib/dms.py deleted file mode 100644 index 20ecaaa06..000000000 --- a/romancal/lib/dms.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Functionality required by DMS outside of core functionality of the package - -Example: Certain tests are required by DMS to log in a specific way. The -decorator for this is defined in this module. -""" - -from stpipe import log as stpipe_log - - -def log_result(requirement, message, result): - """Log individual test results that relate to a requirement - - Parameters - ---------- - requirement : str - The requirement being logged. I.e "DMS363" - - message : str - Message describing what is being tested - - result : bool - The result of the test - """ - logger = stpipe_log.delegator.log - result_text = "PASS" if result else "FAIL" - log_msg = f"{requirement} MSG: {message}.......{result_text}" - logger.info(log_msg) diff --git a/romancal/regtest/test_ramp_fitting.py b/romancal/regtest/test_ramp_fitting.py index e1e79e767..2881808ff 100644 --- a/romancal/regtest/test_ramp_fitting.py +++ b/romancal/regtest/test_ramp_fitting.py @@ -10,14 +10,34 @@ import pytest import roman_datamodels as rdm +from stpipe import log as stpipe_log -from romancal.lib.dms import log_result from romancal.lib.suffix import replace_suffix from romancal.ramp_fitting import RampFitStep from .regtestdata import compare_asdf +def log_result(requirement, message, result): + """Log individual test results that relate to a requirement + + Parameters + ---------- + requirement : str + The requirement being logged. I.e "DMS363" + + message : str + Message describing what is being tested + + result : bool + The result of the test + """ + logger = stpipe_log.delegator.log + result_text = "PASS" if result else "FAIL" + log_msg = f"{requirement} MSG: {message}.......{result_text}" + logger.info(log_msg) + + # ########## # Conditions # ########## From ac98a17c4f620085b1614000f9ecd77edd64c397 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 10 Oct 2024 14:25:29 -0400 Subject: [PATCH 2/5] add changelog --- changes/1433.general.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/1433.general.rst diff --git a/changes/1433.general.rst b/changes/1433.general.rst new file mode 100644 index 000000000..153d72ca5 --- /dev/null +++ b/changes/1433.general.rst @@ -0,0 +1 @@ +Removed now unused lib.dms. From 39b14045932db8ca7b5275b4445a97aaa3d107a7 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 18 Oct 2024 11:01:01 -0400 Subject: [PATCH 3/5] Manual Pre-commit update (#1462) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 10 +++++----- romancal/dark_current/tests/test_dark.py | 6 +++--- romancal/linearity/tests/test_linearity.py | 3 +-- romancal/resample/tests/test_resample.py | 4 +--- romancal/tweakreg/tests/test_tweakreg.py | 22 +++++++++------------- 5 files changed, 19 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bad5af22c..a5e38f155 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ exclude: ".*\\.asdf$" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-ast @@ -28,13 +28,13 @@ repos: - id: text-unicode-replacement-char - repo: https://github.com/asottile/pyupgrade - rev: 'v3.15.1' + rev: 'v3.18.0' hooks: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.2.2' + rev: 'v0.6.9' hooks: - id: ruff args: ["--fix"] @@ -45,12 +45,12 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.10 hooks: - id: bandit args: ["-c", "pyproject.toml"] diff --git a/romancal/dark_current/tests/test_dark.py b/romancal/dark_current/tests/test_dark.py index 316063cb2..296264e0e 100644 --- a/romancal/dark_current/tests/test_dark.py +++ b/romancal/dark_current/tests/test_dark.py @@ -31,7 +31,7 @@ def test_dark_step_interface(instrument, exptype): # Test dark results assert (result.data == ramp_model.data).all() - assert type(result) == RampModel + assert isinstance(result, RampModel) assert result.validate() is None assert result.data.shape == shape assert result.groupdq.shape == shape @@ -99,7 +99,7 @@ def test_dark_step_output_dark_file(tmp_path, instrument, exptype): dark_out_file_model = rdm.open(path) # Test dark file results - assert type(dark_out_file_model) == DarkRefModel + assert isinstance(dark_out_file_model, DarkRefModel) assert dark_out_file_model.validate() is None assert dark_out_file_model.data.shape == shape assert dark_out_file_model.dq.shape == shape[1:] @@ -132,7 +132,7 @@ def test_dark_step_getbestrefs(tmp_path, instrument, exptype): dark_out_file_model = rdm.open(path) # Test dark file results - assert type(dark_out_file_model) == DarkRefModel + assert isinstance(dark_out_file_model, DarkRefModel) assert dark_out_file_model.validate() is None assert dark_out_file_model.data.shape == shape assert dark_out_file_model.dq.shape == shape[1:] diff --git a/romancal/linearity/tests/test_linearity.py b/romancal/linearity/tests/test_linearity.py index 6b1fed838..bd34c6efa 100644 --- a/romancal/linearity/tests/test_linearity.py +++ b/romancal/linearity/tests/test_linearity.py @@ -6,11 +6,10 @@ import numpy as np import pytest -from roman_datamodels import maker_utils +from roman_datamodels import dqflags, maker_utils from roman_datamodels.datamodels import LinearityRefModel, ScienceRawModel from romancal.dq_init import DQInitStep -from romancal.lib import dqflags from romancal.linearity import LinearityStep diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index 78c9f7fbc..b20579f08 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -346,11 +346,9 @@ def test_resampledata_init_default(exposure_1): @pytest.mark.parametrize("input_models", [list()]) def test_resampledata_init_invalid_input(input_models): """Test that ResampleData will raise an exception on invalid inputs.""" - with pytest.raises(Exception) as exec_info: + with pytest.raises(ValueError): ResampleData(input_models) - assert type(exec_info.value) == ValueError - def test_resampledata_do_drizzle_many_to_one_default_no_rotation_single_exposure( exposure_1, diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py index 9929e3393..63dad8e5a 100644 --- a/romancal/tweakreg/tests/test_tweakreg.py +++ b/romancal/tweakreg/tests/test_tweakreg.py @@ -686,7 +686,7 @@ def test_tweakreg_raises_error_on_invalid_abs_refcat(tmp_path, base_image): img = base_image(shift_1=1000, shift_2=1000) add_tweakreg_catalog_attribute(tmp_path, img) - with pytest.raises(Exception) as exec_info: + with pytest.raises(TypeError): trs.TweakRegStep.call( [img], save_abs_catalog=True, @@ -695,8 +695,6 @@ def test_tweakreg_raises_error_on_invalid_abs_refcat(tmp_path, base_image): output_dir=str(tmp_path), ) - assert type(exec_info.value) == TypeError - def test_tweakreg_combine_custom_catalogs_and_asn_file(tmp_path, base_image): """ @@ -737,7 +735,7 @@ def test_tweakreg_combine_custom_catalogs_and_asn_file(tmp_path, base_image): catfile=catfile, ) - assert type(res) == ModelLibrary + assert isinstance(res, ModelLibrary) with res: for i, (model, target) in enumerate(zip(res, [img1, img2, img3])): @@ -750,7 +748,7 @@ def test_tweakreg_combine_custom_catalogs_and_asn_file(tmp_path, base_image): assert model.meta.filename == target.meta.filename - assert type(model) == type(target) + assert type(model) is type(target) assert (model.data == target.data).all() @@ -853,7 +851,7 @@ def test_tweakreg_parses_asn_correctly(tmp_path, base_image): res = trs.TweakRegStep.call(asn_filepath) - assert type(res) == ModelLibrary + assert isinstance(res, ModelLibrary) with res: models = list(res) @@ -872,8 +870,8 @@ def test_tweakreg_parses_asn_correctly(tmp_path, base_image): assert models[0].meta.filename == img_1.meta.filename assert models[1].meta.filename == img_2.meta.filename - assert type(models[0]) == type(img_1) - assert type(models[1]) == type(img_2) + assert type(models[0]) is type(img_1) + assert type(models[1]) is type(img_2) assert (models[0].data == img_1.data).all() assert (models[1].data == img_2.data).all() @@ -891,7 +889,7 @@ def test_fit_results_in_meta(tmp_path, base_image): res = trs.TweakRegStep.call([img]) - assert type(res) == ModelLibrary + assert isinstance(res, ModelLibrary) with res: for i, model in enumerate(res): assert hasattr(model.meta, "wcs_fit_results") @@ -995,11 +993,9 @@ def test_parse_catfile_raises_error_on_invalid_content(tmp_path, catfile_line_co with open(catfile, mode="w") as f: print(catfile_content.getvalue(), file=f) - with pytest.raises(Exception) as exec_info: + with pytest.raises(ValueError): trs._parse_catfile(catfile) - assert type(exec_info.value) == ValueError - def test_update_source_catalog_coordinates(tmp_path, base_image): """Test that TweakReg updates the catalog coordinates with the tweaked WCS.""" @@ -1234,7 +1230,7 @@ def test_tweakreg_skips_invalid_exposure_types(exposure_type, tmp_path, base_ima img2.meta.exposure.type = exposure_type res = trs.TweakRegStep.call([img1, img2]) - assert type(res) == ModelLibrary + assert isinstance(res, ModelLibrary) with res: for i, model in enumerate(res): assert hasattr(model.meta.cal_step, "tweakreg") From cb93bfd563fa0e4eee156cb1509f804440ba8c88 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 21 Oct 2024 11:40:26 -0400 Subject: [PATCH 4/5] vendorize and fix crds workflow --- .github/workflows/contexts.yml | 26 ++++++++++++++++++++++++++ .github/workflows/roman_ci.yml | 2 +- .github/workflows/roman_ci_cron.yaml | 2 +- .github/workflows/tests_devdeps.yml | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/contexts.yml diff --git a/.github/workflows/contexts.yml b/.github/workflows/contexts.yml new file mode 100644 index 000000000..14e4c17b7 --- /dev/null +++ b/.github/workflows/contexts.yml @@ -0,0 +1,26 @@ +name: contexts + +on: + workflow_call: + outputs: + roman: + value: ${{ jobs.contexts.outputs.roman }} + workflow_dispatch: + +jobs: + contexts: + name: retrieve latest CRDS contexts + runs-on: ubuntu-latest + outputs: + roman: ${{ steps.roman_crds_context.outputs.pmap }} + steps: + - id: roman_crds_context + env: + OBSERVATORY: roman + CRDS_SERVER_URL: https://roman-crds.stsci.edu + run: > + echo "pmap=$( + curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["${{ env.OBSERVATORY }}", null], "id": 1}' ${{ env.CRDS_SERVER_URL }}/json/ --retry 8 | + python -c "import sys, json; print(json.load(sys.stdin)['result'])" + )" >> $GITHUB_OUTPUT + - run: if [[ ! -z "${{ steps.roman_crds_context.outputs.pmap }}" ]]; then echo ${{ steps.roman_crds_context.outputs.pmap }}; else exit 1; fi diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml index 5a9f1c1f8..9c4df6bc3 100644 --- a/.github/workflows/roman_ci.yml +++ b/.github/workflows/roman_ci.yml @@ -41,7 +41,7 @@ jobs: with: minimal: true latest_crds_contexts: - uses: spacetelescope/crds/.github/workflows/contexts.yml@master + uses: ./.github/workflows/contexts.yml crds_context: needs: [ latest_crds_contexts ] runs-on: ubuntu-latest diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml index d509ede99..d04830366 100644 --- a/.github/workflows/roman_ci_cron.yaml +++ b/.github/workflows/roman_ci_cron.yaml @@ -41,7 +41,7 @@ jobs: minimal: true latest_crds_contexts: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests'))) - uses: spacetelescope/crds/.github/workflows/contexts.yml@master + uses: ./.github/workflows/contexts.yml crds_context: needs: [ latest_crds_contexts ] runs-on: ubuntu-latest diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml index a48106053..83613ede5 100644 --- a/.github/workflows/tests_devdeps.yml +++ b/.github/workflows/tests_devdeps.yml @@ -42,7 +42,7 @@ jobs: minimal: true latest_crds_contexts: if: (github.repository == 'spacetelescope/romancal' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests'))) - uses: spacetelescope/crds/.github/workflows/contexts.yml@master + uses: ./.github/workflows/contexts.yml crds_context: needs: [ latest_crds_contexts ] runs-on: ubuntu-latest From e215f872fef62775f0eb8473ac836e62fb1e2bc9 Mon Sep 17 00:00:00 2001 From: D Davis <49163225+ddavis-stsci@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:26:21 -0400 Subject: [PATCH 5/5] rcal-936 Update skycell_asn docs and add skycell_asn as a script at install time (#1471) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- changes/1471.associations.rst | 1 + docs/conf.py | 2 +- docs/roman/associations/commands.rst | 1 + docs/roman/associations/skycell_asn.rst | 91 +++++++++++++++++++++++++ pyproject.toml | 1 + 5 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 changes/1471.associations.rst create mode 100644 docs/roman/associations/skycell_asn.rst diff --git a/changes/1471.associations.rst b/changes/1471.associations.rst new file mode 100644 index 000000000..4caeda3a6 --- /dev/null +++ b/changes/1471.associations.rst @@ -0,0 +1 @@ +Update skycell_asn docs and add skycell_asn as a script at install time diff --git a/docs/conf.py b/docs/conf.py index 21365ad0d..cdcfd66ec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -274,7 +274,7 @@ def check_sphinx_version(expected_version): # further. For a list of options available for each theme, see the # documentation. -html_theme_options = {"collapse_navigation": True, "version_selector": True} +html_theme_options = {"collapse_navigation": True} # "nosidebar": "false", # "sidebarbgcolor": "#4db8ff", diff --git a/docs/roman/associations/commands.rst b/docs/roman/associations/commands.rst index 825e37c09..064ecf11c 100644 --- a/docs/roman/associations/commands.rst +++ b/docs/roman/associations/commands.rst @@ -7,3 +7,4 @@ Association Commands :maxdepth: 2 asn_from_list.rst + skycell_asn.rst diff --git a/docs/roman/associations/skycell_asn.rst b/docs/roman/associations/skycell_asn.rst new file mode 100644 index 000000000..459d47e93 --- /dev/null +++ b/docs/roman/associations/skycell_asn.rst @@ -0,0 +1,91 @@ +.. _skycell_asn: + +skycell_asn +=========== + +Create an association using either the command line tool +``skycell_asn`` or through the Python API using either +:class:`romancal.associations.skycell_asn.Main` or +:func:`romancal.associations.skycellasn.skycell_asn` + + +Associations +^^^^^^^^^^^^ + +Refer to TBD for a full description of associations. + +To create an association, use the following command: + +.. code-block:: python + + skycell_asn file_list -o my_program + +where the individual calibrated detector files are in the current directory and the -o is the root +for the output associations. To get a complete list of options you can run the command with the +\-h option + +.. code-block:: python + + skycell_asn -h + + +By knowing the structure of the conventional file name you can generate custom associations based +on the information included in the file names. +The current naming convention for the input files consists of a Visit_ID + Exposure_ID + detector + suffix +Where the Visit_ID is a 19 digit string +PPPPPCCAAASSSOOOVVV, with + +PPPPP = Program number +CC = Execution plan number +AAA = Pass number (within execution plan) +SSS = Segment Number (within pass) +OOO = Observation number (within the leg, not to be confused with the Observation ID) +VVV = Visit Number (within the observation) + +and the Exposure_ID is a four digits designating the exposures + +eeee = Exposure number (within the visit) + +The detector is WFI01, WFI02, ... WFI18 + +The suffix indicates the processing level of the file and to create products based on the +skycells the suffix should generally be 'cal'. + +and the file name is constructed as, +rPPPPPCCAAASSSOOOVVV_eeee_detector_suffix.asdf + +To give a more concrete example we'll use the APT example program for the +High Latitude Wide Angle Survey (hlwas). The program is 00991, the execution plan number is 01, +there are 12 passes, 169 segments, 5 observations and a various number of visits. +For this example we want to select a single filter, say F158, and that is observation 003. +So to generate the visit level associations for observation 001 we would select the files using the bash +command line, + +.. code-block:: text + + skycell_asn r0099101001001003001_*_cal.asdf -o r0099101001001003001 --product-type visit + +Where the wildcard selects all the exposures for visit 001 and generates associations based on the skycells +the exposures will populate. This will generate associations based the skycells that the exposures can +contribute data to. The association files will be json files with names based + +.. code-block:: text + + r0099101001001003001_____i2d_asn.json + +or for the selections above + +.. code-block:: text + + r0099101001001003001_r257dp63x98y83_visit_F158_prompt_i2d_asn.json + +where the skycell name can vary based on the location on the celestial sphere and the i2d indicates +that this is resampled 2-d imaging data. The release product name can be changed from the default +by adding the optional argument --release-product to the command line. + +An analogous command to generate the pass level products, again setting observation to 003 to only select +the F158 filter. + +.. code-block:: text + + skycell_asn r0099101???003*_*_cal.asdf -o r0099101 --product-type pass diff --git a/pyproject.toml b/pyproject.toml index f2bede94d..e5e8107f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ schema_editor = "romancal.scripts.schema_editor:main" schemadoc = "romancal.scripts.schemadoc:main" verify_install_requires = "romancal.scripts.verify_install_requires:main" asn_from_list = "romancal.associations.asn_from_list:Main" +skycell_asn = "romancal.associations.skycell_asn:Main" [build-system] requires = [