diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..acd5061c0 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,27 @@ + +Resolves [CCD-nnnn](https://jira.stsci.edu/browse/CCD-nnnn) + + +Closes # + + +This PR addresses ... + + +## Tasks +- [ ] update or add relevant tests +- [ ] update relevant docstrings and / or `docs/` page +- [ ] Does this PR change any API used downstream? (if not, label with `no-changelog-entry-needed`) + - [ ] write news fragment(s) in `changes/`: `echo "changed something" > changes/..rst` (see below for change types) + +
news fragment change types... + +- ``changes/.hst.rst``: HST reference files +- ``changes/.jwst.rst``: JWST reference files +- ``changes/.roman.rst``: Roman reference files +- ``changes/.doc.rst``: documentation change +- ``changes/.testing.rst``: change to tests or test automation +- ``changes/.general.rst``: infrastructure or miscellaneous change +
+ diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 3d1a679ae..cb5c44f47 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -1,20 +1,30 @@ -name: Changelog +name: changelog on: pull_request: - types: [labeled, unlabeled, opened, synchronize, reopened] - branches: - - master + types: + - labeled + - unlabeled + - opened + - synchronize + - reopened + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - changelog: - name: Confirm changelog entry + check: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog-entry-needed') }} runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: true - - name: Grep for PR number in CHANGES.rst - run: grep -P '\[[^\]]*#${{github.event.number}}[,\]]' CHANGES.rst - if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog-entry-needed') }} + - run: pip install . + - run: pip install towncrier + - run: towncrier check + - run: towncrier build --draft | grep -P '#${{ github.event.number }}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29f8e0a0b..d791956ba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,14 +27,21 @@ jobs: git config user.name "GitHub Actions" git config user.email "<>" - - name: Set changelog release date + - uses: actions/setup-python@v5 + with: + python-version: 3 + + - name: create new change log entry for release run: | - release_scripts/set_release_date ${{ github.event.inputs.version }} + pip install towncrier + pip install . + towncrier check + towncrier build --yes --version ${{ inputs.version }} - name: Commit changelog and tag release run: | git add CHANGES.rst - git commit -m "Set ${{ github.event.inputs.version }} release date" + git commit -m "created change log for ${{ inputs.version }}" git push origin HEAD git tag -a ${{ github.event.inputs.version }} -m "${{ github.event.inputs.description }}" git push origin ${{ github.event.inputs.version }} diff --git a/CHANGES.rst b/CHANGES.rst index 84ebe6159..403265328 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,13 +1,26 @@ -11.18.4 (unreleased) +12.0.0 (unreleased) ==================== General ------- +- Default context changed from "operational" to "latest". For JWST, the default context is the "build" context as determined by locally installed calibration software version. This can be overridden if CRDS_CONTEXT environment variable is explicitly set by user. [#1076] + - Setting environment variable `CRDS_CONTEXT=latest` automatically sets the effective context to the latest operational context found on the CRDS Server. [#1062] - `client.api.get_default_context` by default returns build context for jwst, else latest. This can still be overridden by explicitly passing a value into optional arg `state`. [#1069] + +11.18.4 (2024-09-10) +==================== + +General +------- + +- Replaced deprecated SafeConfigParser with ConfigParser in crds.core.config [#1065] +- moved DMS requirement correlations with tests from ``@metrics_logger`` test decorators to ``test/dms_requirement_tests.json`` [#1064] + + 11.18.3 (2024-09-03) ==================== diff --git a/changes/.gitkeep b/changes/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/changes/.gitkeep @@ -0,0 +1 @@ + diff --git a/changes/1068.doc.rst b/changes/1068.doc.rst new file mode 100644 index 000000000..724817599 --- /dev/null +++ b/changes/1068.doc.rst @@ -0,0 +1 @@ +use ``towncrier`` to handle change log entries diff --git a/crds/core/config.py b/crds/core/config.py index c1efc544d..e4efa7a0e 100644 --- a/crds/core/config.py +++ b/crds/core/config.py @@ -28,7 +28,7 @@ def _get_crds_ini_parser(): """Load and return the environment from the CRDS rc file.""" global CRDS_INI_PARSER if CRDS_INI_PARSER is None: - parser = configparser.SafeConfigParser() + parser = configparser.ConfigParser() with log.warn_on_exception("Failed reading CRDS rc file"): ini_path = _get_crds_ini_path() if os.path.exists(ini_path): diff --git a/pyproject.toml b/pyproject.toml index 9742f8586..a2c0d80dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,3 +193,29 @@ markers = [ [tool.setuptools_scm] write_to = "crds/_version.py" + +[tool.towncrier] +filename = "CHANGES.rst" +directory = "changes" +package = "crds" +title_format = "{version} ({project_date})" +ignore = [".gitkeep"] +wrap = true +issue_format = "`#{issue} `_" + +[tool.towncrier.fragment.hst] +name = "HST" + +[tool.towncrier.fragment.jwst] +name = "JWST" + +[tool.towncrier.fragment.roman] +name = "ROMAN" + +[tool.towncrier.fragment.doc] +name = "Documentation" + +[tool.towncrier.fragment.testing] +name = "Testing / Automation" + +[tool.towncrier.fragment.general] diff --git a/release_scripts/set_release_date b/release_scripts/set_release_date deleted file mode 100755 index 5949724cf..000000000 --- a/release_scripts/set_release_date +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -""" -Set the changelog release date of the specified version to today. -""" -import argparse -from datetime import date -from pathlib import Path -import re - - -UNRELEASED_VERSION_RE = re.compile(r"\A(?P[0-9]+\.[0-9]+\.[0-9]+) \([uU]nreleased\)$", re.MULTILINE) - - -def parse_args(): - parser = argparse.ArgumentParser("set_release_date", "set changelog release date for the specified version") - - parser.add_argument("version", help="version number") - - return parser.parse_args() - - -def main(): - args = parse_args() - - changelog_path = Path(__file__).absolute().parent.parent / "CHANGES.rst" - content = changelog_path.read_text() - - unreleased_version_re = re.compile(r"\A(?P" + re.escape(args.version) + r") \([uU]nreleased\)$", re.MULTILINE) - - match = unreleased_version_re.match(content) - if not match: - raise RuntimeError("cannot find matching unreleased version in CHANGES.rst") - - version = match.group("version") - new_heading = f"{version} ({date.today().isoformat()})" - new_content = unreleased_version_re.sub(new_heading, content) - - with changelog_path.open("w") as f: - f.write(new_content) - -if __name__ == "__main__": - main() diff --git a/test/certify/test_certify.py b/test/certify/test_certify.py index 872decc23..bfd57df6f 100644 --- a/test/certify/test_certify.py +++ b/test/certify/test_certify.py @@ -2,7 +2,6 @@ from pytest import mark, xfail import numpy as np import logging -from metrics_logger.decorators import metrics_logger from crds.core import utils, log, exceptions from crds import data_file, certify from crds.certify import CertifyScript, generic_tpn, validators, mapping_parser @@ -766,7 +765,6 @@ def test_certify_AsdfCertify_valid(jwst_serverless_state, jwst_data, caplog): @mark.roman @mark.certify -@metrics_logger("DMS4") def test_certify_roman_valid_asdf(roman_test_cache_state, roman_data, caplog): """Required Roman test: confirm that a valid asdf file is recognized as such. """ @@ -779,7 +777,6 @@ def test_certify_roman_valid_asdf(roman_test_cache_state, roman_data, caplog): @mark.roman @mark.certify -@metrics_logger("DMS4") def test_certify_roman_invalid_asdf_schema(roman_test_cache_state, roman_data, caplog): """Required Roman test: confirm that an asdf file that does not conform to its schema definition triggers an error in DataModels. @@ -794,7 +791,6 @@ def test_certify_roman_invalid_asdf_schema(roman_test_cache_state, roman_data, c @mark.roman @mark.certify -@metrics_logger("DMS4") def test_certify_roman_invalid_asdf_tpn(roman_test_cache_state, roman_data, caplog): """Required Roman test: confirm that an asdf file that does not conform to its tpn definition triggers an error in crds. Note: as the tpn often replicates schema implementation, this also @@ -810,7 +806,6 @@ def test_certify_roman_invalid_asdf_tpn(roman_test_cache_state, roman_data, capl @mark.roman @mark.certify -@metrics_logger("DMS5") def test_certify_roman_valid_spec_asdf(roman_test_cache_state, roman_data, caplog): """Required Roman test: confirm that a valid spectroscopic asdf file is recognized as such.""" with caplog.at_level(logging.INFO, logger="CRDS"): @@ -822,7 +817,6 @@ def test_certify_roman_valid_spec_asdf(roman_test_cache_state, roman_data, caplo @mark.roman @mark.certify -@metrics_logger("DMS5") def test_certify_roman_invalid_spec_asdf_schema(roman_test_cache_state, roman_data, caplog): """Required Roman test: confirm that a spectroscopic asdf file that does not conform to its schema definition triggers an error in DataModels.""" @@ -836,7 +830,6 @@ def test_certify_roman_invalid_spec_asdf_schema(roman_test_cache_state, roman_da @mark.roman @mark.certify -@metrics_logger("DMS5") def test_certify_roman_invalid_spec_asdf_tpn(roman_test_cache_state, roman_data, caplog): """Required Roman test: confirm that a spectroscopic asdf file that does not conform to its tpn definition triggers an error in crds. Note: as the tpn often replicates schema implementation, @@ -906,7 +899,6 @@ def test_certify_rmap_compare(jwst_serverless_state, caplog): @mark.roman @mark.certify -@metrics_logger("DMS6") def test_certify_roman_rmap_compare(roman_test_cache_state, caplog): """Required Roman test: confirm that a calibration mapping file properly compares to its context.""" with caplog.at_level(logging.INFO, logger="CRDS"): @@ -961,7 +953,6 @@ def test_certify_duplicate_rmap_case_error(hst_serverless_state, hst_data, caplo @mark.roman @mark.certify -@metrics_logger("DMS6") def test_certify_roman_duplicate_rmap_case_error(roman_test_cache_state, roman_data, caplog): """Required Roman test: verify that a calibration mapping file containing duplicate match cases fails.""" @@ -994,7 +985,6 @@ def test_checksum_duplicate_rmap_case_error(hst_serverless_state, hst_data, capl @mark.roman @mark.certify -@metrics_logger("DMS6") def test_checksum_roman_duplicate_rmap_case_error(roman_serverless_state, roman_data, caplog): """Required Roman test: verify that the crds rmap checksum update tool does not silently drop duplicate rmap entries when updating the checksum and rewriting the file.""" @@ -1010,7 +1000,6 @@ def test_checksum_roman_duplicate_rmap_case_error(roman_serverless_state, roman_ @mark.roman @mark.certify -@metrics_logger("DMS6") def test_certify_roman_invalid_rmap_tpn(roman_test_cache_state, roman_data, caplog): """Required Roman test: verify that a calibration mapping file that violates tpn rules produces an error.""" diff --git a/test/dms_requirement_tests.json b/test/dms_requirement_tests.json new file mode 100644 index 000000000..614fea790 --- /dev/null +++ b/test/dms_requirement_tests.json @@ -0,0 +1,35 @@ +{ + "DMS4": [ + "test.certify.test_certify.test_certify_roman_invalid_asdf_schema", + "test.certify.test_certify.test_certify_roman_invalid_asdf_tpn", + "test.certify.test_certify.test_certify_roman_valid_asdf" + ], + "DMS5": [ + "test.certify.test_certify.test_certify_roman_invalid_spec_asdf_schema", + "test.certify.test_certify.test_certify_roman_invalid_spec_asdf_tpn", + "test.certify.test_certify.test_certify_roman_valid_spec_asdf" + ], + "DMS6": [ + "test.certify.test_certify.test_certify_roman_duplicate_rmap_case_error", + "test.certify.test_certify.test_certify_roman_invalid_rmap_tpn", + "test.certify.test_certify.test_certify_roman_rmap_compare", + "test.certify.test_certify.test_checksum_roman_duplicate_rmap_case_error" + ], + "DMS16": [ + "test.roman.test_roman.test_getreferences_with_invalid_header", + "test.roman.test_roman.test_getreferences_with_valid_header_ISOT_fmt", + "test.roman.test_roman.test_getreferences_with_valid_header_ISO_fmt" + ], + "DMS25": [ + "test.roman.test_roman.test_getreferences_with_valid_header_ISOT_fmt", + "test.roman.test_roman.test_getreferences_with_valid_header_ISO_fmt", + "test.roman.test_roman.test_list_references" + ], + "DMS26": [ + "test.roman.test_roman.test_getreferences_with_valid_header_ISO_fmt", + "test.roman.test_roman.test_list_references" + ], + "DMS114": [ + "test.roman.test_roman.test_list_references" + ] +} \ No newline at end of file diff --git a/test/roman/test_roman.py b/test/roman/test_roman.py index 058f31be6..9ec575773 100644 --- a/test/roman/test_roman.py +++ b/test/roman/test_roman.py @@ -4,11 +4,9 @@ import pathlib from crds.core import heavy_client from crds.core.exceptions import CrdsLookupError -from metrics_logger.decorators import metrics_logger @mark.roman -@metrics_logger("DMS16", "DMS25") def test_getreferences_with_valid_header_ISOT_fmt(roman_test_cache_state): """ test_getreferences_with_valid_header: test satisfies Roman 303.1 and 628.1 """ @@ -30,7 +28,6 @@ def test_getreferences_with_valid_header_ISOT_fmt(roman_test_cache_state): @mark.roman -@metrics_logger("DMS16", "DMS25", "DMS26") def test_getreferences_with_valid_header_ISO_fmt(roman_test_cache_state): """ test_getreferences_with_valid_header: test satisfies Roman 303.1 and 628.1 """ @@ -78,7 +75,6 @@ def test_getreferences_with_valid_header_ISO_fmt(roman_test_cache_state): @mark.roman -@metrics_logger("DMS16") def test_getreferences_with_invalid_header(roman_test_cache_state): """ test_getreferences_with_invalid_header: test satisfies Roman 303.1 """ @@ -101,7 +97,6 @@ def test_getreferences_with_invalid_header(roman_test_cache_state): @mark.roman -@metrics_logger("DMS114", "DMS25", "DMS26") def test_list_references(roman_test_cache_state): """ test_list_references: test satisfies Roman 303.2 and 628.2 """ diff --git a/test/test_dms_requirements.py b/test/test_dms_requirements.py new file mode 100644 index 000000000..5d748263d --- /dev/null +++ b/test/test_dms_requirements.py @@ -0,0 +1,34 @@ +import json +import re +from pathlib import Path + +TEST_DIRECTORY = Path(__file__).parent +TEST_REQUIREMENTS_FILENAME = Path(__file__).parent / "dms_requirement_tests.json" + + +def test_requirements(): + test_requirements_filename = TEST_REQUIREMENTS_FILENAME.expanduser().absolute() + test_directory = TEST_DIRECTORY.expanduser().absolute() + + with open(test_requirements_filename) as test_requirements_file: + requirements = json.load(test_requirements_file) + + required_tests = { + test + for requirement_tests in requirements.values() + for test in requirement_tests + } + + existing_tests = set() + test_regex = re.compile(r"def (test_[^\(]+)\(.*\):") + for test_filename in test_directory.glob("**/test_*.py"): + with open(test_filename) as test_file: + test_file_contents = test_file.read() + + for match in re.finditer(test_regex, test_file_contents): + test = f"{test_directory.stem}.{str(test_filename.relative_to(test_directory).parent).replace('/', '.')}.{test_filename.stem}.{match.group(1)}" + if test in required_tests: + existing_tests.add(test) + + missing_tests = required_tests - existing_tests + assert not missing_tests, f"could not find the following tests correlated with DMS requirements: {missing_tests}"