From 9a5f3fc4a048b39b6055f7fcf898b375543c1b8f Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 17 Oct 2024 11:12:57 -0400 Subject: [PATCH 1/2] remove `display_version` from docs config (#1463) Co-authored-by: Brett Graham --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index c390a66fd..191e88acf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -273,7 +273,7 @@ def check_sphinx_version(expected_version): # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = {"collapse_navigation": True, "display_version": True} +html_theme_options = {"collapse_navigation": True, "version_selector": True} # "nosidebar": "false", # "sidebarbgcolor": "#4db8ff", # "sidebartextcolor": "black", From f23b4801187738790e00de7fa4ce5de8b80e2c0f Mon Sep 17 00:00:00 2001 From: mairan Date: Thu, 17 Oct 2024 11:48:37 -0400 Subject: [PATCH 2/2] RCAL-911 & 932: remove units from MOS and ELP pipelines. (#1445) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Brett Co-authored-by: Zach Burnett Co-authored-by: Zach Burnett Co-authored-by: Ned Molter Co-authored-by: Eddie Schlafly Co-authored-by: Paul Huwe <42071634+PaulHuwe@users.noreply.github.com> --- changes/1445.general.rst | 1 + docs/conf.py | 2 + romancal/dark_current/dark_current_step.py | 2 +- romancal/dark_current/tests/test_dark.py | 9 +- romancal/dq_init/tests/test_dq_init.py | 19 +--- romancal/flatfield/flat_field.py | 7 +- romancal/flatfield/tests/test_flatfield.py | 21 +--- romancal/flux/flux_step.py | 24 ++--- romancal/flux/tests/test_flux_step.py | 33 ++---- romancal/jump/jump_step.py | 5 +- romancal/jump/tests/test_jump_step.py | 16 ++- romancal/lib/tests/test_psf.py | 14 +-- romancal/linearity/linearity_step.py | 7 +- romancal/linearity/tests/test_linearity.py | 5 +- romancal/outlier_detection/_fileio.py | 4 +- .../tests/test_outlier_detection.py | 24 ++--- romancal/outlier_detection/utils.py | 10 +- romancal/photom/photom.py | 14 +-- romancal/photom/tests/test_photom.py | 71 ++++++------- romancal/ramp_fitting/ramp_fit_step.py | 16 +-- .../ramp_fitting/tests/test_ramp_fit_cas22.py | 6 +- romancal/refpix/data.py | 34 ++---- romancal/refpix/tests/conftest.py | 9 +- romancal/refpix/tests/test_data.py | 22 ++-- romancal/refpix/tests/test_refpix.py | 8 +- romancal/refpix/tests/test_step.py | 16 +-- romancal/regtest/test_catalog.py | 2 +- romancal/regtest/test_dark_current.py | 14 +-- romancal/regtest/test_jump_det.py | 4 +- romancal/regtest/test_linearity.py | 6 +- romancal/regtest/test_mos_pipeline.py | 3 + romancal/regtest/test_ramp_fitting.py | 8 +- romancal/regtest/test_refpix.py | 4 +- romancal/regtest/test_regtestdata.py | 4 +- romancal/regtest/test_resample.py | 2 +- romancal/regtest/test_skycell_generation.py | 8 +- romancal/regtest/test_tweakreg.py | 10 +- romancal/regtest/test_wfi_dq_init.py | 8 +- romancal/regtest/test_wfi_flat_field.py | 16 +-- .../regtest/test_wfi_grism_16resultants.py | 4 +- romancal/regtest/test_wfi_grism_pipeline.py | 4 +- .../regtest/test_wfi_image_16resultants.py | 4 +- romancal/regtest/test_wfi_image_pipeline.py | 16 +-- romancal/regtest/test_wfi_photom.py | 100 ++++++------------ romancal/regtest/test_wfi_saturation.py | 8 +- romancal/resample/gwcs_drizzle.py | 2 +- romancal/resample/resample.py | 35 +++--- romancal/resample/resample_utils.py | 2 +- romancal/resample/tests/test_resample.py | 28 ++--- romancal/resample/tests/test_resample_step.py | 9 +- romancal/saturation/saturation.py | 2 +- romancal/saturation/tests/test_saturation.py | 89 ++++++++-------- romancal/scripts/make_regtestdata.sh | 92 +++++++++------- romancal/skymatch/skymatch.py | 8 +- romancal/skymatch/skymatch_step.py | 6 +- romancal/skymatch/skystatistics.py | 10 +- romancal/skymatch/tests/test_skymatch.py | 56 +++++----- romancal/source_catalog/detection.py | 8 +- romancal/source_catalog/source_catalog.py | 62 +++-------- .../tests/test_source_catalog.py | 10 +- 60 files changed, 409 insertions(+), 634 deletions(-) create mode 100644 changes/1445.general.rst diff --git a/changes/1445.general.rst b/changes/1445.general.rst new file mode 100644 index 000000000..4471074d0 --- /dev/null +++ b/changes/1445.general.rst @@ -0,0 +1 @@ +Remove units from romancal. diff --git a/docs/conf.py b/docs/conf.py index 191e88acf..21365ad0d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -273,7 +273,9 @@ def check_sphinx_version(expected_version): # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. + html_theme_options = {"collapse_navigation": True, "version_selector": True} + # "nosidebar": "false", # "sidebarbgcolor": "#4db8ff", # "sidebartextcolor": "black", diff --git a/romancal/dark_current/dark_current_step.py b/romancal/dark_current/dark_current_step.py index a8a72658e..ecad8b93b 100644 --- a/romancal/dark_current/dark_current_step.py +++ b/romancal/dark_current/dark_current_step.py @@ -54,7 +54,7 @@ def process(self, input): # Do the dark correction out_model = input_model nresultants = len(input_model.meta.exposure["read_pattern"]) - out_model.data -= dark_model.data[:nresultants] + out_model.data -= dark_model.data[:nresultants].value out_model.pixeldq |= dark_model.dq out_model.meta.cal_step.dark = "COMPLETE" diff --git a/romancal/dark_current/tests/test_dark.py b/romancal/dark_current/tests/test_dark.py index 5c2ce3d21..316063cb2 100644 --- a/romancal/dark_current/tests/test_dark.py +++ b/romancal/dark_current/tests/test_dark.py @@ -5,7 +5,6 @@ import numpy as np import pytest import roman_datamodels as rdm -from astropy import units as u from roman_datamodels import maker_utils from roman_datamodels.datamodels import DarkRefModel, RampModel @@ -60,7 +59,7 @@ def test_dark_step_subtraction(instrument, exptype): # populate data array of science cube for i in range(0, 20): - ramp_model.data[0, 0, i] = i * ramp_model.data.unit + ramp_model.data[0, 0, i] = i darkref_model.data[0, 0, i] = i * 0.1 * darkref_model.data.unit orig_model = ramp_model.copy() @@ -68,12 +67,12 @@ def test_dark_step_subtraction(instrument, exptype): result = DarkCurrentStep.call(ramp_model, override_dark=darkref_model) # check that the dark file is subtracted frame by frame from the science data - diff = orig_model.data.value - darkref_model.data.value + diff = orig_model.data - darkref_model.data.value # test that the output data file is equal to the difference found when subtracting # reffile from sci file np.testing.assert_array_equal( - result.data.value, diff, err_msg="dark file should be subtracted from sci file " + result.data, diff, err_msg="dark file should be subtracted from sci file " ) @@ -148,7 +147,7 @@ def create_ramp_and_dark(shape, instrument, exptype): ramp.meta.instrument.detector = "WFI01" ramp.meta.instrument.optical_element = "F158" ramp.meta.exposure.type = exptype - ramp.data = u.Quantity(np.ones(shape, dtype=np.float32), u.DN, dtype=np.float32) + ramp.data = np.ones(shape, dtype=np.float32) ramp_model = RampModel(ramp) # Create dark model diff --git a/romancal/dq_init/tests/test_dq_init.py b/romancal/dq_init/tests/test_dq_init.py index 87ac2476e..c7ed8b8c0 100644 --- a/romancal/dq_init/tests/test_dq_init.py +++ b/romancal/dq_init/tests/test_dq_init.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from astropy import units as u from roman_datamodels import maker_utils, stnode from roman_datamodels.datamodels import MaskRefModel, ScienceRawModel from roman_datamodels.dqflags import pixel @@ -199,9 +198,7 @@ def test_dqinit_step_interface(instrument, exptype): wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype - wfi_sci_raw.data = u.Quantity( - np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16 - ) + wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw[extra_key] = extra_value wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) @@ -251,9 +248,7 @@ def test_dqinit_refpix(instrument, exptype): wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype - wfi_sci_raw.data = u.Quantity( - np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16 - ) + wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) # Create mask model @@ -272,7 +267,7 @@ def test_dqinit_refpix(instrument, exptype): # check if reference pixels are correct assert result.data.shape == (2, 20, 20) # no pixels should be trimmed - assert result.amp33.value.shape == (2, 4096, 128) + assert result.amp33.shape == (2, 4096, 128) assert result.border_ref_pix_right.shape == (2, 20, 4) assert result.border_ref_pix_left.shape == (2, 20, 4) assert result.border_ref_pix_top.shape == (2, 4, 20) @@ -304,9 +299,7 @@ def test_dqinit_resultantdq(instrument, exptype): wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype wfi_sci_raw.resultantdq[1, 12, 12] = pixel["DROPOUT"] - wfi_sci_raw.data = u.Quantity( - np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16 - ) + wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) # Create mask model @@ -354,9 +347,7 @@ def test_dqinit_getbestref(instrument, exptype): wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype - wfi_sci_raw.data = u.Quantity( - np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16 - ) + wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) # Perform Data Quality application step diff --git a/romancal/flatfield/flat_field.py b/romancal/flatfield/flat_field.py index dd489a926..b74475274 100644 --- a/romancal/flatfield/flat_field.py +++ b/romancal/flatfield/flat_field.py @@ -5,7 +5,6 @@ import logging import numpy as np -from astropy import units as u from roman_datamodels.dqflags import pixel log = logging.getLogger(__name__) @@ -105,9 +104,7 @@ def apply_flat_field(science, flat): flat_data[np.where(flat_bad)] = 1.0 # Now let's apply the correction to science data and error arrays. Rely # on array broadcasting to handle the cubes - science.data = u.Quantity( - (science.data.value / flat_data), u.DN / u.s, dtype=science.data.dtype - ) + science.data = (science.data / flat_data).astype(science.data.dtype) # Update the variances using BASELINE algorithm. For guider data, it has # not gone through ramp fitting so there is no Poisson noise or readnoise @@ -121,8 +118,6 @@ def apply_flat_field(science, flat): science.var_flat = science.data**2 / flat_data_squared * flat_err**2 science.err = np.sqrt(science.var_poisson + science.var_rnoise + science.var_flat) - science.err = science.err.to(science.data.unit) - # Workaround for https://github.com/astropy/astropy/issues/16055 # Combine the science and flat DQ arrays science.dq = np.bitwise_or(science.dq, flat_dq) diff --git a/romancal/flatfield/tests/test_flatfield.py b/romancal/flatfield/tests/test_flatfield.py index 65d4adab8..f246fae2e 100644 --- a/romancal/flatfield/tests/test_flatfield.py +++ b/romancal/flatfield/tests/test_flatfield.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from astropy import units as u from astropy.time import Time from roman_datamodels import maker_utils, stnode from roman_datamodels.datamodels import FlatRefModel, ImageModel @@ -26,22 +25,12 @@ def test_flatfield_step_interface(instrument, exptype): wfi_image.meta.instrument.detector = "WFI01" wfi_image.meta.instrument.optical_element = "F158" wfi_image.meta.exposure.type = exptype - wfi_image.data = u.Quantity( - np.ones(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32 - ) + wfi_image.data = np.ones(shape, dtype=np.float32) wfi_image.dq = np.zeros(shape, dtype=np.uint32) - wfi_image.err = u.Quantity( - np.zeros(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32 - ) - wfi_image.var_poisson = u.Quantity( - np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32 - ) - wfi_image.var_rnoise = u.Quantity( - np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32 - ) - wfi_image.var_flat = u.Quantity( - np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32 - ) + wfi_image.err = np.zeros(shape, dtype=np.float32) + wfi_image.var_poisson = np.zeros(shape, dtype=np.float32) + wfi_image.var_rnoise = np.zeros(shape, dtype=np.float32) + wfi_image.var_flat = np.zeros(shape, dtype=np.float32) wfi_image_model = ImageModel(wfi_image) flatref = stnode.FlatRef() diff --git a/romancal/flux/flux_step.py b/romancal/flux/flux_step.py index 79d820199..c1f333515 100644 --- a/romancal/flux/flux_step.py +++ b/romancal/flux/flux_step.py @@ -2,7 +2,6 @@ import logging -import astropy.units as u from roman_datamodels import datamodels from ..datamodels import ModelLibrary @@ -14,10 +13,6 @@ __all__ = ["FluxStep"] -# Define expected Level 2 units -LV2_UNITS = u.DN / u.s - - class FluxStep(RomanStep): """Apply flux scaling to count-rate data @@ -104,26 +99,19 @@ def apply_flux_correction(model): DATA = ("data", "err") VARIANCES = ("var_rnoise", "var_poisson", "var_flat") - if model.data.unit == model.meta.photometry.conversion_megajanskys.unit: - log.info( - f"Input data is already in flux units of {model.meta.photometry.conversion_megajanskys.unit}." - ) - log.info("Flux correction already applied.") - return - - if model.data.unit != LV2_UNITS: + if model.meta.cal_step["flux"] == "COMPLETE": message = ( - f"Input data units {model.data.unit} are not in the expected units of {LV2_UNITS}" - "\nAborting flux correction" + "Input data is already in flux units of MJy/sr." + "\nFlux correction already applied." ) - [log.error(line) for line in message.splitlines()] - raise ValueError(message) + log.info(message) + return # Apply the correction. # The end goal in units is to have MJy/sr. The scale is in MJy/sr also. # Hence the extra factor of s/DN must be applied to cancel DN/s. log.debug("Flux correction being applied") - c_mj = model.meta.photometry.conversion_megajanskys / model.data.unit + c_mj = model.meta.photometry.conversion_megajanskys for data in DATA: model[data] = model[data] * c_mj for variance in VARIANCES: diff --git a/romancal/flux/tests/test_flux_step.py b/romancal/flux/tests/test_flux_step.py index dcb04dd06..f7209241d 100644 --- a/romancal/flux/tests/test_flux_step.py +++ b/romancal/flux/tests/test_flux_step.py @@ -7,7 +7,6 @@ from romancal.datamodels import ModelLibrary from romancal.flux import FluxStep -from romancal.flux.flux_step import LV2_UNITS @pytest.mark.parametrize( @@ -23,7 +22,7 @@ def test_attributes(flux_step, attr, factor): """Test that the attribute has been scaled by the right factor""" original, result = flux_step - c_unit = 1.0 / LV2_UNITS + c_unit = 1.0 # Handle difference between just a single image and a list. if isinstance(original, datamodels.ImageModel): @@ -86,31 +85,15 @@ def flux_step(request): @pytest.fixture(scope="module") def image_model(): """Product a basic ImageModel""" - # Create a random image and specify a conversion. + # Create a random image and specify a conversion rng = np.random.default_rng() shape = (10, 10) image_model = maker_utils.mk_datamodel(datamodels.ImageModel, shape=shape) - image_model.data = u.Quantity( - rng.poisson(2.5, size=shape).astype(np.float32), - LV2_UNITS, - dtype=np.float32, - ) - image_model.var_rnoise = u.Quantity( - rng.normal(1, 0.05, size=shape).astype(np.float32), - LV2_UNITS**2, - dtype=np.float32, - ) - image_model.var_poisson = u.Quantity( - rng.poisson(1, size=shape).astype(np.float32), - LV2_UNITS**2, - dtype=np.float32, - ) - image_model.var_flat = u.Quantity( - rng.uniform(0, 1, size=shape).astype(np.float32), - LV2_UNITS**2, - dtype=np.float32, - ) - image_model.meta.photometry.conversion_megajanskys = 2.0 * u.MJy / u.sr + image_model.data = rng.poisson(2.5, size=shape).astype(np.float32) + image_model.var_rnoise = rng.normal(1, 0.05, size=shape).astype(np.float32) + image_model.var_poisson = rng.poisson(1, size=shape).astype(np.float32) + image_model.var_flat = rng.uniform(0, 1, size=shape).astype(np.float32) + image_model.meta.photometry.conversion_megajanskys = (2.0 * u.MJy / u.sr).value return image_model @@ -129,6 +112,6 @@ def input_modellibrary(image_model): # Create and return a ModelLibrary image_model1 = image_model.copy() image_model2 = image_model.copy() - image_model2.meta.photometry.conversion_megajanskys = 0.5 * u.MJy / u.sr + image_model2.meta.photometry.conversion_megajanskys = (0.5 * u.MJy / u.sr).value container = ModelLibrary([image_model1, image_model2]) return container diff --git a/romancal/jump/jump_step.py b/romancal/jump/jump_step.py index dfee1592f..74d555863 100644 --- a/romancal/jump/jump_step.py +++ b/romancal/jump/jump_step.py @@ -53,10 +53,10 @@ def process(self, input): # Extract the needed info from the Roman Data Model meta = input_model.meta - r_data = input_model.data.value + r_data = input_model.data r_gdq = input_model.groupdq r_pdq = input_model.pixeldq - r_err = input_model.err.value + r_err = input_model.err result = input_model # If the ramp fitting jump detection is enabled, then skip this step @@ -106,6 +106,7 @@ def process(self, input): self.log.info("Maximum cores to use = %s", max_cores) # Get the gain and readnoise reference files + # TODO: remove units from gain and RN reference files gain_filename = self.get_reference_file(input_model, "gain") self.log.info("Using GAIN reference file: %s", gain_filename) gain_model = rdd.GainRefModel(gain_filename) diff --git a/romancal/jump/tests/test_jump_step.py b/romancal/jump/tests/test_jump_step.py index b34c63958..14d48947b 100644 --- a/romancal/jump/tests/test_jump_step.py +++ b/romancal/jump/tests/test_jump_step.py @@ -106,10 +106,10 @@ def _setup( dm_ramp.meta.instrument.name = "WFI" dm_ramp.meta.instrument.optical_element = "F158" - dm_ramp.data = u.Quantity(data + 6.0, u.DN, dtype=np.float32) + dm_ramp.data = data + 6.0 dm_ramp.pixeldq = pixdq dm_ramp.groupdq = gdq - dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32) + dm_ramp.err = err dm_ramp.meta.exposure.type = "WFI_IMAGE" dm_ramp.meta.exposure.group_time = deltatime @@ -148,7 +148,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs): ) for i in range(ngroups): - model1.data[i, :, :] = deltaDN * i * model1.data.unit + model1.data[i, :, :] = deltaDN * i first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] @@ -161,8 +161,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs): for i in range(len(CR_x_locs)): CR_group = next(CR_pool) model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] - + 5000.0 * model1.data.unit + model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000.0 ) out_model = JumpStep.call( @@ -203,7 +202,7 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs): ) for i in range(ngroups): - model1.data[i, :, :] = deltaDN * i * model1.data.unit + model1.data[i, :, :] = deltaDN * i first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0] @@ -215,11 +214,10 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs): CR_group = next(CR_pool) model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000 * model1.data.unit + model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000 ) model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] - + 700 * model1.data.unit + model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] + 700 ) out_model = JumpStep.call( diff --git a/romancal/lib/tests/test_psf.py b/romancal/lib/tests/test_psf.py index 0738534d1..4eb81d43d 100644 --- a/romancal/lib/tests/test_psf.py +++ b/romancal/lib/tests/test_psf.py @@ -29,21 +29,15 @@ def setup_inputs( Return ImageModel of level 2 image. """ wfi_image = testutil.mk_level2_image(shape=shape) - wfi_image.data = u.Quantity( - np.ones(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32 - ) + wfi_image.data = np.ones(shape, dtype=np.float32) wfi_image.meta.filename = "filename" wfi_image.meta.instrument["optical_element"] = "F087" # add noise to data if noise is not None: setup_rng = np.random.default_rng(seed or 19) - wfi_image.data = u.Quantity( - setup_rng.normal(scale=noise, size=shape), - u.DN / u.s, - dtype=np.float32, - ) - wfi_image.err = noise * np.ones(shape, dtype=np.float32) * u.DN / u.s + wfi_image.data = setup_rng.normal(scale=noise, size=shape).astype("float32") + wfi_image.err = noise * (np.ones(shape, dtype=np.float32) * u.DN / u.s).value # add dq array wfi_image.dq = np.zeros(shape, dtype=np.uint32) @@ -106,7 +100,7 @@ def setup_method(self): def test_psf_fit(self, dx, dy, true_flux): # generate an ImageModel image_model = deepcopy(self.image_model) - init_data_stddev = np.std(image_model.data.value) + init_data_stddev = np.std(image_model.data) # add synthetic sources to the ImageModel: true_x = image_model_shape[0] / 2 + dx diff --git a/romancal/linearity/linearity_step.py b/romancal/linearity/linearity_step.py index 5320ad8e3..217fa6795 100644 --- a/romancal/linearity/linearity_step.py +++ b/romancal/linearity/linearity_step.py @@ -3,7 +3,6 @@ """ import numpy as np -from astropy import units as u from roman_datamodels import datamodels as rdd from roman_datamodels.dqflags import pixel from stcal.linearity.linearity import linearity_correction @@ -56,12 +55,10 @@ def process(self, input): # The third return value is the procesed zero frame which # Roman does not use. new_data, new_pdq, _ = linearity_correction( - input_model.data.value, gdq, pdq, lin_coeffs, lin_dq, pixel + input_model.data, gdq, pdq, lin_coeffs, lin_dq, pixel ) - input_model.data = u.Quantity( - new_data[0, :, :, :], u.DN, dtype=new_data.dtype - ) + input_model.data = new_data[0, :, :, :] input_model.pixeldq = new_pdq # Update the step status diff --git a/romancal/linearity/tests/test_linearity.py b/romancal/linearity/tests/test_linearity.py index b137069fd..6b1fed838 100644 --- a/romancal/linearity/tests/test_linearity.py +++ b/romancal/linearity/tests/test_linearity.py @@ -6,7 +6,6 @@ import numpy as np import pytest -from astropy import units as u from roman_datamodels import maker_utils from roman_datamodels.datamodels import LinearityRefModel, ScienceRawModel @@ -35,9 +34,7 @@ def test_linearity_coeff(instrument, exptype): wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = exptype - wfi_sci_raw.data = u.Quantity( - np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16 - ) + wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw) result = DQInitStep.call(wfi_sci_raw_model) diff --git a/romancal/outlier_detection/_fileio.py b/romancal/outlier_detection/_fileio.py index d1dde8489..0b6fdc857 100644 --- a/romancal/outlier_detection/_fileio.py +++ b/romancal/outlier_detection/_fileio.py @@ -1,7 +1,5 @@ import logging -from astropy.units import Quantity - log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -20,7 +18,7 @@ def save_drizzled(drizzled_model, make_output_path): def _make_median_model(example_model, data, wcs): model = example_model.copy() - model.data = Quantity(data, unit=model.data.unit) + model.data = data model.meta.filename = "drizzled_median.asdf" model.meta.wcs = wcs return model diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py index 4ee3651d9..4f8ecf68a 100644 --- a/romancal/outlier_detection/tests/test_outlier_detection.py +++ b/romancal/outlier_detection/tests/test_outlier_detection.py @@ -1,9 +1,7 @@ import os -import astropy.units as u import numpy as np import pytest -from astropy.units import Quantity from romancal.datamodels import ModelLibrary from romancal.outlier_detection import OutlierDetectionStep @@ -104,10 +102,10 @@ def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_imag """ img_1 = base_image() img_1.meta.filename = "img1_cal.asdf" - img_1.meta.background.level = 0 * u.DN / u.s + img_1.meta.background.level = 0 img_2 = base_image() img_2.meta.filename = "img2_cal.asdf" - img_2.meta.background.level = 0 * u.DN / u.s + img_2.meta.background.level = 0 input_models = ModelLibrary([img_1, img_2]) outlier_step = OutlierDetectionStep( @@ -134,9 +132,9 @@ def test_find_outliers(tmp_path, base_image, on_disk): """ Test that OutlierDetection can find outliers. """ - cr_value = Quantity(100, "DN / s") - source_value = Quantity(10, "DN / s") - err_value = Quantity(10, "DN / s") # snr=1 + cr_value = 100 + source_value = 10 + err_value = 10 # snr=1 imgs = [] for i in range(3): @@ -145,7 +143,7 @@ def test_find_outliers(tmp_path, base_image, on_disk): img.err[:] = err_value img.meta.filename = str(tmp_path / f"img{i}_suffix.asdf") img.meta.observation.exposure = i - img.meta.background.level = 0 * u.DN / u.s + img.meta.background.level = 0 imgs.append(img) # add outliers @@ -204,14 +202,12 @@ def test_identical_images(tmp_path, base_image, caplog): """ img_1 = base_image() img_1.meta.filename = "img1_suffix.asdf" - img_1.meta.background.level = 0 * u.DN / u.s + img_1.meta.background.level = 0 # add outliers img_1_input_coords = np.array( [(5, 45), (25, 25), (45, 85), (65, 65), (85, 5)], dtype=[("x", int), ("y", int)] ) - img_1.data[img_1_input_coords["x"], img_1_input_coords["y"]] = Quantity( - 100000, "DN / s" - ) + img_1.data[img_1_input_coords["x"], img_1_input_coords["y"]] = 100000 img_2 = img_1.copy() img_2.meta.filename = "img2_suffix.asdf" @@ -258,10 +254,10 @@ def test_outlier_detection_always_returns_modelcontainer_with_updated_datamodels os.chdir(tmp_path) img_1 = base_image() img_1.meta.filename = "img_1.asdf" - img_1.data *= img_1.meta.photometry.conversion_megajanskys / img_1.data.unit + img_1.data *= img_1.meta.photometry.conversion_megajanskys / img_1.data img_2 = base_image() img_2.meta.filename = "img_2.asdf" - img_2.data *= img_2.meta.photometry.conversion_megajanskys / img_2.data.unit + img_2.data *= img_2.meta.photometry.conversion_megajanskys / img_2.data library = ModelLibrary([img_1, img_2]) library._save(tmp_path) diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py index 6e529d052..703bd6dbe 100644 --- a/romancal/outlier_detection/utils.py +++ b/romancal/outlier_detection/utils.py @@ -86,7 +86,7 @@ def _median_with_resampling( weight_threshold = compute_weight_threshold(drizzled_model.weight, maskpt) drizzled_model.data[drizzled_model.weight < weight_threshold] = np.nan - computer.append(drizzled_model.data.value, i) + computer.append(drizzled_model.data, i) del drizzled_model # Perform median combination on set of drizzled mosaics @@ -173,7 +173,7 @@ def _median_without_resampling( weight_threshold = compute_weight_threshold(wht, maskpt) - data_copy = model.data.value.copy() + data_copy = model.data.copy() data_copy[wht < weight_threshold] = np.nan computer.append(data_copy, i) @@ -211,13 +211,13 @@ def _flag_resampled_model_crs( and image.meta.background.subtracted is False and image.meta.background.level is not None ): - backg = image.meta.background.level.value + backg = image.meta.background.level log.debug( f"Adding background level {image.meta.background.level} to blotted image" ) cr_mask = flag_resampled_crs( - image.data.value, image.err.value, blot, snr1, snr2, scale1, scale2, backg + image.data, image.err, blot, snr1, snr2, scale1, scale2, backg ) # update the dq flags in-place @@ -226,7 +226,7 @@ def _flag_resampled_model_crs( def _flag_model_crs(image, median_data, snr): - cr_mask = flag_crs(image.data.value, image.err.value, median_data, snr) + cr_mask = flag_crs(image.data, image.err, median_data, snr) # Update dq array in-place image.dq |= cr_mask * np.uint32(pixel.DO_NOT_USE | pixel.OUTLIER) diff --git a/romancal/photom/photom.py b/romancal/photom/photom.py index 75c6243e6..86ce7618d 100644 --- a/romancal/photom/photom.py +++ b/romancal/photom/photom.py @@ -28,20 +28,22 @@ def photom_io(input_model, photom_metadata): # Store the conversion factor in the meta data log.info(f"photmjsr value: {conversion:.6g}") - input_model.meta.photometry.conversion_megajanskys = conversion + input_model.meta.photometry.conversion_megajanskys = conversion.value input_model.meta.photometry.conversion_microjanskys = conversion.to( u.microjansky / u.arcsecond**2 - ) + ).value # Get the scalar conversion uncertainty factor uncertainty_conv = photom_metadata["uncertainty"] # Store the uncertainty conversion factor in the meta data log.info(f"uncertainty value: {uncertainty_conv:.6g}") - input_model.meta.photometry.conversion_megajanskys_uncertainty = uncertainty_conv + input_model.meta.photometry.conversion_megajanskys_uncertainty = ( + uncertainty_conv.value + ) input_model.meta.photometry.conversion_microjanskys_uncertainty = ( uncertainty_conv.to(u.microjansky / u.arcsecond**2) - ) + ).value # Return updated input model return input_model @@ -62,8 +64,8 @@ def save_area_info(input_model, photom_parameters): """ # Load the average pixel area values from the photom reference file header - area_ster = photom_parameters["pixelareasr"] - area_a2 = photom_parameters["pixelareasr"].to(u.arcsecond**2) + area_ster = photom_parameters["pixelareasr"].value + area_a2 = photom_parameters["pixelareasr"].to(u.arcsecond**2).value # Copy the pixel area values to the input model log.debug(f"pixelarea_steradians = {area_ster}, pixelarea_arcsecsq = {area_a2}") diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py index 044a3737a..d5b222a3e 100644 --- a/romancal/photom/tests/test_photom.py +++ b/romancal/photom/tests/test_photom.py @@ -102,13 +102,9 @@ def test_no_photom_match(): input_model.meta.instrument.optical_element = "F146" # Set bad values which would be overwritten by apply_photom - input_model.meta.photometry.pixelarea_steradians = -1.0 * u.sr - input_model.meta.photometry.conversion_megajanskys = ( - -1.0 * u.megajansky / u.steradian - ) - input_model.meta.photometry.conversion_microjanskys_uncertainty = ( - -1.0 * u.microjansky / u.arcsecond**2 - ) + input_model.meta.photometry.pixelarea_steradians = -1.0 + input_model.meta.photometry.conversion_megajanskys = -1.0 + input_model.meta.photometry.conversion_microjanskys_uncertainty = -1.0 with warnings.catch_warnings(record=True) as caught: # Look for now non existent F146 optical element @@ -121,15 +117,9 @@ def test_no_photom_match(): ) # Assert that photom elements are not updated - assert output_model.meta.photometry.pixelarea_steradians == -1.0 * u.sr - assert ( - output_model.meta.photometry.conversion_megajanskys - == -1.0 * u.megajansky / u.steradian - ) - assert ( - output_model.meta.photometry.conversion_microjanskys_uncertainty - == -1.0 * u.microjansky / u.arcsecond**2 - ) + assert output_model.meta.photometry.pixelarea_steradians == -1.0 + assert output_model.meta.photometry.conversion_megajanskys == -1.0 + assert output_model.meta.photometry.conversion_microjanskys_uncertainty == -1.0 def test_apply_photom1(): @@ -153,17 +143,16 @@ def test_apply_photom1(): # Tests for pixel areas assert np.isclose( - output_model.meta.photometry.pixelarea_steradians.value, + output_model.meta.photometry.pixelarea_steradians, area_ster.value, atol=1.0e-7, ) - assert output_model.meta.photometry.pixelarea_steradians.unit == area_ster.unit + # assert output_model.meta.photometry.pixelarea_steradians.unit == area_ster.unit assert np.isclose( - output_model.meta.photometry.pixelarea_arcsecsq.value, + output_model.meta.photometry.pixelarea_arcsecsq, area_a2.value, atol=1.0e-7, ) - assert output_model.meta.photometry.pixelarea_arcsecsq.unit == area_a2.unit # Set reference photometry phot_ster = 3.5 * u.megajansky / u.steradian @@ -171,17 +160,17 @@ def test_apply_photom1(): # Tests for photometry assert np.isclose( - output_model.meta.photometry.conversion_megajanskys.value, + output_model.meta.photometry.conversion_megajanskys, phot_ster.value, atol=1.0e-7, ) - assert output_model.meta.photometry.conversion_megajanskys.unit == phot_ster.unit + # assert output_model.meta.photometry.conversion_megajanskys.unit == phot_ster.unit assert np.isclose( - output_model.meta.photometry.conversion_microjanskys.value, + output_model.meta.photometry.conversion_microjanskys, phot_a2.value, atol=1.0e-7, ) - assert output_model.meta.photometry.conversion_microjanskys.unit == phot_a2.unit + # assert output_model.meta.photometry.conversion_microjanskys.unit == phot_a2.unit # Set reference photometric uncertainty muphot_ster = 0.175 * u.megajansky / u.steradian @@ -189,23 +178,23 @@ def test_apply_photom1(): # Tests for photometric uncertainty assert np.isclose( - output_model.meta.photometry.conversion_megajanskys_uncertainty.value, + output_model.meta.photometry.conversion_megajanskys_uncertainty, muphot_ster.value, atol=1.0e-7, ) - assert ( - output_model.meta.photometry.conversion_megajanskys_uncertainty.unit - == muphot_ster.unit - ) + # assert ( + # output_model.meta.photometry.conversion_megajanskys_uncertainty.unit + # == muphot_ster.unit + # ) assert np.isclose( - output_model.meta.photometry.conversion_microjanskys_uncertainty.value, + output_model.meta.photometry.conversion_microjanskys_uncertainty, muphot_a2.value, atol=1.0e-7, ) - assert ( - output_model.meta.photometry.conversion_microjanskys_uncertainty.unit - == muphot_a2.unit - ) + # assert ( + # output_model.meta.photometry.conversion_microjanskys_uncertainty.unit + # == muphot_a2.unit + # ) def test_apply_photom2(): @@ -287,22 +276,24 @@ def test_photom_step_interface_spectroscopic(instrument, exptype): wfi_image.meta.instrument.optical_element = "PRISM" # Set photometric values for spectroscopic data - wfi_image.meta.photometry.pixelarea_steradians = 2.31307642258977e-14 * u.steradian + wfi_image.meta.photometry.pixelarea_steradians = ( + 2.31307642258977e-14 * u.steradian + ).value wfi_image.meta.photometry.pixelarea_arcsecsq = ( 0.000984102303070964 * u.arcsecond * u.arcsecond - ) + ).value wfi_image.meta.photometry.conversion_megajanskys = ( -99999 * u.megajansky / u.steradian - ) + ).value wfi_image.meta.photometry.conversion_megajanskys_uncertainty = ( -99999 * u.megajansky / u.steradian - ) + ).value wfi_image.meta.photometry.conversion_microjanskys = ( -99999 * u.microjansky / u.arcsecond**2 - ) + ).value wfi_image.meta.photometry.conversion_microjanskys_uncertainty = ( -99999 * u.microjansky / u.arcsecond**2 - ) + ).value # Create input model wfi_image_model = ImageModel(wfi_image) diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py index 0447cfbcc..790bc5098 100644 --- a/romancal/ramp_fitting/ramp_fit_step.py +++ b/romancal/ramp_fitting/ramp_fit_step.py @@ -101,7 +101,7 @@ def ols_cas22(self, input_model, readnoise_model, gain_model, dark_model): if self.threshold_constant is not None: kwargs["threshold_constant"] = self.threshold_constant - resultants = input_model.data.value + resultants = input_model.data dq = input_model.groupdq read_noise = readnoise_model.data.value gain = gain_model.data.value @@ -183,10 +183,6 @@ def create_image_model(input_model, image_info): """ data, dq, var_poisson, var_rnoise, err = image_info - data = u.Quantity(data, u.DN / u.s, dtype=data.dtype) - var_poisson = u.Quantity(var_poisson, u.DN**2 / u.s**2, dtype=var_poisson.dtype) - var_rnoise = u.Quantity(var_rnoise, u.DN**2 / u.s**2, dtype=var_rnoise.dtype) - err = u.Quantity(err, u.DN / u.s, dtype=err.dtype) if dq is None: dq = np.zeros(data.shape, dtype="u4") @@ -198,13 +194,11 @@ def create_image_model(input_model, image_info): meta["photometry"] = maker_utils.mk_photometry() inst = { "meta": meta, - "data": u.Quantity(data, u.DN / u.s, dtype=data.dtype), + "data": data, "dq": dq, - "var_poisson": u.Quantity( - var_poisson, u.DN**2 / u.s**2, dtype=var_poisson.dtype - ), - "var_rnoise": u.Quantity(var_rnoise, u.DN**2 / u.s**2, dtype=var_rnoise.dtype), - "err": u.Quantity(err, u.DN / u.s, dtype=err.dtype), + "var_poisson": var_poisson, + "var_rnoise": var_rnoise, + "err": err, "amp33": input_model.amp33.copy(), "border_ref_pix_left": input_model.border_ref_pix_left.copy(), "border_ref_pix_right": input_model.border_ref_pix_right.copy(), diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py index 042ca3d85..419cd8ec6 100644 --- a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py +++ b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py @@ -105,7 +105,7 @@ def test_fits(fit_ramps, attribute): """Check slopes""" image_model, expected = fit_ramps - value = getattr(image_model, attribute).value + value = getattr(image_model, attribute) np.testing.assert_allclose(value, expected[attribute], 1e-05) @@ -226,10 +226,10 @@ def model_from_resultants(resultants, read_pattern=None): gdq = np.zeros(shape=shape, dtype=np.uint8) dm_ramp = maker_utils.mk_ramp(shape=shape) - dm_ramp.data = u.Quantity(full_wfi, u.DN, dtype=np.float32) + dm_ramp.data = full_wfi dm_ramp.pixeldq = pixdq dm_ramp.groupdq = gdq - dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32) + dm_ramp.err = err dm_ramp.meta.exposure.frame_time = ROMAN_READ_TIME dm_ramp.meta.exposure.ngroups = shape[0] diff --git a/romancal/refpix/data.py b/romancal/refpix/data.py index 383305e48..0a4ee55a8 100644 --- a/romancal/refpix/data.py +++ b/romancal/refpix/data.py @@ -11,7 +11,6 @@ from enum import IntEnum import numpy as np -from astropy import units as u from scipy import fft @@ -82,24 +81,15 @@ class StandardView(BaseView): - right = detector[-Const.REF:] """ - @staticmethod - def extract_value(data): - if isinstance(data, u.Quantity): - if data.unit != u.DN: - raise ValueError(f"Input data must be in units of DN, not {data.unit}") - data = data.value - - return data - @classmethod def from_datamodel(cls, datamodel: RampModel) -> StandardView: """ Read the datamodel into the standard view. """ - detector = cls.extract_value(datamodel.data) + detector = datamodel.data # Extract amp33 - amp33 = cls.extract_value(datamodel.amp33) + amp33 = datamodel.amp33 # amp33 is normally a uint16, but this computation requires it to match # the data type of the detector pixels. amp33 = amp33.astype(detector.dtype) @@ -112,23 +102,11 @@ def update(self, datamodel: RampModel) -> RampModel: - Returns the updated datamodel for a functional approach. """ - datamodel.data = u.Quantity( - self.detector, unit=datamodel.data.unit, dtype=datamodel.data.dtype - ) + datamodel.data = self.detector # ABS to avoid casting negative numbers to uint16 - datamodel.amp33 = u.Quantity( - np.abs(self.amp33), unit=datamodel.amp33.unit, dtype=datamodel.amp33.dtype - ) - datamodel.border_ref_pix_left = u.Quantity( - self.left, - unit=datamodel.border_ref_pix_left.unit, - dtype=datamodel.border_ref_pix_left.dtype, - ) - datamodel.border_ref_pix_right = u.Quantity( - self.right, - unit=datamodel.border_ref_pix_right.unit, - dtype=datamodel.border_ref_pix_right.dtype, - ) + datamodel.amp33 = np.abs(self.amp33).astype(datamodel.amp33.dtype) + datamodel.border_ref_pix_left = self.left + datamodel.border_ref_pix_right = self.right return datamodel diff --git a/romancal/refpix/tests/conftest.py b/romancal/refpix/tests/conftest.py index 3afdd2aab..f70795b9c 100644 --- a/romancal/refpix/tests/conftest.py +++ b/romancal/refpix/tests/conftest.py @@ -2,7 +2,6 @@ import numpy as np import pytest -from astropy import units as u from roman_datamodels.datamodels import RampModel, RefpixRefModel from roman_datamodels.maker_utils import mk_ramp, mk_refpix @@ -48,12 +47,8 @@ def datamodel(data): assert detector.shape == (Dims.N_FRAMES, Dims.N_ROWS, Const.N_COLUMNS) assert amp33.shape == (Dims.N_FRAMES, Dims.N_ROWS, Const.CHAN_WIDTH) - datamodel.data = u.Quantity( - detector, unit=datamodel.data.unit, dtype=datamodel.data.dtype - ) - datamodel.amp33 = u.Quantity( - amp33, unit=datamodel.amp33.unit, dtype=datamodel.amp33.dtype - ) + datamodel.data = detector + datamodel.amp33 = amp33.astype(datamodel.amp33.dtype) datamodel.border_ref_pix_left = datamodel.data[:, :, : Const.REF] datamodel.border_ref_pix_right = datamodel.data[:, :, -Const.REF :] diff --git a/romancal/refpix/tests/test_data.py b/romancal/refpix/tests/test_data.py index 92555a08d..75fa2b854 100644 --- a/romancal/refpix/tests/test_data.py +++ b/romancal/refpix/tests/test_data.py @@ -57,13 +57,13 @@ def test_construct_from_datamodel(self, datamodel): assert standard.data.base is None # Check the relationship between the standard view and the datamodel - assert (standard.detector == datamodel.data.value).all() - assert (standard.left == datamodel.border_ref_pix_left.value).all() - assert (standard.right == datamodel.border_ref_pix_right.value).all() + assert (standard.detector == datamodel.data).all() + assert (standard.left == datamodel.border_ref_pix_left).all() + assert (standard.right == datamodel.border_ref_pix_right).all() # The amp33's dtype changes because it needs to be promoted to match that # of the rest of the data - assert (standard.amp33 == datamodel.amp33.value.astype(np.float32)).all() + assert (standard.amp33 == datamodel.amp33.astype(np.float32)).all() def test_update(self, datamodel): standard = StandardView.from_datamodel(datamodel) @@ -97,20 +97,14 @@ def test_update(self, datamodel): assert new.border_ref_pix_left.dtype == old_left.dtype assert new.border_ref_pix_right.dtype == old_right.dtype - # Check the unit has been preserved - assert new.data.unit == old_detector.unit - assert new.amp33.unit == old_amp33.unit - assert new.border_ref_pix_left.unit == old_left.unit - assert new.border_ref_pix_right.unit == old_right.unit - # Check the data has been updated correctly - assert (new.data.value == standard.detector).all() - assert (new.border_ref_pix_left.value == standard.left).all() - assert (new.border_ref_pix_right.value == standard.right).all() + assert (new.data == standard.detector).all() + assert (new.border_ref_pix_left == standard.left).all() + assert (new.border_ref_pix_right == standard.right).all() # The amp33's dtype changes because it needs to be shifted to match the # original data's dtype - assert (new.amp33.value == standard.amp33.astype(old_amp33.dtype)).all() + assert (new.amp33 == standard.amp33.astype(old_amp33.dtype)).all() def test_create_standard_view(self, data): """ diff --git a/romancal/refpix/tests/test_refpix.py b/romancal/refpix/tests/test_refpix.py index e51181359..5fd6e04f4 100644 --- a/romancal/refpix/tests/test_refpix.py +++ b/romancal/refpix/tests/test_refpix.py @@ -1,5 +1,3 @@ -from astropy import units as u - from romancal.refpix.data import StandardView from romancal.refpix.refpix import run_steps @@ -14,9 +12,5 @@ def test_run_steps_regression(datamodel, ref_pix_ref): result = run_steps(datamodel, ref_pix_ref, True, True, True, True) - assert (result.data.value == regression_out).all() + assert (result.data == regression_out).all() # regression_out does not return amp33 data - - # Check the units - assert result.data.unit == u.DN - assert result.amp33.unit == u.DN diff --git a/romancal/refpix/tests/test_step.py b/romancal/refpix/tests/test_step.py index 2bd95e49b..7aaf28867 100644 --- a/romancal/refpix/tests/test_step.py +++ b/romancal/refpix/tests/test_step.py @@ -43,21 +43,13 @@ def test_refpix_step( assert result is not regression # Check the data - assert (result.data.value == regression.data.value).all() - assert result.data.unit == regression.data.unit + assert (result.data == regression.data).all() # Check the amp33 - assert (result.amp33.value == regression.amp33.value).all() - assert result.amp33.unit == regression.amp33.unit + assert (result.amp33 == regression.amp33).all() # Check left ref pix - assert ( - result.border_ref_pix_left.value == regression.border_ref_pix_left.value - ).all() - assert result.border_ref_pix_left.unit == regression.border_ref_pix_left.unit + assert (result.border_ref_pix_left == regression.border_ref_pix_left).all() # Check right ref pix - assert ( - result.border_ref_pix_right.value == regression.border_ref_pix_right.value - ).all() - assert result.border_ref_pix_right.unit == regression.border_ref_pix_right.unit + assert (result.border_ref_pix_right == regression.border_ref_pix_right).all() # # Run the step with reffile = N/A result = RefPixStep.call( diff --git a/romancal/regtest/test_catalog.py b/romancal/regtest/test_catalog.py index 6ba270577..3c461af1c 100644 --- a/romancal/regtest/test_catalog.py +++ b/romancal/regtest/test_catalog.py @@ -13,7 +13,7 @@ scope="module", params=[ "r0099101001001001001_F158_visit_i2d.asdf", - "r0000101001001001001_01101_0001_WFI01_cal.asdf", + "r0000101001001001001_0001_WFI01_cal.asdf", ], ids=["L3", "L2"], ) diff --git a/romancal/regtest/test_dark_current.py b/romancal/regtest/test_dark_current.py index 3e31501a4..86200bc71 100644 --- a/romancal/regtest/test_dark_current.py +++ b/romancal/regtest/test_dark_current.py @@ -12,13 +12,13 @@ def test_dark_current_subtraction_step(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Note: This should include tests for overrides etc.""" - input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile args = ["romancal.step.DarkCurrentStep", rtdata.input] RomanStep.from_cmdline(args) - output = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf" + output = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) @@ -29,7 +29,7 @@ def test_dark_current_subtraction_step(rtdata, ignore_asdf_paths): def test_dark_current_outfile_step(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Here the test is for renaming the output file.""" - input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile @@ -50,7 +50,7 @@ def test_dark_current_outfile_step(rtdata, ignore_asdf_paths): def test_dark_current_outfile_suffix(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Here the test is for renaming the output file.""" - input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile @@ -73,10 +73,10 @@ def test_dark_current_output(rtdata, ignore_asdf_paths): """Function to run and compare Dark Current subtraction files. Here the test for overriding the CRDS dark reference file.""" - input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf" + input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile - dark_output_name = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf" + dark_output_name = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" args = [ "romancal.step.DarkCurrentStep", @@ -84,7 +84,7 @@ def test_dark_current_output(rtdata, ignore_asdf_paths): f"--dark_output={dark_output_name}", ] RomanStep.from_cmdline(args) - output = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf" + output = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) diff --git a/romancal/regtest/test_jump_det.py b/romancal/regtest/test_jump_det.py index 7d892de1f..4c014d161 100644 --- a/romancal/regtest/test_jump_det.py +++ b/romancal/regtest/test_jump_det.py @@ -13,7 +13,7 @@ def test_jump_detection_step(rtdata, ignore_asdf_paths): """Function to run and compare Jump Detection files. Note: This should include tests for overrides etc.""" - input_file = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf" + input_file = "r0000101001001001001_0001_WFI01_darkcurrent.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -29,7 +29,7 @@ def test_jump_detection_step(rtdata, ignore_asdf_paths): "--run_ramp_jump_detection=False", ] RomanStep.from_cmdline(args) - output = "r0000101001001001001_01101_0001_WFI01_jump.asdf" + output = "r0000101001001001001_0001_WFI01_jump.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) diff --git a/romancal/regtest/test_linearity.py b/romancal/regtest/test_linearity.py index aad0a88a2..b6c9b5cdc 100644 --- a/romancal/regtest/test_linearity.py +++ b/romancal/regtest/test_linearity.py @@ -10,13 +10,13 @@ @pytest.mark.bigdata def test_linearity_step(rtdata, ignore_asdf_paths): """Function to run and compare linearity correction files.""" - input_file = "r0000101001001001001_01101_0001_WFI01_refpix.asdf" + input_file = "r0000101001001001001_0001_WFI01_refpix.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file args = ["romancal.step.LinearityStep", rtdata.input] RomanStep.from_cmdline(args) - output = "r0000101001001001001_01101_0001_WFI01_linearity.asdf" + output = "r0000101001001001001_0001_WFI01_linearity.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) @@ -27,7 +27,7 @@ def test_linearity_step(rtdata, ignore_asdf_paths): def test_linearity_outfile_step(rtdata, ignore_asdf_paths): """Function to run and linearity correction files. Here the test is for renaming the output file.""" - input_file = "r0000101001001001001_01101_0001_WFI01_refpix.asdf" + input_file = "r0000101001001001001_0001_WFI01_refpix.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file diff --git a/romancal/regtest/test_mos_pipeline.py b/romancal/regtest/test_mos_pipeline.py index 5fd49c339..1e22438ac 100644 --- a/romancal/regtest/test_mos_pipeline.py +++ b/romancal/regtest/test_mos_pipeline.py @@ -16,16 +16,19 @@ @pytest.fixture(scope="module") def run_mos(rtdata_module): rtdata = rtdata_module + rtdata.get_asn("WFI/image/L3_regtest_asn.json") # Test Pipeline output = "r0099101001001001001_F158_visit_i2d.asdf" rtdata.output = output + args = [ "roman_mos", rtdata.input, ] MosaicPipeline.from_cmdline(args) + rtdata.get_truth(f"truth/WFI/image/{output}") return rtdata diff --git a/romancal/regtest/test_ramp_fitting.py b/romancal/regtest/test_ramp_fitting.py index e1e79e767..6b76f8d20 100644 --- a/romancal/regtest/test_ramp_fitting.py +++ b/romancal/regtest/test_ramp_fitting.py @@ -100,22 +100,22 @@ def cond_science_verification( params=[ ( "DMS362", - Path("WFI/image/r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf"), + Path("WFI/image/r0000101001001001001_0001_WFI01_darkcurrent.asdf"), CONDITIONS_FULL, ), ( "DMS366", - Path("WFI/grism/r0000201001001001001_01101_0001_WFI01_darkcurrent.asdf"), + Path("WFI/grism/r0000201001001001001_0001_WFI01_darkcurrent.asdf"), CONDITIONS_FULL, ), ( "DMS363", - Path("WFI/image/r0000101001001001001_01101_0003_WFI01_darkcurrent.asdf"), + Path("WFI/image/r0000101001001001001_0003_WFI01_darkcurrent.asdf"), CONDITIONS_TRUNC, ), ( "DMS367", - Path("WFI/grism/r0000201001001001001_01101_0003_WFI01_darkcurrent.asdf"), + Path("WFI/grism/r0000201001001001001_0003_WFI01_darkcurrent.asdf"), CONDITIONS_TRUNC, ), ], diff --git a/romancal/regtest/test_refpix.py b/romancal/regtest/test_refpix.py index ec87d0cbc..b8c7c44d3 100644 --- a/romancal/regtest/test_refpix.py +++ b/romancal/regtest/test_refpix.py @@ -10,7 +10,7 @@ @pytest.mark.bigdata def test_refpix_step(rtdata, ignore_asdf_paths): # I have no idea what this is supposed to be - input_datafile = "r0000101001001001001_01101_0001_WFI01_saturation.asdf" + input_datafile = "r0000101001001001001_0001_WFI01_saturation.asdf" rtdata.get_data(f"WFI/image/{input_datafile}") rtdata.input = input_datafile @@ -18,7 +18,7 @@ def test_refpix_step(rtdata, ignore_asdf_paths): RomanStep.from_cmdline(args) # Again I have no idea here - output = "r0000101001001001001_01101_0001_WFI01_refpix.asdf" + output = "r0000101001001001001_0001_WFI01_refpix.asdf" rtdata.output = output rtdata.get_truth(f"truth/WFI/image/{output}") diff --git a/romancal/regtest/test_regtestdata.py b/romancal/regtest/test_regtestdata.py index 4b71c3f2f..d3d768be7 100644 --- a/romancal/regtest/test_regtestdata.py +++ b/romancal/regtest/test_regtestdata.py @@ -33,9 +33,9 @@ def test_compare_asdf(tmp_path, modification): l2.save(file0) atol = 0.0001 if modification == "small": - l2.data += (atol / 2) * l2.data.unit + l2.data += atol / 2 elif modification == "large": - l2.data += (atol * 2) * l2.data.unit + l2.data += atol * 2 l2.save(file1) diff = compare_asdf(file0, file1, atol=atol) if modification == "large": diff --git a/romancal/regtest/test_resample.py b/romancal/regtest/test_resample.py index bdbf48170..d669eecc6 100644 --- a/romancal/regtest/test_resample.py +++ b/romancal/regtest/test_resample.py @@ -12,7 +12,7 @@ def test_resample_single_file(rtdata, ignore_asdf_paths): output_data = "mosaic_resamplestep.asdf" - rtdata.get_asn("WFI/image/mosaic_asn.json") + rtdata.get_asn("WFI/image/L3_mosaic_asn.json") rtdata.get_truth(f"truth/WFI/image/{output_data}") rtdata.output = output_data diff --git a/romancal/regtest/test_skycell_generation.py b/romancal/regtest/test_skycell_generation.py index 60e943ba0..cc08de547 100644 --- a/romancal/regtest/test_skycell_generation.py +++ b/romancal/regtest/test_skycell_generation.py @@ -13,13 +13,13 @@ def test_skycell_asn_generation(rtdata): # This test should generate seven json files args = [ - "r0000101001001001001_01101_0002_WFI01_cal.asdf", - "r0000101001001001001_01101_0002_WFI10_cal.asdf", + "r0000101001001001001_0002_WFI01_cal.asdf", + "r0000101001001001001_0002_WFI10_cal.asdf", "-o", "r512", ] - rtdata.get_data("WFI/image/r0000101001001001001_01101_0002_WFI01_cal.asdf") - rtdata.get_data("WFI/image/r0000101001001001001_01101_0002_WFI10_cal.asdf") + rtdata.get_data("WFI/image/r0000101001001001001_0002_WFI01_cal.asdf") + rtdata.get_data("WFI/image/r0000101001001001001_0002_WFI10_cal.asdf") skycell_asn.Main(args) diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py index 7adcc2ce3..9402c16d7 100644 --- a/romancal/regtest/test_tweakreg.py +++ b/romancal/regtest/test_tweakreg.py @@ -16,12 +16,14 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path): # ``shifted'' version is created in make_regtestdata.sh; cal file is taken, # the wcsinfo is perturbed, and AssignWCS is run to update the WCS with the # perturbed information - orig_uncal = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" - input_data = "r0000101001001001001_01101_0001_WFI01_shift_cal.asdf" - output_data = "r0000101001001001001_01101_0001_WFI01_shift_tweakregstep.asdf" - truth_data = "r0000101001001001001_01101_0001_WFI01_shift_tweakregstep.asdf" + orig_uncal = "r0000101001001001001_0001_WFI01_uncal.asdf" + orig_catfile = "r0000101001001001001_0001_WFI01_cat.asdf" + input_data = "r0000101001001001001_0001_WFI01_shift_cal.asdf" + output_data = "r0000101001001001001_0001_WFI01_shift_tweakregstep.asdf" + truth_data = "r0000101001001001001_0001_WFI01_shift_tweakregstep.asdf" rtdata.get_data(f"WFI/image/{input_data}") + rtdata.get_data(f"WFI/image/{orig_catfile}") rtdata.get_data(f"WFI/image/{orig_uncal}") rtdata.get_truth(f"truth/WFI/image/{truth_data}") diff --git a/romancal/regtest/test_wfi_dq_init.py b/romancal/regtest/test_wfi_dq_init.py index b05eabce9..5f9ba08dd 100644 --- a/romancal/regtest/test_wfi_dq_init.py +++ b/romancal/regtest/test_wfi_dq_init.py @@ -16,7 +16,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths): """DMS25 Test: Testing retrieval of best ref file for image data, and creation of a ramp file with CRDS selected mask file applied.""" - input_file = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" + input_file = "r0000101001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -40,7 +40,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths): assert "roman_wfi_mask" in ref_file_name # Test DQInitStep - output = "r0000101001001001001_01101_0001_WFI01_dqinit.asdf" + output = "r0000101001001001001_0001_WFI01_dqinit.asdf" rtdata.output = output args = ["romancal.step.DQInitStep", rtdata.input] step.log.info( @@ -71,7 +71,7 @@ def test_dq_init_grism_step(rtdata, ignore_asdf_paths): """DMS25 Test: Testing retrieval of best ref file for grism data, and creation of a ramp file with CRDS selected mask file applied.""" - input_file = "r0000201001001001001_01101_0001_WFI01_uncal.asdf" + input_file = "r0000201001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_file}") rtdata.input = input_file @@ -95,7 +95,7 @@ def test_dq_init_grism_step(rtdata, ignore_asdf_paths): assert "roman_wfi_mask" in ref_file_name # Test DQInitStep - output = "r0000201001001001001_01101_0001_WFI01_dqinit.asdf" + output = "r0000201001001001001_0001_WFI01_dqinit.asdf" rtdata.output = output args = ["romancal.step.DQInitStep", rtdata.input] step.log.info( diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py index 323e2d17e..f9b01a1a1 100644 --- a/romancal/regtest/test_wfi_flat_field.py +++ b/romancal/regtest/test_wfi_flat_field.py @@ -14,7 +14,7 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths): """Test for the flat field step using imaging data.""" - input_data = "r0000101001001001001_01101_0001_WFI01_assignwcs.asdf" + input_data = "r0000101001001001001_0001_WFI01_assignwcs.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data @@ -25,7 +25,7 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths): ref_file_name = os.path.split(ref_file_path)[-1] assert "roman_wfi_flat" in ref_file_name # Test FlatFieldStep - output = "r0000101001001001001_01101_0001_WFI01_flat.asdf" + output = "r0000101001001001001_0001_WFI01_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] RomanStep.from_cmdline(args) @@ -40,7 +40,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths): the grism and prism data should be None, only testing the grism case here.""" - input_file = "r0000201001001001001_01101_0001_WFI01_uncal.asdf" + input_file = "r0000201001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_file}") rtdata.input = input_file @@ -59,7 +59,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths): assert ref_file_name == "N/A" # Test FlatFieldStep - output = "r0000201001001001001_01101_0001_WFI01_flat.asdf" + output = "r0000201001001001001_0001_WFI01_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] RomanStep.from_cmdline(args) @@ -75,7 +75,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): flat files and successfully make level 2 output""" # First file - input_l2_file = "r0000101001001001001_01101_0001_WFI01_assignwcs.asdf" + input_l2_file = "r0000101001001001001_0001_WFI01_assignwcs.asdf" rtdata.get_data(f"WFI/image/{input_l2_file}") rtdata.input = input_l2_file @@ -102,7 +102,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): ) # Test FlatFieldStep - output = "r0000101001001001001_01101_0001_WFI01_flat.asdf" + output = "r0000101001001001001_0001_WFI01_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] step.log.info( @@ -126,7 +126,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): # to separate flat files in CRDS. # Second file - input_file = "r0000101001001001001_01101_0001_WFI01_changetime_assignwcs.asdf" + input_file = "r0000101001001001001_0001_WFI01_changetime_assignwcs.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -149,7 +149,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths): ) # Test FlatFieldStep - output = "r0000101001001001001_01101_0001_WFI01_changetime_flat.asdf" + output = "r0000101001001001001_0001_WFI01_changetime_flat.asdf" rtdata.output = output args = ["romancal.step.FlatFieldStep", rtdata.input] step.log.info( diff --git a/romancal/regtest/test_wfi_grism_16resultants.py b/romancal/regtest/test_wfi_grism_16resultants.py index b76c38e38..4e222ed09 100644 --- a/romancal/regtest/test_wfi_grism_16resultants.py +++ b/romancal/regtest/test_wfi_grism_16resultants.py @@ -16,12 +16,12 @@ def run_elp(rtdata_module): # The input data is from INS for stress testing at some point this should be generated # by INS every time new data is needed. - input_data = "r0000201001001001001_01101_0004_WFI01_uncal.asdf" + input_data = "r0000201001001001001_0004_WFI01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000201001001001001_01101_0004_WFI01_cal.asdf" + output = "r0000201001001001001_0004_WFI01_cal.asdf" rtdata.output = output args = [ "roman_elp", diff --git a/romancal/regtest/test_wfi_grism_pipeline.py b/romancal/regtest/test_wfi_grism_pipeline.py index 259cd5a49..2ad63c58c 100644 --- a/romancal/regtest/test_wfi_grism_pipeline.py +++ b/romancal/regtest/test_wfi_grism_pipeline.py @@ -19,12 +19,12 @@ def run_elp(rtdata_module): rtdata = rtdata_module - input_data = "r0000201001001001001_01101_0001_WFI01_uncal.asdf" + input_data = "r0000201001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/grism/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000201001001001001_01101_0001_WFI01_cal.asdf" + output = "r0000201001001001001_0001_WFI01_cal.asdf" rtdata.output = output args = [ "roman_elp", diff --git a/romancal/regtest/test_wfi_image_16resultants.py b/romancal/regtest/test_wfi_image_16resultants.py index 4e7add78e..8b2a997a5 100644 --- a/romancal/regtest/test_wfi_image_16resultants.py +++ b/romancal/regtest/test_wfi_image_16resultants.py @@ -16,12 +16,12 @@ def run_elp(rtdata_module): # The input data is from INS for stress testing at some point this should be generated # every time new data is needed. - input_data = "r0000101001001001001_01101_0004_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0004_WFI01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000101001001001001_01101_0004_WFI01_cal.asdf" + output = "r0000101001001001001_0004_WFI01_cal.asdf" rtdata.output = output args = [ "roman_elp", diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 2dfd56d76..3cf8b4b26 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -20,12 +20,12 @@ def run_elp(rtdata_module): rtdata = rtdata_module - input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000101001001001001_01101_0001_WFI01_cal.asdf" + output = "r0000101001001001001_0001_WFI01_cal.asdf" rtdata.output = output args = [ "roman_elp", @@ -206,12 +206,12 @@ def test_repointed_wcs_differs(repointed_filename_and_delta, output_model): def test_elp_input_dm(rtdata, ignore_asdf_paths): """Test for input roman Datamodel to exposure level pipeline""" - input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") dm_input = rdm.open(rtdata.input) # Test Pipeline with input datamodel - output = "r0000101001001001001_01101_0001_WFI01_cal.asdf" + output = "r0000101001001001001_0001_WFI01_cal.asdf" rtdata.output = output ExposurePipeline.call(dm_input, save_results=True) rtdata.get_truth(f"truth/WFI/image/{output}") @@ -235,12 +235,12 @@ def test_processing_pipeline_all_saturated(rtdata, ignore_asdf_paths): Note that this test mimics how the pipeline is run in OPS. Any changes to this test should be coordinated with OPS. """ - input_data = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_uncal.asdf" + input_data = "r0000101001001001001_0001_WFI01_ALL_SATURATED_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data # Test Pipeline - output = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_cal.asdf" + output = "r0000101001001001001_0001_WFI01_ALL_SATURATED_cal.asdf" rtdata.output = output args = [ "roman_elp", @@ -273,10 +273,10 @@ def test_pipeline_suffix(rtdata, ignore_asdf_paths): Any changes to this test should be coordinated with OPS. """ - input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" + input_data = "r0000101001001001001_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") - output = "r0000101001001001001_01101_0001_WFI01_star.asdf" + output = "r0000101001001001001_0001_WFI01_star.asdf" rtdata.output = output args = [ diff --git a/romancal/regtest/test_wfi_photom.py b/romancal/regtest/test_wfi_photom.py index 81f888dc1..7af8734c7 100644 --- a/romancal/regtest/test_wfi_photom.py +++ b/romancal/regtest/test_wfi_photom.py @@ -4,7 +4,6 @@ import pytest import roman_datamodels as rdm -from astropy import units as u from romancal.step import PhotomStep from romancal.stpipe import RomanStep @@ -17,7 +16,7 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): """DMS140 Test: Testing application of photometric correction using CRDS selected photom file.""" - input_data = "r0000101001001001001_01101_0001_WFI01_flat.asdf" + input_data = "r0000101001001001001_0001_WFI01_flat.asdf" rtdata.get_data(f"WFI/image/{input_data}") rtdata.input = input_data @@ -40,7 +39,7 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): # photom match from CRDS. Values come from roman_wfi_photom_0034.asdf # Test PhotomStep - output = "r0000101001001001001_01101_0001_WFI01_photom.asdf" + output = "r0000101001001001001_0001_WFI01_photom.asdf" rtdata.output = output args = ["romancal.step.PhotomStep", rtdata.input] step.log.info( @@ -60,40 +59,29 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): step.log.info( "DMS140 MSG: Photom megajansky conversion calculated? : " + str( - (photom_out.meta.photometry.conversion_megajanskys.unit == u.MJy / u.sr) - and math.isclose( - photom_out.meta.photometry.conversion_megajanskys.value, + math.isclose( + photom_out.meta.photometry.conversion_megajanskys, 0.3324, abs_tol=0.0001, ) ) ) - assert photom_out.meta.photometry.conversion_megajanskys.unit == u.MJy / u.sr assert math.isclose( - photom_out.meta.photometry.conversion_megajanskys.value, 0.3324, abs_tol=0.0001 + photom_out.meta.photometry.conversion_megajanskys, 0.3324, abs_tol=0.0001 ) step.log.info( "DMS140 MSG: Photom microjanskys conversion calculated? : " + str( - ( - photom_out.meta.photometry.conversion_microjanskys.unit - == u.uJy / u.arcsec**2 - ) - and ( - math.isclose( - photom_out.meta.photometry.conversion_microjanskys.value, - 7.81320, - abs_tol=0.0001, - ) + math.isclose( + photom_out.meta.photometry.conversion_microjanskys, + 7.81320, + abs_tol=0.0001, ) ) ) - assert ( - photom_out.meta.photometry.conversion_microjanskys.unit == u.uJy / u.arcsec**2 - ) assert math.isclose( - photom_out.meta.photometry.conversion_microjanskys.value, + photom_out.meta.photometry.conversion_microjanskys, 7.81320, abs_tol=0.0001, ) @@ -101,19 +89,15 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): step.log.info( "DMS140 MSG: Pixel area in steradians calculated? : " + str( - (photom_out.meta.photometry.pixelarea_steradians.unit == u.sr) - and ( - math.isclose( - photom_out.meta.photometry.pixelarea_steradians.value, - 2.8083e-13, - abs_tol=1.0e-17, - ) + math.isclose( + photom_out.meta.photometry.pixelarea_steradians, + 2.8083e-13, + abs_tol=1.0e-17, ) ) ) - assert photom_out.meta.photometry.pixelarea_steradians.unit == u.sr assert math.isclose( - photom_out.meta.photometry.pixelarea_steradians.value, + photom_out.meta.photometry.pixelarea_steradians, 2.8083e-13, abs_tol=1.0e-17, ) @@ -121,43 +105,29 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): step.log.info( "DMS140 MSG: Pixel area in square arcseconds calculated? : " + str( - (photom_out.meta.photometry.pixelarea_arcsecsq.unit == u.arcsec**2) - and ( - math.isclose( - photom_out.meta.photometry.pixelarea_arcsecsq.value, - 0.011948, - abs_tol=1.0e-6, - ) + math.isclose( + photom_out.meta.photometry.pixelarea_arcsecsq, + 0.011948, + abs_tol=1.0e-6, ) ) ) - assert photom_out.meta.photometry.pixelarea_arcsecsq.unit == u.arcsec**2 assert math.isclose( - photom_out.meta.photometry.pixelarea_arcsecsq.value, 0.011948, abs_tol=1.0e-6 + photom_out.meta.photometry.pixelarea_arcsecsq, 0.011948, abs_tol=1.0e-6 ) step.log.info( "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : " + str( - ( - photom_out.meta.photometry.conversion_megajanskys_uncertainty.unit - == u.MJy / u.sr - ) - and ( - math.isclose( - photom_out.meta.photometry.conversion_megajanskys_uncertainty.value, - 0.0, - abs_tol=1.0e-6, - ) + math.isclose( + photom_out.meta.photometry.conversion_megajanskys_uncertainty, + 0.0, + abs_tol=1.0e-6, ) ) ) - assert ( - photom_out.meta.photometry.conversion_megajanskys_uncertainty.unit - == u.MJy / u.sr - ) assert math.isclose( - photom_out.meta.photometry.conversion_megajanskys_uncertainty.value, + photom_out.meta.photometry.conversion_megajanskys_uncertainty, 0.0, abs_tol=1.0e-6, ) @@ -165,25 +135,15 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths): step.log.info( "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : " + str( - ( - photom_out.meta.photometry.conversion_microjanskys_uncertainty.unit - == u.uJy / u.arcsec**2 - ) - and ( - math.isclose( - photom_out.meta.photometry.conversion_microjanskys_uncertainty.value, - 0.0, - abs_tol=1.0e-6, - ) + math.isclose( + photom_out.meta.photometry.conversion_microjanskys_uncertainty, + 0.0, + abs_tol=1.0e-6, ) ) ) - assert ( - photom_out.meta.photometry.conversion_microjanskys_uncertainty.unit - == u.uJy / u.arcsec**2 - ) assert math.isclose( - photom_out.meta.photometry.conversion_microjanskys_uncertainty.value, + photom_out.meta.photometry.conversion_microjanskys_uncertainty, 0.0, abs_tol=1.0e-6, ) diff --git a/romancal/regtest/test_wfi_saturation.py b/romancal/regtest/test_wfi_saturation.py index 0180008c1..1484af520 100644 --- a/romancal/regtest/test_wfi_saturation.py +++ b/romancal/regtest/test_wfi_saturation.py @@ -16,7 +16,7 @@ def test_saturation_image_step(rtdata, ignore_asdf_paths): """Testing retrieval of best ref file for image data, and creation of a ramp file with CRDS selected saturation file applied.""" - input_file = "r0000101001001001001_01101_0001_WFI01_dqinit.asdf" + input_file = "r0000101001001001001_0001_WFI01_dqinit.asdf" rtdata.get_data(f"WFI/image/{input_file}") rtdata.input = input_file @@ -30,7 +30,7 @@ def test_saturation_image_step(rtdata, ignore_asdf_paths): assert "roman_wfi_saturation" in ref_file_name # Test SaturationStep - output = "r0000101001001001001_01101_0001_WFI01_saturation.asdf" + output = "r0000101001001001001_0001_WFI01_saturation.asdf" rtdata.output = output args = ["romancal.step.SaturationStep", rtdata.input] @@ -49,7 +49,7 @@ def test_saturation_grism_step(rtdata, ignore_asdf_paths): """Testing retrieval of best ref file for grism data, and creation of a ramp file with CRDS selected saturation file applied.""" - input_file = "r0000201001001001001_01101_0001_WFI01_dqinit.asdf" + input_file = "r0000201001001001001_0001_WFI01_dqinit.asdf" rtdata.get_data(f"WFI/grism/{input_file}") rtdata.input = input_file @@ -63,7 +63,7 @@ def test_saturation_grism_step(rtdata, ignore_asdf_paths): assert "roman_wfi_saturation" in ref_file_name # Test SaturationStep - output = "r0000201001001001001_01101_0001_WFI01_saturation.asdf" + output = "r0000201001001001001_0001_WFI01_saturation.asdf" rtdata.output = output args = ["romancal.step.SaturationStep", rtdata.input] diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py index 12cf8b75b..96e1018f8 100644 --- a/romancal/resample/gwcs_drizzle.py +++ b/romancal/resample/gwcs_drizzle.py @@ -431,7 +431,7 @@ def dodrizzle( log.info(f"Drizzling {insci.shape} --> {outsci.shape}") _vers, nmiss, nskip = cdrizzle.tdriz( - insci.astype(np.float32).value, + insci.astype(np.float32), inwht.astype(np.float32), pixmap, outsci, diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py index 55bc277be..d2a9b0b2e 100644 --- a/romancal/resample/resample.py +++ b/romancal/resample/resample.py @@ -386,18 +386,15 @@ def resample_many_to_one(self): exptime_tot = self.resample_exposure_time(output_model) # TODO: fix unit here - output_model.err = u.Quantity( - np.sqrt( - np.nansum( - [ - output_model.var_rnoise, - output_model.var_poisson, - output_model.var_flat, - ], - axis=0, - ) - ), - unit=output_model.err.unit, + output_model.err = np.sqrt( + np.nansum( + [ + output_model.var_rnoise, + output_model.var_poisson, + output_model.var_flat, + ], + axis=0, + ) ) self.update_exposure_times(output_model, exptime_tot) @@ -416,7 +413,7 @@ def resample_variance_array(self, name, output_model): This modifies ``output_model`` in-place. """ output_wcs = self.output_wcs - inverse_variance_sum = np.full_like(output_model.data.value, np.nan) + inverse_variance_sum = np.full_like(output_model.data, np.nan) log.info(f"Resampling {name}") with self.input_models: @@ -482,9 +479,7 @@ def resample_variance_array(self, name, output_model): # We now have a sum of the inverse resampled variances. We need the # inverse of that to get back to units of variance. # TODO: fix unit here - output_variance = u.Quantity( - np.reciprocal(inverse_variance_sum), unit=u.MJy**2 / u.sr**2 - ) + output_variance = np.reciprocal(inverse_variance_sum) setattr(output_model, name, output_variance) @@ -537,7 +532,7 @@ def resample_exposure_time(self, output_model): ymax=ymax, ) - exptime_tot += resampled_exptime.value + exptime_tot += resampled_exptime self.input_models.shelve(model, i, modify=False) return exptime_tot @@ -723,11 +718,11 @@ def drizzle_arrays( log.info(f"Drizzling {insci.shape} --> {outsci.shape}") _vers, _nmiss, _nskip = cdrizzle.tdriz( - insci.astype(np.float32).value, + insci.astype(np.float32), inwht, pixmap, - outsci.value, - outwht.value, + outsci, + outwht, outcon, uniqid=uniqid, xmin=xmin, diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py index 0a08460cf..0b399e68b 100644 --- a/romancal/resample/resample_utils.py +++ b/romancal/resample/resample_utils.py @@ -151,7 +151,7 @@ def build_driz_weight( and model.var_rnoise.shape == model.data.shape ): with np.errstate(divide="ignore", invalid="ignore"): - inv_variance = model.var_rnoise.value**-1 + inv_variance = model.var_rnoise**-1 inv_variance[~np.isfinite(inv_variance)] = 1 else: warnings.warn( diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py index b7eda3088..78c9f7fbc 100644 --- a/romancal/resample/tests/test_resample.py +++ b/romancal/resample/tests/test_resample.py @@ -66,7 +66,7 @@ def create_image(self): datamodels.ImageModel An L2 ImageModel datamodel. """ - rng = np.random.default_rng() + rng = np.random.default_rng(seed=13) l2 = maker_utils.mk_level2_image( shape=self.shape, **{ @@ -94,26 +94,10 @@ def create_image(self): "exposure": 1, }, }, - "data": u.Quantity( - rng.poisson(2.5, size=self.shape).astype(np.float32), - u.MJy / u.sr, - dtype=np.float32, - ), - "var_rnoise": u.Quantity( - rng.normal(1, 0.05, size=self.shape).astype(np.float32), - u.MJy**2 / u.sr**2, - dtype=np.float32, - ), - "var_poisson": u.Quantity( - rng.poisson(1, size=self.shape).astype(np.float32), - u.MJy**2 / u.sr**2, - dtype=np.float32, - ), - "var_flat": u.Quantity( - rng.uniform(0, 1, size=self.shape).astype(np.float32), - u.MJy**2 / u.sr**2, - dtype=np.float32, - ), + "data": rng.poisson(2.5, size=self.shape).astype(np.float32), + "var_rnoise": rng.normal(1, 0.05, size=self.shape).astype(np.float32), + "var_poisson": rng.poisson(1, size=self.shape).astype(np.float32), + "var_flat": rng.uniform(0, 1, size=self.shape).astype(np.float32), }, ) # data from WFISim simulation of SCA #01 @@ -634,7 +618,7 @@ def test_resample_variance_array(wfi_sca1, wfi_sca4, name): expected_combined_variance_value = np.nanmean(mean_data) / len(input_models) np.isclose( - np.nanmean(getattr(output_model, name)).value, + np.nanmean(getattr(output_model, name)), expected_combined_variance_value, atol=0.01, ) diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py index 74beb7a8e..a221cc1f0 100644 --- a/romancal/resample/tests/test_resample_step.py +++ b/romancal/resample/tests/test_resample_step.py @@ -4,7 +4,6 @@ from astropy import coordinates as coord from astropy import units as u from astropy.modeling import models -from astropy.units import Quantity from gwcs import WCS from gwcs import coordinate_frames as cf from roman_datamodels import datamodels, maker_utils @@ -318,7 +317,7 @@ def test_build_driz_weight_multiple_good_bits( data_shape = dq_array.shape img1 = base_image() img1.dq = dq_array - img1.data = Quantity(np.ones(data_shape), unit="DN / s") + img1.data = np.ones(data_shape) result = resample_utils.build_driz_weight( img1, weight_type=None, good_bits=good_bits @@ -346,7 +345,7 @@ def test_set_good_bits_in_resample_meta(base_image, good_bits): img = datamodels.ImageModel(model) - img.data *= img.meta.photometry.conversion_megajanskys / img.data.unit + img.data *= img.meta.photometry.conversion_megajanskys / img.data step = ResampleStep @@ -361,14 +360,14 @@ def test_build_driz_weight_different_weight_type(base_image, weight_type): img1 = base_image() # update attributes that will be used in building the weight array img1.meta.exposure.exposure_time = 10 - img1.var_rnoise = Quantity(rng.normal(1, 0.1, size=img1.shape), unit="DN2 / s2") + img1.var_rnoise = rng.normal(1, 0.1, size=img1.shape) # build the drizzle weight array result = resample_utils.build_driz_weight( img1, weight_type=weight_type, good_bits="~DO_NOT_USE+NON_SCIENCE" ) expected_results = { - "ivm": img1.var_rnoise.value**-1, + "ivm": img1.var_rnoise**-1, "exptime": np.ones(img1.shape, dtype=img1.data.dtype) * img1.meta.exposure.exposure_time, None: np.ones(img1.shape, dtype=img1.data.dtype), diff --git a/romancal/saturation/saturation.py b/romancal/saturation/saturation.py index 87314e578..81b6a0378 100644 --- a/romancal/saturation/saturation.py +++ b/romancal/saturation/saturation.py @@ -34,7 +34,7 @@ def flag_saturation(input_model, ref_model): The input model is modified in place and returned as the output model. """ - data = input_model.data[np.newaxis, :].value + data = input_model.data[np.newaxis, :] # Modify input_model in place. gdq = input_model.groupdq[np.newaxis, :] diff --git a/romancal/saturation/tests/test_saturation.py b/romancal/saturation/tests/test_saturation.py index 76c0eee3d..f6bbd3df9 100644 --- a/romancal/saturation/tests/test_saturation.py +++ b/romancal/saturation/tests/test_saturation.py @@ -6,7 +6,6 @@ import numpy as np import pytest -from astropy import units as u from roman_datamodels import maker_utils from roman_datamodels.datamodels import ScienceRawModel from roman_datamodels.dqflags import group, pixel @@ -27,11 +26,11 @@ def test_basic_saturation_flagging(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 0 * ramp.data.unit - ramp.data[1, 5, 5] = 20000 * ramp.data.unit - ramp.data[2, 5, 5] = 40000 * ramp.data.unit - ramp.data[3, 5, 5] = 60000 * ramp.data.unit # Signal reaches saturation limit - ramp.data[4, 5, 5] = 62000 * ramp.data.unit + ramp.data[0, 5, 5] = 0 + ramp.data[1, 5, 5] = 20000 + ramp.data[2, 5, 5] = 40000 + ramp.data[3, 5, 5] = 60000 # Signal reaches saturation limit + ramp.data[4, 5, 5] = 62000 # Set saturation value in the saturation model satmap.data[5, 5] = satvalue * satmap.data.unit @@ -40,7 +39,7 @@ def test_basic_saturation_flagging(setup_wfi_datamodels): output = flag_saturation(ramp, satmap) # Make sure that groups with signal > saturation limit get flagged - satindex = np.argmax(output.data.value[:, 5, 5] == satvalue) + satindex = np.argmax(output.data[:, 5, 5] == satvalue) assert np.all(output.groupdq[satindex:, 5, 5] == group.SATURATED) @@ -56,11 +55,11 @@ def test_read_pattern_saturation_flagging(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 0 * ramp.data.unit - ramp.data[1, 5, 5] = 20000 * ramp.data.unit - ramp.data[2, 5, 5] = 40000 * ramp.data.unit - ramp.data[3, 5, 5] = 60000 * ramp.data.unit # Signal reaches saturation limit - ramp.data[4, 5, 5] = 62000 * ramp.data.unit + ramp.data[0, 5, 5] = 0 + ramp.data[1, 5, 5] = 20000 + ramp.data[2, 5, 5] = 40000 + ramp.data[3, 5, 5] = 60000 # Signal reaches saturation limit + ramp.data[4, 5, 5] = 62000 # set read_pattern to have many reads in the third resultant, so that # its mean exposure time is much smaller than its last read time @@ -100,11 +99,11 @@ def test_ad_floor_flagging(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 0 * ramp.data.unit # Signal at bottom rail - low saturation - ramp.data[1, 5, 5] = 0 * ramp.data.unit # Signal at bottom rail - low saturation - ramp.data[2, 5, 5] = 20 * ramp.data.unit - ramp.data[3, 5, 5] = 40 * ramp.data.unit - ramp.data[4, 5, 5] = 60 * ramp.data.unit + ramp.data[0, 5, 5] = 0 # Signal at bottom rail - low saturation + ramp.data[1, 5, 5] = 0 # Signal at bottom rail - low saturation + ramp.data[2, 5, 5] = 20 + ramp.data[3, 5, 5] = 40 + ramp.data[4, 5, 5] = 60 # frames that should be flagged as saturation (low) satindxs = [0, 1] @@ -134,11 +133,11 @@ def test_ad_floor_and_saturation_flagging(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 0 * ramp.data.unit # Signal at bottom rail - low saturation - ramp.data[1, 5, 5] = 0 * ramp.data.unit # Signal at bottom rail - low saturation - ramp.data[2, 5, 5] = 20 * ramp.data.unit - ramp.data[3, 5, 5] = 40 * ramp.data.unit - ramp.data[4, 5, 5] = 61000 * ramp.data.unit # Signal above the saturation threshold + ramp.data[0, 5, 5] = 0 # Signal at bottom rail - low saturation + ramp.data[1, 5, 5] = 0 # Signal at bottom rail - low saturation + ramp.data[2, 5, 5] = 20 + ramp.data[3, 5, 5] = 40 + ramp.data[4, 5, 5] = 61000 # Signal above the saturation threshold # frames that should be flagged as ad_floor floorindxs = [0, 1] @@ -171,11 +170,11 @@ def test_signal_fluctuation_flagging(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 10 * ramp.data.unit - ramp.data[1, 5, 5] = 20000 * ramp.data.unit - ramp.data[2, 5, 5] = 40000 * ramp.data.unit - ramp.data[3, 5, 5] = 60000 * ramp.data.unit # Signal reaches saturation limit - ramp.data[4, 5, 5] = 40000 * ramp.data.unit # Signal drops below saturation limit + ramp.data[0, 5, 5] = 10 + ramp.data[1, 5, 5] = 20000 + ramp.data[2, 5, 5] = 40000 + ramp.data[3, 5, 5] = 60000 # Signal reaches saturation limit + ramp.data[4, 5, 5] = 40000 # Signal drops below saturation limit # Set saturation value in the saturation model satmap.data[5, 5] = satvalue * satmap.data.unit @@ -184,7 +183,7 @@ def test_signal_fluctuation_flagging(setup_wfi_datamodels): output = flag_saturation(ramp, satmap) # Make sure that all groups after first saturated group are flagged - satindex = np.argmax(output.data.value[:, 5, 5] == satvalue) + satindex = np.argmax(output.data[:, 5, 5] == satvalue) assert np.all(output.groupdq[satindex:, 5, 5] == group.SATURATED) @@ -200,11 +199,11 @@ def test_all_groups_saturated(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values at or above saturation limit - ramp.data[0, 5, 5] = 60000 * ramp.data.unit - ramp.data[1, 5, 5] = 62000 * ramp.data.unit - ramp.data[2, 5, 5] = 62000 * ramp.data.unit - ramp.data[3, 5, 5] = 60000 * ramp.data.unit - ramp.data[4, 5, 5] = 62000 * ramp.data.unit + ramp.data[0, 5, 5] = 60000 + ramp.data[1, 5, 5] = 62000 + ramp.data[2, 5, 5] = 62000 + ramp.data[3, 5, 5] = 60000 + ramp.data[4, 5, 5] = 62000 # Set saturation value in the saturation model satmap.data[5, 5] = satvalue * satmap.data.unit @@ -252,11 +251,11 @@ def test_no_sat_check(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 10 * ramp.data.unit - ramp.data[1, 5, 5] = 20000 * ramp.data.unit - ramp.data[2, 5, 5] = 40000 * ramp.data.unit - ramp.data[3, 5, 5] = 60000 * ramp.data.unit - ramp.data[4, 5, 5] = 62000 * ramp.data.unit # Signal reaches saturation limit + ramp.data[0, 5, 5] = 10 + ramp.data[1, 5, 5] = 20000 + ramp.data[2, 5, 5] = 40000 + ramp.data[3, 5, 5] = 60000 + ramp.data[4, 5, 5] = 62000 # Signal reaches saturation limit # Set saturation value in the saturation model & DQ value for NO_SAT_CHECK satmap.data[5, 5] = satvalue * satmap.data.unit @@ -291,11 +290,11 @@ def test_nans_in_mask(setup_wfi_datamodels): ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols) # Add ramp values up to the saturation limit - ramp.data[0, 5, 5] = 10 * ramp.data.unit - ramp.data[1, 5, 5] = 20000 * ramp.data.unit - ramp.data[2, 5, 5] = 40000 * ramp.data.unit - ramp.data[3, 5, 5] = 60000 * ramp.data.unit - ramp.data[4, 5, 5] = 62000 * ramp.data.unit + ramp.data[0, 5, 5] = 10 + ramp.data[1, 5, 5] = 20000 + ramp.data[2, 5, 5] = 40000 + ramp.data[3, 5, 5] = 60000 + ramp.data[4, 5, 5] = 62000 # Set saturation value for pixel to NaN satmap.data[5, 5] = np.nan * satmap.data.unit @@ -324,9 +323,7 @@ def test_saturation_getbestref(setup_wfi_datamodels): wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012 wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16 wfi_sci_raw.meta.exposure.type = "WFI_IMAGE" - wfi_sci_raw.data = u.Quantity( - np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16 - ) + wfi_sci_raw.data = np.ones(shape, dtype=np.uint16) wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw, dq=True) # Run the pipeline diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh index 348e1269a..ec0d82098 100644 --- a/romancal/scripts/make_regtestdata.sh +++ b/romancal/scripts/make_regtestdata.sh @@ -3,18 +3,21 @@ # takes one argument: the directory into which to start putting the regtest files. # input files needed: -# r0000101001001001001_01101_0001_WFI01 - default for most steps -# r0000201001001001001_01101_0001_WFI01 - equivalent for spectroscopic data -# r0000101001001001001_01101_0002_WFI01 - a second resample exposure, only cal step needed -# r0000101001001001001_01101_0003_WFI01 - for ramp fitting; truncated image -# r0000201001001001001_01101_0003_WFI01 - for ramp fitting; truncated spectroscopic +# r0000101001001001001_0001_WFI01 - default for most steps +# r0000201001001001001_0001_WFI01 - equivalent for spectroscopic data +# r0000101001001001001_0002_WFI01 - a second resample exposure, only cal step needed +# r0000101001001001001_0003_WFI01 - for ramp fitting; truncated image +# r0000201001001001001_0003_WFI01 - for ramp fitting; truncated spectroscopic # we need only darkcurrent & ramp fit for these # -# r00r1601001001001001_01101_0001_WFI01 - special 16 resultant file, imaging, only need cal file -# r10r1601001001001001_01101_0001_WFI01 - special 16 resultant file, spectroscopy, only need cal file -# roman_dark_WFI01_IMAGE_STRESS_TEST_16_MA_TABLE_998_D1 - special dark for 16 resultant file +# r00r1601001001001001_0001_WFI01 - special 16 resultant file, imaging, only need cal file +# r10r1601001001001001_0001_WFI01 - special 16 resultant file, spectroscopy, only need cal file outdir=$1 +logfile=$outdir/make_regtestdata.log + +# Redirect all output to the logfile +exec > $logfile 2>&1 # set up the directory structure mkdir -p $outdir/roman-pipeline/dev/WFI/image @@ -24,7 +27,7 @@ mkdir -p $outdir/roman-pipeline/dev/truth/WFI/grism # most regtests; run the pipeline and save a lot of results. -for fn in r0000101001001001001_01101_0001_WFI01 r0000201001001001001_01101_0001_WFI01 +for fn in r0000101001001001001_0001_WFI01 r0000201001001001001_0001_WFI01 do echo "Running pipeline on ${fn}..." strun roman_elp ${fn}_uncal.asdf --steps.dq_init.save_results True --steps.saturation.save_results True --steps.linearity.save_results True --steps.dark_current.save_results True --steps.rampfit.save_results True --steps.assign_wcs.save_results True --steps.photom.save_results True --steps.refpix.save_results True --steps.flatfield.save_results True --steps.assign_wcs.save_results True @@ -38,7 +41,7 @@ done # truncated files for ramp fit regtests -for fn in r0000101001001001001_01101_0003_WFI01 r0000201001001001001_01101_0003_WFI01 +for fn in r0000101001001001001_0003_WFI01 r0000201001001001001_0003_WFI01 do echo "Running pipeline on ${fn}..." strun roman_elp ${fn}_uncal.asdf --steps.dark_current.save_results True --steps.rampfit.save_results True @@ -46,29 +49,28 @@ do cp ${fn}_darkcurrent.asdf $outdir/roman-pipeline/dev/WFI/$dirname/ cp ${fn}_rampfit.asdf $outdir/roman-pipeline/dev/truth/WFI/$dirname/ done -cp r0000101001001001001_01101_0003_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ +cp r0000101001001001001_0003_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ # second imaging exposure -strun roman_elp r0000101001001001001_01101_0002_WFI01_uncal.asdf -cp r0000101001001001001_01101_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ +strun roman_elp r0000101001001001001_0002_WFI01_uncal.asdf +cp r0000101001001001001_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/ -# resample regtest; needs r0000101001001001001_01101_000{1,2}_WFI01_cal.asdf +# resample regtest; needs r0000101001001001001_000{1,2}_WFI01_cal.asdf # builds the appropriate asn file and calls strun with it echo "Creating regtest files for resample..." -asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf -o mosaic_asn.json --product-name mosaic -strun romancal.step.ResampleStep mosaic_asn.json --rotation=0 --output_file=mosaic.asdf -cp mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/ +asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic +strun romancal.step.ResampleStep L3_mosaic_asn.json --rotation=0 --output_file=mosaic.asdf +cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/ cp mosaic_resamplestep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ - -# CRDS test needs the "usual" r00001..._01101_0001_WFI01 files. -# It also needs a hacked r00001..._01101_0001_WFI01 file, with the time changed. +# CRDS test needs the "usual" r00001..._0001_WFI01 files. +# It also needs a hacked r00001..._0001_WFI01 file, with the time changed. # this makes the hacked version. echo "Creating regtest files for CRDS tests..." -basename="r0000101001001001001_01101_0001_WFI01" +basename="r0000101001001001001_0001_WFI01" python -c " import asdf from roman_datamodels import stnode @@ -86,14 +88,14 @@ cp ${basename}_changetime_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/image # need to make a special ALL_SATURATED file for the all saturated test. echo "Creating regtest files for all saturated tests..." -basename="r0000101001001001001_01101_0001_WFI01" +basename="r0000101001001001001_0001_WFI01" python -c " import asdf from roman_datamodels import stnode basename = '$basename' f = asdf.open(f'{basename}_uncal.asdf') data = f['roman']['data'].copy() -data[...] = 65535 * f['roman']['data'].unit +data[...] = 65535 f['roman']['data'] = data f['roman']['meta']['filename'] = stnode.Filename(f'{basename}_ALL_SATURATED_uncal.asdf') f.write_to(f'{basename}_ALL_SATURATED_uncal.asdf')" @@ -103,19 +105,19 @@ cp ${basename}_ALL_SATURATED_cal.asdf $outdir/roman-pipeline/dev/truth/WFI/image # make a special file dark file with a different name -strun romancal.step.DarkCurrentStep r0000101001001001001_01101_0001_WFI01_linearity.asdf --output_file=Test_dark +strun romancal.step.DarkCurrentStep r0000101001001001001_0001_WFI01_linearity.asdf --output_file=Test_dark cp Test_darkcurrent.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # make a special linearity file with a different suffix -strun romancal.step.LinearityStep r0000101001001001001_01101_0001_WFI01_refpix.asdf --output_file=Test_linearity +strun romancal.step.LinearityStep r0000101001001001001_0001_WFI01_refpix.asdf --output_file=Test_linearity cp Test_linearity.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ # we have a test that runs the flat field step directly on an _L1_ spectroscopic # file and verifies that it gets skipped. # I don't really understand that but we can duplicate it for now. -basename="r0000201001001001001_01101_0001_WFI01" +basename="r0000201001001001001_0001_WFI01" strun romancal.step.FlatFieldStep ${basename}_uncal.asdf cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/ @@ -124,7 +126,7 @@ cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/ # we haven't updated the filename in these files, but the regtest mechanism # also doesn't # update them, and we need to match. -for basename in r0000101001001001001_01101_0001_WFI01 r0000201001001001001_01101_0001_WFI01 +for basename in r0000101001001001001_0001_WFI01 r0000201001001001001_0001_WFI01 do python -c " import asdf @@ -146,7 +148,7 @@ model.to_asdf(f'${basename}_cal_repoint.asdf')" done # Test tweakreg with repointed file, only shifted by 1" -for basename in r0000101001001001001_01101_0001_WFI01 +for basename in r0000101001001001001_0001_WFI01 do python -c " import asdf @@ -167,13 +169,13 @@ model.to_asdf(f'${basename}_shift_cal.asdf')" done -strun roman_elp r0000101001001001001_01101_0004_WFI01_uncal.asdf -strun roman_elp r0000201001001001001_01101_0004_WFI01_uncal.asdf -cp r0000101001001001001_01101_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/ -cp r0000201001001001001_01101_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/ +strun roman_elp r0000101001001001001_0004_WFI01_uncal.asdf +strun roman_elp r0000201001001001001_0004_WFI01_uncal.asdf +cp r0000101001001001001_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/ +cp r0000201001001001001_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/ l3name="r0099101001001001001_F158_visit" -asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_regtest_asn.json --product-name $l3name +asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_regtest_asn.json --product-name $l3name strun roman_mos L3_regtest_asn.json cp L3_regtest_asn.json $outdir/roman-pipeline/dev/WFI/image/ cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/ @@ -183,11 +185,29 @@ cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ strun romancal.step.SourceCatalogStep ${l3name}_i2d.asdf cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ + +l3name="r0099101001001001001_r274dp63x31y81_prompt_F158" +asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name $l3name --target r274dp63x31y81 +strun roman_mos L3_mosaic_asn.json +cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/ +cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/ +cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ + +# L3 catalog +strun romancal.step.SourceCatalogStep ${l3name}_i2d.asdf +cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ + + # L2 catalog -strun romancal.step.SourceCatalogStep r0000101001001001001_01101_0001_WFI01_cal.asdf -cp r0000101001001001001_01101_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ +strun romancal.step.SourceCatalogStep r0000101001001001001_0001_WFI01_cal.asdf +cp r0000101001001001001_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ + l3name="r0099101001001001001_F158_visit_r274dp63x31y81" -asn_from_list --product-name=$l3name r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_m1_asn.json +asn_from_list --product-name=$l3name r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_m1_asn.json strun roman_mos L3_m1_asn.json cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ + +# tests passing suffix to the pipeline +strun roman_elp r0000101001001001001_0001_WFI01_uncal.asdf --steps.tweakreg.skip=True --suffix=star +cp r0000101001001001001_0001_WFI01_star.asdf $outdir/roman-pipeline/dev/truth/WFI/image/ diff --git a/romancal/skymatch/skymatch.py b/romancal/skymatch/skymatch.py index 80a41a3ff..38390b05f 100644 --- a/romancal/skymatch/skymatch.py +++ b/romancal/skymatch/skymatch.py @@ -460,8 +460,7 @@ def _overlap_matrix(images, apply_sky=True): # TODO: to improve performance, the nested loops could be parallelized # since _calc_sky() here can be called independently from previous steps. ns = len(images) - data_unit = images[0].image.unit - A = np.zeros((ns, ns), dtype=float) * data_unit + A = np.zeros((ns, ns), dtype=float) W = np.zeros((ns, ns), dtype=float) for i in range(ns): for j in range(i + 1, ns): @@ -482,7 +481,6 @@ def _overlap_matrix(images, apply_sky=True): def _find_optimum_sky_deltas(images, apply_sky=True): ns = len(images) A, W = _overlap_matrix(images, apply_sky=apply_sky) - data_unit = images[0].image.unit def is_valid(i, j): return W[i, j] > 0 and W[j, i] > 0 @@ -514,7 +512,7 @@ def is_valid(i, j): if is_valid(i, j): K[ieq, i] = Wm[i, j] K[ieq, j] = -Wm[i, j] - F[ieq] = Wm[i, j] * (A[j, i] - A[i, j]).value + F[ieq] = Wm[i, j] * (A[j, i] - A[i, j]) invalid[i] = False invalid[j] = False ieq += 1 @@ -538,4 +536,4 @@ def is_valid(i, j): deltas = np.dot(invK, F) deltas[np.asarray(invalid, dtype=bool)] = np.nan - return deltas * data_unit + return deltas diff --git a/romancal/skymatch/skymatch_step.py b/romancal/skymatch/skymatch_step.py index cef846ecd..309f99fbc 100644 --- a/romancal/skymatch/skymatch_step.py +++ b/romancal/skymatch/skymatch_step.py @@ -173,15 +173,13 @@ def _imodel2skyim(self, image_model): def _set_sky_background(self, sky_image, step_status): image = sky_image.meta["image_model"] - sky = sky_image.sky - if sky == 0 or sky is None: - sky = 0 * image.data.unit + sky = sky_image.sky if sky_image.sky is not None else 0 image.meta.background.method = str(self.skymethod) image.meta.background.subtracted = self.subtract # In numpy 2, the dtypes are more carefully controlled, so to match the # schema the data type needs to be re-cast to float64. - image.meta.background.level = sky.astype(np.float64) + image.meta.background.level = sky if step_status == "COMPLETE" and self.subtract: image.data[...] = sky_image.image[...] diff --git a/romancal/skymatch/skystatistics.py b/romancal/skymatch/skystatistics.py index a61aca7f3..ce22eee2b 100644 --- a/romancal/skymatch/skystatistics.py +++ b/romancal/skymatch/skystatistics.py @@ -126,14 +126,8 @@ def calc_sky(self, data): in `skyvalue`. """ - imstat = ImageStats(image=data.value, fields=self._fields, **(self._kwargs)) - stat = self._skystat(imstat) # dict or scalar - - # re-attach units: - if hasattr(stat, "__len__"): - self.skyval = {k: value * data.unit for k, value in stat.items()} - else: - self.skyval = stat * data.unit + imstat = ImageStats(image=data, fields=self._fields, **(self._kwargs)) + self.skyval = self._skystat(imstat) # dict or scalar self.npix = imstat.npix return self.skyval, self.npix diff --git a/romancal/skymatch/tests/test_skymatch.py b/romancal/skymatch/tests/test_skymatch.py index b9aa61fd5..fc272bb91 100644 --- a/romancal/skymatch/tests/test_skymatch.py +++ b/romancal/skymatch/tests/test_skymatch.py @@ -73,12 +73,9 @@ def mk_image_model( ): l2 = mk_level2_image(shape=image_shape) l2_im = ImageModel(l2) - l2_im.data = u.Quantity( - rng.normal(loc=rate_mean, scale=rate_std, size=l2_im.data.shape).astype( - np.float32 - ), - l2_im.data.unit, - ) + l2_im.data = rng.normal( + loc=rate_mean, scale=rate_std, size=l2_im.data.shape + ).astype(np.float32) l2_im.meta["wcs"] = mk_gwcs(image_shape, sky_offset=sky_offset, rotate=rotation) @@ -125,11 +122,10 @@ def _add_bad_pixels(im, sat_val, dont_use_val): mask = np.ones(im.data.shape, dtype=bool) # Add some "bad" pixels: # corners - im_unit = im.data.unit - im.data[:5, :5] = sat_val * im_unit - im.data[-5:, :5] = sat_val * im_unit - im.data[-5:, -5:] = sat_val * im_unit - im.data[:5, -5:] = sat_val * im_unit + im.data[:5, :5] = sat_val + im.data[-5:, :5] = sat_val + im.data[-5:, -5:] = sat_val + im.data[:5, -5:] = sat_val im.dq[:5, :5] = pixel.SATURATED im.dq[-5:, :5] = pixel.SATURATED @@ -146,7 +142,7 @@ def _add_bad_pixels(im, sat_val, dont_use_val): cy -= 5 # center - im.data[cx : cx + 10, cy : cy + 10] = dont_use_val * im_unit + im.data[cx : cx + 10, cy : cy + 10] = dont_use_val im.dq[cx : cx + 10, cy : cy + 10] = pixel.DO_NOT_USE mask[cx : cx + 10, cy : cy + 10] = False @@ -183,7 +179,7 @@ def test_skymatch(wfi_rate, skymethod, subtract, skystat, match_down): with library: for i, (im, lev) in enumerate(zip(library, levels)): - im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape) * im.data.unit + im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape) library.shelve(im, i) # exclude central DO_NOT_USE and corner SATURATED pixels @@ -219,14 +215,14 @@ def test_skymatch(wfi_rate, skymethod, subtract, skystat, match_down): assert im.meta.background.subtracted == subtract # test computed/measured sky values if level is set: - if not np.isclose(im.meta.background.level.value, 0): - assert abs(im.meta.background.level.value - rlev) < 0.01 + if not np.isclose(im.meta.background.level, 0): + assert abs(im.meta.background.level - rlev) < 0.01 # test if subtract: - assert abs(np.mean(im.data[dq_mask]).value - slev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - slev) < 0.01 else: - assert abs(np.mean(im.data[dq_mask]).value - lev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01 result.shelve(im, i, modify=False) @@ -248,7 +244,7 @@ def test_skymatch_overlap(mk_sky_match_image_models, skymethod, subtract, skysta with library: for i, (im, lev) in enumerate(zip(library, levels)): - im.data = rng.normal(loc=lev, scale=0.01, size=im.data.shape) * im.data.unit + im.data = rng.normal(loc=lev, scale=0.01, size=im.data.shape) library.shelve(im, i) # We do not exclude SATURATED pixels. They should be ignored because @@ -287,23 +283,23 @@ def test_skymatch_overlap(mk_sky_match_image_models, skymethod, subtract, skysta # These two sky methods must fail because they do not take # into account (do not compute) overlap regions and use # entire images: - assert abs(im.meta.background.level.value - rlev) < 0.1 + assert abs(im.meta.background.level - rlev) < 0.1 # test if subtract: - assert abs(np.mean(im.data[dq_mask]).value - slev) < 0.1 + assert abs(np.mean(im.data[dq_mask]) - slev) < 0.1 else: - assert abs(np.mean(im.data[dq_mask]).value - lev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01 else: # test computed/measured sky values if level is nonzero: - if not np.isclose(im.meta.background.level.value, 0): - assert abs(im.meta.background.level.value - rlev) < 0.01 + if not np.isclose(im.meta.background.level, 0): + assert abs(im.meta.background.level - rlev) < 0.01 # test if subtract: - assert abs(np.mean(im.data[dq_mask].value) - slev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - slev) < 0.01 else: - assert abs(np.mean(im.data[dq_mask].value) - lev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01 result.shelve(im, i, modify=False) @@ -330,7 +326,7 @@ def test_skymatch_2x(wfi_rate, skymethod, subtract): with library: for i, (im, lev) in enumerate(zip(library, levels)): - im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape) * im.data.unit + im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape) library.shelve(im, i) # We do not exclude SATURATED pixels. They should be ignored because @@ -385,14 +381,14 @@ def test_skymatch_2x(wfi_rate, skymethod, subtract): assert im.meta.background.subtracted == step.subtract # test computed/measured sky values: - if not np.isclose(im.meta.background.level.value, 0, atol=1e-6): - assert abs(im.meta.background.level.value - rlev) < 0.01 + if not np.isclose(im.meta.background.level, 0, atol=1e-6): + assert abs(im.meta.background.level - rlev) < 0.01 # test if subtract: - assert abs(np.mean(im.data[dq_mask]).value - slev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - slev) < 0.01 else: - assert abs(np.mean(im.data[dq_mask]).value - lev) < 0.01 + assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01 result2.shelve(im, i, modify=False) diff --git a/romancal/source_catalog/detection.py b/romancal/source_catalog/detection.py index 384919511..f83421296 100644 --- a/romancal/source_catalog/detection.py +++ b/romancal/source_catalog/detection.py @@ -7,7 +7,6 @@ import warnings from astropy.convolution import convolve -from astropy.units import Quantity from astropy.utils.exceptions import AstropyUserWarning from photutils.segmentation import SourceFinder, make_2dgaussian_kernel from photutils.utils.exceptions import NoDetectionsWarning @@ -23,8 +22,8 @@ def convolve_data(data, kernel_fwhm, size=None, mask=None): Parameters ---------- - data : 2D `astropy.units.Quantity` - The input 2D Quantity array. The data is assumed to be + data : 2D `numpy.ndarray` + The input 2D array. The data is assumed to be background subtracted. kernel_fwhm : float The FWHM of the Gaussian kernel. @@ -40,9 +39,6 @@ def convolve_data(data, kernel_fwhm, size=None, mask=None): convolved_data : `numpy.ndarray` The convolved data array. """ - if not isinstance(data, Quantity): - raise ValueError("Input model must be a Quantity array.") - size = math.ceil(kernel_fwhm * 3) size = size + 1 if size % 2 == 0 else size # make size be odd kernel = make_2dgaussian_kernel(kernel_fwhm, size=size) # normalized to 1 diff --git a/romancal/source_catalog/source_catalog.py b/romancal/source_catalog/source_catalog.py index e80bc3d07..66dadaf82 100644 --- a/romancal/source_catalog/source_catalog.py +++ b/romancal/source_catalog/source_catalog.py @@ -103,9 +103,7 @@ def __init__( self.sb_unit = "MJy/sr" self.l2_unit = "DN/s" - self.l2_conv_factor = ( - self.model.meta.photometry.conversion_megajanskys / self.l2_unit - ) + self.l2_conv_factor = self.model.meta.photometry.conversion_megajanskys if len(ci_star_thresholds) != 2: raise ValueError("ci_star_thresholds must contain only 2 items") @@ -161,7 +159,7 @@ def pixel_area(self): """ pixel_area = self.model.meta.photometry.pixelarea_steradians if pixel_area < 0: - pixel_area = (self._pixel_scale**2).to(u.sr) + pixel_area = (self._pixel_scale**2).to(u.sr).value return pixel_area def convert_l2_to_sb(self): @@ -169,10 +167,6 @@ def convert_l2_to_sb(self): Convert a level-2 image from units of DN/s to MJy/sr (surface brightness). """ - if self.model.data.unit != self.l2_unit or self.model.err.unit != self.l2_unit: - raise ValueError( - f"data and err are expected to be in units of {self.l2_unit}" - ) # the conversion in done in-place to avoid making copies of the data; # use a dictionary to set the value to avoid on-the-fly validation @@ -182,15 +176,11 @@ def convert_l2_to_sb(self): def convert_sb_to_l2(self): """ - Convert the data and error Quantity arrays from MJy/sr (surface + Convert the data and error arrays from MJy/sr (surface brightness) to DN/s (level-2 units). This is the inverse operation of `convert_l2_to_sb`. """ - if self.model.data.unit != self.sb_unit or self.model.err.unit != self.sb_unit: - raise ValueError( - f"data and err are expected to be in units of {self.sb_unit}" - ) # the conversion in done in-place to avoid making copies of the data; # use a dictionary to set the value to avoid on-the-fly validation @@ -200,46 +190,31 @@ def convert_sb_to_l2(self): def convert_sb_to_flux_density(self): """ - Convert the data and error Quantity arrays from MJy/sr (surface + Convert the data and error arrays from MJy/sr (surface brightness) to flux density units. The flux density unit is defined by self.flux_unit. """ - if self.model.data.unit != self.sb_unit or self.model.err.unit != self.sb_unit: - raise ValueError( - f"data and err are expected to be in units of {self.sb_unit}" - ) # the conversion in done in-place to avoid making copies of the data; # use a dictionary to set the value to avoid on-the-fly validation self.model["data"] *= self.pixel_area - self.model["data"] <<= self.flux_unit + self.model["err"] *= self.pixel_area - self.model["err"] <<= self.flux_unit + self.convolved_data *= self.pixel_area - self.convolved_data <<= self.flux_unit def convert_flux_density_to_sb(self): """ - Convert the data and error Quantity arrays from flux density units to + Convert the data and error arrays from flux density units to MJy/sr (surface brightness). This is the inverse operation of `convert_sb_to_flux_density`. """ - if ( - self.model.data.unit != self.flux_unit - or self.model.err.unit != self.flux_unit - ): - raise ValueError( - f"data and err are expected to be in units of {self.flux_unit}" - ) self.model["data"] /= self.pixel_area - self.model["data"] <<= self.sb_unit self.model["err"] /= self.pixel_area - self.model["err"] <<= self.sb_unit self.convolved_data /= self.pixel_area - self.convolved_data <<= self.sb_unit def convert_flux_to_abmag(self, flux, flux_err): """ @@ -247,7 +222,7 @@ def convert_flux_to_abmag(self, flux, flux_err): Parameters ---------- - flux, flux_err : `~astropy.unit.Quantity` + flux, flux_err : `~numpy.ndarray` The input flux and error arrays. Returns @@ -260,15 +235,14 @@ def convert_flux_to_abmag(self, flux, flux_err): warnings.simplefilter("ignore", category=RuntimeWarning) # exact AB mag zero point - flux_zpt = 10 ** (-0.4 * 48.60) * u.erg / u.s / u.cm**2 / u.Hz - flux_zpt <<= self.flux_unit + flux_zpt = 10 ** (-0.4 * 48.60) - abmag_zpt = 2.5 * np.log10(flux_zpt.value) - abmag = -2.5 * np.log10(flux.value) + abmag_zpt - abmag_err = 2.5 * np.log10(1.0 + (flux_err.value / flux.value)) + abmag_zpt = 2.5 * np.log10(flux_zpt) + abmag = -2.5 * np.log10(flux) + abmag_zpt + abmag_err = 2.5 * np.log10(1.0 + (flux_err / flux)) # handle negative fluxes - idx = flux.value < 0 + idx = flux < 0 abmag[idx] = np.nan abmag_err[idx] = np.nan @@ -560,7 +534,7 @@ def _aper_local_background(self): bkg_median = [] bkg_std = [] for mask in bkg_aper_masks: - bkg_data = mask.get_values(self.model.data.value) + bkg_data = mask.get_values(self.model.data) values = sigclip(bkg_data, masked=False) nvalues.append(values.size) bkg_median.append(np.median(values)) @@ -571,9 +545,6 @@ def _aper_local_background(self): # standard error of the median bkg_median_err = np.sqrt(np.pi / (2.0 * nvalues)) * np.array(bkg_std) - bkg_median <<= self.model.data.unit - bkg_median_err <<= self.model.data.unit - return bkg_median, bkg_median_err @lazyproperty @@ -783,7 +754,7 @@ def _daofind_convolved_data(self): The DAOFind convolved data. """ return ndimage.convolve( - self.model.data.value, self._daofind_kernel, mode="constant", cval=0.0 + self.model.data, self._daofind_kernel, mode="constant", cval=0.0 ) @lazyproperty @@ -1219,8 +1190,7 @@ def catalog(self): self._update_metadata() catalog.meta.update(self.meta) - # convert QTable to Table to change Quantity columns to regular - # columns with units + # convert QTable to Table catalog = Table(catalog) # split SkyCoord columns into separate RA and Dec columns diff --git a/romancal/source_catalog/tests/test_source_catalog.py b/romancal/source_catalog/tests/test_source_catalog.py index f1dfd15bf..13580f474 100644 --- a/romancal/source_catalog/tests/test_source_catalog.py +++ b/romancal/source_catalog/tests/test_source_catalog.py @@ -73,12 +73,9 @@ def mosaic_model(): wfi_mosaic = mk_level3_mosaic(shape=(101, 101)) model = MosaicModel(wfi_mosaic) data, err = make_test_image() - units = u.MJy / u.sr - data <<= units - err <<= units model.data = data model.err = err - model.weight = 1.0 / err.value + model.weight = 1.0 / err return model @@ -87,12 +84,9 @@ def image_model(): wfi_image = mk_level2_image(shape=(101, 101)) model = ImageModel(wfi_image) data, err = make_test_image() - units = u.DN / u.s - data <<= units - err <<= units model.data = data model.err = err - model.meta.photometry.conversion_megajanskys = 0.3324 * u.MJy / u.sr + model.meta.photometry.conversion_megajanskys = (0.3324 * u.MJy / u.sr).value return model