diff --git a/.gitignore b/.gitignore index 75d400001..c32d780f2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,10 @@ develop-eggs .installed.cfg distribute-*.tar.gz +# emacs save files +\#* +.#* + # Other .cache .tox diff --git a/changes/1534.general.rst b/changes/1534.general.rst new file mode 100644 index 000000000..e336983a2 --- /dev/null +++ b/changes/1534.general.rst @@ -0,0 +1 @@ +Remove the jump code for the deprecated jump detection for even ramps and update the documentation diff --git a/docs/images/r0000501001001001001_01101_0001_WFI01_cal.png b/docs/images/r0000501001001001001_0001_wfi01_cal.png similarity index 100% rename from docs/images/r0000501001001001001_01101_0001_WFI01_cal.png rename to docs/images/r0000501001001001001_0001_wfi01_cal.png diff --git a/docs/images/r0000501001001001001_01101_0001_WFI01_cal_thumb.png b/docs/images/r0000501001001001001_0001_wfi01_cal_thumb.png similarity index 100% rename from docs/images/r0000501001001001001_01101_0001_WFI01_cal_thumb.png rename to docs/images/r0000501001001001001_0001_wfi01_cal_thumb.png diff --git a/docs/roman/associations/asn_from_list.rst b/docs/roman/associations/asn_from_list.rst index 3b848a7f5..d32ff2fa7 100644 --- a/docs/roman/associations/asn_from_list.rst +++ b/docs/roman/associations/asn_from_list.rst @@ -19,7 +19,7 @@ To create an association, use the following command: import romancal.associations.asn_from_list as asn_from_list product_name = 'test_product' - items = {'r0000101001001001001_01101_0001_WFI01_uncal.asdf': 'science', 'r0000101001001001001_3_01101_0001_WFI01_uncal.asdf': 'guide_window', 'c': 'somethingelse'} + items = {'r0000101001001001001_0001_wfi01_uncal.asdf': 'science', 'r0000101001001001001_3_0001_wfi01_uncal.asdf': 'guide_window', 'c': 'somethingelse'} asn = asn_from_list.asn_from_list([(item, type_) for item, type_ in items.items()], product_name=product_name, with_exptype=True) asn['asn_rule'] 'DMS_ELPP_Base' @@ -29,9 +29,9 @@ an example product that has both a science and guide window exposures would look like the following:: asn['products'] - [ { 'members': [ { 'expname': 'r0000101001001001001_01101_0001_WFI01_uncal.asdf', + [ { 'members': [ { 'expname': 'r0000101001001001001_0001_wfi01_uncal.asdf', 'exptype': 'science'}, - { 'expname': 'r0000101001001001001_3_01101_0001_WFI01_uncal.asdf', + { 'expname': 'r0000101001001001001_3_0001_wfi01_uncal.asdf', 'exptype': 'guide_window'}, {'expname': 'c', 'exptype': 'somethingelse'}], 'name': 'test_product'}] @@ -41,7 +41,7 @@ To create a association with all the detectors for a given exposure from the com .. code-block:: python - asn_from_list -o detector_asn.json --product-name r0000101001001001001_01101_0001_WFI data/*_cal.asdf + asn_from_list -o detector_asn.json --product-name r0000101001001001001_0001_wfi data/*_cal.asdf where the individual calibrated detector files are in a data subdirectory. diff --git a/docs/roman/data_products/file_naming.rst b/docs/roman/data_products/file_naming.rst index 5df0fde3e..dfe30270a 100644 --- a/docs/roman/data_products/file_naming.rst +++ b/docs/roman/data_products/file_naming.rst @@ -11,7 +11,7 @@ The names of the exposure level data are constructed with information from the science header of the exposure, allowing users to map it to the observation in their corresponding APT files. The ASDF file naming scheme for the Exposure products is:: - r____.asdf + r___.asdf where @@ -25,7 +25,7 @@ where - s: sequence ID (1=prime, >1 parallel) - aa: activity number (base 36) - eeee: exposure number - - detector: detector name (e.g. WFI01, WFI02, ...) + - detector: detector name (e.g. wfi01, wfi02, ...) - prodType: product type identifier (e.g. 'uncal', 'cal') The standard for the pipeline are uncal and cal, for the input products and resulting @@ -35,8 +35,8 @@ linearity, jump, darkcurrent, rampfit, assignwcs, flat, and photom. An example Exposure Level 1 upcalibrated ASDF file name is:: - r0000101001001001001_01101_0001_WFI01_uncal.asdf + r0000101001001001001_0001_wfi01_uncal.asdf An example Exposure Level 2 product ASDF file name is:: - r0000101001001001001_01101_0001_WFI01_cal.asdf + r0000101001001001001_0001_wfi01_cal.asdf diff --git a/docs/roman/datamodels/datamodels_asdf.rst b/docs/roman/datamodels/datamodels_asdf.rst index b75971ad1..eb76a0561 100644 --- a/docs/roman/datamodels/datamodels_asdf.rst +++ b/docs/roman/datamodels/datamodels_asdf.rst @@ -14,7 +14,7 @@ If you have installed the roman calibration pipeline you should also have access to the standalone tool asdfinfo which allows access to `ASDF `_ (and roman) files from the terminal prompt,:: - asdftool info r0000101001001001001_01101_0001_WFI16_cal.asdf + asdftool info r0000101001001001001_0001_wfi16_cal.asdf root (AsdfObject) ├─asdf_library (Software) │ ├─author (str): The ASDF Developers @@ -62,10 +62,10 @@ To access the files via a python session, import roman_datamodels as rdm import asdf - with rdm.open('r0000101001001001001_01101_0001_WFI16_cal.asdf') as model: + with rdm.open('r0000101001001001001_0001_wfi16_cal.asdf') as model: - with asdf.open('r0000101001001001001_01101_0001_WFI16_cal.asdf', memmap=False) as model: + with asdf.open('r0000101001001001001_0001_wfi16_cal.asdf', memmap=False) as model: Once the files are loaded you can access various attributes. Below is a table @@ -146,7 +146,7 @@ asdf tree. .. code:: python >>> from roman_datamodels import datamodels as rdm # doctest: +SKIP - >>> d_uncal = rdm.open('r0000101001001001001_01101_0001_WFI01_uncal.asdf') # doctest: +SKIP + >>> d_uncal = rdm.open('r0000101001001001001_0001_wfi01_uncal.asdf') # doctest: +SKIP >>> d_uncal.info() # doctest: +SKIP root (AsdfObject) ├─asdf_library (Software) diff --git a/docs/roman/datamodels/models.rst b/docs/roman/datamodels/models.rst index d46f30161..47beb3260 100644 --- a/docs/roman/datamodels/models.rst +++ b/docs/roman/datamodels/models.rst @@ -36,7 +36,7 @@ using python. .. testsetup:: >>> from roman_datamodels import datamodels as rdm >>> from roman_datamodels.maker_utils import mk_datamodel - >>> fn = 'r0019106003005004023_03203_0034_WFI01_cal.asdf' + >>> fn = 'r0019106003005004023_0034_wfi01_cal.asdf' >>> image_model = mk_datamodel(rdm.ImageModel) # set some metadata for the below tests @@ -46,7 +46,7 @@ using python. .. code-block:: python >>> from roman_datamodels import datamodels as rdm - >>> fn = 'r0019106003005004023_03203_0034_WFI01_cal.asdf' + >>> fn = 'r0019106003005004023_0034_wfi01_cal.asdf' >>> image_model = rdm.open(fn) # doctest: +SKIP >>> type(image_model) @@ -145,7 +145,7 @@ You can examine the contents of your model from within python using meta.calibration_software_name RomanCAL meta.calibration_software_version 9.9.0 meta.product_type l2 - meta.filename r0019106003005004023_03203_0034_WFI01_cal.asdf + meta.filename r0019106003005004023_0034_wfi01_cal.asdf meta.file_date 2020-01-01T00:00:00.000 meta.model_type ImageModel meta.origin STSCI/SOC diff --git a/docs/roman/jump/arguments.rst b/docs/roman/jump/arguments.rst deleted file mode 100644 index f77fafe80..000000000 --- a/docs/roman/jump/arguments.rst +++ /dev/null @@ -1,50 +0,0 @@ -Arguments -========= - -The ``jump`` step has five optional arguments that can be set by the user: - -* ``--rejection_threshold``: A floating-point value that sets the sigma - threshold for jump detection for ramps having 5 or more groups.. In the code - sigma is determined using the read noise from the read noise reference file - and the Poisson noise (based on the median difference between samples, and the - gain reference file). Note that any noise source beyond these two that may be - present in the data will lead to an increase in the false positive rate and - thus may require an increase in the value of this parameter. The default value - of 4.0 for the rejection threshold will yield 6200 false positives for every - million pixels, if the noise model is correct. - -* ``--three_group_rejection_threshold``: A floating-point value that sets the - sigma threshold for jump detection for ramps having exactly 3 groups. The - default value is 6.0 - -* ``--four_group_rejection_threshold``: A floating-point value that sets the - sigma threshold for jump detection for ramps having exactly 4 groups. The - default value is 5.0 - -* ``--maximum_cores``: The fraction of available cores that will be - used for multi-processing in this step. The default value is 'none' which does not use - multi-processing. The other options are 'quarter', 'half', and 'all'. Note that these - fractions refer to the total available cores and on most CPUs these include physical - and virtual cores. The clock time for the step is reduced - almost linearly by the number of physical cores used on all machines. For example, on an Intel CPU with - six real cores and 6 virtual cores setting maximum_cores to 'half' results in a - decrease of a factor of six in the clock time for the step to run. Depending on the system - the clock time can also decrease even more with maximum_cores is set to 'all'. - -* ``--flag_4_neighbors``: If set to True (default is True) it will cause the four perpendicular - neighbors of all detected jumps to be flagged as a jump. This is needed because of - the inter-pixel capacitance (IPC) causing a small jump in the neighbors. The small jump - might be below the rejection threshold but will affect the slope determination of - the pixel. The step will take about 40% longer to run when this is set to True. - -* ``--max_jump_to_flag_neighbors``: A floating point value in units of sigma that limits - the flagging of neighbors. Any jump above this cutoff will not have its neighbors flagged. - The concept is that the jumps in neighbors will be above the rejection-threshold and thus - be flagged as primary jumps. The default value is 1000. - -* ``--min_jump_to_flag_neighbors``: A floating point value in units of sigma that limits - the flagging of neighbors of marginal detections. Any primary jump below this value will - not have its neighbors flagged. The goal is to prevent flagging jumps that would be too - small to significantly affect the slope determination. The default value is 10. - -* ``--use_ramp_jump_detection``: See the description in :ref:`ramp fitting `. diff --git a/docs/roman/jump/description.rst b/docs/roman/jump/description.rst deleted file mode 100644 index 295863460..000000000 --- a/docs/roman/jump/description.rst +++ /dev/null @@ -1,77 +0,0 @@ -Description -=========== - -Assumptions ------------ -We assume that the ``saturation`` step has already been applied to the input -science exposure, so that saturated values are appropriately flagged in the -input GROUPDQ array. We also assume that steps such as the reference pixel -correction (``refpix``) and non-linearity correction (``linearity``) have been applied, so -that the input data ramps do not have any non-linearities or noise above the modeled Poission -and read noise due to instrumental effects. The absence of any of these preceding corrections -or residual non-linearities or noise can lead to the false detection of jumps in the ramps, -due to departure from linearity. - -The ``jump`` step will automatically skip execution if the input data contain fewer -than 3 groups in the integration, because the baseline algorithm requires two first -differences to work. - -Algorithm ---------- -This routine detects jumps in an exposure by looking for outliers -in the up-the-ramp signal for each pixel in the integration within -an input exposure. On output, the GROUPDQ array is updated with the DQ flag -"JUMP_DET" to indicate the location of each jump that was found. -In addition, any pixels that have non-positive or NaN values in the gain -reference file will have DQ flags "NO_GAIN_VALUE" and "DO_NOT_USE" set in the -output PIXELDQ array. -The SCI and ERR arrays of the input data are not modified. - -The current implementation uses the two-point difference method described -in Anderson&Gordon2011_. - -Two-Point Difference Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The two-point difference method is applied to the integration as follows: - -* Compute the first differences for each pixel (the difference between - adjacent groups) -* Compute the clipped (dropping the largest difference) median of the first differences for each pixel. -* Use the median to estimate the Poisson noise for each group and combine it - with the read noise to arrive at an estimate of the total expected noise for - each difference. -* Compute the "difference ratio" as the difference between the first differences - of each group and the median, divided by the expected noise. -* If the largest "difference ratio" is greater than the rejection threshold, - flag the group corresponding to that ratio as having a jump. -* If a jump is found in a given pixel, iterate the above steps with the - jump-impacted group excluded, looking for additional lower-level jumps - that still exceed the rejection threshold. -* Stop iterating on a given pixel when no new jumps are found or only one difference remains. -* If the there are only three differences (four groups), the standard median is used rather than - the clipped median. -* If there are only two differences (three groups), the smallest one is compared to the larger - one and if the larger one is above a threshold, it is flagged as a jump. - -Note that any ramp values flagged as SATURATED in the input GROUPDQ array -are not used in any of the above calculations and hence will never be -marked as containing a jump. - -Multiprocessing -=============== -This step has the option of running in multiprocessing mode. In that mode it will -split the input data cube into a number of row slices based on the number of available -cores on the host computer and the value of the max_cores input parameter. By -default the step runs on a single processor. At the other extreme if max_cores is -set to 'all', it will use all available cores (real and virtual). Testing has shown -a reduction in the elapsed time for the step proportional to the number of real -cores used. Using the virtual cores also reduces the elapsed time but at a slightly -lower rate than the real cores. - -If multiprocessing is requested the input cube will be divided into a number of -slices in the row dimension (with the last slice being slightly larger, if needed). -The slices are then sent to twopoint_difference.py by detect_jumps. After all the -slices have finished processing, detect_jumps assembles the output group_dq cube -from the slices. - -.. _Anderson&Gordon2011: https://ui.adsabs.harvard.edu/abs/2011PASP..123.1237A diff --git a/docs/roman/jump/even_ramp.rst b/docs/roman/jump/even_ramp.rst deleted file mode 100644 index 8748b8ff5..000000000 --- a/docs/roman/jump/even_ramp.rst +++ /dev/null @@ -1,17 +0,0 @@ -============================= -Jump Detection for Even Ramps -============================= - -.. warning:: - - This step can only be run on evenly-spaced ramps. Using this step - will turn off the jump detection algorithm integrated into ramp fitting. - -.. toctree:: - :maxdepth: 2 - - description.rst - arguments.rst - reference_files.rst - -.. automodapi:: romancal.jump diff --git a/docs/roman/jump/index.rst b/docs/roman/jump/index.rst deleted file mode 100644 index fa1850c8f..000000000 --- a/docs/roman/jump/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _jump_step: - -============== -Jump Detection -============== - -.. toctree:: - :maxdepth: 3 - - uneven_ramp.rst - even_ramp.rst diff --git a/docs/roman/jump/reference_files.rst b/docs/roman/jump/reference_files.rst deleted file mode 100644 index de77cc0f2..000000000 --- a/docs/roman/jump/reference_files.rst +++ /dev/null @@ -1,15 +0,0 @@ -Reference File Types -===================== - -The ``jump`` step uses two reference files: :ref:`GAIN ` -and :ref:`READNOISE `. -The GAIN reference file is used to temporarily convert pixel values in -the ``jump`` step from units of DN to electrons. -The READNOISE reference file is used in estimating the expected noise -in each pixel. -Both are necessary for proper computation of noise estimates within the -``jump`` step. - -:ref:`GAIN ` - -:ref:`READNOISE ` diff --git a/docs/roman/jump/uneven_ramp.rst b/docs/roman/jump/uneven_ramp.rst deleted file mode 100644 index 95165f17a..000000000 --- a/docs/roman/jump/uneven_ramp.rst +++ /dev/null @@ -1,13 +0,0 @@ -=============================== -Jump Detection for Uneven Ramps -=============================== - -.. note:: - - This version of jump detection will work on both unevenly-spaced and - evenly-spaced ramps. So there is no need to worry about which type of - of ramp you have. - -Description ------------ -This "step" is actually integrated into the :ref:`unevenly-spaced ramp fitting algorithm`. diff --git a/docs/roman/outlier_detection/main.rst b/docs/roman/outlier_detection/main.rst index 765c07378..62126e4a3 100644 --- a/docs/roman/outlier_detection/main.rst +++ b/docs/roman/outlier_detection/main.rst @@ -4,11 +4,13 @@ Description :Classes: `romancal.outlier_detection.OutlierDetectionStep` :Aliases: outlier_detection, outlier_detection_scaled, outlier_detection_stack -Processing multiple datasets together allows for the identification of bad pixels -or cosmic-rays that remain in each of the input images, many times at levels which -were not detectable by the :ref:`jump ` step. The ``outlier_detection`` step -implements the following algorithm to identify and flag any remaining cosmic-rays or -other artifacts left over from previous calibrations: +Processing multiple datasets together allows for the identification of +bad pixels or cosmic-rays that remain in each of the input images, +many times at levels which were not detectable by jump detection in +:ref:`ramp_fitting ` step. The +``outlier_detection`` step implements the following algorithm to +identify and flag any remaining cosmic-rays or other artifacts left +over from previous calibrations: - build a stack of input data diff --git a/docs/roman/package_index.rst b/docs/roman/package_index.rst index bc03ed8ce..34a31b3fd 100644 --- a/docs/roman/package_index.rst +++ b/docs/roman/package_index.rst @@ -7,7 +7,6 @@ dq_init/index.rst flatfield/index.rst flux/index.rst - jump/index.rst linearity/index.rst outlier_detection/index.rst pipeline/index.rst diff --git a/docs/roman/pipeline/exposure_pipeline.rst b/docs/roman/pipeline/exposure_pipeline.rst index dda8ee7a9..30f81d863 100644 --- a/docs/roman/pipeline/exposure_pipeline.rst +++ b/docs/roman/pipeline/exposure_pipeline.rst @@ -43,11 +43,11 @@ The ``exposure`` pipeline has an optional argument:: --use_ramp_jump_detection boolean default=True -When set to ``True``, the pipeline will perform :ref:`jump ` detection as a part of the ramp +When set to ``True``, the pipeline will perform jump detection as a part of the ramp fitting step. The data at this stage of the pipeline are still in the form of the original 3D ramps ( ngroups x ncols x nrows ) and have had all of the detector-level correction steps applied to it, up to but not including the detection and flagging of -Cosmic-Ray (CR) hits within each ramp (integration). For this case the :ref:`jump ` +Cosmic-Ray (CR) hits within each ramp (integration). For this case the jump detection module in :ref:`ramp_fitting ` will update the dq array with the CR hits (jumps) that are identified in the step. @@ -62,7 +62,7 @@ Inputs :File suffix: _uncal The input to the ``ExposurePipeline`` can be a single raw exposure, -e.g. "r0008308002010007027_06311_0019_WFI01_uncal.asdf", which contains the +e.g. "r0008308002010007027_0019_wfi01_uncal.asdf", which contains the original raw data from all of the detector readouts in the exposure ( ngroups x ncols x nrows ). The raw data may also be input using an association file. diff --git a/docs/roman/pipeline/mosaic_pipeline.rst b/docs/roman/pipeline/mosaic_pipeline.rst index 6e53179d0..262373a33 100644 --- a/docs/roman/pipeline/mosaic_pipeline.rst +++ b/docs/roman/pipeline/mosaic_pipeline.rst @@ -54,7 +54,7 @@ An association of 2D calibrated image data :File suffix: _cal The input to the ``MosaicPipeline`` is a group of calibrated exposures, -e.g. "r0008308002010007027_06311_0019_WFI01_cal.asdf", which contains the +e.g. "r0008308002010007027_0019_wfi01_cal.asdf", which contains the calibrated data for the the exposures. The most convenient way to pass the list of exposures to be processed with the mosaic level pipeline is to use an association. Instructions on how to create an input association an be found at :ref:`asn-from-list`. diff --git a/docs/roman/pipeline_naming_conventions.rst b/docs/roman/pipeline_naming_conventions.rst index 27aba40bf..0540a7af1 100644 --- a/docs/roman/pipeline_naming_conventions.rst +++ b/docs/roman/pipeline_naming_conventions.rst @@ -47,10 +47,10 @@ name. If the input file name already has a known suffix, that suffix will be replaced. For example: :: - $ strun romancal.dq_init.DQInitStep r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun romancal.dq_init.DQInitStep r0008308002010007027_0019_wfi01_uncal.asdf produces an output file named -``r0008308002010007027_06311_0019_WFI01_dq_init.asdf``. +``r0008308002010007027_0019_wfi01_dq_init.asdf``. .. _pipeline_step_suffix_definitions: diff --git a/docs/roman/pipeline_parameters.rst b/docs/roman/pipeline_parameters.rst index 3531e4fca..b2ff0a694 100644 --- a/docs/roman/pipeline_parameters.rst +++ b/docs/roman/pipeline_parameters.rst @@ -17,7 +17,7 @@ To set a parameter, simply specify it on the command line. For example, to have ``strun`` command would be as follows: :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf --save_calibrated_ramp=true + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --save_calibrated_ramp=true To specify parameter values for an individual step when running a pipeline use the syntax ``--steps..=value``. @@ -25,7 +25,7 @@ For example, to override the default selection of a dark current reference file from CRDS when running a pipeline: :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --steps.dark_current.override_dark='my_dark.asdf' Universal Parameters @@ -46,7 +46,7 @@ have all output from ``roman_elp``, including any saved intermediate steps, appear in the sub-directory ``calibrated``, use :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --output_dir=calibrated ``output_dir`` can be specified at the step level, overriding what was @@ -54,7 +54,7 @@ specified for the pipeline. From the example above, to change the name and location of the ``dark_current`` step, use the following :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --output_dir=calibrated --steps.dark_current.output_file='dark_sub.asdf' --steps.dark_current.output_dir='dark_calibrated' diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index 2697fdb74..295164ceb 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -21,7 +21,7 @@ For example, the exposure level pipeline is implemented by the class run this pipeline is: :: - $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_0019_wfi01_uncal.asdf Pipeline classes also have a **pipeline name**, or **alias**, that can be used @@ -71,7 +71,8 @@ From the Python Prompt ------------------------------ You can execute a pipeline or a step from within python by using the -``call`` method of the class. +``call`` method of the class. This creates a new instance of the class +and runs the pipeline or step. The ``call`` method creates a new instance of the class and runs the pipeline or step. Optional parameter settings can be specified by via keyword arguments or @@ -82,10 +83,10 @@ For the exposure pipeline and steps, :: from romancal.pipeline import ExposurePipeline - result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + result = ExposurePipeline.call('r0000101001001001001_0001_wfi01_uncal.asdf') from romancal.linearity import LinearityStep - result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + result = LinearityStep.call('r0000101001001001001_0001_wfi01_uncal.asdf') One difference between the mosaic level pipeline and the exposure level pipeline is that the mosaic level pipeline is generally designed to run on multiple overlapping exposures. To achieve diff --git a/docs/roman/pipeline_static_preview.rst b/docs/roman/pipeline_static_preview.rst index a572a3a61..42feaeb89 100644 --- a/docs/roman/pipeline_static_preview.rst +++ b/docs/roman/pipeline_static_preview.rst @@ -67,16 +67,16 @@ Examples .. code-block:: shell - roman_static_preview preview r0000501001001001001_01101_0001_WFI01_cal.asdf r0000501001001001001_01101_0001_WFI01_cal.png 400 400 + roman_static_preview preview r0000501001001001001_0001_wfi01_cal.asdf r0000501001001001001_0001_wfi01_cal.png 400 400 -.. image:: ../images/r0000501001001001001_01101_0001_WFI01_cal.png +.. image:: ../images/r0000501001001001001_0001_wfi01_cal.png :alt: preview of Roman imagery, with compass rose showing orientation .. code-block:: shell - roman_static_preview thumbnail r0000501001001001001_01101_0001_WFI01_cal.asdf r0000501001001001001_01101_0001_WFI01_cal_thumb.png + roman_static_preview thumbnail r0000501001001001001_0001_wfi01_cal.asdf r0000501001001001001_0001_wfi01_cal_thumb.png -.. image:: ../images/r0000501001001001001_01101_0001_WFI01_cal_thumb.png +.. image:: ../images/r0000501001001001001_0001_wfi01_cal_thumb.png :alt: thumbnail of Roman imagery using ``stpreview`` directly diff --git a/docs/roman/pipeline_steps.rst b/docs/roman/pipeline_steps.rst index 93e1e7185..f10d94f05 100644 --- a/docs/roman/pipeline_steps.rst +++ b/docs/roman/pipeline_steps.rst @@ -8,7 +8,6 @@ Pipeline Steps refpix/index.rst linearity/index.rst dark_current/index.rst - jump/index.rst ramp_fitting/index.rst assign_wcs/index.rst flatfield/index.rst diff --git a/docs/roman/references_general/references_general.rst b/docs/roman/references_general/references_general.rst index 44593fded..de4261a53 100644 --- a/docs/roman/references_general/references_general.rst +++ b/docs/roman/references_general/references_general.rst @@ -51,9 +51,7 @@ documentation on each reference file. +---------------------------------------------+--------------------------------------------------+ | :ref:`flatfield ` | :ref:`FLAT ` | +---------------------------------------------+--------------------------------------------------+ -| :ref:`jump_detection ` | :ref:`GAIN ` | -+ +--------------------------------------------------+ -| | :ref:`READNOISE ` | +| :ref:`GAIN ` | :ref:`ramp_fitting ` | +---------------------------------------------+--------------------------------------------------+ | :ref:`linearity ` | :ref:`LINEARITY ` | +---------------------------------------------+--------------------------------------------------+ @@ -76,19 +74,13 @@ documentation on each reference file. +--------------------------------------------------+---------------------------------------------+ | :ref:`FLAT ` | :ref:`flatfield ` | +--------------------------------------------------+---------------------------------------------+ -| :ref:`GAIN ` | :ref:`jump_detection ` | -+ +---------------------------------------------+ -| | :ref:`ramp_fitting ` | -+--------------------------------------------------+---------------------------------------------+ | :ref:`LINEARITY ` | :ref:`linearity ` | +--------------------------------------------------+---------------------------------------------+ | :ref:`MASK ` | :ref:`dq_init ` | +--------------------------------------------------+---------------------------------------------+ | :ref:`PHOTOM ` | :ref:`photom ` | +--------------------------------------------------+---------------------------------------------+ -| :ref:`READNOISE ` | :ref:`jump_detection ` | -+ +---------------------------------------------+ -| | :ref:`ramp_fitting ` | +| :ref:`READNOISE ` | :ref:`ramp_fitting ` | +--------------------------------------------------+---------------------------------------------+ | :ref:`SATURATION ` | :ref:`saturation ` | +--------------------------------------------------+---------------------------------------------+ diff --git a/docs/roman/stpipe/call_via_run.rst b/docs/roman/stpipe/call_via_run.rst index 014ef62f8..9fad623b3 100644 --- a/docs/roman/stpipe/call_via_run.rst +++ b/docs/roman/stpipe/call_via_run.rst @@ -6,8 +6,8 @@ Executing a pipeline or pipeline step directly, or via run() When calling a pipeline or step instance directly, or using the ``run`` method, you can specify individual parameter values manually. In this case, parameter files are not used. If you use ``run`` after instantiating with a parameter -file (as is done when using the :ref:`call` method), the -parameter file will be ignored. +file (as is done when using the :ref:`call` method), +the parameter file will be ignored. :: @@ -16,27 +16,26 @@ parameter file will be ignored. # Manually set any desired non-default parameter values pipe.assign_wcs.skip = True - pipe.jump.rejection_threshold = 5 pipe.ramp_fit.override_gain = 'my_gain_file.asdf' pipe.save_result = True pipe.output_dir = '/my/data/pipeline_outputs' # Run the pipeline - result = pipe.run('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + result = pipe.run('r0000101001001001001_0001_wfi01_uncal.asdf') To run a single step: :: - from romancal.jump import JumpStep + from romancal.ramp_fitting import RampFitStep # Instantiate the step - step = JumpStep() + step = RampFitStep() # Set parameter values - step.rejection_threshold = 5 + step.override_gain = 'my_gain_file.asdf' step.save_results = True step.output_dir = '/my/data/jump_data' # Execute using the run method - result = step.run('r0000101001001001001_01101_0001_WFI01_linearity.asdf') + result = step.run('r0000101001001001001_0001_wfi01_linearity.asdf') diff --git a/docs/roman/stpipe/config_asdf.rst b/docs/roman/stpipe/config_asdf.rst index 5a526e9ba..923d517b9 100644 --- a/docs/roman/stpipe/config_asdf.rst +++ b/docs/roman/stpipe/config_asdf.rst @@ -16,13 +16,13 @@ class, Step class, or already existing .asdf or .cfg file, and run that step using the ``--save-parameters`` option. For example, to get the parameters for the ``ExposurePipeline`` pipeline, do the following: :: - $ strun --save-parameters=exp_pars.asdf roman_elp r0000101001001001001_01101_0001_WFI01_uncal.asdf + $ strun --save-parameters=exp_pars.asdf roman_elp r0000101001001001001_0001_wfi01_uncal.asdf Once created and modified as necessary, the file can now be used by ``strun`` to run the step/pipeline with the desired parameters: :: - $ strun exp_pars.asdf r0000101001001001001_01101_0001_WFI01_uncal.asdf + $ strun exp_pars.asdf r0000101001001001001_0001_wfi01_uncal.asdf The remaining sections will describe the file format and contents. @@ -30,7 +30,8 @@ File Contents ------------- To describe the contents of an ASDF file, the configuration for the step -``roman_elp`` will be used as the example: +``roman_elp`` will be used as the example which only shows some of the +entries for clarity: .. code-block:: @@ -40,16 +41,18 @@ To describe the contents of an ASDF file, the configuration for the step %TAG ! tag:stsci.edu:asdf/ --- !core/asdf-1.1.0 asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', - name: asdf, version: 2.13.0} + name: asdf, version: 3.3.0} history: - extensions: - - !core/extension_metadata-1.0.0 - extension_class: asdf.extension.BuiltinExtension - software: !core/software-1.0.0 {name: asdf, version: 2.13.0} + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.5.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 3.3.0} class: romancal.pipeline.exposure_pipeline.ExposurePipeline meta: author: - date: '2022-09-15T13:59:54' + date: '2024-11-20T19:41:22' description: Parameters for calibration step romancal.pipeline.exposure_pipeline.ExposurePipeline instrument: {name: } origin: @@ -59,7 +62,7 @@ To describe the contents of an ASDF file, the configuration for the step useafter: name: ExposurePipeline parameters: - input_dir: '' + input_dir: rsim output_dir: null output_ext: .asdf output_file: null @@ -71,33 +74,37 @@ To describe the contents of an ASDF file, the configuration for the step save_results: true search_output_file: true skip: false - suffix: null + suffix: cal steps: - - class: romancal.jump.jump_step.JumpStep - name: jump + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: source_catalog parameters: - flag_4_neighbors: true - four_group_rejection_threshold: 190.0 - input_dir: '' - max_jump_to_flag_neighbors: 1000.0 - maximum_cores: none - min_jump_to_flag_neighbors: 10.0 - output_dir: null - output_ext: .asdf - output_file: null - output_use_index: true - output_use_model: false - post_hooks: [] - pre_hooks: [] - rejection_threshold: 180.0 - save_results: false - search_output_file: true - skip: false - suffix: null - three_group_rejection_threshold: 185.0 - ... - -Required Components + aperture_ee1: 30 + aperture_ee2: 50 + aperture_ee3: 70 + bkg_boxsize: 1000 + ci1_star_threshold: 2.0 + ci2_star_threshold: 1.8 + deblend: false + fit_psf: true + input_dir: rsim + kernel_fwhm: 2.0 + npixels: 25 + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + snr_threshold: 3.0 + suffix: cat + ... + +REQUIRED Components ~~~~~~~~~~~~~~~~~~~ Preamble @@ -146,11 +153,8 @@ For example, the ``parameters`` block above could also have been formatted as: .. code-block:: - parameters: {flag_4_neighbors: true, four_group_rejection_threshold: 190.0, - max_jump_to_flag_neighbors: 1000.0, maximum_cores: none, - min_jump_to_flag_neighbors: 10.0, output_dir: null, output_ext: .asdf, - output_file: null, output_use_index: true, output_use_model: false, - rejection_threshold: 180.0, three_group_rejection_threshold: 185.0} + parameters:{ aperture_ee1: 30, aperture_ee2: 50, aperture_ee3: 70, + bkg_boxsize: 1000, ci1_star_threshold: 2.0, ci2_star_threshold: 1.8} Optional Components ~~~~~~~~~~~~~~~~~~~ @@ -173,14 +177,14 @@ step is executed. Remember that parameter values can come from numerous sources. Refer to :ref:`Parameter Precedence` for a full listing of how parameters can be set. -From the ``JumpStep`` example, if all that needed to change is the -``rejection_threshold`` parameter with a setting of ``80.0``, +From the ``SourceCatalogStep`` example, if all that needed to change is the +``aperture_ee1`` parameter with a setting of ``30``, the ``parameters`` block need only contain the following: .. code-block:: parameters: - rejection_threshold: 80.0 + aperture_ee1: 30 Pipeline Configuration @@ -190,8 +194,8 @@ Pipelines are essentially steps that refer to sub-steps. As in the original cfg format, parameters for sub-steps can also be specified. All sub-step parameters appear in a key called `steps`. Sub-step parameters are specified by using the sub-step name as the key, then underneath and indented, the parameters to change -for that sub-step. For example, to define the ``rejection_threshold`` of the -``JumpStep`` step in a ``ExposurePipeline`` parameter file, the parameter +for that sub-step. For example, to define the ``aperture_ee1`` of the +``SourceCatalogStep`` step in a ``ExposurePipeline`` parameter file, the parameter block would look as follows: .. code-block:: @@ -199,9 +203,9 @@ block would look as follows: class: romancal.pipeline.exposure_pipeline.ExposurePipeline parameters: {} steps: - - class: romancal.jump.jump_step.JumpStep + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep parameters: - rejection_threshold: 80.0 + aperture_ee1: 30 As with step parameter files, not all sub-steps need to be specified. If left unspecified, the sub-steps will be run with their default parameter sets. For @@ -236,14 +240,14 @@ There are a number of ways to create an ASDF parameter file. From the command line utility ``strun``, the option ``--save-parameters`` can be used. Within a Python script, the method ``Step.export_config(filename: str)`` can be -used. For example, to create a parameter file for ``JumpStep``, use the +used. For example, to create a parameter file for ``SourceCatalogStep``, use the following: .. doctest-skip:: - >>> from romancal.jump import JumpStep - >>> step = JumpStep() - >>> step.export_config('jump_step.asdf') + >>> from romancal..source_catalog import SourceCatalogStep + >>> step = SourceCatalogStep() + >>> step.export_config('source_catalog_step.asdf') diff --git a/docs/roman/stpipe/parameter_files.rst b/docs/roman/stpipe/parameter_files.rst index 50cc5bcfe..359de9622 100644 --- a/docs/roman/stpipe/parameter_files.rst +++ b/docs/roman/stpipe/parameter_files.rst @@ -20,14 +20,14 @@ For example, to save the parameters used for a run of the ``ExposurePipeline`` pipeline, use: :: -$ strun roman_elp r0000101001001001001_01101_0001_WFI01_uncal.asdf --save-parameters my_exppars.asdf +$ strun roman_elp r0000101001001001001_0001_wfi01_uncal.asdf --save-parameters my_exppars.asdf Once saved, the file can be edited, removing parameters that should be left at their default values, and setting the remaining parameters to the desired values. Once modified, the new parameter file can be used: :: -$ strun my_exppars2.asdf r0000101001001001001_01101_0001_WFI01_uncal.asdf +$ strun my_exppars2.asdf r0000101001001001001_0001_wfi01_uncal.asdf Note that the parameter values will reflect whatever was set on the command-line, or through a specified local parameter file. In short, the diff --git a/docs/roman/stpipe/user_pipeline.rst b/docs/roman/stpipe/user_pipeline.rst index 9176baf8c..5c7dde510 100644 --- a/docs/roman/stpipe/user_pipeline.rst +++ b/docs/roman/stpipe/user_pipeline.rst @@ -105,35 +105,37 @@ parameter file that contains: .. code-block:: yaml - - class: romancal.jump.jump_step.JumpStep - name: jump - parameters: - flag_4_neighbors: true - four_group_rejection_threshold: 190.0 - input_dir: '' - max_jump_to_flag_neighbors: 1000.0 - maximum_cores: none - min_jump_to_flag_neighbors: 10.0 + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: SourceCatalogStep + parameters: + aperture_ee1: 30 + aperture_ee2: 50 + aperture_ee3: 70 + bkg_boxsize: 1000 + ci1_star_threshold: 2.0 + ci2_star_threshold: 1.8 + deblend: false + fit_psf: true is equivalent to: .. code-block:: yaml steps: - - class: romancal.jump.jump_step.JumpStep - name: jump + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: SourceCatalogStep parameters: - config_file = myjump.asdf + config_file = myconfig.asdf -with the file ``myjump.asdf.`` in the same directory: +with the file ``myconfig.asdf.`` in the same directory: .. code-block:: yaml - class: romancal.jump.jump_step.JumpStep - name: jump + class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: SourceCatalogStep parameters: - flag_4_neighbors: true - four_group_rejection_threshold: 190.0 + aperture_ee1: 30 + aperture_ee2: 50 If both a ``config_file`` and additional parameters are specified, the ``config_file`` is loaded, and then the local parameters override @@ -154,31 +156,29 @@ equivalent of the parameter file above: .. code-block:: python - from stpipe.pipeline import Image2Pipeline + from romancal.pipeline import ExposurePipeline steps = { - 'jump':{'rejection_threshold': 180., - 'three_group_rejection_threshold': 190., - 'four_group_rejection_threshold':195. - } + 'source_catalog':{ 'aperture_ee1': 3, 'aperture_ee2': 5, 'aperture_ee3': 7} + } pipe = ExposurePipeline(steps=steps) Running a Pipeline ================== -From the commandline --------------------- +From the command line +--------------------- -The same ``strun`` script used to run Steps from the commandline can +The same ``strun`` script used to run Steps from the command line can also run Pipelines. The only wrinkle is that any parameters overridden from the -commandline use dot notation to specify the parameter name. For -example, to override the ``rejection_threshold`` value on the ``jump`` +command line use dot notation to specify the parameter name. For +example, to override the ``aperture_ee1`` value on the ``source_catalog`` step in the example above, one can do:: - > strun romancal.pipeline.ExposurePipeline --steps.jump.rejection_threshold=180. + > strun romancal.pipeline.ExposurePipeline --steps.source_catalog.aperture_ee1=3 From Python ----------- diff --git a/docs/roman/stpipe/user_step.rst b/docs/roman/stpipe/user_step.rst index b50878603..26eaf125c 100644 --- a/docs/roman/stpipe/user_step.rst +++ b/docs/roman/stpipe/user_step.rst @@ -172,15 +172,14 @@ for ``strun``. The call signature is:: For example, given the following command-line:: - $ strun romancal.pipeline.ExposurePipeline r0000101001001001001_01101_0001_WFI01_uncal.asdf \ - --steps.jump.override_gain=roman_wfi_gain_0033.asdf + $ strun romancal.pipeline.ExposurePipeline r0000101001001001001_0001_wfi01_uncal.asdf \ + --steps.ramp_fit.override_gain=roman_wfi_gain_0033.asdf the equivalent `from_cmdline` call would be:: from romancal.pipeline import ExposurePipeline - ExposurePipeline.from_cmdline([' r0000101001001001001_01101_0001_WFI01_uncal.asdf', - 'steps.jump.override_gain', 'roman_wfi_gain_0033.asdf']) - + ExposurePipeline.from_cmdline([' r0000101001001001001_0001_wfi01_uncal.asdf', + 'steps.ramp_fit.override_gain', 'roman_wfi_gain_0033.asdf']) call() `````` @@ -202,19 +201,19 @@ particular step accepts. The method returns the result of the step. A basic example is:: from romancal.jump import JumpStep - output = JumpStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf') makes a new instance of `JumpStep` and executes using the specified exposure file. `JumpStep` has a parameter ``rejection_threshold``. To use a different value than the default, the statement would be:: - output = JumpStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', + output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf', rejection_threshold=42.0) If one wishes to use a :ref:`parameter file`, specify the path to it using the ``config_file`` argument:: - output = JumpStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', + output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf', config_file='my_jumpstep_config.asdf') run() diff --git a/romancal/jump/__init__.py b/romancal/jump/__init__.py deleted file mode 100644 index 87f39f59a..000000000 --- a/romancal/jump/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .jump_step import JumpStep - -__all__ = ["JumpStep"] diff --git a/romancal/jump/jump_step.py b/romancal/jump/jump_step.py deleted file mode 100644 index 2dc1a3d80..000000000 --- a/romancal/jump/jump_step.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -Detect jumps in a science image -""" - -import logging -import time - -import numpy as np -from roman_datamodels import datamodels as rdd -from roman_datamodels.dqflags import group, pixel -from stcal.jump.jump import detect_jumps - -from romancal.stpipe import RomanStep - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["JumpStep"] - - -class JumpStep(RomanStep): - """ - JumpStep: Performs CR/jump detection. The 2-point difference method is - applied. - """ - - class_alias = "jump" - - spec = """ - rejection_threshold = float(default=180.0,min=0) # CR sigma rej thresh - three_group_rejection_threshold = float(default=185.0,min=0) # CR sigma rej thresh - four_group_rejection_threshold = float(default=190.0,min=0) # CR sigma rej thresh - maximum_cores = option('none', 'quarter', 'half', 'all', default='none') # max number of processes to create - flag_4_neighbors = boolean(default=True) # flag the four perpendicular neighbors of each CR - max_jump_to_flag_neighbors = float(default=1000) # maximum jump sigma that will trigger neighbor flagging - min_jump_to_flag_neighbors = float(default=10) # minimum jump sigma that will trigger neighbor flagging - min_sat_area = float(default=1.0) # minimum required area for the central saturation of snowballs - min_jump_area = float(default=5.0) # minimum area to trigger large events processing - expand_factor = float(default=2.0) # The expansion factor for the enclosing circles or ellipses - use_ellipses = boolean(default=False) # Use an enclosing ellipse rather than a circle for MIRI showers - sat_required_snowball = boolean(default=True) # Require the center of snowballs to be saturated - expand_large_events = boolean(default=False) # must be True to trigger snowball and shower flagging - use_ramp_jump_detection = boolean(default=True) # Use jump detection during ramp fitting - """ # noqa: E501 - - reference_file_types = ["gain", "readnoise"] - - def process(self, input): - # Open input as a Roman DataModel (single integration; 3D arrays) - if isinstance(input, rdd.DataModel): - input_model = input - else: - input_model = rdd.open(input) - - # Extract the needed info from the Roman Data Model - r_data = input_model.data - r_gdq = input_model.groupdq - r_pdq = input_model.pixeldq - r_err = input_model.err - result = input_model - - # If the ramp fitting jump detection is enabled, then skip this step - if self.use_ramp_jump_detection: - result.meta.cal_step.jump = "SKIPPED" - return result - - # FIXME: since frames_per_group => meta.exposure.nframes has been removed, - # we need to fix stcal.jump.jump to remove it from there too - frames_per_group = 1 - - # Modify the arrays for input into the 'common' jump (4D) - data = np.copy(r_data[np.newaxis, :]) - gdq = r_gdq[np.newaxis, :] - pdq = r_pdq[np.newaxis, :] - err = np.copy(r_err[np.newaxis, :]) - - tstart = time.time() - - # Check for an input model with nresultants<=2 - nresultants = data.shape[1] - - if nresultants <= 2: - self.log.warning("Cannot apply jump detection as nresultants<=2;") - self.log.warning("Jump step will be skipped") - - result = input_model - - result.meta.cal_step.jump = "SKIPPED" - return result - - # Retrieve the parameter values - rej_thresh = self.rejection_threshold - three_grp_rej_thresh = self.three_group_rejection_threshold - four_grp_rej_thresh = self.four_group_rejection_threshold - max_cores = self.maximum_cores - max_jump_to_flag_neighbors = self.max_jump_to_flag_neighbors - min_jump_to_flag_neighbors = self.min_jump_to_flag_neighbors - flag_4_neighbors = self.flag_4_neighbors - min_sat_area = self.min_sat_area - min_jump_area = self.min_jump_area - expand_factor = self.expand_factor - use_ellipses = self.use_ellipses - sat_required_snowball = self.sat_required_snowball - expand_large_events = self.expand_large_events - - self.log.info("CR rejection threshold = %g sigma", rej_thresh) - if self.maximum_cores != "none": - self.log.info("Maximum cores to use = %s", max_cores) - - # Get the gain and readnoise reference files - # TODO: remove units from gain and RN reference files - gain_filename = self.get_reference_file(input_model, "gain") - self.log.info("Using GAIN reference file: %s", gain_filename) - gain_model = rdd.GainRefModel(gain_filename) - gain_2d = gain_model.data.value - - readnoise_filename = self.get_reference_file(input_model, "readnoise") - self.log.info("Using READNOISE reference file: %s", readnoise_filename) - readnoise_model = rdd.ReadnoiseRefModel(readnoise_filename) - # This is to clear the WRITEABLE=False flag? - readnoise_2d = np.copy(readnoise_model.data.value) - - # DG 0810/21: leave for now; make dqflags changes in a later, - # separate PR - dqflags_d = {} # Dict of DQ flags - dqflags_d = { - "GOOD": group.GOOD, - "DO_NOT_USE": group.DO_NOT_USE, - "SATURATED": group.SATURATED, - "JUMP_DET": group.JUMP_DET, - "NO_GAIN_VALUE": pixel.NO_GAIN_VALUE, - } - - gdq, pdq, *_ = detect_jumps( - frames_per_group, - data, - gdq, - pdq, - err, - gain_2d, - readnoise_2d, - rej_thresh, - three_grp_rej_thresh, - four_grp_rej_thresh, - max_cores, - max_jump_to_flag_neighbors, - min_jump_to_flag_neighbors, - flag_4_neighbors, - dqflags_d, - min_sat_area=min_sat_area, - min_jump_area=min_jump_area, - expand_factor=expand_factor, - use_ellipses=use_ellipses, - sat_required_snowball=sat_required_snowball, - expand_large_events=expand_large_events, - ) - - gdq = gdq[0, :, :, :] - pdq = pdq[0, :, :] - result.groupdq = gdq - result.pixeldq = pdq - gain_model.close() - readnoise_model.close() - tstop = time.time() - self.log.info("The execution time in seconds: %f", tstop - tstart) - - result.meta.cal_step.jump = "COMPLETE" - - if self.save_results: - try: - self.suffix = "jump" - except AttributeError: - self["suffix"] = "jump" - - return result diff --git a/romancal/jump/tests/__init__.py b/romancal/jump/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/romancal/jump/tests/test_jump_step.py b/romancal/jump/tests/test_jump_step.py deleted file mode 100644 index dc06be40b..000000000 --- a/romancal/jump/tests/test_jump_step.py +++ /dev/null @@ -1,322 +0,0 @@ -""" - Unit tests for the Roman jump step code -""" - -from itertools import cycle - -import numpy as np -import pytest -import roman_datamodels.stnode as rds -from astropy import units as u -from astropy.time import Time -from roman_datamodels import datamodels as rdm -from roman_datamodels import maker_utils -from roman_datamodels.datamodels import GainRefModel, ReadnoiseRefModel - -from romancal.jump import JumpStep - -RNG = np.random.default_rng(42) - -MAXIMUM_CORES = ["none", "quarter", "half", "all"] - - -@pytest.fixture(scope="module") -def generate_wfi_reffiles(tmp_path_factory): - gainfile = str(tmp_path_factory.mktemp("ndata") / "gain.asdf") - readnoisefile = str(tmp_path_factory.mktemp("ndata") / "readnoise.asdf") - - ingain = 6 - xsize = 20 - ysize = 20 - - shape = (ysize, xsize) - - # Create temporary gain reference file - gain_ref = rds.GainRef() - meta = maker_utils.mk_ref_common("GAIN") - meta["instrument"]["detector"] = "WFI01" - meta["instrument"]["name"] = "WFI" - meta["author"] = "John Doe" - meta["pedigree"] = "DUMMY" - meta["description"] = "DUMMY" - meta["useafter"] = Time("2022-01-01T11:11:11.111") - - gain_ref["meta"] = meta - gain_ref["data"] = u.Quantity( - np.ones(shape, dtype=np.float32) * ingain, u.electron / u.DN, dtype=np.float32 - ) - gain_ref["dq"] = np.zeros(shape, dtype=np.uint16) - gain_ref["err"] = u.Quantity( - (RNG.uniform(size=shape) * 0.05).astype(np.float64), - u.electron / u.DN, - dtype=np.float64, - ) - - gain_ref_model = GainRefModel(gain_ref) - gain_ref_model.save(gainfile) - - # Create temporary readnoise reference file - rn_ref = rds.ReadnoiseRef() - meta = maker_utils.mk_ref_common("READNOISE") - meta["instrument"]["detector"] = "WFI01" - meta["instrument"]["name"] = "WFI" - meta["author"] = "John Doe" - meta["pedigree"] = "DUMMY" - meta["description"] = "DUMMY" - meta["useafter"] = Time("2022-01-01T11:11:11.111") - meta["exposure"] = {} - meta["exposure"]["type"] = "WFI_IMAGE" - meta["exposure"]["p_exptype"] = "WFI_IMAGE|WFI_GRISM|WFI_PRISM|" - - rn_ref["meta"] = meta - rn_ref["data"] = u.Quantity( - np.ones(shape, dtype=np.float32), u.DN, dtype=np.float32 - ) - rn_ref["dq"] = np.zeros(shape, dtype=np.uint16) - rn_ref["err"] = u.Quantity( - (RNG.uniform(size=shape) * 0.05).astype(np.float64), u.DN, dtype=np.float64 - ) - - rn_ref_model = ReadnoiseRefModel(rn_ref) - rn_ref_model.save(readnoisefile) - - return gainfile, readnoisefile - - -@pytest.fixture -def setup_inputs(): - def _setup( - nresultants=10, - readnoise=10, - nrows=20, - ncols=20, - gain=1, - deltatime=1, - ): - err = np.ones(shape=(nresultants, nrows, ncols), dtype=np.float32) - data = np.zeros(shape=(nresultants, nrows, ncols), dtype=np.float32) - gdq = np.zeros(shape=(nresultants, nrows, ncols), dtype=np.uint8) - pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32) - - csize = (nresultants, nrows, ncols) - dm_ramp = rdm.RampModel(maker_utils.mk_ramp(shape=csize)) - - dm_ramp.meta.instrument.name = "WFI" - dm_ramp.meta.instrument.optical_element = "F158" - - dm_ramp.data = data + 6.0 - dm_ramp.pixeldq = pixdq - dm_ramp.groupdq = gdq - dm_ramp.err = err - - dm_ramp.meta.exposure.type = "WFI_IMAGE" - dm_ramp.meta.exposure.exposure_time = deltatime - dm_ramp.meta.exposure.frame_time = deltatime - dm_ramp.meta.exposure.nresultants = nresultants - dm_ramp.meta.cal_step["dq_init"] = "INCOMPLETE" - dm_ramp.meta.cal_step["jump"] = "INCOMPLETE" - - return dm_ramp - - return _setup - - -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs): - override_gain, override_readnoise = generate_wfi_reffiles - - grouptime = 3.0 - deltaDN = 5 - ingain = 200 - inreadnoise = 7.0 - nresultants = 100 - CR_fraction = 3 - xsize = 20 - ysize = 20 - - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - for i in range(nresultants): - model1.data[i, :, :] = deltaDN * i - - first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] - - CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0] - CR_x_locs = [x % ysize for x in CR_locs] - CR_y_locs = [int(x / xsize) for x in CR_locs] - CR_pool = cycle(first_CR_group_locs) - - # Add CRs to the SCI data - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000.0 - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - use_ramp_jump_detection=False, - ) - - CR_pool = cycle(first_CR_group_locs) - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - - assert 4 == np.max(out_model.groupdq[CR_group, CR_y_locs[i], CR_x_locs[i]]) - - -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs): - override_gain, override_readnoise = generate_wfi_reffiles - - grouptime = 3.0 - deltaDN = 5 - ingain = 6 - inreadnoise = 7.0 - nresultants = 100 - CR_fraction = 5 - xsize = 20 - ysize = 20 - - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - for i in range(nresultants): - model1.data[i, :, :] = deltaDN * i - - first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] - CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0] - CR_x_locs = [x % ysize for x in CR_locs] - CR_y_locs = [int(x / xsize) for x in CR_locs] - CR_pool = cycle(first_CR_group_locs) - - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000 - ) - model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] + 700 - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - use_ramp_jump_detection=False, - ) - - CR_pool = cycle(first_CR_group_locs) - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - - assert 4 == np.max(out_model.groupdq[CR_group, CR_y_locs[i], CR_x_locs[i]]) - assert 4 == np.max(out_model.groupdq[CR_group + 8, CR_y_locs[i], CR_x_locs[i]]) - - -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_two_group_integration(generate_wfi_reffiles, max_cores, setup_inputs): - override_gain, override_readnoise = generate_wfi_reffiles - grouptime = 3.0 - ingain = 6 - inreadnoise = 7 - nresultants = 2 - xsize = 20 - ysize = 20 - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - ) - - assert out_model.meta.cal_step["jump"] == "SKIPPED" - - -@pytest.mark.parametrize("use_jump", [False, True]) -def test_four_group_integration(generate_wfi_reffiles, setup_inputs, use_jump): - override_gain, override_readnoise = generate_wfi_reffiles - ingain = 6 - inreadnoise = 7 - nresultants = 4 - xsize = 20 - ysize = 20 - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores="none", - use_ramp_jump_detection=use_jump, - ) - - if use_jump: - assert out_model.meta.cal_step["jump"] == "SKIPPED" - else: - assert out_model.meta.cal_step["jump"] == "COMPLETE" - - -@pytest.mark.parametrize("use_jump", [False, True]) -def test_three_group_integration(generate_wfi_reffiles, setup_inputs, use_jump): - override_gain, override_readnoise = generate_wfi_reffiles - grouptime = 3.0 - ingain = 6 - inreadnoise = 7 - nresultants = 3 - xsize = 20 - ysize = 20 - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores="none", - use_ramp_jump_detection=use_jump, - ) - - if use_jump: - assert out_model.meta.cal_step.jump == "SKIPPED" - else: - assert out_model.meta.cal_step.jump == "COMPLETE" diff --git a/romancal/lib/suffix.py b/romancal/lib/suffix.py index 0eef48b74..9f8e9728d 100644 --- a/romancal/lib/suffix.py +++ b/romancal/lib/suffix.py @@ -51,7 +51,6 @@ "dqinit", "assign_wcs", "linearity", - "jump", "rampfit", "saturation", "dark_current", @@ -73,7 +72,6 @@ _calculated_suffixes = { "saturationstep", "darkcurrentstep", - "jumpstep", "rampfit", "dark_current", "assignwcsstep", @@ -89,7 +87,6 @@ "dq_init", "linearitystep", "dark_current", - "jump", "tweakregstep", "outlierdetectionstep", "skymatchstep", diff --git a/romancal/regtest/test_jump_det.py b/romancal/regtest/test_jump_det.py deleted file mode 100644 index 768ee6888..000000000 --- a/romancal/regtest/test_jump_det.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Regression test for the Jump detection step.""" - -import pytest - -from romancal.stpipe import RomanStep - -from .regtestdata import compare_asdf - - -@pytest.mark.bigdata -@pytest.mark.skip(reason="Jump detection step is being removed from the pipeline") -def test_jump_detection_step(rtdata, ignore_asdf_paths): - """Function to run and compare Jump Detection files. Note: This should - include tests for overrides etc.""" - - input_file = "r0000101001001001001_0001_wfi01_darkcurrent.asdf" - rtdata.get_data(f"WFI/image/{input_file}") - rtdata.input = input_file - - # Note: the thresholds should be reset to the defaults once we have better - # input data - args = [ - "romancal.step.JumpStep", - rtdata.input, - "--rejection_threshold=180.", - "--three_group_rejection_threshold=185.", - "--four_group_rejection_threshold=190.", - # disable the jump detection in the rampfit step so this one will run - "--run_ramp_jump_detection=False", - ] - RomanStep.from_cmdline(args) - output = "r0000101001001001001_0001_wfi01_jump.asdf" - rtdata.output = output - rtdata.get_truth(f"truth/WFI/image/{output}") - diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) - assert diff.identical, diff.report() diff --git a/romancal/step.py b/romancal/step.py index 34692c33e..58df99475 100644 --- a/romancal/step.py +++ b/romancal/step.py @@ -8,7 +8,6 @@ from .dq_init.dq_init_step import DQInitStep from .flatfield.flat_field_step import FlatFieldStep from .flux import FluxStep -from .jump.jump_step import JumpStep from .linearity.linearity_step import LinearityStep from .multiband_catalog.multiband_catalog_step import MultibandCatalogStep from .outlier_detection.outlier_detection_step import OutlierDetectionStep @@ -27,7 +26,6 @@ "DQInitStep", "FlatFieldStep", "FluxStep", - "JumpStep", "LinearityStep", "OutlierDetectionStep", "PhotomStep", diff --git a/romancal/stpipe/integration.py b/romancal/stpipe/integration.py index e90f9921b..1251bda7c 100644 --- a/romancal/stpipe/integration.py +++ b/romancal/stpipe/integration.py @@ -27,7 +27,6 @@ def get_steps(): ("romancal.step.DQInitStep", "dq_init", False), ("romancal.step.FlatFieldStep", "flat_field", False), ("romancal.step.FluxStep", "flux", False), - ("romancal.step.JumpStep", "jump", False), ("romancal.step.LinearityStep", "linearity", False), ("romancal.step.PhotomStep", "photom", False), ("romancal.step.RampFitStep", "ramp_fit", False),