From 1788fae1087adfaa212b260dfe55336db09cb6af Mon Sep 17 00:00:00 2001 From: dulude Date: Thu, 5 Oct 2023 13:31:00 -0400 Subject: [PATCH 01/18] Added notebook-level requirements.txt file. --- notebooks/WFC3/calwf3_v1.0_cte/requirements.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 notebooks/WFC3/calwf3_v1.0_cte/requirements.txt diff --git a/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt b/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt new file mode 100644 index 000000000..743605bac --- /dev/null +++ b/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt @@ -0,0 +1,7 @@ +astropy +astroquery +ginga +numpy +photutils +scipy +wfc3tools \ No newline at end of file From 92dd234a7827cc49fde1ba74189770c5fcc2816e Mon Sep 17 00:00:00 2001 From: dulude Date: Thu, 5 Oct 2023 13:31:27 -0400 Subject: [PATCH 02/18] calwf3_with_v1.0_PCTE.ipynb: cleared notebook outputs --- notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index aafb0f594..a60be4def 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -924,7 +924,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.16" + "version": "3.8.12" } }, "nbformat": 4, From faa67a04721db629a44040a7b29502c39d73fea0 Mon Sep 17 00:00:00 2001 From: dulude Date: Thu, 5 Oct 2023 13:31:53 -0400 Subject: [PATCH 03/18] background_median.py: added trailing space --- notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py b/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py index 7e9c6a88d..33df56641 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py +++ b/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py @@ -96,3 +96,4 @@ def calc_aperture_mmm(data, mask, sigma_clip): mode = 3 * median - 2 * mean actual_area = (~np.isnan(values)).sum() return (mean, median, mode, std, actual_area) + From a86c5bad94b74159acb4c47ae325e7bbec0440ba Mon Sep 17 00:00:00 2001 From: dulude Date: Thu, 5 Oct 2023 13:32:42 -0400 Subject: [PATCH 04/18] _config.yml: removed calwf3_with_v1.0_PCTE.ipynb notebook from exclude_list --- _config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/_config.yml b/_config.yml index c2ba711e2..7331af255 100644 --- a/_config.yml +++ b/_config.yml @@ -45,7 +45,6 @@ exclude_patterns: [notebooks/DrizzlePac/align_mosaics/align_mosaics.ipynb, notebooks/DrizzlePac/sky_matching/sky_matching.ipynb, notebooks/DrizzlePac/use_ds9_regions_in_tweakreg/use_ds9_regions_in_tweakreg.ipynb, notebooks/WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb, - notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb, notebooks/WFC3/dash/dash.ipynb, notebooks/WFC3/exception_report/wfc3_exception_report.ipynb, notebooks/WFC3/filter_transformations/filter_transformations.ipynb, From bbea0fc8f526e196ac77919f14fec1854b698601 Mon Sep 17 00:00:00 2001 From: dulude Date: Thu, 5 Oct 2023 13:33:23 -0400 Subject: [PATCH 05/18] _toc.yml: uncommented line pointing to calwf3_with_v1.0_PCTE.ipynb notebook. --- _toc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_toc.yml b/_toc.yml index b381f5262..b3ffb2db1 100644 --- a/_toc.yml +++ b/_toc.yml @@ -60,7 +60,7 @@ parts: chapters: - file: notebooks/WFC3/README.md # - file: notebooks/WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb -# - file: notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb + - file: notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb # - file: notebooks/WFC3/dash/dash.ipynb # - file: notebooks/WFC3/exception_report/wfc3_exception_report.ipynb # - file: notebooks/WFC3/filter_transformations/filter_transformations.ipynb From d2057e97a76679f1abb955f7e2c2c3ce092b45fd Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Fri, 3 Nov 2023 15:22:17 -0400 Subject: [PATCH 06/18] Create pre-requirements.sh This notebook relies on an old version of `hstcal` and therefore an old version of `calwf3`. Every `hstcal` version > 2.5.0 and every corresponding `calwf3` version > 3.5.2 will employ the v2.0 PCTE correction. Since this notebook's purpose is to demonstrate how to use the v1.0 PCTE correction, we must ensure `hstcal` version 2.5.0 is installed and used. I don't believe this version (2.5.0) is offered on conda-forge. --- notebooks/WFC3/calwf3_v1.0_cte/pre-requirements.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 notebooks/WFC3/calwf3_v1.0_cte/pre-requirements.sh diff --git a/notebooks/WFC3/calwf3_v1.0_cte/pre-requirements.sh b/notebooks/WFC3/calwf3_v1.0_cte/pre-requirements.sh new file mode 100644 index 000000000..82294ac5e --- /dev/null +++ b/notebooks/WFC3/calwf3_v1.0_cte/pre-requirements.sh @@ -0,0 +1 @@ +conda install --yes -c http://ssb.stsci.edu/astroconda hstcal==2.5.0 From fe255cf5fcc1c27bc858dc6af22e313ab4913970 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Fri, 3 Nov 2023 15:32:48 -0400 Subject: [PATCH 07/18] Update README.md changing environment creation command to use the `requirements.txt` file --- notebooks/WFC3/calwf3_v1.0_cte/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/README.md b/notebooks/WFC3/calwf3_v1.0_cte/README.md index 0e3469566..1f1b65f9e 100755 --- a/notebooks/WFC3/calwf3_v1.0_cte/README.md +++ b/notebooks/WFC3/calwf3_v1.0_cte/README.md @@ -4,13 +4,13 @@ called `archived_drkcfiles.txt` and a subdirectory `example/`. **In order to run this Jupyter Notebook you must have created a virtual conda environment that includes `calwf3` v3.5.2.** Version 3.5.2 of `calwf3` -is available in HSTCAL release 2.5.0. To create an environment with +is available in `hstcal` release 2.5.0. To create an environment with `calwf3` v3.5.2 try this from the terminal: ``` $ conda config --add channels http://ssb.stsci.edu/astroconda -$ conda create -n v1_PCTE hstcal==2.5.0 python=3.7 ginga stsci-hst notebook +$ conda create -n v1_PCTE hstcal==2.5.0 --file requirements.txt ``` In general, users wanting to use the v1.0 pixel-based CTE correction From e7ad4962ba642f69230f4788a801f1ed64bba838 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Fri, 3 Nov 2023 15:35:55 -0400 Subject: [PATCH 08/18] Update background_median.py PEP8 compliance --- .../calwf3_v1.0_cte/example/background_median.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py b/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py index 33df56641..38d93d424 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py +++ b/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py @@ -16,13 +16,12 @@ See the docstring of aperture_stats_tbl for more info. """ import numpy as np - # WAY faster than astropy.stats.sigma_clipped_stats from scipy.stats import sigmaclip from astropy.table import Table -def aperture_stats_tbl(data, apertures, - method='exact', sigma_clip=True): +def aperture_stats_tbl(data, apertures, method='exact', sigma_clip=True): + """Computes mean/median/mode/std in Photutils apertures. Compute statistics for custom local background methods. This is primarily intended for estimating backgrounds @@ -63,23 +62,23 @@ def aperture_stats_tbl(data, apertures, aperture_stats = np.array(aperture_stats) - # Place the array of the x y positions alongside the stats stacked = np.hstack([apertures.positions, aperture_stats]) # Name the columns - names = ['X','Y','aperture_mean','aperture_median','aperture_mode', - 'aperture_std', 'aperture_area'] + names = ['X', 'Y', 'aperture_mean', 'aperture_median', + 'aperture_mode', 'aperture_std', 'aperture_area'] # Make the table stats_tbl = Table(data=stacked, names=names) - return stats_tbl def calc_aperture_mmm(data, mask, sigma_clip): + """Helper function to actually calculate the stats for pixels falling within some Photutils aperture mask on some array of data. """ + cutout = mask.cutout(data, fill_value=np.nan) if cutout is None: return (np.nan, np.nan, np.nan, np.nan, np.nan) @@ -95,5 +94,5 @@ def calc_aperture_mmm(data, mask, sigma_clip): mode = 3 * median - 2 * mean actual_area = (~np.isnan(values)).sum() + return (mean, median, mode, std, actual_area) - From 0cc1a197b3ad4a3a372cef827039910be385a021 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Fri, 3 Nov 2023 15:39:12 -0400 Subject: [PATCH 09/18] Update requirements.txt pinning the packages to the versions in my environment where the notebook runs successfully --- notebooks/WFC3/calwf3_v1.0_cte/requirements.txt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt b/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt index 743605bac..2be162929 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt +++ b/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt @@ -1,7 +1,8 @@ -astropy -astroquery -ginga -numpy -photutils -scipy -wfc3tools \ No newline at end of file +astropy==4.3.1 +astroquery==0.4 +ginga==3.2.0 +matplotlib==3.5.3 +numpy==1.21.5 +photutils==1.1.0 +scipy==1.7.3 +wfc3tools==1.3.5 From d6abf2f8026cdae7775ec83b78f955054f7ad531 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Fri, 3 Nov 2023 17:19:18 -0400 Subject: [PATCH 10/18] Update calwf3_with_v1.0_PCTE.ipynb mostly minor edits to update the notebook for being hosted in hst_notebooks repo --- .../calwf3_with_v1.0_PCTE.ipynb | 69 ++++++++++++++----- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index a60be4def..583d3456f 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -50,7 +50,8 @@ "\n", "[Additional Resources](#resources)
\n", "[About the Notebook](#about)
\n", - "[Citations](#cite) \n" + "[Citations](#cite) \n", + "\n" ] }, { @@ -61,11 +62,10 @@ "\n", "## Introduction \n", "\n", - "The v1.0 pixel-based Charge Transfer Efficiency (CTE) correction was first implemented into `calwf3` `v3.3` in 2016 ([Ryan et al. 2016](https://ui.adsabs.harvard.edu/abs/2016wfc..rept....1R/abstract),
[Anderson & Bedin 2010](https://ui.adsabs.harvard.edu/abs/2010PASP..122.1035A/abstract), [HSTCAL release notes](https://github.com/spacetelescope/hstcal/releases/tag/1.0.0)). This also marked the first time users could directly download CTE-corrected `flc & drc`
files from [MAST](https://mast.stsci.edu/search/hst/ui/#/). While the v1.0 correction was sufficient for many years, the degradation of CTE over time reduced the efficacy of the model
in treating low-level pixels. The v1.0 correction adversely impacts (overcorrects) both the image background and faint sources. In April 2021
the v2.0 pixel-based CTE correction was implemented in `calwf3` `v3.6.0` ([Anderson et al. 2021](https://ui.adsabs.harvard.edu/abs/2021wfc..rept....9A/abstract), [Kuhn & Anderson 2021](https://ui.adsabs.harvard.edu/abs/2021wfc..rept....6K/abstract), [HSTCAL release
notes](https://github.com/spacetelescope/hstcal/releases/tag/2.7.0)). Since MAST uses the latest release of `calwf3` for calibration, any WFC3/UVIS CTE corrected data retrieved from MAST, regardless of
observation date, will be calibrated with the v2.0 pixel-based CTE correction. Although v1.0 pixel-based CTE-corrected `flc & drc` files are
no longer accessable through MAST, this notebook steps through the procedure required to calibrate WFC3/UVIS images using the v1.0 CTE
correction.
\n", + "The v1.0 pixel-based Charge Transfer Efficiency (CTE) correction was first implemented into `calwf3` `v3.3` in 2016
([Ryan et al. 2016](https://ui.adsabs.harvard.edu/abs/2016wfc..rept....1R/abstract), [Anderson & Bedin 2010](https://ui.adsabs.harvard.edu/abs/2010PASP..122.1035A/abstract), [HSTCAL release notes](https://github.com/spacetelescope/hstcal/releases/tag/1.0.0)). This also marked the first time users could directly
download CTE-corrected `flc & drc` files from [MAST](https://mast.stsci.edu/search/hst/ui/#/). While the v1.0 correction was sufficient for many years, the
degradation of CTE over time reduced the efficacy of the model in treating low-level pixels. The v1.0 correction adversely
impacts (overcorrects) both the image background and faint sources. In April 2021 the v2.0 pixel-based CTE correction
was implemented in `calwf3` `v3.6.0` ([Anderson et al. 2021](https://ui.adsabs.harvard.edu/abs/2021wfc..rept....9A/abstract), [Kuhn & Anderson 2021](https://ui.adsabs.harvard.edu/abs/2021wfc..rept....6K/abstract), [HSTCAL release notes](https://github.com/spacetelescope/hstcal/releases/tag/2.7.0)). Since
MAST uses the latest release of `calwf3` for calibration, any WFC3/UVIS CTE corrected data retrieved from MAST,
regardless of observation date, will be calibrated with the v2.0 pixel-based CTE correction. Although v1.0 pixel-based
CTE-corrected `flc & drc` files are no longer accessable through MAST, this notebook steps through the procedure
required to calibrate WFC3/UVIS images using the v1.0 CTE correction.
\n", "\n", "\n", - "One of the limiting factors of using the v1.0 CTE correction are the CTE corrected dark current reference files (`DRKCFILE`). These dark
reference files are delivered to MAST by the WFC3 team and use the same pixel-based CTE correction within `calwf3`. Now that we
have switched to the v2.0 CTE correction there is a cut off for dark current reference files that use the v1.0 correction. Observations taken
after February 2021 will not have\n", - "CTE corrected dark files using the v1.0 algorithm, which means **applying the v1.0 CTE correction works
best for observations taken between May 2009 - February 2021.** If the observation being calibrated was taken after February 2021 there
are two options: **1)** use the last v1.0 CTE corrected dark reference file from February 2021 or **2)** use the v2.0 CTE corrected dark with the
most appropriate `USEAFTER` for the science exposure's observation date. " + "One of the limiting factors of using the v1.0 CTE correction are the CTE corrected dark current reference files (`DRKCFILE`).
These dark reference files are delivered to MAST by the WFC3 team and use the same pixel-based CTE correction within
`calwf3`. Now that we have switched to the v2.0 CTE correction there is a cut off for dark current reference files that use
the v1.0 correction. Observations taken after February 2021 will not have CTE corrected dark files using the v1.0 algorithm,
which means **applying the v1.0 CTE correction works best for observations taken between May 2009 - February 2021.**
If the observation being calibrated was taken after February 2021 there are two options: **1)** use the last v1.0 CTE corrected
dark reference file from February 2021 or **2)** use the v2.0 CTE corrected dark with the most appropriate `USEAFTER` for the
science exposure's observation date. " ] }, { @@ -75,7 +75,9 @@ "\n", "## 1. Imports\n", "\n", - "This notebook assumes you have created the virtual environment in [WFC3 Library's](https://github.com/spacetelescope/WFC3Library) installation instructions.\n", + "
This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository..
\n", + "\n", + "\n", "\n", "We import:
\n", "
\n", @@ -105,15 +107,15 @@ "outputs": [], "source": [ "import glob\n", - "import numpy as np\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "import os\n", "import shutil\n", "\n", "from astropy.io import fits\n", - "from astroquery.mast import Observations\n", "from astropy.table import Table\n", "from astropy.time import Time\n", + "from astroquery.mast import Observations\n", "from ginga.util import zscale\n", "from photutils.aperture import aperture_photometry, CircularAperture, CircularAnnulus\n", "from wfc3tools import calwf3\n", @@ -127,7 +129,7 @@ "source": [ "\n", "## 2. Verify `archived_drkcfiles.txt` is in CWD\n", - "When you cloned/downloaded this notebook from [WFC3 Library](https://github.com/spacetelescope/WFC3Library), a .txt file should have been included. The file name is
`archived_drkcfiles.txt` and it is used later on in the notebook. This .txt file includes the file name, delivery date, activation
date, and USEAFTER date for every v1.0 CTE corrected dark reference file between May 2009 - February 2021. Below, we will use
this file in conjunction with the observation date of the file(s) being calibrated to pick out the most appropriate v1.0 CTE corrected
dark reference file(s).
\n", + "When you cloned/downloaded this notebook from [hst_notebooks](https://github.com/spacetelescope/hst_notebooks/), a .txt file should have been included. The file name is
`archived_drkcfiles.txt` and it is used later on in the notebook. This .txt file includes the file name, delivery date, activation
date, and USEAFTER date for every v1.0 CTE corrected dark reference file between May 2009 - February 2021. Below, we will use
this file in conjunction with the observation date of the file(s) being calibrated to pick out the most appropriate v1.0 CTE corrected
dark reference file(s).
\n", "\n", "Please make sure the file is in the current working directory before continuing." ] @@ -149,10 +151,10 @@ "\n", "## 3. Check that `calwf3` Version is `v3.5.2`\n", "In April 2021, a new `calwf3` version was released that contains the v2.0 CTE-correction.\n", - "If you would like to use the v1.0 correction, your
current environment must be using `calwf3` versions equal to or between `3.3` - `3.5.2`. However, in order to get the best v1.0 calibrated
images we must use `calwf3` `v3.5.2`. This version of `calwf3` includes the recent ([~Jan 2021](https://github.com/spacetelescope/hstcal/releases/tag/2.5.0)) update that added MJD as a parameterized
variable for the `PHOTMODE` keyword, which enables a time-dependent photometric correction and zeropoint. If your version is `3.6.0` or higher,
you must downgrade the [`hstcal` package](https://github.com/spacetelescope/hstcal). The safer option, however, is to create a new environment such as:
\n", + "If you would like to use the v1.0 correction, your
current environment must be using `calwf3` versions equal to or between `3.3` - `3.5.2`. However, in order to get the best v1.0 calibrated
images we must use `calwf3` `v3.5.2`. This version of `calwf3` includes the recent ([~Jan 2021](https://github.com/spacetelescope/hstcal/releases/tag/2.5.0)) update that added MJD as a parameterized
variable for the `PHOTMODE` keyword, which enables a time-dependent photometric correction and zeropoint. If your version is `3.6.0` or higher,
you must downgrade the [`hstcal` package](https://github.com/spacetelescope/hstcal). The safer option, however, is to create a new environment using the requirements file provided in the [notebook's repository](https://github.com/spacetelescope/hst_notebooks/tree/main/notebooks/WFC3/calwf3_v1.0_cte):
\n", "\n", "* `$ conda config --add channels http://ssb.stsci.edu/astroconda`\n", - "* `$ conda create -n v1_PCTE hstcal==2.5.0 python=3.7 ginga stsci-hst notebook`\n", + "* `$ conda create -n v1_PCTE hstcal==2.5.0 --file requirements.txt`\n", "\n", "`hstcal` `v2.5.0` provides version `3.5.2` of `calwf3`, which is the last version that offers the v1.0 pixel-based CTE correction.\n" ] @@ -317,7 +319,7 @@ "source": [ "# Generate table of filename, date-obs, drkc-filename, corresponding useafter\n", "raw_dkc_tab = Table([rawfiles, obsdates, dkcfiles, uafters, active_dates],\n", - " names=('filename','date-obs','dkc-filename','dkc-useafter','dkc-activation'))\n", + " names=('filename', 'date-obs', 'dkc-filename', 'dkc-useafter', 'dkc-activation'))\n", "# Display table\n", "raw_dkc_tab\n" ] @@ -341,7 +343,7 @@ "metadata": {}, "outputs": [], "source": [ - " for file in glob.glob('i*raw.fits'):\n", + "for file in glob.glob('i*raw.fits'):\n", " # Using raw_dkc_tab from above, grab appropriate drkcfile\n", " ctecorr_dark = 'iref$'+raw_dkc_tab[raw_dkc_tab['filename']==file]['dkc-filename'][0]\n", " fits.setval(file, 'DRKCFILE', value = ctecorr_dark)\n" @@ -372,7 +374,7 @@ "\n", "## 7. Re-Inspect Image Header \n", "Now with the headers modified, we inspect the keywords one last time to verify the file was updated properly before we process
it through `calwf3`. At this point you should see:
\n", - "- `PCTETAB` set to `iref$zcv2057mi_cte.fits\t`
\n", + "- `PCTETAB` set to `iref$zcv2057mi_cte.fits`
\n", "- `DRKCFILE` set to `iref$3961719li_dkc.fits`
" ] }, @@ -619,8 +621,7 @@ "source": [ "
Animated GIF of the v1.0 and v2.0 FLC image subsections:
\n", "\n", - "\n", - "\n" + "\"An\n" ] }, { @@ -886,9 +887,9 @@ "\n", "## About this Notebook\n", "\n", - "**Author:** Benjamin Kuhn; WFC3 Instrument Team\n", - "\n", - "**Updated on:** January 19, 2023\n", + "**Author:** Benjamin Kuhn; WFC3 Instrument Team
\n", + "**Created:** February 22, 2022
\n", + "**Last updated on:** November 3, 2023\n", "\n", "\n", "## Citations\n", @@ -901,6 +902,7 @@ "* [Citing `matplotlib`](https://matplotlib.org/stable/users/project/citing.html)\n", "* [Citing `numpy`](https://numpy.org/citing-numpy/)\n", "* [Citing `photutils`](https://photutils.readthedocs.io/en/stable/citation.html)\n", + "* [Citing `scipy`](https://scipy.org/citing-scipy/)\n", "
\n", "***\n", "[Top of Page](#top)\n", @@ -924,7 +926,36 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.7.16" + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false } }, "nbformat": 4, From cd2c01cda4fb6140f287ca5cd1376c9ed16a4c8d Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Mon, 6 Nov 2023 16:59:04 -0500 Subject: [PATCH 11/18] Update calwf3_with_v1.0_PCTE.ipynb updates for PEP8 compliance --- .../calwf3_with_v1.0_PCTE.ipynb | 245 +++++++++--------- 1 file changed, 118 insertions(+), 127 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index 583d3456f..c7e823143 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -75,29 +75,24 @@ "\n", "## 1. Imports\n", "\n", - "
This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository..
\n", - "\n", - "\n", + "
This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository.
\n", "\n", "We import:
\n", - "
\n", - "**•** *glob* for creating list of files
\n", - "**•** *matplotlib.pyplot* for plotting and displaying images
\n", - "**•** *numpy* for finding indices and concatenating arrays
\n", - "**•** *os* for setting environment variables
\n", - "**•** *shutil* removing an empty directory
\n", - "\n", - "**•** *astropy.io.fits* for opening and modifying fits files
\n", - "**•** *astroquery.mast.Observations* for downloading data from MAST
\n", - "**•** *astropy.table.Table* for creating and manipulating data tables
\n", - "**•** *astropy.time.Time* for converting between time formats
\n", - "**•** *ginga.util.zscale* for finding scale limits when displaying images
\n", - "**•** *photutils.aperture.aperture_photometry* for performing aperture photometry
\n", - "**•** *photutils.aperture.CircularAperture* for creating circular apertures
\n", - "**•** *photutils.aperture.CircularAnnulus* for creating circular annuli
\n", - "**•** *wfc3tools.calwf3* for verifying the version and running pipeline
\n", - "\n", - "**•** *background_median.aperture_stats_tbl* for measuring background values within annuli
" + "| Package Name | Purpose |\n", + "|:---------------------------------------|:--------------------------------------------|\n", + "| `glob` | creating list of files |\n", + "| `matplotlib.pyplot` | plotting and displaying images |\n", + "| `numpy` | finding indices and concatenating arrays |\n", + "| `os` | directory maintenance and setting environment variables|\n", + "| `astropy.io.fits` | opening and modifying fits files |\n", + "| `astroquery.mast.Observations` | downloading data from MAST |\n", + "| `astropy.table.Table` | creating and manipulating data tables |\n", + "| `ginga.util.zscale` | finding scale limits when displaying images |\n", + "| `photutils.aperture.aperture_photometry` | performing aperture photometry |\n", + "| `photutils.aperture.CircularAperture` | creating circular apertures |\n", + "| `photutils.aperture.CircularAnnulus` | creating circular annuli |\n", + "| `wfc3tools.calwf3` | verifying the version and running pipeline |\n", + "| `background_median.aperture_stats_tbl` | measuring background values within annuli |" ] }, { @@ -110,11 +105,9 @@ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", - "import shutil\n", "\n", "from astropy.io import fits\n", "from astropy.table import Table\n", - "from astropy.time import Time\n", "from astroquery.mast import Observations\n", "from ginga.util import zscale\n", "from photutils.aperture import aperture_photometry, CircularAperture, CircularAnnulus\n", @@ -141,7 +134,7 @@ "outputs": [], "source": [ "# list cwd to verify txt file is there\n", - "!ls -l archived_drkcfiles.txt\n" + "!ls -l archived_drkcfiles.txt" ] }, { @@ -191,7 +184,7 @@ "source": [ "# Edit this cell's first line if you would to download your own file(s)\n", "# Get the observation records\n", - "obs_table = Observations.query_criteria(obs_id='idv404axq*',proposal_id=15576)\n", + "obs_table = Observations.query_criteria(obs_id='idv404axq*', proposal_id=15576)\n", "\n", "# Get the listing of data products\n", "products = Observations.get_product_list(obs_table)\n", @@ -209,8 +202,7 @@ " os.rmdir('mastDownload/HST/'+filename[:9])\n", " \n", "os.rmdir('mastDownload/HST/')\n", - "os.rmdir('mastDownload/')\n", - " " + "os.rmdir('mastDownload/')" ] }, { @@ -220,7 +212,7 @@ "outputs": [], "source": [ "# show list of current dir to verify fits file is there\n", - "!ls -l *raw.fits\n" + "!ls -l *raw.fits" ] }, { @@ -243,7 +235,7 @@ "outputs": [], "source": [ "# Collect header keyword info from raw file\n", - "file, date, expstart, pctetab, drkcfile, pctecorr = [],[],[],[],[],[]\n", + "file, date, expstart, pctetab, drkcfile, pctecorr = [], [], [], [], [], []\n", "for f in glob.glob('*raw.fits'):\n", " h = fits.getheader(f)\n", " file.append(h['filename'])\n", @@ -254,7 +246,7 @@ " pctecorr.append(h['pctecorr'])\n", "\n", "image_table = Table([file, date, expstart, pctetab, drkcfile, pctecorr],\n", - " names=('file','date-obs', 'expstart', 'pctetab', 'drkcfile', 'pctecorr'))\n", + " names=('file', 'date-obs', 'expstart', 'pctetab', 'drkcfile', 'pctecorr'))\n", "image_table['expstart'].format = '5.6f'\n", "\n", "# Sort and display the table\n", @@ -281,10 +273,10 @@ "outputs": [], "source": [ "# Generate astropy table from `archived_drkcfiles.txt`\n", - "drkc_table = Table.read('archived_drkcfiles.txt',format='ascii.commented_header')\n", + "drkc_table = Table.read('archived_drkcfiles.txt', format='ascii.commented_header')\n", "\n", "# Create empty lists for final astropy table\n", - "rawfiles, obsdates, dkcfiles, uafters, active_dates = [],[],[],[], []\n" + "rawfiles, obsdates, dkcfiles, uafters, active_dates = [], [], [], [], []\n" ] }, { @@ -297,18 +289,17 @@ "for expstart in image_table['expstart']:\n", " table_idx = np.where(abs(drkc_table['useafter-mjd']-expstart) == abs(drkc_table['useafter-mjd']-expstart).min())[0][0]\n", "\n", - " rawfile = image_table[image_table['expstart']==expstart]['file'][0]\n", + " rawfile = image_table[image_table['expstart'] == expstart]['file'][0]\n", " \n", " # if drkcfile has useafter date > rawfile expstart use previous drkcfile\n", - " if drkc_table[table_idx]['useafter-mjd'] > image_table[image_table['file']==rawfile]['expstart'][0]:\n", + " if drkc_table[table_idx]['useafter-mjd'] > image_table[image_table['file'] == rawfile]['expstart'][0]:\n", " table_idx -= 1\n", - " #append info\n", + " # append info\n", " rawfiles.append(rawfile)\n", - " obsdates.append(image_table[image_table['file']==rawfile]['date-obs'][0])\n", + " obsdates.append(image_table[image_table['file'] == rawfile]['date-obs'][0])\n", " dkcfiles.append(drkc_table[table_idx]['drkcfile'])\n", " uafters.append(drkc_table[table_idx]['useafter'])\n", - " active_dates.append(drkc_table[table_idx]['activation-date'])\n", - " " + " active_dates.append(drkc_table[table_idx]['activation-date'])" ] }, { @@ -321,7 +312,7 @@ "raw_dkc_tab = Table([rawfiles, obsdates, dkcfiles, uafters, active_dates],\n", " names=('filename', 'date-obs', 'dkc-filename', 'dkc-useafter', 'dkc-activation'))\n", "# Display table\n", - "raw_dkc_tab\n" + "raw_dkc_tab" ] }, { @@ -345,8 +336,8 @@ "source": [ "for file in glob.glob('i*raw.fits'):\n", " # Using raw_dkc_tab from above, grab appropriate drkcfile\n", - " ctecorr_dark = 'iref$'+raw_dkc_tab[raw_dkc_tab['filename']==file]['dkc-filename'][0]\n", - " fits.setval(file, 'DRKCFILE', value = ctecorr_dark)\n" + " ctecorr_dark = 'iref$'+raw_dkc_tab[raw_dkc_tab['filename'] == file]['dkc-filename'][0]\n", + " fits.setval(file, 'DRKCFILE', value=ctecorr_dark)" ] }, { @@ -364,7 +355,7 @@ "metadata": {}, "outputs": [], "source": [ - "fits.setval('idv404axq_raw.fits', 'PCTETAB', value = 'iref$zcv2057mi_cte.fits') \n" + "fits.setval('idv404axq_raw.fits', 'PCTETAB', value='iref$zcv2057mi_cte.fits') " ] }, { @@ -385,7 +376,7 @@ "outputs": [], "source": [ "# Recollect and display header keywords\n", - "file, date, expstart, pctetab, drkcfile, pctecorr = [],[],[],[],[],[]\n", + "file, date, expstart, pctetab, drkcfile, pctecorr = [], [], [], [], [], []\n", "for f in glob.glob('*raw.fits'):\n", " h = fits.getheader(f)\n", " file.append(h['filename'])\n", @@ -396,7 +387,7 @@ " pctecorr.append(h['pctecorr'])\n", "\n", "updated_table = Table([file, date, expstart, pctetab, drkcfile, pctecorr],\n", - " names=('file','date-obs', 'expstart', 'pctetab', 'drkcfile', 'pctecorr'))\n", + " names=('file', 'date-obs', 'expstart', 'pctetab', 'drkcfile', 'pctecorr'))\n", "updated_table['expstart'].format = '5.6f'\n", "\n", "# Sort and display the table\n", @@ -430,7 +421,7 @@ "os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'\n", "os.environ['CRDS_SERVER'] = 'https://hst-crds.stsci.edu'\n", "os.environ['CRDS_PATH'] = '~/crds_cache'\n", - "os.environ['iref'] = '~/crds_cache/references/hst/iref/'\n" + "os.environ['iref'] = '~/crds_cache/references/hst/iref/'" ] }, { @@ -450,7 +441,7 @@ }, "outputs": [], "source": [ - "calwf3('idv404axq_raw.fits')\n" + "calwf3('idv404axq_raw.fits')" ] }, { @@ -460,7 +451,7 @@ "outputs": [], "source": [ "# show list of cwd to verify calibrated files were made\n", - "!ls -ltr *.fits\n" + "!ls -ltr *.fits" ] }, { @@ -478,8 +469,8 @@ "metadata": {}, "outputs": [], "source": [ - "#Recollect and display FLC header keywords\n", - "file, pctetab, drkcfile, pctecorr, calver, ctename, ctever = [],[],[],[],[],[],[]\n", + "# Recollect and display FLC header keywords\n", + "file, pctetab, drkcfile, pctecorr, calver, ctename, ctever = [], [], [], [], [], [], []\n", "for f in glob.glob('*flc.fits'):\n", " h = fits.getheader(f)\n", " file.append(h['filename'])\n", @@ -491,7 +482,7 @@ " ctever.append(h['cte_ver'])\n", "\n", "final_table = Table([file, pctetab, drkcfile, pctecorr, calver, ctename, ctever],\n", - " names=('file','pctetab', 'drkcfile', 'pctecorr','cal_ver','cte_name','cte_ver'))\n", + " names=('file', 'pctetab', 'drkcfile', 'pctecorr', 'cal_ver', 'cte_name', 'cte_ver'))\n", "\n", "final_table" ] @@ -516,13 +507,13 @@ "outputs": [], "source": [ "# Get the observation records\n", - "obs_table = Observations.query_criteria(obs_id='idv404axq*',proposal_id=15576)\n", + "obs_table = Observations.query_criteria(obs_id='idv404axq*', proposal_id=15576)\n", "\n", "# Get the listing of data products\n", "products = Observations.get_product_list(obs_table)\n", "\n", "# Filter the products for the RAW files\n", - "filtered_products = Observations.filter_products(products, productSubGroupDescription='FLC',project='CALWF3')\n", + "filtered_products = Observations.filter_products(products, productSubGroupDescription='FLC', project='CALWF3')\n", "\n", "# Download all the images above\n", "download_table = Observations.download_products(filtered_products, mrp_only=False)\n", @@ -568,9 +559,9 @@ "PAM_uvis2 = fits.getdata('example/UVIS2wfc3_map.fits')\n", "\n", "# Stich UVIS1 and 2 together and multiply by pixel area map\n", - "v2sci = np.concatenate([v2_uvis2*PAM_uvis2 ,v2_uvis1*PAM_uvis1])\n", - "v1sci = np.concatenate([v1_uvis2*PAM_uvis2 ,v1_uvis1*PAM_uvis1])\n", - "fltsci = np.concatenate([flt_uvis2*PAM_uvis2,flt_uvis1*PAM_uvis1])\n" + "v2sci = np.concatenate([v2_uvis2*PAM_uvis2, v2_uvis1*PAM_uvis1])\n", + "v1sci = np.concatenate([v1_uvis2*PAM_uvis2, v1_uvis1*PAM_uvis1])\n", + "fltsci = np.concatenate([flt_uvis2*PAM_uvis2, flt_uvis1*PAM_uvis1])" ] }, { @@ -589,30 +580,30 @@ "outputs": [], "source": [ "# Generate subplots\n", - "fig, [ax1,ax2,ax3] = plt.subplots(1,3,figsize=(15,10),dpi=150)\n", + "fig, [ax1, ax2, ax3] = plt.subplots(1, 3, figsize=(15, 10), dpi=150)\n", "\n", "# Generate background subsections\n", - "flt_bkg = fltsci[2070:2120,2180:2230]\n", - "v1_bkg = v1sci[2070:2120,2180:2230]\n", - "v2_bkg = v2sci[2070:2120,2180:2230]\n", + "flt_bkg = fltsci[2070:2120, 2180:2230]\n", + "v1_bkg = v1sci[2070:2120, 2180:2230]\n", + "v2_bkg = v2sci[2070:2120, 2180:2230]\n", "\n", "# Calculate min and max values for image scaling \n", - "v1z1,v1z2 = zscale.zscale(v1_bkg)\n", + "v1z1, v1z2 = zscale.zscale(v1_bkg)\n", "\n", "# Display background subsection\n", - "im1 = ax1.imshow(flt_bkg,origin='lower',cmap='Greys_r',vmin=v1z1,vmax=v1z2)\n", - "im2 = ax2.imshow(v1_bkg,origin='lower',cmap='Greys_r',vmin=v1z1,vmax=v1z2)\n", - "im3 = ax3.imshow(v2_bkg,origin='lower',cmap='Greys_r',vmin=v1z1,vmax=v1z2)\n", + "im1 = ax1.imshow(flt_bkg, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", + "im2 = ax2.imshow(v1_bkg, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", + "im3 = ax3.imshow(v2_bkg, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", "\n", "# Formatting\n", - "fig.colorbar(im1,ax=ax1,shrink=0.35,pad=0.02)\n", - "fig.colorbar(im2,ax=ax2,shrink=0.35,pad=0.02)\n", - "fig.colorbar(im3,ax=ax3,shrink=0.35,pad=0.02)\n", - "ax1.set_title('FLT File BKG Subsection',size=14)\n", - "ax2.set_title('v1.0 PCTE FLC BKG Subsection',size=14)\n", - "ax3.set_title('v2.0 PCTE FLC BKG Subsection',size=14)\n", - "ax1.axis('off'),ax2.axis('off'),ax3.axis('off')\n", - "fig.tight_layout()\n" + "fig.colorbar(im1, ax=ax1, shrink=0.35, pad=0.02)\n", + "fig.colorbar(im2, ax=ax2, shrink=0.35, pad=0.02)\n", + "fig.colorbar(im3, ax=ax3, shrink=0.35, pad=0.02)\n", + "ax1.set_title('FLT File BKG Subsection', size=14)\n", + "ax2.set_title('v1.0 PCTE FLC BKG Subsection', size=14)\n", + "ax3.set_title('v2.0 PCTE FLC BKG Subsection', size=14)\n", + "ax1.axis('off'), ax2.axis('off'), ax3.axis('off')\n", + "fig.tight_layout()" ] }, { @@ -640,31 +631,31 @@ "outputs": [], "source": [ "# Generate subplots\n", - "fig, [ax1,ax2] = plt.subplots(2,1,figsize=(7,10),dpi=120)\n", + "fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(7, 10), dpi=120)\n", "\n", "# Plot background subsection histograms\n", - "ax1.hist(flt_bkg.ravel(),bins=100,range=(-30,100),histtype='step',color='C3',label='FLT')\n", - "ax1.hist(v1_bkg.ravel(),bins=100,range=(-30,100),histtype='step',color='C0',label='v1.0 FLC')\n", - "ax1.hist(v2_bkg.ravel(),bins=100,range=(-30,100),histtype='step',color='k',label='v2.0 FLC')\n", + "ax1.hist(flt_bkg.ravel(), bins=100, range=(-30, 100), histtype='step', color='C3', label='FLT')\n", + "ax1.hist(v1_bkg.ravel(), bins=100, range=(-30, 100), histtype='step', color='C0', label='v1.0 FLC')\n", + "ax1.hist(v2_bkg.ravel(), bins=100, range=(-30, 100), histtype='step', color='k', label='v2.0 FLC')\n", "\n", "# Plot background subsection differential histograms\n", - "ax2.hist((v1_bkg-flt_bkg).ravel(),bins=100,range=(-25,50),histtype='step',color='magenta',label='v1.0 FLC $-$ FLT')\n", - "ax2.hist((v2_bkg-flt_bkg).ravel(),bins=100,range=(-25,50),histtype='step',color='limegreen',label='v2.0 FLC $-$ FLT')\n", - "ax2.hist((v1_bkg-v2_bkg).ravel(),bins=100,range=(-25,50),histtype='step',color='C9',label='v1.0 $-$ v2.0 FLC')\n", + "ax2.hist((v1_bkg-flt_bkg).ravel(), bins=100, range=(-25, 50), histtype='step', color='magenta', label='v1.0 FLC $-$ FLT')\n", + "ax2.hist((v2_bkg-flt_bkg).ravel(), bins=100, range=(-25, 50), histtype='step', color='limegreen', label='v2.0 FLC $-$ FLT')\n", + "ax2.hist((v1_bkg-v2_bkg).ravel(), bins=100, range=(-25, 50), histtype='step', color='C9', label='v1.0 $-$ v2.0 FLC')\n", "\n", "# Formatting\n", - "ax1.set_title('Background Subsection Histogram',size=14)\n", - "ax2.set_title('Background Subsection Differential Histogram',size=14)\n", - "ax1.set_xlabel('Pixel Value [e-]',size=12)\n", - "ax1.set_ylabel('Frequency',size=12)\n", - "ax2.set_xlabel('Pixel Value [e-]',size=12)\n", - "ax2.set_ylabel('Frequency',size=12)\n", - "ax1.grid(alpha=0.5),ax2.grid(alpha=0.5)\n", + "ax1.set_title('Background Subsection Histogram', size=14)\n", + "ax2.set_title('Background Subsection Differential Histogram', size=14)\n", + "ax1.set_xlabel('Pixel Value [e-]', size=12)\n", + "ax1.set_ylabel('Frequency', size=12)\n", + "ax2.set_xlabel('Pixel Value [e-]', size=12)\n", + "ax2.set_ylabel('Frequency', size=12)\n", + "ax1.grid(alpha=0.5), ax2.grid(alpha=0.5)\n", "ax1.legend(), ax2.legend()\n", "ax1.set_yscale('log')\n", "ax2.set_yscale('log')\n", "\n", - "fig.tight_layout()\n" + "fig.tight_layout()" ] }, { @@ -685,23 +676,23 @@ "outputs": [], "source": [ "# Generate subplots\n", - "fig, [ax1,ax2] = plt.subplots(2,1,figsize=(8,8),dpi=150)\n", + "fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(8, 8), dpi=150)\n", "\n", "# Calculate min and max values for image scaling \n", - "v1z1,v1z2 = zscale.zscale(v1_uvis1)\n", + "v1z1, v1z2 = zscale.zscale(v1_uvis1)\n", "\n", "# Display subsection\n", - "im1 = ax1.imshow(v1_uvis1,origin='lower',cmap='Greys_r',vmin=v1z1,vmax=v1z2)\n", - "im2 = ax2.imshow(v2_uvis1,origin='lower',cmap='Greys_r',vmin=v1z1,vmax=v1z2)\n", + "im1 = ax1.imshow(v1_uvis1, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", + "im2 = ax2.imshow(v2_uvis1, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", "\n", "# Formatting\n", - "ax1.set_xlim(85,325),ax2.set_xlim(85,325)\n", - "ax1.set_ylim(0,149),ax2.set_ylim(0,149)\n", - "fig.colorbar(im1,ax=ax1,shrink=0.95,pad=0.01)\n", - "fig.colorbar(im2,ax=ax2,shrink=0.95,pad=0.01)\n", - "ax1.set_title('v1.0 PCTE FLC UVIS1 Subsection',size=14)\n", - "ax2.set_title('v2.0 PCTE FLC UVIS1 Subsection',size=14)\n", - "fig.tight_layout()\n" + "ax1.set_xlim(85, 325), ax2.set_xlim(85, 325)\n", + "ax1.set_ylim(0, 149), ax2.set_ylim(0, 149)\n", + "fig.colorbar(im1, ax=ax1, shrink=0.95, pad=0.01)\n", + "fig.colorbar(im2, ax=ax2, shrink=0.95, pad=0.01)\n", + "ax1.set_title('v1.0 PCTE FLC UVIS1 Subsection', size=14)\n", + "ax2.set_title('v2.0 PCTE FLC UVIS1 Subsection', size=14)\n", + "fig.tight_layout()" ] }, { @@ -743,7 +734,7 @@ "metadata": {}, "outputs": [], "source": [ - "def get_flux(data,aperture,annulus_aperture):\n", + "def get_flux(data, aperture, annulus_aperture):\n", " \"\"\"\n", " Function to calculate background subtracted aperture sum \n", " \n", @@ -766,7 +757,7 @@ " phot = aperture_photometry(data, aperture)\n", " \n", " # Measure background around sources. aperture_stats_tbl() comes from background_median.py\n", - " bkg_phot = aperture_stats_tbl(data, annulus_aperture, method = 'exact', sigma_clip = True)\n", + " bkg_phot = aperture_stats_tbl(data, annulus_aperture, method='exact', sigma_clip=True)\n", " \n", " # Calculate background subtracted aperture sum\n", " flux = phot['aperture_sum'] - bkg_phot['aperture_median'] * aperture.area\n", @@ -774,12 +765,12 @@ " return flux\n", "\n", "# Approximate x,y pixel locations of each star in the 4Kx4K array\n", - "positions = [(299.4,2135.6),\n", - " (114.7,2093.4),\n", - " (171.3,2074.9),\n", - " (262.6,2164.9),\n", - " (289.1,2085.6),\n", - " (204.8,2073.1)]\n", + "positions = [(299.4, 2135.6),\n", + " (114.7, 2093.4),\n", + " (171.3, 2074.9),\n", + " (262.6, 2164.9),\n", + " (289.1, 2085.6),\n", + " (204.8, 2073.1)]\n", "\n", "# Photutils cirular aperture object with small radius\n", "aperture = CircularAperture(positions, r=3)\n", @@ -789,9 +780,9 @@ "\n", "# Call function to calculate flux of stars\n", "# THE RUNTIME WARNING MAY BE IGNORED\n", - "fltflux = get_flux(fltsci,aperture,annulus_aperture)\n", - "v1flux = get_flux(v1sci,aperture,annulus_aperture)\n", - "v2flux = get_flux(v2sci,aperture,annulus_aperture)\n" + "fltflux = get_flux(fltsci, aperture, annulus_aperture)\n", + "v1flux = get_flux(v1sci, aperture, annulus_aperture)\n", + "v2flux = get_flux(v2sci, aperture, annulus_aperture)" ] }, { @@ -820,33 +811,33 @@ "outputs": [], "source": [ "# Generate subplots\n", - "fig, [ax1,ax2] = plt.subplots(2,1,figsize=(8,13),dpi=120)\n", - "ax1.grid(alpha=0.5,which='both'),ax2.grid(alpha=0.5)\n", + "fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(8, 13), dpi=120)\n", + "ax1.grid(alpha=0.5), ax2.grid(alpha=0.5)\n", "\n", "# Find median flux values between products\n", - "medflux = np.median([fltflux,v1flux,v2flux],axis=0)\n", + "medflux = np.median([fltflux, v1flux, v2flux], axis=0)\n", "\n", "# Scatter plot of measured flux\n", - "ax1.scatter(medflux,fltflux,25,marker='o',c='C3',label='FLT')\n", - "ax1.scatter(medflux,v1flux,30,marker='^',c='C0',label='v1.0 FLC')\n", - "ax1.scatter(medflux,v2flux,45,marker='*',c='k',label='v2.0 FLC')\n", + "ax1.scatter(medflux, fltflux, 25, marker='o', c='C3', label='FLT')\n", + "ax1.scatter(medflux, v1flux, 30, marker='^', c='C0', label='v1.0 FLC')\n", + "ax1.scatter(medflux, v2flux, 45, marker='*', c='k', label='v2.0 FLC')\n", "\n", "# Scatter plot of percentage difference\n", - "ax2.scatter(medflux,abs((fltflux-v1flux))/((fltflux+v1flux)/2)*100,30,\n", - " marker='^',c='magenta',label=r'$\\frac{|FLT - v1.0|}{(FLT + v1.0) ÷ 2}$')\n", - "ax2.scatter(medflux,abs((fltflux-v2flux))/((fltflux+v2flux)/2)*100,45,\n", - " marker='*',c='limegreen',label=r'$\\frac{|FLT - v2.0|}{(FLT + v2.0) ÷ 2}$')\n", - "ax2.scatter(medflux,abs((v1flux-v2flux))/((v1flux+v2flux)/2)*100,25,\n", - " marker='s',c='C9',label=r'$\\frac{|v1.0 - v2.0|}{(v1.0 + v2.0) ÷ 2}$')\n", + "ax2.scatter(medflux, abs((fltflux-v1flux))/((fltflux+v1flux)/2)*100, 30,\n", + " marker='^', c='magenta', label=r'$\\frac{|FLT - v1.0|}{(FLT + v1.0) ÷ 2}$')\n", + "ax2.scatter(medflux, abs((fltflux-v2flux))/((fltflux+v2flux)/2)*100, 45,\n", + " marker='*', c='limegreen', label=r'$\\frac{|FLT - v2.0|}{(FLT + v2.0) ÷ 2}$')\n", + "ax2.scatter(medflux, abs((v1flux-v2flux))/((v1flux+v2flux)/2)*100, 25,\n", + " marker='s', c='C9', label=r'$\\frac{|v1.0 - v2.0|}{(v1.0 + v2.0) ÷ 2}$')\n", "\n", "# Formatting \n", - "ax1.set_title('Measured Flux Within 3-pix Radius Aperture',size=14)\n", - "ax1.set_xlabel('Median Flux [e-]',size=12)\n", - "ax1.set_ylabel('Flux [e-]',size=12)\n", - "ax2.set_xlabel('Median Flux [e-]',size=12)\n", - "ax2.set_ylabel('Percent Difference [%]',size=12)\n", - "ax1.legend(prop={'size':11}),ax2.legend(prop={'size':15})\n", - "ax1.set_yscale('log')\n" + "ax1.set_title('Measured Flux Within 3-pix Radius Aperture', size=14)\n", + "ax1.set_xlabel('Median Flux [e-]', size=12)\n", + "ax1.set_ylabel('Flux [e-]', size=12)\n", + "ax2.set_xlabel('Median Flux [e-]', size=12)\n", + "ax2.set_ylabel('Percent Difference [%]', size=12)\n", + "ax1.legend(prop={'size' : 11}), ax2.legend(prop={'size' : 15})\n", + "ax1.set_yscale('log')" ] }, { From 1babfc1843ec8e06e542a0caeafdd1fc0b1cc8a7 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Tue, 7 Nov 2023 11:25:03 -0500 Subject: [PATCH 12/18] Update background_median.py adding blank lines for PEP8 compliance --- notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py b/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py index 38d93d424..3ae6d72f7 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py +++ b/notebooks/WFC3/calwf3_v1.0_cte/example/background_median.py @@ -20,6 +20,7 @@ from scipy.stats import sigmaclip from astropy.table import Table + def aperture_stats_tbl(data, apertures, method='exact', sigma_clip=True): """Computes mean/median/mode/std in Photutils apertures. @@ -72,6 +73,7 @@ def aperture_stats_tbl(data, apertures, method='exact', sigma_clip=True): return stats_tbl + def calc_aperture_mmm(data, mask, sigma_clip): """Helper function to actually calculate the stats for pixels From b493d686074e20cc027b3b129fba703f0ff1c334 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Tue, 7 Nov 2023 11:31:48 -0500 Subject: [PATCH 13/18] Update calwf3_with_v1.0_PCTE.ipynb more PEP8 compliance changes --- .../calwf3_with_v1.0_PCTE.ipynb | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index c7e823143..144ce3d33 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -78,21 +78,22 @@ "
This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository.
\n", "\n", "We import:
\n", - "| Package Name | Purpose |\n", - "|:---------------------------------------|:--------------------------------------------|\n", - "| `glob` | creating list of files |\n", - "| `matplotlib.pyplot` | plotting and displaying images |\n", - "| `numpy` | finding indices and concatenating arrays |\n", + "\n", + "| Package Name | Purpose |\n", + "|:-----------------------------------------|:-------------------------------------------------------|\n", + "| `glob` | creating list of files |\n", + "| `matplotlib.pyplot` | plotting and displaying images |\n", + "| `numpy` | finding indices and concatenating arrays |\n", "| `os` | directory maintenance and setting environment variables|\n", - "| `astropy.io.fits` | opening and modifying fits files |\n", - "| `astroquery.mast.Observations` | downloading data from MAST |\n", - "| `astropy.table.Table` | creating and manipulating data tables |\n", - "| `ginga.util.zscale` | finding scale limits when displaying images |\n", - "| `photutils.aperture.aperture_photometry` | performing aperture photometry |\n", - "| `photutils.aperture.CircularAperture` | creating circular apertures |\n", - "| `photutils.aperture.CircularAnnulus` | creating circular annuli |\n", - "| `wfc3tools.calwf3` | verifying the version and running pipeline |\n", - "| `background_median.aperture_stats_tbl` | measuring background values within annuli |" + "| `astropy.io.fits` | opening and modifying fits files |\n", + "| `astroquery.mast.Observations` | downloading data from MAST |\n", + "| `astropy.table.Table` | creating and manipulating data tables |\n", + "| `ginga.util.zscale` | finding scale limits when displaying images |\n", + "| `photutils.aperture.aperture_photometry` | performing aperture photometry |\n", + "| `photutils.aperture.CircularAperture` | creating circular apertures |\n", + "| `photutils.aperture.CircularAnnulus` | creating circular annuli |\n", + "| `wfc3tools.calwf3` | verifying the version and running pipeline |\n", + "| `background_median.aperture_stats_tbl` | measuring background values within annuli |" ] }, { @@ -276,7 +277,7 @@ "drkc_table = Table.read('archived_drkcfiles.txt', format='ascii.commented_header')\n", "\n", "# Create empty lists for final astropy table\n", - "rawfiles, obsdates, dkcfiles, uafters, active_dates = [], [], [], [], []\n" + "rawfiles, obsdates, dkcfiles, uafters, active_dates = [], [], [], [], []" ] }, { @@ -764,6 +765,7 @@ " \n", " return flux\n", "\n", + "\n", "# Approximate x,y pixel locations of each star in the 4Kx4K array\n", "positions = [(299.4, 2135.6),\n", " (114.7, 2093.4),\n", @@ -836,7 +838,7 @@ "ax1.set_ylabel('Flux [e-]', size=12)\n", "ax2.set_xlabel('Median Flux [e-]', size=12)\n", "ax2.set_ylabel('Percent Difference [%]', size=12)\n", - "ax1.legend(prop={'size' : 11}), ax2.legend(prop={'size' : 15})\n", + "ax1.legend(prop={'size': 11}), ax2.legend(prop={'size': 15})\n", "ax1.set_yscale('log')" ] }, From 4fbf6024e38e1f2cfe18be83bb74a0d0d45a467f Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Wed, 15 Nov 2023 15:44:38 -0500 Subject: [PATCH 14/18] Update README.md slight change to the workflow for creating the virtual environment --- notebooks/WFC3/calwf3_v1.0_cte/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/README.md b/notebooks/WFC3/calwf3_v1.0_cte/README.md index 1f1b65f9e..cab8df072 100755 --- a/notebooks/WFC3/calwf3_v1.0_cte/README.md +++ b/notebooks/WFC3/calwf3_v1.0_cte/README.md @@ -1,19 +1,20 @@ This directory, once cloned from the repository, should contain this `README.md`, the Jupyter Notebook `calwf3_with_v1.0_PCTE.ipynb`, a file -called `archived_drkcfiles.txt` and a subdirectory `example/`. +called `archived_drkcfiles.txt`, a `requirements.txt` file, and a subdirectory `example/`. -**In order to run this Jupyter Notebook you must have created a virtual +**To run this Jupyter Notebook you must have created a virtual conda environment that includes `calwf3` v3.5.2.** Version 3.5.2 of `calwf3` is available in `hstcal` release 2.5.0. To create an environment with `calwf3` v3.5.2 try this from the terminal: ``` $ conda config --add channels http://ssb.stsci.edu/astroconda - -$ conda create -n v1_PCTE hstcal==2.5.0 --file requirements.txt +$ conda create -n v1_PCTE hstcal==2.5.0 python=3.11 +$ conda activate v1_PCTE +$ pip install -r requirements.txt ``` -In general, users wanting to use the v1.0 pixel-based CTE correction +In most cases, users wanting to use the v1.0 pixel-based CTE correction within `calwf3` should use `calwf3` v3.5.2. This version will provide the most up-to-date calibration procedures such as time-dependent photometric corrections and zeropoints, while also including the v1.0 correction. From 371fde3c527b3870931b79de59cc3ef6ab442bd3 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Wed, 15 Nov 2023 15:46:10 -0500 Subject: [PATCH 15/18] Update requirements.txt removing version pins and adding `crds` --- notebooks/WFC3/calwf3_v1.0_cte/requirements.txt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt b/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt index 2be162929..71a33b71f 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt +++ b/notebooks/WFC3/calwf3_v1.0_cte/requirements.txt @@ -1,8 +1,9 @@ -astropy==4.3.1 -astroquery==0.4 -ginga==3.2.0 -matplotlib==3.5.3 -numpy==1.21.5 -photutils==1.1.0 -scipy==1.7.3 -wfc3tools==1.3.5 +astropy +astroquery +crds +jupyter +matplotlib +numpy +photutils +scipy +wfc3tools From cb3a70841413dc72dc1f4657c65939de7edf7538 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Wed, 15 Nov 2023 15:48:29 -0500 Subject: [PATCH 16/18] Update calwf3_with_v1.0_PCTE.ipynb adding `crds bestref` to pull ref files and `crds sync` to pull necessary archived files --- .../calwf3_with_v1.0_PCTE.ipynb | 227 ++++++++++++------ 1 file changed, 150 insertions(+), 77 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index 144ce3d33..060bd417e 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -21,23 +21,25 @@ "- Compare v1.0 and v2.0 products. \n", "\n", "\n", - "Please make sure you have read the `README.md` file before continuing.\n", + "**Please make sure you have read the `README.md` file before continuing.**\n", "\n", "## Table of Contents\n", " [Introduction](#intro)
\n", " \n", " [1. Imports](#imports)
\n", - " [2. Verify `archived_drkcfiles.txt` is in CWD](#txtfile)
\n", - " [3. Check that `calwf3` Version is `v3.5.2`](#checkcalver)
\n", + " [2. Verify `archived_drkcfiles.txt` is in CWD](#txtfile)
\n", + " [3. Check that `calwf3` Version is `v3.5.2`](#checkcalver)
\n", " [4. Query MAST and Download a WFC3 `raw.fits` Image](#download)
\n", - "       [4.1 Inspect Image Headers](#imageheaders)
\n", - " [5. Find the Correct `DRKCFILE`](#finddark)
\n", - " [6 Modify Image Header](#modhead)
\n", - "       [6.1 `DRKCFILE`](#drkcfile)
\n", - "       [6.2 `PCTETAB`](#pctetab)
\n", - "[7. Re-Inspect Image Header](#reinspect)
\n", - "[8. Set Environment Variables](#envvar)
\n", - "[9. Run `calwf3`](#runcal)
\n", + " [5. Set CRDS Environment Variable and Download Reference Files](#env_var)
\n", + "       [5.1 Run `crds bestrefs`](#bestrefs)
\n", + "       [5.2 Inspect Image Header](#imageheaders)
\n", + " [6. Find the Correct v1.0 DRKCFILE](#finddark)
\n", + "       [6.1 Download the Correct DRKCFILE from CRDS](#sync_dkc)
\n", + "       [6.2 Modify the Image Header Keyword, `DRKCFILE`](#mod_dkc)
\n", + " [7. Download the v1.0 PCTETAB](#sync_pctetab)
\n", + "       [7.1 Modify the Image Header Keyword `PCTETAB`](#mod_pcte)
\n", + " [8. Re-Inspect Image Header](#reinspect)
\n", + " [9. Run `calwf3`](#runcal)
\n", "[10. Inspect `FLC` Image Header](#inspectflc)
\n", "[11. Investigate v1.0 and v2.0 Differences](#verdiffs)
\n", "      [11.1 Download the v2.0 FLC File](#downloadv2.0)
\n", @@ -50,8 +52,7 @@ "\n", "[Additional Resources](#resources)
\n", "[About the Notebook](#about)
\n", - "[Citations](#cite) \n", - "\n" + "[Citations](#cite) " ] }, { @@ -82,13 +83,13 @@ "| Package Name | Purpose |\n", "|:-----------------------------------------|:-------------------------------------------------------|\n", "| `glob` | creating list of files |\n", - "| `matplotlib.pyplot` | plotting and displaying images |\n", - "| `numpy` | finding indices and concatenating arrays |\n", "| `os` | directory maintenance and setting environment variables|\n", "| `astropy.io.fits` | opening and modifying fits files |\n", "| `astroquery.mast.Observations` | downloading data from MAST |\n", "| `astropy.table.Table` | creating and manipulating data tables |\n", - "| `ginga.util.zscale` | finding scale limits when displaying images |\n", + "| `astropy.visualization.ZScaleInterval` | finding z-scale limits when displaying images |\n", + "| `matplotlib.pyplot` | plotting and displaying images |\n", + "| `numpy` | finding indices and concatenating arrays |\n", "| `photutils.aperture.aperture_photometry` | performing aperture photometry |\n", "| `photutils.aperture.CircularAperture` | creating circular apertures |\n", "| `photutils.aperture.CircularAnnulus` | creating circular annuli |\n", @@ -103,14 +104,15 @@ "outputs": [], "source": [ "import glob\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", "import os\n", "\n", "from astropy.io import fits\n", "from astropy.table import Table\n", "from astroquery.mast import Observations\n", - "from ginga.util import zscale\n", + "from astropy.stats import sigma_clipped_stats\n", + "from astropy.visualization import ZScaleInterval\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "from photutils.aperture import aperture_photometry, CircularAperture, CircularAnnulus\n", "from wfc3tools import calwf3\n", "\n", @@ -125,7 +127,7 @@ "## 2. Verify `archived_drkcfiles.txt` is in CWD\n", "When you cloned/downloaded this notebook from [hst_notebooks](https://github.com/spacetelescope/hst_notebooks/), a .txt file should have been included. The file name is
`archived_drkcfiles.txt` and it is used later on in the notebook. This .txt file includes the file name, delivery date, activation
date, and USEAFTER date for every v1.0 CTE corrected dark reference file between May 2009 - February 2021. Below, we will use
this file in conjunction with the observation date of the file(s) being calibrated to pick out the most appropriate v1.0 CTE corrected
dark reference file(s).
\n", "\n", - "Please make sure the file is in the current working directory before continuing." + "Please make sure the `archived_drkcfiles.txt` file is in the current working directory before continuing." ] }, { @@ -145,10 +147,12 @@ "\n", "## 3. Check that `calwf3` Version is `v3.5.2`\n", "In April 2021, a new `calwf3` version was released that contains the v2.0 CTE-correction.\n", - "If you would like to use the v1.0 correction, your
current environment must be using `calwf3` versions equal to or between `3.3` - `3.5.2`. However, in order to get the best v1.0 calibrated
images we must use `calwf3` `v3.5.2`. This version of `calwf3` includes the recent ([~Jan 2021](https://github.com/spacetelescope/hstcal/releases/tag/2.5.0)) update that added MJD as a parameterized
variable for the `PHOTMODE` keyword, which enables a time-dependent photometric correction and zeropoint. If your version is `3.6.0` or higher,
you must downgrade the [`hstcal` package](https://github.com/spacetelescope/hstcal). The safer option, however, is to create a new environment using the requirements file provided in the [notebook's repository](https://github.com/spacetelescope/hst_notebooks/tree/main/notebooks/WFC3/calwf3_v1.0_cte):
\n", + "If you would like to use the v1.0 correction, your
current environment must be using `calwf3` versions equal to or between `3.3` - `3.5.2`. However, in order to get the best v1.0 calibrated
images we must use `calwf3` `v3.5.2`. This version of `calwf3` includes the recent ([~Jan 2021](https://github.com/spacetelescope/hstcal/releases/tag/2.5.0)) update that added MJD as a parameterized
variable for the `PHOTMODE` keyword, which enables a time-dependent photometric correction and zeropoint. If your version is `3.6.0` or higher,
you must downgrade the [`hstcal` package](https://github.com/spacetelescope/hstcal). The safer option, however, is to create a new environment using the requirements file provided in
the [notebook's repository](https://github.com/spacetelescope/hst_notebooks/tree/main/notebooks/WFC3/calwf3_v1.0_cte):
\n", "\n", "* `$ conda config --add channels http://ssb.stsci.edu/astroconda`\n", - "* `$ conda create -n v1_PCTE hstcal==2.5.0 --file requirements.txt`\n", + "* `$ conda create -n v1_PCTE hstcal==2.5.0 python=3.11`\n", + "* `$ conda activate v1_PCTE`\n", + "* `$ pip install -r requirements.txt`\n", "\n", "`hstcal` `v2.5.0` provides version `3.5.2` of `calwf3`, which is the last version that offers the v1.0 pixel-based CTE correction.\n" ] @@ -216,12 +220,70 @@ "!ls -l *raw.fits" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## 5. Set CRDS Environment Variable and Download Reference Files" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
If you already have the absolute paths set for CRDS, please skip the code cell immediately below and proceed to the crds bestrefs command in Section 4.1.1.
\n", + "\n", + "Before we run `crds bestfefs` and `calwf3`, we need to [set environment variables](https://hst-crds.stsci.edu/docs/cmdline_bestrefs/) for several subsequent calibration tasks. We will point to a
subdirectory within the main `crds_cache/` using the `IREF` environment variable. The `IREF` variable is used for WFC3 reference files. Other
instruments use other variables, e.g., `JREF` for ACS. You have the option to permanently add these environment variables to your user profile by
adding the path in your shell's configuration file. If you're using bash, you would edit the `~/.bash_profile` file with lines such as:\n", + "\n", + "* `export CRDS_SERVER_URL=\"https://hst-crds.stsci.edu\"`\n", + "* `export CRDS_SERVER=\"https://hst-crds.stsci.edu\"`\n", + "* `export CRDS_PATH=\"$HOME/crds_cache\"`\n", + "* `export iref=\"${CRDS_PATH}/references/hst/wfc3/\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'\n", + "os.environ['CRDS_SERVER'] = 'https://hst-crds.stsci.edu'\n", + "os.environ['CRDS_PATH'] = 'crds_cache'\n", + "os.environ['iref'] = 'crds_cache/references/hst/wfc3/'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## 5.1 Run `crds bestrefs` \n", + "\n", + "The cell below calls [CRDS bestref](https://hst-crds.stsci.edu/static/users_guide/basic_use.html), which will copy the necessary reference files from CRDS over to your local machine, if you do not already have
them. Without running this command we would not be able to calibrate the image with `calwf3`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "cl_input = 'crds bestrefs --update-bestrefs --sync-references=1 --files idv404axq_raw.fits'\n", + "cl = os.system(cl_input)\n", + "if cl != 0:\n", + " print(f\"bestref failed with exit code: {cl}\")" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "\n", - "## 4.1 Inspect Image Header\n", + "## 5.2 Inspect Image Header\n", "When processing a raw file through `calwf3` , the pipeline uses a few different header keywords to initiate and run the pixel-based CTE
correction. Here, we inspect the important header keywords from the raw file just downloaded.
\n", "At this step, you should see:
\n", "- `pctetab` set to `iref$54l1347ei_cte.fits`
\n", @@ -260,7 +322,7 @@ "metadata": {}, "source": [ "\n", - "## 5. Find the Correct `DRKCFILE` \n", + "## 6. Find the Correct v1.0 `DRKCFILE` \n", "Below, we open the .txt file containing a list of all the `DRKCFILE` reference files created with the v1.0 pixel-based CTE correction.
`DRKCFILE` reference files are CTE corrected files used by the pipeline to perform the dark current subtraction during the generation
of the `flc` file. The `DRKCFILE` files listed in `archived_drkcfiles.txt` have been archived on the CRDS database and while
they are still accessible for use and download, they are not being actively used by MAST.
\n", "\n", "In the first cell, we generate an `astropy.Table` ( `drkc_table` ) using the data from the file `archived_drkcfiles.txt`, mentioned
in [Section 2](#txtfile), and create empty lists for the final table. Then, in the second cell, we index the `drkc_table` table for the best `DRKCFILE`
that corresponds to the `DATE-OBS` of the `raw` file being calibrated. Lastly, in the third cell, we create and display the final `astropy.table`
that contains just the necessary `DRKCFILE`.
\n", @@ -320,13 +382,31 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "## 6. Modify Image Header\n", - "Now that we know which v1.0 CTE corrected dark current reference file corresponds to our `raw` science file, we're ready to edit the header
keywords the v1.0 CTE corrected reference files and tables. \n", - "\n", - "\n", - "## 6.1 `DRKCFILE`\n", - "First, we will edit `raw` file's header with the v1.0 CTE corrected dark current reference file, `DRKCFILE`, that we just found in Section 6. \n" + "\n", + "## 6.1 Download the Correct `DRKCFILE` from CRDS\n", + "Now that we know the name of the correct `DRKCFILE`, it must be retrieved from CRDS and stored on your local machine so that it can be used
during calibration. To copy the file from CRDS we use the [crds sync](https://hst-crds.stsci.edu/static/users_guide/command_line_tools.html#crds-sync) command. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for dkc in raw_dkc_tab['dkc-filename']:\n", + " cl_input = f\"crds sync --hst --files {dkc} --output-dir {os.environ['iref']} \"\n", + " cl = os.system(cl_input)\n", + " if cl != 0:\n", + " print(f\"crds sync failed with exit code: {cl}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## 6.2 Modify the Image Header Keyword, `DRKCFILE`\n", + "With the v1.0 CTE-corrected dark current reference file that corresponds to our `raw` science file copied to our local machine, we're ready to
edit the header keyword with the proper `DRKCFILE`. " ] }, { @@ -345,9 +425,30 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - "## 6.2 `PCTETAB`\n", - "The next keyword we're going to modify is the `PCTETAB`. This is the pixel-based correction reference table and without it the algorithm will
not work. In order to use the v1.0 pixel based correction, `PCTETAB` will need to be set to the proper reference file. In the below cell, we set
the `raw` file's `PCTETAB` to the v1.0 reference table, [zcv2057mi_cte.fits](https://hst-crds.stsci.edu/browse/zcv2057mi_cte.fits).\n" + "\n", + "## 7. Download the v1.0 `PCTETAB`\n", + "\n", + "The next reference file we're going to download from CRDS is the `PCTETAB`. This is the pixel-based correction reference table and without it
the algorithm will not work. In order to use the v1.0 pixel based correction, we must retrieve the v1.0 `PCTETAB` and set the header keyword to
the proper reference file. In the cells below, we use the `crds sync` command again and then set the `raw` file's `PCTETAB` to the v1.0
reference table, [zcv2057mi_cte.fits](https://hst-crds.stsci.edu/browse/zcv2057mi_cte.fits).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cl_input = f\"crds sync --hst --files zcv2057mi_cte.fits --output-dir {os.environ['iref']}\"\n", + "cl = os.system(cl_input)\n", + "if cl != 0:\n", + " print(f\"crds sync failed: {cl}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## 7.1 Modify the Image Header Keyword, `PCTETAB`" ] }, { @@ -364,7 +465,7 @@ "metadata": {}, "source": [ "\n", - "## 7. Re-Inspect Image Header \n", + "## 8. Re-Inspect Image Header \n", "Now with the headers modified, we inspect the keywords one last time to verify the file was updated properly before we process
it through `calwf3`. At this point you should see:
\n", "- `PCTETAB` set to `iref$zcv2057mi_cte.fits`
\n", "- `DRKCFILE` set to `iref$3961719li_dkc.fits`
" @@ -396,35 +497,6 @@ "updated_table" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## 8. Set Environment Variables\n", - "
If you already have the absolute paths set for CRDS, please skip this step and proceed to Section 9.
\n", - " \n", - "Before we run `calwf3`, we need to [set environment variables](https://hst-crds.stsci.edu/docs/cmdline_bestrefs/) for several subsequent calibration tasks. We will point to a subdirectory called
`crds_cache/` using the `IREF` environment variable. The `IREF` variable is used for WFC3 reference files. Other instruments use other
variables, e.g., `JREF` for ACS. You have the option to permanently add these environment variables to your user profile by adding the path
in your shell's configuration file. If you're using bash, you would edit the `~/.bash_profile` file with lines such as:\n", - "\n", - "* `export CRDS_PATH=\"$HOME/crds_cache\"`\n", - "* `export CRDS_SERVER_URL=\"https://hst-crds.stsci.edu\"`\n", - "* `export iref=\"${CRDS_PATH}/references/hst/iref/\"`\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'\n", - "os.environ['CRDS_SERVER'] = 'https://hst-crds.stsci.edu'\n", - "os.environ['CRDS_PATH'] = '~/crds_cache'\n", - "os.environ['iref'] = '~/crds_cache/references/hst/iref/'" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -498,7 +570,7 @@ "\n", "\n", "## 11.1 Download the v2.0 FLC File\n", - "First, we need to download the same FLC file from MAST that is corrected with the v2.0 pixel-based CTE correction so that we can
compare it to the v1.0 FLC file we just created in the notebook." + "First, we need to download the same FLC file from MAST that is corrected with the v2.0 pixel-based CTE correction so that we can
compare it to the v1.0 FLC file we just created in the notebook. In this step we rename the downloaded FLC to `idv404axq_v2.0_flc.fits`." ] }, { @@ -589,12 +661,13 @@ "v2_bkg = v2sci[2070:2120, 2180:2230]\n", "\n", "# Calculate min and max values for image scaling \n", - "v1z1, v1z2 = zscale.zscale(v1_bkg)\n", + "z = ZScaleInterval()\n", + "z1, z2 = z.get_limits(v1_bkg)\n", "\n", "# Display background subsection\n", - "im1 = ax1.imshow(flt_bkg, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", - "im2 = ax2.imshow(v1_bkg, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", - "im3 = ax3.imshow(v2_bkg, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", + "im1 = ax1.imshow(flt_bkg, origin='lower', cmap='Greys_r', vmin=z1, vmax=z2)\n", + "im2 = ax2.imshow(v1_bkg, origin='lower', cmap='Greys_r', vmin=z1, vmax=z2)\n", + "im3 = ax3.imshow(v2_bkg, origin='lower', cmap='Greys_r', vmin=z1, vmax=z2)\n", "\n", "# Formatting\n", "fig.colorbar(im1, ax=ax1, shrink=0.35, pad=0.02)\n", @@ -655,7 +728,6 @@ "ax1.legend(), ax2.legend()\n", "ax1.set_yscale('log')\n", "ax2.set_yscale('log')\n", - "\n", "fig.tight_layout()" ] }, @@ -680,11 +752,12 @@ "fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(8, 8), dpi=150)\n", "\n", "# Calculate min and max values for image scaling \n", - "v1z1, v1z2 = zscale.zscale(v1_uvis1)\n", + "z = ZScaleInterval()\n", + "z1, z2 = z.get_limits(v1_uvis1)\n", "\n", "# Display subsection\n", - "im1 = ax1.imshow(v1_uvis1, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", - "im2 = ax2.imshow(v2_uvis1, origin='lower', cmap='Greys_r', vmin=v1z1, vmax=v1z2)\n", + "im1 = ax1.imshow(v1_uvis1, origin='lower', cmap='Greys_r', vmin=z1, vmax=z2)\n", + "im2 = ax2.imshow(v2_uvis1, origin='lower', cmap='Greys_r', vmin=z1, vmax=z2)\n", "\n", "# Formatting\n", "ax1.set_xlim(85, 325), ax2.set_xlim(85, 325)\n", @@ -874,7 +947,7 @@ "- [WFC3 Website](https://www.stsci.edu/hst/instrumentation/wfc3)\n", "- [WFC3 Instrument Handbook](https://hst-docs.stsci.edu/wfc3ihb)\n", "- [WFC3 Data Handbook](https://hst-docs.stsci.edu/wfc3dhb)\n", - "- [STScI Jupyter-notebooks](https://github.com/spacetelescope/notebooks/tree/master/notebooks)\n", + "- [STScI Jupyter-notebooks](https://github.com/spacetelescope/hst_notebooks)\n", "- [STScI Astroconda Channel](http://ssb.stsci.edu/astroconda)\n", "\n", "\n", @@ -882,12 +955,12 @@ "\n", "**Author:** Benjamin Kuhn; WFC3 Instrument Team
\n", "**Created:** February 22, 2022
\n", - "**Last updated on:** November 3, 2023\n", + "**Last updated on:** November 15, 2023\n", "\n", "\n", "## Citations\n", "\n", - "If you use `numpy`, `astropy`, `astroquery`, `matplotlib`, or photutils for published research, please cite the authors.
\n", + "If you use `astropy`, `astroquery`, `numpy`, `matplotlib`, `photutils`, or `scipy` for published research, please cite the authors.
\n", "Follow these links for more information about citing the libraries below:\n", "\n", "* [Citing `astropy`](https://www.astropy.org/acknowledging.html)\n", @@ -919,7 +992,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.16" + "version": "3.11.0" }, "varInspector": { "cols": { From 7fce79b9889de6a04eb740f584eb51276fe1e212 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Wed, 15 Nov 2023 16:32:18 -0500 Subject: [PATCH 17/18] Update calwf3_with_v1.0_PCTE.ipynb remove astropy sigma clip stats since it isn't used --- notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index 060bd417e..defe51938 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -109,7 +109,6 @@ "from astropy.io import fits\n", "from astropy.table import Table\n", "from astroquery.mast import Observations\n", - "from astropy.stats import sigma_clipped_stats\n", "from astropy.visualization import ZScaleInterval\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", From 77ac32dd97ad1c7fd18b66123e11fcc8f852ade3 Mon Sep 17 00:00:00 2001 From: bjkuhn Date: Fri, 17 Nov 2023 15:04:37 -0500 Subject: [PATCH 18/18] Update calwf3_with_v1.0_PCTE.ipynb made better variable names and added logic to check if the FLT file already exists --- .../calwf3_with_v1.0_PCTE.ipynb | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb index defe51938..efc08eb9e 100644 --- a/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb +++ b/notebooks/WFC3/calwf3_v1.0_cte/calwf3_with_v1.0_PCTE.ipynb @@ -20,9 +20,6 @@ "- Run `calwf3` `v3.5.2` to calibrate the raw image with the v1.0 pixel based CTE-correction.\n", "- Compare v1.0 and v2.0 products. \n", "\n", - "\n", - "**Please make sure you have read the `README.md` file before continuing.**\n", - "\n", "## Table of Contents\n", " [Introduction](#intro)
\n", " \n", @@ -76,7 +73,7 @@ "\n", "## 1. Imports\n", "\n", - "
This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository.
\n", + "
This notebook assumes you have created and activated a virtual environment using the requirements file in this notebook's repository. Please make sure you have read the README file before continuing.
\n", "\n", "We import:
\n", "\n", @@ -231,7 +228,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "
If you already have the absolute paths set for CRDS, please skip the code cell immediately below and proceed to the crds bestrefs command in Section 4.1.1.
\n", + "
If you already have the absolute paths set for CRDS, please skip the code cell immediately below and proceed to the crds bestrefs command in Section 5.1.
\n", "\n", "Before we run `crds bestfefs` and `calwf3`, we need to [set environment variables](https://hst-crds.stsci.edu/docs/cmdline_bestrefs/) for several subsequent calibration tasks. We will point to a
subdirectory within the main `crds_cache/` using the `IREF` environment variable. The `IREF` variable is used for WFC3 reference files. Other
instruments use other variables, e.g., `JREF` for ACS. You have the option to permanently add these environment variables to your user profile by
adding the path in your shell's configuration file. If you're using bash, you would edit the `~/.bash_profile` file with lines such as:\n", "\n", @@ -271,10 +268,10 @@ }, "outputs": [], "source": [ - "cl_input = 'crds bestrefs --update-bestrefs --sync-references=1 --files idv404axq_raw.fits'\n", - "cl = os.system(cl_input)\n", - "if cl != 0:\n", - " print(f\"bestref failed with exit code: {cl}\")" + "bestref_input = 'crds bestrefs --update-bestrefs --sync-references=1 --files idv404axq_raw.fits'\n", + "run_bestref = os.system(bestref_input)\n", + "if run_bestref != 0:\n", + " print(f\"bestref failed with exit code: {run_bestref}\")" ] }, { @@ -393,10 +390,10 @@ "outputs": [], "source": [ "for dkc in raw_dkc_tab['dkc-filename']:\n", - " cl_input = f\"crds sync --hst --files {dkc} --output-dir {os.environ['iref']} \"\n", - " cl = os.system(cl_input)\n", - " if cl != 0:\n", - " print(f\"crds sync failed with exit code: {cl}\")" + " crds_sync = f\"crds sync --hst --files {dkc} --output-dir {os.environ['iref']} \"\n", + " run_sync = os.system(crds_sync)\n", + " if run_sync != 0:\n", + " print(f\"crds sync failed with exit code: {run_sync}\")" ] }, { @@ -436,10 +433,10 @@ "metadata": {}, "outputs": [], "source": [ - "cl_input = f\"crds sync --hst --files zcv2057mi_cte.fits --output-dir {os.environ['iref']}\"\n", - "cl = os.system(cl_input)\n", - "if cl != 0:\n", - " print(f\"crds sync failed: {cl}\")" + "crds_sync = f\"crds sync --hst --files zcv2057mi_cte.fits --output-dir {os.environ['iref']}\"\n", + "run_sync = os.system(crds_sync)\n", + "if run_sync != 0:\n", + " print(f\"crds sync failed: {run_sync}\")" ] }, { @@ -513,7 +510,8 @@ }, "outputs": [], "source": [ - "calwf3('idv404axq_raw.fits')" + "if not os.path.exists('idv404axq_flt.fits'):\n", + " calwf3('idv404axq_raw.fits')" ] }, {