From fdf5077b735c887b4cf61696048e110f97c79be1 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 27 Sep 2024 12:03:03 -0400 Subject: [PATCH 01/11] Calwf3 update to remove sys call --- .../WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/notebooks/WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb b/notebooks/WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb index dab095657..a7612cb8c 100755 --- a/notebooks/WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb +++ b/notebooks/WFC3/calwf3_recalibration/calwf3_recal_tvb.ipynb @@ -89,7 +89,8 @@ "from drizzlepac import astrodrizzle\n", "from stwcs import updatewcs\n", "\n", - "from wfc3tools import calwf3, pstat" + "from wfc3tools import calwf3, pstat\n", + "from IPython.display import clear_output" ] }, { @@ -222,8 +223,8 @@ "raw_files = glob.glob('*_raw.fits')\n", "\n", "for file in raw_files:\n", - " command_line_input = f'crds bestrefs --files {file} --sync-references=1 --update-bestrefs'\n", - " os.system(command_line_input)" + " !crds bestrefs --files {file} --sync-references=1 --update-bestrefs\n", + "clear_output()" ] }, { @@ -768,7 +769,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 34a3ee223c6c09483cd93ee42c05f6ccb3c1fb25 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 27 Sep 2024 16:55:36 -0400 Subject: [PATCH 02/11] Filter transformation nb update --- .../WFC3/filter_transformations/filter_transformations.ipynb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/notebooks/WFC3/filter_transformations/filter_transformations.ipynb b/notebooks/WFC3/filter_transformations/filter_transformations.ipynb index 92c8f6abc..068a6a075 100644 --- a/notebooks/WFC3/filter_transformations/filter_transformations.ipynb +++ b/notebooks/WFC3/filter_transformations/filter_transformations.ipynb @@ -110,8 +110,7 @@ "metadata": {}, "outputs": [], "source": [ - "cmd_input = 'curl -O https://archive.stsci.edu/hlsps/reference-atlases/hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar'\n", - "os.system(cmd_input)" + "!curl -O https://archive.stsci.edu/hlsps/reference-atlases/hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar" ] }, { @@ -553,7 +552,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.6" } }, "nbformat": 4, From 80bfe985bcfbf0b3b61f95770b374dd9fbfbec4b Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Thu, 3 Oct 2024 22:11:20 -0400 Subject: [PATCH 03/11] Safely extract tar files --- .../filter_transformations.ipynb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/notebooks/WFC3/filter_transformations/filter_transformations.ipynb b/notebooks/WFC3/filter_transformations/filter_transformations.ipynb index 068a6a075..5bed0a8de 100644 --- a/notebooks/WFC3/filter_transformations/filter_transformations.ipynb +++ b/notebooks/WFC3/filter_transformations/filter_transformations.ipynb @@ -130,10 +130,17 @@ "source": [ "tar_archive = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar'\n", "extract_to = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed'\n", - "with tarfile.open(tar_archive, 'r') as tar:\n", - " tar.extractall(path=extract_to)\n", + "abs_extract_to = os.path.abspath(extract_to)\n", "\n", - "os.environ['PYSYN_CDBS'] = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed/grp/redcat/trds/'" + "with tarfile.open(tar_archive, 'r') as tar:\n", + " for member in tar.getmembers():\n", + " member_path = os.path.abspath(os.path.join(abs_extract_to, member.name))\n", + " if member_path.startswith(abs_extract_to):\n", + " tar.extract(member, path=extract_to) \n", + " else:\n", + " print(f\"Skipped {member.name} due to potential security risk\")\n", + "\n", + "os.environ['PYSYN_CDBS'] = os.path.join(abs_extract_to, 'grp/redcat/trds/')" ] }, { From 62b79679df7c84b01bc6b899c0e93fdcefb441ae Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Thu, 3 Oct 2024 22:33:57 -0400 Subject: [PATCH 04/11] Safely extract tar files, remove sys call in flux conv nb --- .../flux_conversion_tool.ipynb | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/notebooks/WFC3/flux_conversion_tool/flux_conversion_tool.ipynb b/notebooks/WFC3/flux_conversion_tool/flux_conversion_tool.ipynb index a6f187d8d..cdaed62d5 100644 --- a/notebooks/WFC3/flux_conversion_tool/flux_conversion_tool.ipynb +++ b/notebooks/WFC3/flux_conversion_tool/flux_conversion_tool.ipynb @@ -116,8 +116,7 @@ "metadata": {}, "outputs": [], "source": [ - "cmd_input = 'curl -O https://archive.stsci.edu/hlsps/reference-atlases/hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar'\n", - "os.system(cmd_input)" + "!curl -O https://archive.stsci.edu/hlsps/reference-atlases/hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar" ] }, { @@ -137,10 +136,17 @@ "source": [ "tar_archive = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar'\n", "extract_to = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed'\n", + "abs_extract_to = os.path.abspath(extract_to)\n", + "\n", "with tarfile.open(tar_archive, 'r') as tar:\n", - " tar.extractall(path=extract_to)\n", + " for member in tar.getmembers():\n", + " member_path = os.path.abspath(os.path.join(abs_extract_to, member.name))\n", + " if member_path.startswith(abs_extract_to):\n", + " tar.extract(member, path=extract_to) \n", + " else:\n", + " print(f\"Skipped {member.name} due to potential security risk\")\n", "\n", - "os.environ['PYSYN_CDBS'] = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed/grp/redcat/trds/'" + "os.environ['PYSYN_CDBS'] = os.path.join(abs_extract_to, 'grp/redcat/trds/')" ] }, { @@ -803,14 +809,6 @@ "[Top of Page](#title)\n", "\"Space " ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f708c513", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -829,7 +827,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.6" } }, "nbformat": 4, From 7af2e9a945570c895cce2bf57e811e5873c2b4e9 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 11:55:25 -0400 Subject: [PATCH 05/11] Safely extract files, remove sys call in phot nb --- .../WFC3/photometry_examples/phot_examples.ipynb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/notebooks/WFC3/photometry_examples/phot_examples.ipynb b/notebooks/WFC3/photometry_examples/phot_examples.ipynb index 4b9d25873..4fa80be32 100644 --- a/notebooks/WFC3/photometry_examples/phot_examples.ipynb +++ b/notebooks/WFC3/photometry_examples/phot_examples.ipynb @@ -122,8 +122,7 @@ "metadata": {}, "outputs": [], "source": [ - "cmd_input = 'curl -O https://archive.stsci.edu/hlsps/reference-atlases/hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar'\n", - "os.system(cmd_input)" + "!curl -O https://archive.stsci.edu/hlsps/reference-atlases/hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar" ] }, { @@ -143,10 +142,17 @@ "source": [ "tar_archive = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed.tar'\n", "extract_to = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed'\n", + "abs_extract_to = os.path.abspath(extract_to)\n", + "\n", "with tarfile.open(tar_archive, 'r') as tar:\n", - " tar.extractall(path=extract_to)\n", + " for member in tar.getmembers():\n", + " member_path = os.path.abspath(os.path.join(abs_extract_to, member.name))\n", + " if member_path.startswith(abs_extract_to):\n", + " tar.extract(member, path=extract_to) \n", + " else:\n", + " print(f\"Skipped {member.name} due to potential security risk\")\n", "\n", - "os.environ['PYSYN_CDBS'] = 'hlsp_reference-atlases_hst_multi_everything_multi_v11_sed/grp/redcat/trds/'" + "os.environ['PYSYN_CDBS'] = os.path.join(abs_extract_to, 'grp/redcat/trds/')" ] }, { @@ -874,7 +880,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 64e03a2dc40214d8e843da59bcca3868ee83ed28 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 12:12:43 -0400 Subject: [PATCH 06/11] Remove sys call in the align_sparse_field nn --- .../DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb b/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb index d1a75cb74..136328cb9 100755 --- a/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb +++ b/notebooks/DrizzlePac/align_sparse_fields/align_sparse_fields.ipynb @@ -801,7 +801,7 @@ "# The following lines of code find and download the MDRIZTAB reference file.\n", "mdz = fits.getval(input_files[0], 'MDRIZTAB', ext=0).split('$')[1]\n", "print('Searching for the MDRIZTAB file:', mdz)\n", - "get_mdriztab = os.system('crds sync --hst --files ' + mdz + ' --output-dir '+os.environ['jref'])" + "!crds sync --hst --files {mdz} --output-dir {os.environ['jref']}" ] }, { @@ -1013,7 +1013,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 37d3e827009d397063caa20b7beaa18090cddcde Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 12:19:21 -0400 Subject: [PATCH 07/11] Remove sys call in align catalogs nb --- .../DrizzlePac/align_to_catalogs/align_to_catalogs.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/DrizzlePac/align_to_catalogs/align_to_catalogs.ipynb b/notebooks/DrizzlePac/align_to_catalogs/align_to_catalogs.ipynb index e955261ae..70f3b2a7b 100644 --- a/notebooks/DrizzlePac/align_to_catalogs/align_to_catalogs.ipynb +++ b/notebooks/DrizzlePac/align_to_catalogs/align_to_catalogs.ipynb @@ -789,7 +789,7 @@ "\n", "print('Searching for the MDRIZTAB file:', mdz)\n", "\n", - "get_mdriztab = os.system('crds sync --hst --files '+mdz+' --output-dir '+os.environ['iref'])" + "!crds sync --hst --files {mdz} --output-dir {os.environ['iref']}" ] }, { @@ -965,7 +965,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 22cb89b8e3a9add6a43b2dac01aebd6a5550bbac Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 13:04:29 -0400 Subject: [PATCH 08/11] remove explicit sys call in the reduction nb --- notebooks/ACS/acs_reduction/acs_reduction.ipynb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/notebooks/ACS/acs_reduction/acs_reduction.ipynb b/notebooks/ACS/acs_reduction/acs_reduction.ipynb index 8b760323a..475803a7e 100644 --- a/notebooks/ACS/acs_reduction/acs_reduction.ipynb +++ b/notebooks/ACS/acs_reduction/acs_reduction.ipynb @@ -330,8 +330,7 @@ "outputs": [], "source": [ "for file in raw_files:\n", - " command_line_input = 'crds bestrefs --files {:} --sync-references=1 --update-bestrefs'.format(file)\n", - " os.system(command_line_input)" + " !crds bestrefs --files {file} --sync-references=1 --update-bestrefs" ] }, { @@ -357,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "os.system('calacs.e j9l960010_asn.fits')" + "!calacs.e j9l960010_asn.fits" ] }, { @@ -475,7 +474,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 24acb65d32a294daf1560cad047d14f9d9cbb877 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 14:49:31 -0400 Subject: [PATCH 09/11] Rmv explicit sys call on the optimize img nb --- .../optimize_image_sampling/optimize_image_sampling.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/DrizzlePac/optimize_image_sampling/optimize_image_sampling.ipynb b/notebooks/DrizzlePac/optimize_image_sampling/optimize_image_sampling.ipynb index ff914a47c..a1812f6f6 100644 --- a/notebooks/DrizzlePac/optimize_image_sampling/optimize_image_sampling.ipynb +++ b/notebooks/DrizzlePac/optimize_image_sampling/optimize_image_sampling.ipynb @@ -349,7 +349,7 @@ "source": [ "mdz = fits.getval(flt_files[0], 'MDRIZTAB', ext=0).split('$')[1]\n", "print('Searching for the MDRIZTAB file:', mdz)\n", - "get_mdriztab = os.system('crds sync --hst --files '+mdz+' --output-dir '+os.environ['iref'])" + "!crds sync --hst --files {mdz} --output-dir {os.environ['iref']}" ] }, { @@ -735,7 +735,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 05aba8c2d4b8e296cd36591975702f467c226ecc Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 14:53:33 -0400 Subject: [PATCH 10/11] remove explicit sys call in astrometry solutions nb --- .../using_updated_astrometry_solutions.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.ipynb b/notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.ipynb index e8c3ad1e7..0f170f322 100644 --- a/notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.ipynb +++ b/notebooks/DrizzlePac/using_updated_astrometry_solutions/using_updated_astrometry_solutions.ipynb @@ -739,7 +739,7 @@ "input_images_f336w = sorted(glob.glob('iepw14g[46]q_flc.fits'))\n", "mdz = fits.getval(input_images_f336w[0], 'MDRIZTAB', ext=0).split('$')[1]\n", "print('Searching for the MDRIZTAB file:', mdz)\n", - "get_mdriztab = os.system('crds sync --hst --files '+mdz+' --output-dir '+os.environ['iref'])" + "!crds sync --hst --files {mdz} --output-dir {os.environ['iref']}" ] }, { @@ -981,7 +981,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, From 95d921c1c95eec9bcf69dedf221f8e739ac0a3d0 Mon Sep 17 00:00:00 2001 From: Hatice Karatay Date: Fri, 4 Oct 2024 15:00:53 -0400 Subject: [PATCH 11/11] Update mask_satellite nb --- .../DrizzlePac/mask_satellite/mask_satellite.ipynb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/notebooks/DrizzlePac/mask_satellite/mask_satellite.ipynb b/notebooks/DrizzlePac/mask_satellite/mask_satellite.ipynb index bb0209d7f..038d09f86 100644 --- a/notebooks/DrizzlePac/mask_satellite/mask_satellite.ipynb +++ b/notebooks/DrizzlePac/mask_satellite/mask_satellite.ipynb @@ -200,9 +200,7 @@ " os.rename(file, os.path.basename(file))\n", " for file in flc_download['Local Path']:\n", " os.rename(file, os.path.basename(file))\n", - " shutil.rmtree('mastDownload')\n", - "else:\n", - " pass" + " shutil.rmtree('mastDownload')" ] }, { @@ -273,9 +271,7 @@ " os.rename(file, os.path.basename(file))\n", " for file in flc_download['Local Path']:\n", " os.rename(file, os.path.basename(file))\n", - " shutil.rmtree('mastDownload')\n", - "else:\n", - " pass" + " shutil.rmtree('mastDownload')" ] }, { @@ -533,7 +529,7 @@ "\n", "mdz = fits.getval(input_images[0], 'MDRIZTAB', ext=0).split('$')[1]\n", "print('Searching for the MDRIZTAB file:', mdz)\n", - "get_mdriztab = os.system('crds sync --hst --files ' + mdz + ' --output-dir '+os.environ['jref'])" + "!crds sync --hst --files {mdz} --output-dir {os.environ['jref']}" ] }, { @@ -998,7 +994,7 @@ "\n", "mdz = fits.getval(input_images[0], 'MDRIZTAB', ext=0).split('$')[1]\n", "print('Searching for the MDRIZTAB file:', mdz)\n", - "get_mdriztab = os.system('crds sync --hst --files '+mdz+' --output-dir '+os.environ['iref'])" + "!crds sync --hst --files {mdz} --output-dir {os.environ['iref']}" ] }, { @@ -1162,7 +1158,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4,