From f71197b4ffb57a1aa373e21b2c7617367a84557a Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 25 Oct 2022 12:02:43 -0400 Subject: [PATCH 1/3] Created pipeline-specific environment file --- environment_pipeline.yml | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 environment_pipeline.yml diff --git a/environment_pipeline.yml b/environment_pipeline.yml new file mode 100644 index 000000000..179a46a6f --- /dev/null +++ b/environment_pipeline.yml @@ -0,0 +1,70 @@ +# This file describes a conda environment that can be to install jwql +# +# Run the following command to set up this environment: +# $ conda env create -f environment_python_3_8.yml +# +# The environment name can be overridden with the following command: +# $ conda env create -n -f environment_python_3_8.yml +# +# Run the following command to activate the environment: +# $ source activate jwql-3.8 +# +# To deactivate the environment run the following command: +# $ source deactivate +# +# To remove the environment entirely, run the following command: +# $ conda env remove -n jwql-3.8 + +name: jwql-3.8 + +channels: + - defaults + +dependencies: + - astropy=5.0.4 + - bokeh=2.4.2 + - beautifulsoup4=4.11.1 + - celery=5.0.5 + - codecov=2.1.11 + - cryptography=36.0.0 + - django=3.2.5 + - flake8=4.0.1 + - inflection=0.5.1 + - ipython=8.2.0 + - jinja2=3.0.2 + - jsonschema=3.2.0 + - matplotlib=3.4.2 + - nodejs=10.13.0 + - numpy=1.21.5 + - numpydoc=1.2 + - pandas=1.4.2 + - pip=21.0.1 + - postgresql=12.2 + - psycopg2=2.8.6 + - pytest=7.1.1 + - pytest-cov=3.0.0 + - python=3.8.5 + - pyyaml=6.0 + - redis + - scipy=1.7.3 + - setuptools=52.0.0 + - sphinx=4.2.0 + - sphinx_rtd_theme=0.4.3 + - sqlalchemy=1.4.32 + - twine=3.7.1 + - wtforms=2.3.3 + + - pip: + - astroquery==0.4.6 + - bandit==1.7.4 + - jwst==1.4.6 + - pysiaf==0.15.0 + - pysqlite3==0.4.7 + - pyvo==1.3 + - redis + - stsci_rtd_theme==0.0.2 + - stcal==0.6.4 + - git+https://github.com/spacetelescope/jwst_reffiles + + # Current package + - -e . From 5094a2e582a0162718286400f451c98e93dcbd9e Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Mon, 31 Oct 2022 15:24:48 -0400 Subject: [PATCH 2/3] Adding files to actually create the pipeline environment. --- environment_pipeline.sh | 29 +++++++++++++++++++++++++ environment_pipeline.yml | 37 ++++++++++++-------------------- environment_pipeline_version.txt | 1 + 3 files changed, 44 insertions(+), 23 deletions(-) create mode 100644 environment_pipeline.sh create mode 100644 environment_pipeline_version.txt diff --git a/environment_pipeline.sh b/environment_pipeline.sh new file mode 100644 index 000000000..449830d3b --- /dev/null +++ b/environment_pipeline.sh @@ -0,0 +1,29 @@ +export env_name="jwql-pipeline" +export PIPELINE_VERSION=$( /dev/null + then + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o installer.sh + ./installer.sh -b -p ./miniconda3_tmp + conda init zsh + fi + conda create -n ${env_name} --file https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/conda_python_macos-stable-deps.txt + conda activate ${env_name} + pip install -r https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/reqs_macos-stable-deps.txt + ;; + Linux) + if ! command -v conda &> /dev/null + then + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o installer.sh + ./installer.sh -b -p ./miniconda3_tmp + conda init zsh + fi + conda create -n ${env_name} --file https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/conda_python_stable-deps.txt + conda activate ${env_name} + pip install -r https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/reqs_stable-deps.txt + ;; +esac + +conda env update --name ${env_name} --file environment_pipeline.yml --prune diff --git a/environment_pipeline.yml b/environment_pipeline.yml index 179a46a6f..612bc2c1b 100644 --- a/environment_pipeline.yml +++ b/environment_pipeline.yml @@ -1,52 +1,44 @@ -# This file describes a conda environment that can be to install jwql +# This file describes a conda environment that can be used to install the jwql +# pipeline processing environment. It is not a standalone environment, but instead is +# intended as a supplement to the JWST pipeline environments stored online at STScI. # # Run the following command to set up this environment: -# $ conda env create -f environment_python_3_8.yml +# $ source environment_pipeline.sh # -# The environment name can be overridden with the following command: -# $ conda env create -n -f environment_python_3_8.yml +# Note that this environment takes the pipeline version from the file +# +# environment_pipeline_version.txt # # Run the following command to activate the environment: -# $ source activate jwql-3.8 +# $ source activate jwql-pipeline # # To deactivate the environment run the following command: # $ source deactivate # # To remove the environment entirely, run the following command: -# $ conda env remove -n jwql-3.8 +# $ conda env remove -n jwql-pipeline -name: jwql-3.8 +name: jwql-pipeline channels: - defaults dependencies: - - astropy=5.0.4 - bokeh=2.4.2 - beautifulsoup4=4.11.1 - - celery=5.0.5 - - codecov=2.1.11 + - celery - cryptography=36.0.0 - django=3.2.5 - - flake8=4.0.1 - inflection=0.5.1 - ipython=8.2.0 - jinja2=3.0.2 - - jsonschema=3.2.0 - - matplotlib=3.4.2 - nodejs=10.13.0 - - numpy=1.21.5 - - numpydoc=1.2 + - numpydoc - pandas=1.4.2 - pip=21.0.1 - postgresql=12.2 - psycopg2=2.8.6 - - pytest=7.1.1 - - pytest-cov=3.0.0 - - python=3.8.5 - - pyyaml=6.0 - redis - - scipy=1.7.3 - setuptools=52.0.0 - sphinx=4.2.0 - sphinx_rtd_theme=0.4.3 @@ -57,14 +49,13 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.4 - - jwst==1.4.6 - pysiaf==0.15.0 - pysqlite3==0.4.7 - pyvo==1.3 - - redis - stsci_rtd_theme==0.0.2 - - stcal==0.6.4 - git+https://github.com/spacetelescope/jwst_reffiles + - vine==1.3.0 + - redis # Current package - -e . diff --git a/environment_pipeline_version.txt b/environment_pipeline_version.txt new file mode 100644 index 000000000..c514bd85c --- /dev/null +++ b/environment_pipeline_version.txt @@ -0,0 +1 @@ +1.4.6 From 9c2773334aab933d79cd3a04614c026d77fd8459 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Mon, 31 Oct 2022 21:46:55 -0400 Subject: [PATCH 3/3] changed bad pixel monitor so that it won't copy uncalibrated file if it exists locally --- .../common_monitors/bad_pixel_monitor.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 86a8d78e4..b8855d2cf 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -790,8 +790,11 @@ def process(self, illuminated_raw_files, illuminated_slope_files, dark_raw_files short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) logging.info('Calling pipeline for {}'.format(uncal_file)) - logging.info("Copying raw file to {}".format(self.data_dir)) - copy_files([uncal_file], self.data_dir) + if os.path.isfile(local_uncal_file): + logging.info("Local uncalibrated file exists") + else: + logging.info("Copying raw file to {}".format(self.data_dir)) + copy_files([uncal_file], self.data_dir) if hasattr(self, 'nints') and self.nints > 1: out_exts[short_name] = ['jump', '1_ramp_fit'] in_files.append(local_uncal_file) @@ -828,9 +831,12 @@ def process(self, illuminated_raw_files, illuminated_slope_files, dark_raw_files out_exts = defaultdict(lambda: ['jump', 'fitopt', '0_ramp_fit']) for uncal_file, rate_file in zip(dark_raw_files, dark_slope_files): logging.info('Calling pipeline for {} {}'.format(uncal_file, rate_file)) - logging.info("Copying raw file to {}".format(self.data_dir)) - copy_files([uncal_file], self.data_dir) local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) + if os.path.isfile(local_uncal_file): + logging.info("Local uncalibrated file exists") + else: + logging.info("Copying raw file to {}".format(self.data_dir)) + copy_files([uncal_file], self.data_dir) short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') if hasattr(self, 'nints') and self.nints > 1: out_exts[short_name] = ['jump', 'fitopt', '1_ramp_fit']