diff --git a/environment_pipeline.sh b/environment_pipeline.sh new file mode 100644 index 000000000..449830d3b --- /dev/null +++ b/environment_pipeline.sh @@ -0,0 +1,29 @@ +export env_name="jwql-pipeline" +export PIPELINE_VERSION=$( /dev/null + then + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o installer.sh + ./installer.sh -b -p ./miniconda3_tmp + conda init zsh + fi + conda create -n ${env_name} --file https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/conda_python_macos-stable-deps.txt + conda activate ${env_name} + pip install -r https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/reqs_macos-stable-deps.txt + ;; + Linux) + if ! command -v conda &> /dev/null + then + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o installer.sh + ./installer.sh -b -p ./miniconda3_tmp + conda init zsh + fi + conda create -n ${env_name} --file https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/conda_python_stable-deps.txt + conda activate ${env_name} + pip install -r https://ssb.stsci.edu/releases/jwstdp/${PIPELINE_VERSION}/reqs_stable-deps.txt + ;; +esac + +conda env update --name ${env_name} --file environment_pipeline.yml --prune diff --git a/environment_pipeline.yml b/environment_pipeline.yml new file mode 100644 index 000000000..612bc2c1b --- /dev/null +++ b/environment_pipeline.yml @@ -0,0 +1,61 @@ +# This file describes a conda environment that can be used to install the jwql +# pipeline processing environment. It is not a standalone environment, but instead is +# intended as a supplement to the JWST pipeline environments stored online at STScI. +# +# Run the following command to set up this environment: +# $ source environment_pipeline.sh +# +# Note that this environment takes the pipeline version from the file +# +# environment_pipeline_version.txt +# +# Run the following command to activate the environment: +# $ source activate jwql-pipeline +# +# To deactivate the environment run the following command: +# $ source deactivate +# +# To remove the environment entirely, run the following command: +# $ conda env remove -n jwql-pipeline + +name: jwql-pipeline + +channels: + - defaults + +dependencies: + - bokeh=2.4.2 + - beautifulsoup4=4.11.1 + - celery + - cryptography=36.0.0 + - django=3.2.5 + - inflection=0.5.1 + - ipython=8.2.0 + - jinja2=3.0.2 + - nodejs=10.13.0 + - numpydoc + - pandas=1.4.2 + - pip=21.0.1 + - postgresql=12.2 + - psycopg2=2.8.6 + - redis + - setuptools=52.0.0 + - sphinx=4.2.0 + - sphinx_rtd_theme=0.4.3 + - sqlalchemy=1.4.32 + - twine=3.7.1 + - wtforms=2.3.3 + + - pip: + - astroquery==0.4.6 + - bandit==1.7.4 + - pysiaf==0.15.0 + - pysqlite3==0.4.7 + - pyvo==1.3 + - stsci_rtd_theme==0.0.2 + - git+https://github.com/spacetelescope/jwst_reffiles + - vine==1.3.0 + - redis + + # Current package + - -e . diff --git a/environment_pipeline_version.txt b/environment_pipeline_version.txt new file mode 100644 index 000000000..c514bd85c --- /dev/null +++ b/environment_pipeline_version.txt @@ -0,0 +1 @@ +1.4.6 diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 86a8d78e4..b8855d2cf 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -790,8 +790,11 @@ def process(self, illuminated_raw_files, illuminated_slope_files, dark_raw_files short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) logging.info('Calling pipeline for {}'.format(uncal_file)) - logging.info("Copying raw file to {}".format(self.data_dir)) - copy_files([uncal_file], self.data_dir) + if os.path.isfile(local_uncal_file): + logging.info("Local uncalibrated file exists") + else: + logging.info("Copying raw file to {}".format(self.data_dir)) + copy_files([uncal_file], self.data_dir) if hasattr(self, 'nints') and self.nints > 1: out_exts[short_name] = ['jump', '1_ramp_fit'] in_files.append(local_uncal_file) @@ -828,9 +831,12 @@ def process(self, illuminated_raw_files, illuminated_slope_files, dark_raw_files out_exts = defaultdict(lambda: ['jump', 'fitopt', '0_ramp_fit']) for uncal_file, rate_file in zip(dark_raw_files, dark_slope_files): logging.info('Calling pipeline for {} {}'.format(uncal_file, rate_file)) - logging.info("Copying raw file to {}".format(self.data_dir)) - copy_files([uncal_file], self.data_dir) local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) + if os.path.isfile(local_uncal_file): + logging.info("Local uncalibrated file exists") + else: + logging.info("Copying raw file to {}".format(self.data_dir)) + copy_files([uncal_file], self.data_dir) short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') if hasattr(self, 'nints') and self.nints > 1: out_exts[short_name] = ['jump', 'fitopt', '1_ramp_fit']