Merge branch 'main' into update_packaging

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit = *test*

.github/workflows/python-package.yml

@@ -5,35 +5,35 @@ name: Goodman Pipeline
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest wheel
+        pip install flake8 pytest-cov wheel
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Compile DCR Binaries
       run: |
-        SOURCE_DIR=${{ github.workspace }}/goodman_pipeline/data/dcr-source/dcr
+        SOURCE_DIR=${{ github.workspace }}/goodman_pipeline/data/dcr_source/dcr
         make --directory $SOURCE_DIR
         chmod +x $SOURCE_DIR/dcr
-        echo "${{ github.workspace }}/goodman_pipeline/data/dcr-source/dcr" >> $GITHUB_PATH
+        echo "${{ github.workspace }}/goodman_pipeline/data/dcr_source/dcr" >> $GITHUB_PATH
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -42,4 +42,11 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        pytest
+        pytest --cov=goodman_pipeline
+
+    - uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+        fail_ci_if_error: true # optional (default = false)
+
+

.github/workflows/python-publish.yml

@@ -2,29 +2,43 @@
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
 name: Upload to PYPI
-on:
-  release:
-    types: [created]
+on: push
 
 jobs:
-  deploy:
-
+  release-build:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+        python3 -m pip install build
+    - name: Build a binary wheel and source tarball
+      run: python3 -m build
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: release-dists
+        path: dist/
+
+  pypi-publish:
+    name: Publish Package to PyPI
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - release-build
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Retrieve release distributions
+      uses: actions/download-artifact@v4
+      with:
+        name: release-dists
+        path: dist/
+    - name: Publish release distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -7,10 +7,12 @@ __pycache__/*
 .ipynb_checkpoints/*
 files.txt
 docs/_build/*
-goodman_pipeline/data/dcr-source/dcr
-goodman_pipeline/data/dcr-source/*.o
+goodman_pipeline/data/dcr_source/dcr
+goodman_pipeline/data/dcr_source/*.o
 /dev-tools/
-.tox/*
-.tmp/*
+.cache/*
 .eggs/*
-.cache/*
+.tmp/*
+.tox/*
+goodman_pipeline.egg-info/*
+*.DS_Store

.readthedocs.yml

@@ -1,14 +1,17 @@
 version: 2
 
+build:
+  os: ubuntu-lts-latest
+  tools:
+    python: "mambaforge-22.9"
+
+
 sphinx:
     configuration: docs/conf.py
 
 formats: all
 
-
-
 python:
-    version: 3.5
     install:
         - method: pip
           path: .

README.md

@@ -1,19 +1,20 @@
 # Goodman High Throughput Spectrograph Data Reduction Pipeline
 
-![Goodman Pipeline](https://github.com/soar-telescope/goodman_pipeline/workflows/Goodman%20Pipeline/badge.svg?branch=master)
-[![Coverage Status](https://coveralls.io/repos/github/soar-telescope/goodman_pipeline/badge.svg?branch=master)](https://coveralls.io/github/soar-telescope/goodman_pipeline?branch=master)
+![Goodman Pipeline](https://github.com/soar-telescope/goodman_pipeline/actions/workflows/python-package.yml/badge.svg)
+[![codecov](https://codecov.io/gh/soar-telescope/goodman_pipeline/branch/main/graph/badge.svg)](https://codecov.io/gh/soar-telescope/goodman_pipeline)
 [![Documentation Status](https://readthedocs.org/projects/goodman/badge/?version=latest)](http://goodman.readthedocs.io/en/latest/?badge=latest)
 [![pypi](https://img.shields.io/pypi/v/goodman_pipeline.svg?style=flat)](https://pypi.org/project/goodman-pipeline/)
 [![astropy](http://img.shields.io/badge/powered%20by-AstroPy-orange.svg?style=flat)](http://www.astropy.org/)
 
 
 ![Upload to PYPI](https://github.com/soar-telescope/goodman_pipeline/workflows/Upload%20to%20PYPI/badge.svg)
 
+
 ## Overview
 The Goodman High Throughput Spectrograph (Goodman HTS) Data-Reduction Pipeline
 is the SOAR Telescope's official data reduction pipeline for *Goodman HTS*.
 
-It has been fully developed in Python 3.5 and uses mostly astropy affiliated packages
+It has been fully developed in Python and uses mostly astropy affiliated packages
 with the exception of [dcr](http://users.camk.edu.pl/pych/DCR/) which is an external tool
 that does cosmic ray identification and correction. The reason for using it
 instead of LACosmic is that it works very well for spectroscopic data and the
@@ -41,7 +42,7 @@ open a [new Issue](https://github.com/soar-telescope/goodman_pipeline/issues/new
 
 ## Development Team
 
-- [Simón Torres](https://github.com/simontorres) (SOAR Telescope Data Analyst - main code developer)
+- [Simón Torres](https://github.com/simontorres) (NOIRLab Software Engineer - main code developer)
 - [César Briceño](https://github.com/cbaorion) (SOAR Telescope Scientist - team lead)
 - [Bruno Quint](https://github.com/b1quint) (Brazil Support Astronomer - code development adviser)
 

docs/_common_issues.rst

@@ -0,0 +1,48 @@
+.. _common_issues:
+
+Common issues
+*************
+
+No comparison lamps were found
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The latest version may introduce changes to wavelength solutions. To
+view the current list of available modes and lamps, please refer to the
+`GitHub repository <https://github.com/soar-telescope/goodman_pipeline/tree/main/goodman_pipeline/data/ref_comp>`_.
+
+If your lamp, filter, or mode is not included in the repository mentioned
+above, ``redspec`` will not work as expected. This is particularly common
+with custom modes of the |pipeline full name|. You may encounter the following
+error after running ``redspec``:
+
+.. code-block:: console
+
+    $ redspec
+    ፧
+    [15:02:49][W]: No comparison lamps were provided for file ecfzst_0001_science.fits
+
+This error occurs because the |pipeline full name| relies on specific keywords to
+extract spectra, such as:
+
+.. _`table lamp key`:
+
+.. table:: Keywords that are used to produce a wavelength solution.
+
+    ========== =============================================================
+     Keyword    Purpose
+    ========== =============================================================
+     LAMP_HGA   Indicates if HgAr lamp is used.
+     LAMP_NE    Indicates if Ne lamp is used.
+     LAMP_AR    Indicates if Ar lamp is used.
+     LAMP_FE    Indicates if Fe lamp is used.
+     LAMP_CU    Indicates if Cu lamp is used.
+     WAVMODE    Slit and mode configuration.
+    ========== =============================================================
+
+Multiple spectra output
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+When taking multiple ``ARC`` images during your observation run, they will be linked
+with your science data. This means that if you capture several lamp files,
+they will be processed alongside the science images, potentially resulting
+in multiple outputs of the same spectrum.

docs/_cosmic_ray_removal.rst

@@ -46,4 +46,9 @@ methods.
 Asymetric binnings have not been tested but the pipeline only takes in
 consideration the dispersion axis to decide. This does not mean that the spatial
 binning does not impact the performance of any of the methods, we just don't
-know it yet.
+know it yet.
+
+.. note::
+
+  The prefix ``c`` is added to all the comparison lamps, despite they not being
+  affected by cosmic rays.

docs/_file_prefixes.rst

@@ -3,19 +3,25 @@
 File Prefixes
 *************
 
+.. note::
+
+  Overscan is no longer performed by default. Since it caused a double bias level substraction.
+  Fixed since release :ref:`v1.3.3`.
+
+
 There are several ways one can do this but we selected adding prefixes to the
 file name because is easier to add and also easy to filter using a terminal,
 for instance.
 
-  ``ls cfzsto*fits``
+  ``ls cfzst*fits``
 
 or in python
 
 .. code-block:: python
 
   import glob
 
-  file_list = glob.glob('cfzsto*fits')
+  file_list = glob.glob('cfzst*fits')
 
 So what does all those letter mean? Here is a table to explain it.
 
@@ -39,19 +45,15 @@ So what does all those letter mean? Here is a table to explain it.
 
 So, for an original file named ``file.fits``:
 
-  ``o_file.fits``
-
-Means the file have been overscan corrected while
-
-  ``eczsto_file.fits``
+  ``eczst_file.fits``
 
 Means the spectrum has been extracted to a 1D  file but the file has not been
 flat fielded (``f`` missing).
 
 Ideally after running ``redccd`` the file should be named:
 
-  ``cfzsto_file.fits``
+  ``cfzst_file.fits``
 
 And after running ``redspec``:
 
-  ``wecfzsto_file.fits``
+  ``wecfzst_file.fits``

docs/_file_suffixes.rst

@@ -25,4 +25,33 @@ There are two scenarios where this can happen:
     ================ ========= ========================================
 
 
-Assuming the two targets in `sci_file.fits` are extracted we'll end up with
+Assuming the two targets in `sci_file.fits` are extracted and they are approximately at the position
+400 and 600 (pixels in spatial axis), after extraction we'll end up with:
+
+.. code-block:: bash
+
+  esci_file_target_1.fits
+  esci_file_target_2.fits
+  elamp_001_390-410.fits
+  elamp_001_590-610.fits
+  elamp_002_390-410.fits
+  elamp_002_590-610.fits
+
+
+The default prefix for extraction is ``e`` and does not have an underscore to separate it from the
+file name.
+
+After wavelength calibration, since there are two suitable lamps and due to the fact that the
+pipeline does not combine solutions, it will save two wavelength calibrated files with each one
+solved by the respective lamp. Then:
+
+.. code-block:: bash
+
+  wesci_file_target_1_ws_1.fits
+  wesci_file_target_1_ws_2.fits
+  wesci_file_target_2_ws_1.fits
+  wesci_file_target_2_ws_2.fits
+  welamp_001_390-410.fits
+  welamp_001_590-610.fits
+  welamp_002_390-410.fits
+  welamp_002_590-610.fits

docs/_install_dcr.rst

@@ -16,7 +16,7 @@ of LACosmic.
 The latest version of the Goodman Spectroscopic Pipeline uses a modified version
 of ``dcr`` to help with the pipeline's workflow. It is included under
 
-  ``<path_to_download_location>/goodman_pipeline/goodman_pipeline/data/dcr-source/dcr/``
+  ``<path_to_download_location>/goodman_pipeline/goodman_pipeline/data/dcr_source/dcr/``
 
 ``goodman_pipeline-<version>`` is the folder that will be created once you untar or unzip the latest
 release of the |pipeline name|.
@@ -38,7 +38,7 @@ Compiling DCR
 
 Compiling ``dcr`` is actually very simple.
 
-  ``cd <path_to_download_location>/goodman_pipeline/goodman_pipeline/data/dcr-source/dcr/``
+  ``cd <path_to_download_location>/goodman_pipeline/goodman_pipeline/data/dcr_source/dcr/``
 
 Then simply type: