Restructure. Enhance unit testing and use poetry (#18)

* Use poetry. Import files directly, re-structure

* More rewrite and use pytest and mockings

* Finally linting ok. Resstructuring. More unit tests. Gridpp only in interpolation

* Much more testing and re-writing

* Improve coverage and unit testing

* Linting mostly

* Lost changes

* Add a general obs2json entry point to be used for e.g obsoul

* Need all values in ncgen now

* Correct linting

* Correct name and increase version

* Split dependencies

* Adapt to pep517

* Try commenting out groups

* Correct workflow

* Correct typo

* Correct typo

* Assume yes

* Add deps

* Install ncgen

* Use only toml and update README/INSTALL

* Install coveralls

* Disable auto

* Move coveralls group

* Do as I'm told

* Change coveralls tool

* Make is not in poetry

* Try docs

* Try docs

* Try docs

* Call sphinx-build from poetry

.flakeheaven.toml

@@ -0,0 +1,49 @@
+[tool.flakeheaven]
+    exclude = [".*/", "tmp/", "*/tmp/", "*.ipynb"]
+    # Group output by file. Colored.
+    format = "grouped"
+    # Show line of source code in output, with syntax highlighting
+    show_source = true
+    # flake8-darglint params
+    docstring_style = "google"
+    strictness = "short"
+
+    # list of plugins and rules for them
+    [tool.flakeheaven.plugins]
+        # Activate all rules for all plugins by default
+        "*" = ["+*"]
+        # Remove from flake8-bandit:
+        #    "S403": Consider possible security implications associated with pickle
+        #    "S404": Consider possible security implications associated with subprocess
+        #    "S603": To allow using subprocess.call/run
+        #    "S606": To allow using os.startfile
+        flake8-bandit = ["+*", "-S403", "-S404", "-S603", "-S606"]
+        # Remove C408 from flake8-comprehensions because I think sometimes the "dict" syntax
+        # looks cleaner than literal "{}". Dict creation performance is not an issue here.
+        flake8-comprehensions = ["+*", "-C408"]
+        flake8-docstrings = ["+*", "-D105"] # Remove "D105: Missing docstring in magic method"
+        # Exclude some errors from pycodestyle for compatibility with black.
+        # "E501" is for max_line_length violations. Leave this for black to handle.
+        # For the other excluded errors, see:
+        # <https://black.readthedocs.io/en/stable/the_black_code_style.html#slices>
+        # <https://black.readthedocs.io/en/stable/the_black_code_style.html#line-breaks-binary-operators>
+        pycodestyle = ["+*", "-W503", "-E203", "-E501"]
+        # Disable pylint plugin at the moment. pylint will be run separately.
+        pylint = ["-*"]
+
+    [tool.flakeheaven.exceptions."*/wsgi.py"]
+        # Ignore "F401 (imported but unused)" in this case
+        pyflakes = ["+*", "-F401"]
+
+    [tool.flakeheaven.exceptions."tests/*.py"]
+        # Disable some flake8-bandit checks in tests:
+        # "S101": To allow assert use
+        # "S301": To Allow testing pickle/unpickle
+        flake8-bandit = ["+*", "-S101", "-S301"]
+        # Ignore "-D105" and "-D102" (Missing docstring in public class/method) in unit tests.
+        # The unit tests class and method names are supposed to be self-explanatory.
+        flake8-docstrings = ["+*", "-D105", "-D101", "-D102", "-D103"]
+
+    [tool.flakeheaven.exceptions."deode/templates/*.py"]
+        # "E265" is for block comments starting with "# ". It may break ecflow envsubst.
+        pycodestyle = ["-E265"]

.github/workflows/linting.yaml

@@ -0,0 +1,69 @@
+#.github/workflows/linting.yaml
+name: Linting Checks
+
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+    paths:
+      - '**.py'
+  push:
+    branches:
+      - '**' # Every branch
+    paths:
+      - '**.py'
+
+jobs:
+  linting:
+    if: github.repository_owner == 'metno'
+    name: Run Linters
+    runs-on: ubuntu-latest
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      #----------------------------------------------
+      #  --- configure poetry & install project  ----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv (if cache exists)
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+
+      - name: Install dependencies (if venv cache is not found)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root --only linting
+
+      - name: Install the project itself
+        run: poetry install --no-interaction --only-root
+
+      #----------------------------------------------
+      #         Run the linting checks
+      #----------------------------------------------
+      - name: isort
+        run: |
+          poetry run isort --check-only .
+      - name: black
+        run: |
+          poetry run black --check .
+      - name: flakeheaven
+        run: |
+          export FLAKEHEAVEN_CACHE="${TMPDIR:-${TEMP:-${TMP:-/tmp}}}/flakeheaven_cache.$(openssl rand -hex 12)"
+          poetry run flakeheaven lint .

.github/workflows/python-package.yml

@@ -11,104 +11,92 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    if: github.repository_owner == 'metno'
     strategy:
+      fail-fast: true
       matrix:
-        python-version: [3.8]
+        os: [ "ubuntu-latest" ]
+        env: [ "pytest" ]
+        python-version: [ "3.8" ]
+
+    name: "${{ matrix.os }}, python=${{ matrix.python-version }}"
+    runs-on: ${{ matrix.os }}
+
+    container:
+      image: python:${{ matrix.python-version }}-bullseye
+      env:
+        COVERAGE_FILE: ".coverage.${{ matrix.env }}.${{ matrix.python-version }}"
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Check config
-      run: |
-        cat > sshd_config <<EOF
-          SyslogFacility AUTHPRIV
-          PermitRootLogin yes
-          AuthorizedKeysFile	.ssh/authorized_keys
-          PasswordAuthentication yes
-          ChallengeResponseAuthentication no
-          UsePAM yes
-          UseDNS no
-          X11Forwarding no
-          PrintMotd no
-        EOF
-        sudo mv sshd_config /etc/ssh/sshd_config
-        sudo systemctl restart ssh
-    - name: Create ssh key
-      run: >
-        ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa
-    - name: Add key to auth file
-      run: >
-        cat ~/.ssh/id_rsa.pub | tee -a ~/.ssh/authorized_keys
-    - name: Ensure the owner of the key is correct
-      run: |
-        chmod 600 ~/.ssh/authorized_keys
-        chmod 700 ~/.ssh
-        sudo chmod -c 0755 ~/
-        ls -la ~/.ssh
-    - name: Test SSH connection to localhost
-      run: >
-        ssh -vvv -i ~/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no $(whoami)@localhost
-    - name: Install dependencies
-      run: |
-        sudo apt update && sudo apt install libudunits2-dev libboost-dev libproj-dev libeccodes0 libeccodes-dev \
-        libarmadillo-dev libgsl-dev python3-setuptools python3-nose python3-numpy python3-pyproj
+      #----------------------------------------------
+      #           check-out repo
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y libudunits2-dev libproj-dev libeccodes0 libeccodes-dev libnetcdf-dev netcdf-bin
+
+      #----------------------------------------------
+      #  --- configure poetry & install project  ----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv (if cache exists)
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+
+      - name: Install dependencies (if venv cache is not found)
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root --only main,test
+
+      - name: Install the project itself
+        run: poetry install --no-interaction --only-root
+
+      #----------------------------------------------
+      #      run test suite and report coverage
+      #----------------------------------------------
+      - name: Run tests
+        run: |
+          poetry run pytest
+
+      - name: Coveralls
+        if: ${{ matrix.python-version == 3.8 }}
+        run: |
+          export COVERALLS_REPO_TOKEN=${{secrets.COVERALLS_REPO_TOKEN}}
+          git config --global --add safe.directory $PWD
+          poetry run coveralls
 
-        python -m pip install --upgrade pip
-        python --version
-        pip install numpy
-        python -c "import numpy"
-        pip install flake8 pytest
-        pip install sphinx
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    #- name: Lint with flake8
-    #  run: |
-    #    # stop the build if there are Python syntax errors or undefined names
-    #    flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-    #    # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-    #   flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Install package
-      run: |
-        pip install -e .[test,plot]
-    - name: Test and create coverage
-      run: |
-        wget --no-check-certificate --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=16XcmpkaIRki2-F5D16j133kbYdZo_Ws9' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=16XcmpkaIRki2-F5D16j133kbYdZo_Ws9" -O testdata.zip && rm -rf /tmp/cookies.txt
-        unzip -o testdata.zip
-        echo "Testdata"
-        ./prepare_testdata.sh
-        export PATH=$PWD/test/bin:/usr/bin/:$PATH
-        export CLIENTID=${{secrets.CLIENTID}}
-        coverage run --source=. -m unittest discover
-    - name: Coveralls
-      if: ${{ matrix.python-version == 3.8 }}
-      run: |
-        export CLIENTID=${{ secrets.CLIENTID }}
-        # coverage html -d coverage
-        COVERALLS_REPO_TOKEN=${{ secrets.COVERALLS_REPO_TOKEN }} coveralls
-    - name: Create documentation
-      if: ${{ matrix.python-version == 3.8 }}
-      run: |
-        cd docs
-        make html
-    - name: Commit documentation changes
-      if: ${{ matrix.python-version == 3.8 }}
-      run: |
-        git clone https://github.com/metno/pysurfex.git --branch gh-pages --single-branch gh-pages
-        cp -r docs/build/html/* gh-pages/
-        cd gh-pages
-        git config --local user.email "[email protected]"
-        git config --local user.name "GitHub Action"
-        git add .
-        git commit -m "Update documentation" -a || true
-        # The above command will fail if no changes were present, so we ignore
-        # the return code.
-    - name: Push changes
-      if: ${{ matrix.python-version == 3.8 }} && github.repository == 'metno/pysurfex'
-      uses: ad-m/github-push-action@master
-      with:
-        branch: gh-pages
-        directory: gh-pages
-        github_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create documentation
+        if: ${{ matrix.python-version == 3.8 }}
+        run: |
+          cd docs
+          make html
+      - name: Commit documentation changes
+        if: ${{ matrix.python-version == 3.8 }}
+        run: |
+          git clone https://github.com/metno/pysurfex.git --branch gh-pages --single-branch gh-pages
+          cp -r docs/build/html/* gh-pages/
+          cd gh-pages
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add .
+          git commit -m "Update documentation" -a || true
+          # The above command will fail if no changes were present, so we ignore
+          # the return code.
+      - name: Push changes
+        if: ${{ matrix.python-version == 3.8 && github.event_name != 'pull_request' }}
+        uses: ad-m/github-push-action@master
+        with:
+          branch: gh-pages
+          directory: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}

INSTALL

@@ -1,17 +1,3 @@
 
-
-# PPI centos7
-module load Python/3.6.8 
-mv ~/.local ~/.local.old
-pip3 install titanlib==0.2.1 --user --global-option=build_ext --global-option="-I/modules/centos7/boost/1.69.0/include:/modules/centos7/gsl/2.5/include" --global-option=build_ext --global-option="-L/modules/centos7/gsl/2.5/lib"
-pip3 install pysurfex --user
-pip3 install tomlkit --user
-# Remove enum
-mkdir /modules/centos7/user-apps/suv/pysurfex/0.0.1a8
-
-# Create module
-mv ~/.local/* /modules/centos7/user-apps/suv/pysurfex/0.0.1a8/.
-cp -i /modules/MET/centos7/user-modules/suv/pysurfex/0.0.1-dev /modules/MET/centos7/user-modules/suv/pysurfex/0.0.1a8
-# Modify /modules/MET/centos7/user-modules/suv/pysurfex/0.0.1a8
-
+poetry install