Merge pull request #193 from BritishGeologicalSurvey/for_v1

Merge long-running for_v1 branch.

.dockerignore

@@ -1,4 +1,3 @@
-.git
 .gitlab
 bin
 docs

.flake8

@@ -1,2 +1,15 @@
 [flake8]
 max-line-length = 120
+extend-ignore =
+    # Do not require self and cls to have type hints
+    # See: https://github.com/sco1/flake8-annotations/issues/75
+    ANN101, ANN102
+per-file-ignores =
+    # Do not check annotations in these files
+    abort.py: A,
+    connect.py: A,
+    db_helper_factory.py: A,
+    db_params.py: A,
+    row_factories.py: A
+    utils.py: A,
+    etlhelper/db_helpers/*: A,

.github/workflows/gh-pages.yml

@@ -0,0 +1,24 @@
+name: Deploy Sphinx documentation to Pages
+
+on:
+  push:
+    branches: 
+      - for_v1
+    paths:
+      - 'docs/**'  
+
+jobs:
+  pages:
+    runs-on: ubuntu-20.04
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    permissions:
+      pages: write
+      id-token: write
+    steps:
+    - name: Install RTD Theme
+      run: |
+          pip install sphinx sphinx_rtd_theme
+    - id: deployment
+      uses: sphinx-notes/pages@v3

.github/workflows/test.yml

@@ -1,5 +1,12 @@
 name: Lint and test
 
+# Global environment variables for all jobs
+env:
+  POSTGRES_DB: etlhelper
+  POSTGRES_USER: etlhelper_user
+  POSTGRES_PASSWORD: etlhelper_pw
+  TEST_PG_PASSWORD: etlhelper_pw
+
 on:
   push:
     paths:
@@ -24,12 +31,27 @@ jobs:
 
   pytest:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        # Accessed directly from host at localhost, because tests are not
+        # running in a container
+        image: postgis/postgis:15-3.4
+        ports:
+          - 5432:5432
+        # Copy env vars from above - substitution didn't work
+        env:
+          POSTGRES_DB: etlhelper
+          POSTGRES_USER: etlhelper_user
+          POSTGRES_PASSWORD: etlhelper_pw
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     strategy:
       matrix:
-        include:
-          - python-version: 3.9
-          - python-version: '3.10'
-          - python-version: 3.11
+        python-version: [3.9, '3.10', 3.11]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -38,6 +60,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install requirements
-        run: pip install -r requirements.txt
+        run: pip install -r requirements-dev.txt
       - name: Run pytest
-        run: pytest -vvs test/unit/
+        run: pytest -vrsx --cov=etlhelper --cov-report term test/

.gitignore

@@ -107,3 +107,7 @@ setup_env.sh
 # mypy
 .mypy_cache/
 bin/setup_local_env.sh
+.vscode
+
+# Databases created by test scripts
+*.db

CONTRIBUTING.md

@@ -6,10 +6,10 @@
 All main features are complete, however contributions and suggestions are
 welcome, particularly in the following areas:
 
-+ support for more database types (MySQL, Informix)
++ support for more database types (MySQL, IBM Informix)
 + additional recipes / case studies
-+ add type annotations
-+ run more integration tests in Travis CI (or port to GitHub Actions)
++ improved type annotations
++ run more integration tests in GitHub Actions
 + performance optimisation
 + improved documentation
 
@@ -47,7 +47,7 @@ Pull requests and issues should be targeted at the GitHub repository.
 
 ### Prerequisites
 
-+ Python 3.6+ virtual environment
++ Python 3.9+ virtual environment
 + Git
 + Docker
 
@@ -58,19 +58,13 @@ Install locally for development by cloning repository and running the following
 in the root:
 
 ```bash
-pip install -r requirements.txt
+pip install -r requirements-dev.txt
 pip install -e .
 ```
 
 Development files will be linked to the virtual environment/system folder
 hierarchy allowing changes to take effect directly.
 
-Proprietary Oracle Instant Client drivers are required.
-These can be installed with:
-
-```bash
-source $(setup_oracle_client)
-```
 
 ### Running tests
 
@@ -84,7 +78,7 @@ export TEST_PG_PORT=5432
 export TEST_PG_PASSWORD=etlhelper_pw
 docker run -e POSTGRES_USER=etlhelper_user -e POSTGRES_DB=etlhelper \
   -e POSTGRES_PASSWORD=$TEST_PG_PASSWORD --name etlhelper_postgis \
-  -d --rm -p $TEST_PG_PORT:5432 mdillon/postgis:11-alpine
+  -d --rm -p $TEST_PG_PORT:5432 postgis/postgis:15-3.4
 ```
 
 Tests are run with:
@@ -96,7 +90,6 @@ bash bin/run_tests_for_developer.sh
 The test-runner script will run tests within a dedicated container and provide
 HTML coverage output.  It can be viewed with `firefox htmlcov/index.html`.
 
-
 #### Running additional BGS integration tests
 
 Additional integration tests can be run against internal BGS Oracle and SQL Server
@@ -110,7 +103,19 @@ To run these:
 + Paste into the terminal to set environment variables
 + Run tests as before
 
-### Creating a new release
+### Building documentation
+
+The documentation is created using Sphinx.
+To locally build the documentation, run the following:
+
+```bash
+sphinx-build -a docs docs/_build
+```
+
+The documentation can then be viewed at `docs/_build/index.html`
+
+
+## Creating a new release
 
 Releases are created manually from the main branch via tags.
 This should be done via the GitHub web interface.

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6.9-slim
+FROM python:3.9-slim-bullseye
 
 # Install package dependencies
 RUN apt-get update -y && \
@@ -12,7 +12,7 @@ RUN apt-get update -y && \
 # Add repo for Microsoft ODBC driver for SQL Server
 RUN curl https://packages.microsoft.com/keys/microsoft.asc > microsoft.asc && \
     apt-key add microsoft.asc && \
-    curl https://packages.microsoft.com/config/debian/9/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
+    curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
     apt-get update -y && \
     ACCEPT_EULA=y apt-get install -y \
       msodbcsql17 \
@@ -25,21 +25,23 @@ WORKDIR $APP
 RUN mkdir etlhelper
 
 # Install requirements
-COPY requirements.txt $APP/
+COPY requirements-dev.txt $APP/
 RUN pip install --upgrade pip
-RUN pip install -r requirements.txt
+RUN pip install -r requirements-dev.txt
+
+# Copy files required to package for PyPI
+COPY .git/ $APP/.git
+COPY pyproject.toml README.md $APP/
+
+# Copy files required for testing
+COPY .flake8 pytest.ini $APP/
+COPY test/ $APP/test
 
 # Copy app files to container
-COPY setup.py versioneer.py setup.cfg .flake8 .coveragerc README.md pytest.ini $APP/
 COPY etlhelper/ $APP/etlhelper
-COPY test/ $APP/test
 
 # Clear old caches, if present
 RUN find . -regextype posix-egrep -regex '.*/__pycache__.*' -delete
 
-# Set up Oracle Client
-ARG INSTANT_CLIENT_ZIP
-RUN python -m pip install .
-RUN setup_oracle_client -v
-# Have to hard-code oracle_lib_export as ENV can't use result of command
-ENV LD_LIBRARY_PATH=/app/etlhelper/oracle_instantclient
+# Install ETL Helper
+RUN python -m pip install .