Update FastAPI container and dependencies

.github/workflows/lint_and_test.yml

@@ -23,6 +23,12 @@ jobs:
         python-version: 3.11
         architecture: x64
 
+    - name: Ensure Nexus mirror configuration is not committed
+      run: |
+        # Grep checks for Nexus mirror in the index-url parameter,
+        # the ! negates the grep exit code, which would be 0 if URL was present
+        ! grep -E '.*index-url.*nexus.*' requirements*.txt
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

Dockerfile

@@ -1,4 +1,4 @@
-FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim-2023-02-20
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.11-slim-2024-03-04
 
 ## Install python-ags4
 COPY requirements.txt .

README.md

@@ -93,6 +93,20 @@ pytest -vs test
 
 The test environment is configured so that adding `--pdb` to the test command will start an IPython debugger session in the event of test failure.
 
+### Updating dependencies
+
+We are using [pip-tools](https://pip-tools.readthedocs.io/en/stable/) to create a pinned list of all dependencies from the ones that we need to specify.
+
+To refresh the dependency list, update `requirements.in` and `requirements-dev.in` then run the following:
+
+```bash
+pip-compile -o requirements.txt requirements.in
+pip-compile -o requirements_dev.txt requirements_dev.in
+```
+
+The updated requirements files must be edited to remove reference to the Nexus mirror before they can then be committed.
+
+
 ### GUI Customisation
 
 To ammend the GUI HTML we recommend running via `Docker` using your own `Dockerfile` like the below to `COPY` in your own templates.
@@ -108,15 +122,15 @@ COPY content/templates /app/app/templates
 
 Containers for the application are hosted in the GitHub Container Registry
 
-Every push to `Main` branch commits builds `pyagsapi:latest`.
+Every push to `main` branch commits builds `pyagsapi:latest`.
 
 Push Tagged Releases with `^v?[0-9]+[.][0-9]+([.][0-9])?` (v* == v2.0) builds `pyagsapi:2.0` (the "v" gets dropped for the tag).
 
 You can also push release candidates using the format `/^v?[0-9]+[.][0-9]+([.][0-9])?\-rc/` e.g. v3.1.1-rc builds `pyagsapi:3.1.1-rc`
 
 ## Example Files
 
-Files in [https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real](https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real) are a random collection of real AGS files which have been submitted to the BGS and are available under OGL, we have included them here as example files for testing pyagsapi.
+Files in [https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real](https://github.com/BritishGeologicalSurvey/pyagsapi/tree/main/test/files/real) are a collection of real AGS files which have been submitted to the BGS and are available under OGL, we have included them here as example files for testing pyagsapi.
 
 ## Licence
 

app/checkers.py

@@ -47,6 +47,9 @@ def check_ags(filename: Path, standard_AGS4_dictionary: Optional[str] = None) ->
         errors = {'File read error': [{'line': '-', 'group': '', 'desc': description}]}
         dictionary = ''
 
+    # Discard unecessary summary from errors dictionary
+    errors.pop('Summary of data', None)
+
     return dict(checker=f'python_ags4 v{python_ags4.__version__}',
                 errors=errors, dictionary=dictionary)
 

app/conversion.py

@@ -10,7 +10,7 @@
 logger = logging.getLogger(__name__)
 
 
-def convert(filename: Path, results_dir: Path, sort_tables=False) -> Tuple[Optional[Path], dict]:
+def convert(filename: Path, results_dir: Path, sorting_strategy: Optional[str] = None) -> Tuple[Optional[Path], dict]:
     """
     Convert filename between .ags and .xlsx.  Write output to file in
     results_dir and return path alongside job status data in dictionary."""
@@ -28,7 +28,7 @@ def convert(filename: Path, results_dir: Path, sort_tables=False) -> Tuple[Optio
     success = False
     if filename.suffix.lower() == '.ags':
         try:
-            AGS4.AGS4_to_excel(filename, converted_file, sort_tables=sort_tables)
+            AGS4.AGS4_to_excel(filename, converted_file, sorting_strategy=sorting_strategy)
             success = True
         except IndexError:
             error_message = "ERROR: File does not have AGS4 format layout"

app/routes.py

@@ -110,7 +110,7 @@ class ResponseType(StrEnum):
 )
 
 sort_tables_form = Form(
-    default=False,
+    default='default',
     title='Sort worksheets',
     description=('Sort the worksheets into alphabetical order '
                  'or leave in the order found in the AGS file. '
@@ -240,7 +240,7 @@ def prepare_validation_response(request, data):
                           " sort worksheets in .xlsx file in alphabetical order."))
 async def convert(background_tasks: BackgroundTasks,
                   files: List[UploadFile] = conversion_file,
-                  sort_tables: bool = sort_tables_form,
+                  sort_tables: str = sort_tables_form,
                   request: Request = None):
     """
     Convert files between .ags and .xlsx format. Option to sort worksheets in .xlsx file in alphabetical order.
@@ -258,6 +258,8 @@ async def convert(background_tasks: BackgroundTasks,
     :raises Exception: If the conversion fails or an unexpected error occurs.
     """
 
+    if sort_tables == 'default':
+        sort_tables = None
     if not files[0].filename:
         raise InvalidPayloadError(request)
     RESULTS = 'results'
@@ -271,7 +273,7 @@ async def convert(background_tasks: BackgroundTasks,
             contents = await file.read()
             local_file = tmp_dir / file.filename
             local_file.write_bytes(contents)
-            converted, result = conversion.convert(local_file, results_dir, sort_tables=sort_tables)
+            converted, result = conversion.convert(local_file, results_dir, sorting_strategy=sort_tables)
             log = validation.to_plain_text(result)
             f.write(log)
             f.write('\n' + '=' * 80 + '\n')

app/schemas.py

@@ -15,6 +15,8 @@
     'AGS Format Rule 13', 'AGS Format Rule 14', 'AGS Format Rule 15', 'AGS Format Rule 16',
     'AGS Format Rule 17', 'AGS Format Rule 18', 'AGS Format Rule 19', 'AGS Format Rule 19a',
     'AGS Format Rule 19b', 'AGS Format Rule 20', 'General',
+    # Warnings and FYIs
+    'Warning (Related to Rule 16)', 'FYI (Related to Rule 1)',
     # Errors
     'File read error'
 ]

app/static/js/htmlView.js

@@ -207,9 +207,6 @@ agsHtml.displayFileResult=function(fileResult){
         if(fileResult.additional_metadata.bgs_loca_rows){
             xhtml=xhtml + "<li>" + fileResult.additional_metadata.bgs_loca_rows + "</li>";
             };
-        if(fileResult.additional_metadata.bgs_loca_rows){
-            xhtml=xhtml + "<li>" + fileResult.additional_metadata.bgs_loca_rows + "</li>";
-            };
         xhtml=xhtml + "</ul>";
         };
 

app/templates/_base.html

@@ -409,7 +409,7 @@
 		<hr>
 		<br>
       <div>
-            Powered by <a title="pyagsapi" href="https://github.com/BritishGeologicalSurvey/pyagsapi">pyagsapi 4.6</a>. pyagsapi was created by and is maintained by the <a href="https://www.bgs.ac.uk/">British Geological Survey</a> and  is distributed under the <a title="license" href="https://github.com/BritishGeologicalSurvey/pyagsapi/blob/main/LICENSE">LGPL v3.0 licence</a>, code is available on <a title="github" href="https://github.com/BritishGeologicalSurvey/pyagsapi">GitHub</a>. <a title="agspython" href="https://github.com/BritishGeologicalSurvey/pyagsapi">pyagsapi</a> uses the <a href="https://gitlab.com/ags-data-format-wg/ags-python-library.">Official AGS Python Library</a>.
+            Powered by <a title="pyagsapi" href="https://github.com/BritishGeologicalSurvey/pyagsapi">pyagsapi 5.0</a>. pyagsapi was created by and is maintained by the <a href="https://www.bgs.ac.uk/">British Geological Survey</a> and  is distributed under the <a title="license" href="https://github.com/BritishGeologicalSurvey/pyagsapi/blob/main/LICENSE">LGPL v3.0 licence</a>, code is available on <a title="github" href="https://github.com/BritishGeologicalSurvey/pyagsapi">GitHub</a>. <a title="agspython" href="https://github.com/BritishGeologicalSurvey/pyagsapi">pyagsapi</a> uses the <a href="https://gitlab.com/ags-data-format-wg/ags-python-library.">Official AGS Python Library v0.5.0</a>.
         </div>
 
 		<div id="extentModal">

app/templates/landing_page.html

@@ -30,7 +30,7 @@ <h2>AGS Data Submission</h2>
     <h2>AGS4 Schema & Data Validation</h2>
     <br>
     <h3>AGS4 Validation </h3>
-      <p>Performs validation using the <a href="https://gitlab.com/ags-data-format-wg/ags-python-library.">Official AGS Python Library</a> <strong>version 0.4.1</strong>, this implements checks of the rules as written in the <strong>AGS data format standard v4.x.</strong></p>
+      <p>Performs validation using the <a href="https://gitlab.com/ags-data-format-wg/ags-python-library.">Official AGS Python Library</a> <strong>version 0.5.0</strong>, this implements checks of the rules as written in the <strong>AGS data format standard v4.x.</strong></p>
       <p style="color:red">If you're using AGS Data Format Standard v3.x use our legacy <a href="https://webapps.bgs.ac.uk/data/ags/validation/index.cfm">AGS Validator</a></p>
     <br>
     <h3>BGS Data Validation</h3>
@@ -165,11 +165,15 @@ <h2>AGS Converter</h2>
   <br>
   <form action="/convert/" enctype="multipart/form-data" method="post" id="convertForm">
     <fieldset>
-      <legend>Sort worksheets in .xlsx file in alphabetical order <strong>(Warning: .ags to .xlsx only. The original group order will be lost)</strong></legend>
-      <input type="radio" id="alpha" name="sort_tables" value="True">
-      <label for="alpha">True</label>
-      <input type="radio" id="default" name="sort_tables" value="False" checked>
-      <label for="default">False</label><br>
+      <legend>Sort worksheets in .xlsx file using sorting strategy<strong>(Warning: .ags to .xlsx only. The original group order will be lost)</strong></legend>
+      <input type="radio" id="dictionary" name="sort_tables" value="dictionary">
+      <label for="dictionary">Dictionary</label>
+      <input type="radio" id="alphabetical" name="sort_tables" value="alphabetical">
+      <label for="alphabetical">Alphabetical</label>
+      <input type="radio" id="hierarchical" name="sort_tables" value="hierarchical">
+      <label for="hierarchical">Hierarchical</label>
+      <input type="radio" id="default" name="sort_tables" value="default" checked>
+      <label for="default">None (Maintain Input File Order)</label><br>
     </fieldset>
     <br>
     <fieldset>

requirements.in

@@ -0,0 +1,18 @@
+Fiona
+Jinja2
+Shapely
+aiofiles
+colorlog
+geopandas
+numpy
+pyproj
+python-ags4==0.5.0
+requests
+shortuuid
+# These libraries are already in FastAPI container but need updated
+fastapi==0.88.0
+h11==0.14.0
+pydantic==1.10.14
+python-multipart==0.0.9
+starlette==0.22.0
+uvicorn==0.20.0

requirements.txt

@@ -1,16 +1,123 @@
-aiofiles==23.1.0
-numpy==1.24.2
-python-ags4==0.4.1
-python-multipart==0.0.5
-colorlog==6.7.0
-shortuuid==1.0.11
-Jinja2==3.1.2
-Fiona==1.9.1
-Shapely==2.0.1
-pyproj==3.4.1
-geopandas==0.12.2
-requests==2.28.2
-# These libraries are already in FastAPI container but need updated
-fastapi==0.92.0
-uvicorn==0.20.0
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --output-file=requirements.txt requirements.in
+#
+
+aiofiles==23.2.1
+    # via -r requirements.in
+anyio==4.3.0
+    # via starlette
+attrs==23.2.0
+    # via fiona
+certifi==2024.2.2
+    # via
+    #   fiona
+    #   pyproj
+    #   requests
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via
+    #   click-plugins
+    #   cligj
+    #   fiona
+    #   python-ags4
+    #   uvicorn
+click-plugins==1.1.1
+    # via fiona
+cligj==0.7.2
+    # via fiona
+colorlog==6.8.2
+    # via -r requirements.in
+defusedxml==0.7.1
+    # via python-ags4
+et-xmlfile==1.1.0
+    # via openpyxl
+fastapi==0.88.0
+    # via -r requirements.in
+fiona==1.9.5
+    # via
+    #   -r requirements.in
+    #   geopandas
+geopandas==0.14.3
+    # via -r requirements.in
 h11==0.14.0
+    # via
+    #   -r requirements.in
+    #   uvicorn
+idna==3.6
+    # via
+    #   anyio
+    #   requests
+jinja2==3.1.3
+    # via -r requirements.in
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+numpy==1.26.4
+    # via
+    #   -r requirements.in
+    #   pandas
+    #   shapely
+openpyxl==3.1.2
+    # via python-ags4
+packaging==23.2
+    # via geopandas
+pandas==2.2.1
+    # via
+    #   geopandas
+    #   python-ags4
+pydantic==1.10.14
+    # via
+    #   -r requirements.in
+    #   fastapi
+pygments==2.17.2
+    # via rich
+pyproj==3.6.1
+    # via
+    #   -r requirements.in
+    #   geopandas
+python-ags4==0.5.0
+    # via -r requirements.in
+python-dateutil==2.9.0.post0
+    # via pandas
+python-multipart==0.0.9
+    # via -r requirements.in
+pytz==2024.1
+    # via pandas
+requests==2.31.0
+    # via -r requirements.in
+rich==13.7.1
+    # via python-ags4
+shapely==2.0.3
+    # via
+    #   -r requirements.in
+    #   geopandas
+shortuuid==1.0.12
+    # via -r requirements.in
+six==1.16.0
+    # via
+    #   fiona
+    #   python-dateutil
+sniffio==1.3.1
+    # via anyio
+starlette==0.22.0
+    # via
+    #   -r requirements.in
+    #   fastapi
+typing-extensions==4.10.0
+    # via pydantic
+tzdata==2024.1
+    # via pandas
+urllib3==2.2.1
+    # via requests
+uvicorn==0.20.0
+    # via -r requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements_dev.in

@@ -0,0 +1,9 @@
+ipython
+ipdb
+pytest==8.0.2  # 8.1.0 contains breaking changes
+flake8
+requests
+httpx
+pytest-asyncio
+requests-toolbelt
+freezegun