diff --git a/app/ags.py b/app/ags.py index 5feb9867..17391ec7 100644 --- a/app/ags.py +++ b/app/ags.py @@ -10,13 +10,25 @@ import python_ags4 from python_ags4 import AGS4 + from app.response_templates import PLAIN_TEXT_TEMPLATE, RESPONSE_TEMPLATE logger = logging.getLogger(__name__) +# Collect full paths of dictionaries installed alongside python_ags4 +_dictionary_files = list(Path(python_ags4.__file__).parent.glob('Standard_dictionary*.ags')) +STANDARD_DICTIONARIES = {f.name: f.absolute() for f in _dictionary_files} + +logger = logging.getLogger(__name__) + + +def validate(filename: Path, standard_AGS4_dictionary: Optional[str] = None) -> dict: + """ + Validate filename (against optional dictionary) and respond in + dictionary suitable for converting to JSON. -def validate(filename: Path) -> dict: - """Validate filename and respond in dictionary.""" + :raises ValueError: Raised if dictionary provided is not available. + """ logger.info("Validate called for %", filename.name) # Prepare response with metadata @@ -25,9 +37,20 @@ def validate(filename: Path) -> dict: 'checker': f'python_ags4 v{python_ags4.__version__}', 'time': dt.datetime.now(tz=dt.timezone.utc)} + # Select dictionary file if exists + if standard_AGS4_dictionary: + try: + dictionary_file = STANDARD_DICTIONARIES[standard_AGS4_dictionary] + except KeyError: + msg = (f"{standard_AGS4_dictionary} not available. " + f"Installed dictionaries: {STANDARD_DICTIONARIES.keys()}") + raise ValueError(msg) + else: + dictionary_file = None + # Get error information from file try: - errors = AGS4.check_file(filename) + errors = AGS4.check_file(filename, standard_AGS4_dictionary=dictionary_file) try: metadata = errors.pop('Metadata') # This also removes it from returned errors dictionary = [d['desc'] for d in metadata @@ -106,11 +129,11 @@ def convert(filename: Path, results_dir: Path) -> Tuple[Optional[Path], str]: return (converted_file, log) -def is_valid(filename: Path) -> bool: +def is_valid(filename: Path, standard_AGS4_dictionary: Optional[str] = None) -> bool: """ Validate filename and parse returned log to determine if file is valid. """ - return validate(filename)['valid'] + return validate(filename, standard_AGS4_dictionary=standard_AGS4_dictionary)['valid'] def get_unicode_message(stderr: str, filename: str) -> str: diff --git a/app/main.py b/app/main.py index 7c9b267f..27e5cd74 100644 --- a/app/main.py +++ b/app/main.py @@ -72,8 +72,8 @@ def setup_logging(logging_level=logging.INFO): @app.get("/", response_class=HTMLResponse, include_in_schema=False) -async def homepage(request: Request): - return templates.TemplateResponse('index.html', {'request': request}) +async def landing_page(request: Request): + return templates.TemplateResponse('landing_page.html', {'request': request}) def custom_openapi(): diff --git a/app/routes.py b/app/routes.py index 5b9c6ac3..6887b0a1 100644 --- a/app/routes.py +++ b/app/routes.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import List -from fastapi import APIRouter, BackgroundTasks, File, Query, Request, UploadFile -from fastapi.responses import FileResponse, StreamingResponse +from fastapi import APIRouter, BackgroundTasks, File, Form, Request, UploadFile +from fastapi.responses import FileResponse, HTMLResponse, StreamingResponse from app import ags from app.errors import error_responses, InvalidPayloadError @@ -16,8 +16,8 @@ log_responses = dict(error_responses) log_responses['200'] = { - "content": {"text/plain": {}}, - "description": "Return a log file"} + "content": {"application/json": {}, "text/plain": {}}, + "description": "Return a log in json or text"} zip_responses = dict(error_responses) zip_responses['200'] = { @@ -31,10 +31,35 @@ class Format(str, Enum): JSON = "json" -format_query = Query( +# Enum for search logic +class Dictionary(str, Enum): + V4_0_3 = "v4_0_3" + V4_0_4 = "v4_0_4" + V4_1 = "v4_1" + + +format_form = Form( default=Format.JSON, - title='Format', - description='Response format, text or json', + title='Response Format', + description='Response format: json or text', +) + +dictionary_form = Form( + default=None, + title='Validation Dictionary', + description='Version of AGS dictionary to validate against', +) + +validation_file = File( + ..., + title='File to validate', + description='An AGS file ending in .ags', +) + +conversion_file = File( + ..., + title='File to convert', + description='An AGS or XLSX file', ) @@ -42,16 +67,20 @@ class Format(str, Enum): response_model=ValidationResponse, responses=log_responses) async def is_valid(background_tasks: BackgroundTasks, - file: UploadFile = File(...), + file: UploadFile = validation_file, + std_dictionary: Dictionary = dictionary_form, request: Request = None): if not file.filename: raise InvalidPayloadError(request) tmp_dir = Path(tempfile.mkdtemp()) background_tasks.add_task(shutil.rmtree, tmp_dir) + dictionary = None + if std_dictionary: + dictionary = f'Standard_dictionary_{std_dictionary}.ags' contents = await file.read() local_ags_file = tmp_dir / file.filename local_ags_file.write_bytes(contents) - valid = ags.is_valid(local_ags_file) + valid = ags.is_valid(local_ags_file, standard_AGS4_dictionary=dictionary) data = [valid] response = prepare_validation_response(request, data) return response @@ -61,17 +90,21 @@ async def is_valid(background_tasks: BackgroundTasks, response_model=ValidationResponse, responses=log_responses) async def validate(background_tasks: BackgroundTasks, - file: UploadFile = File(...), - fmt: Format = format_query, + file: UploadFile = validation_file, + std_dictionary: Dictionary = dictionary_form, + fmt: Format = format_form, request: Request = None): if not file.filename: raise InvalidPayloadError(request) tmp_dir = Path(tempfile.mkdtemp()) background_tasks.add_task(shutil.rmtree, tmp_dir) + dictionary = None + if std_dictionary: + dictionary = f'Standard_dictionary_{std_dictionary}.ags' contents = await file.read() local_ags_file = tmp_dir / file.filename local_ags_file.write_bytes(contents) - result = ags.validate(local_ags_file) + result = ags.validate(local_ags_file, standard_AGS4_dictionary=dictionary) if fmt == Format.TEXT: log = ags.to_plain_text(result) logfile = tmp_dir / 'results.log' @@ -87,13 +120,17 @@ async def validate(background_tasks: BackgroundTasks, response_model=ValidationResponse, responses=log_responses) async def validate_many(background_tasks: BackgroundTasks, - files: List[UploadFile] = File(...), - fmt: Format = format_query, + files: List[UploadFile] = validation_file, + std_dictionary: Dictionary = dictionary_form, + fmt: Format = format_form, request: Request = None): if not files[0].filename: raise InvalidPayloadError(request) tmp_dir = Path(tempfile.mkdtemp()) background_tasks.add_task(shutil.rmtree, tmp_dir) + dictionary = None + if std_dictionary: + dictionary = f'Standard_dictionary_{std_dictionary}.ags' if fmt == Format.TEXT: full_logfile = tmp_dir / 'results.log' with full_logfile.open('wt') as f: @@ -101,7 +138,8 @@ async def validate_many(background_tasks: BackgroundTasks, contents = await file.read() local_ags_file = tmp_dir / file.filename local_ags_file.write_bytes(contents) - log = ags.to_plain_text(ags.validate(local_ags_file)) + result = ags.validate(local_ags_file, standard_AGS4_dictionary=dictionary) + log = ags.to_plain_text(result) f.write(log) f.write('=' * 80 + '\n') response = FileResponse(full_logfile, media_type="text/plain") @@ -111,8 +149,8 @@ async def validate_many(background_tasks: BackgroundTasks, contents = await file.read() local_ags_file = tmp_dir / file.filename local_ags_file.write_bytes(contents) - log = ags.validate(local_ags_file) - data.append(log) + result = ags.validate(local_ags_file, standard_AGS4_dictionary=dictionary) + data.append(result) response = prepare_validation_response(request, data) return response @@ -121,8 +159,7 @@ async def validate_many(background_tasks: BackgroundTasks, response_class=StreamingResponse, responses=zip_responses) async def convert_many(background_tasks: BackgroundTasks, - files: List[UploadFile] = File(...), - fmt: Format = format_query, + files: List[UploadFile] = conversion_file, request: Request = None): if not files[0].filename: raise InvalidPayloadError(request) @@ -159,4 +196,4 @@ def prepare_validation_response(request, data): 'self': str(request.url), 'data': data, } - return ValidationResponse(**response_data) + return ValidationResponse(**response_data, media_type="application/json") diff --git a/app/static/css/styles.css b/app/static/css/styles.css index f772da1e..36491f98 100644 --- a/app/static/css/styles.css +++ b/app/static/css/styles.css @@ -1,10 +1,21 @@ +main { + min-height: 100%; + margin-bottom: -100px; +} + +main:after { + content: ""; + display: block; + height: 100px; +} + body { + height: 100%; color: #002E40; text-align: left; font-family: Arial, Helvetica, sans-serif; } - h1 { color: #002E40; text-align: center; @@ -17,9 +28,112 @@ h2 { font-family: Arial, Helvetica, sans-serif; } -img { - display: block; - margin-left: auto; - margin-right: auto; - width: 20%; -} \ No newline at end of file +/* Popup text style start */ + +.tooltip { + position: relative; + display: inline-block; + border-bottom: 1px dotted black; + } + + .tooltip .tooltiptext { + visibility: hidden; + width: 120px; + background-color: black; + color: #fff; + text-align: center; + border-radius: 6px; + padding: 5px 0; + position: absolute; + z-index: 1; + top: -5px; + left: 110%; + } + + .tooltip .tooltiptext::after { + content: ""; + position: absolute; + top: 50%; + right: 100%; + margin-top: -5px; + border-width: 5px; + border-style: solid; + border-color: transparent black transparent transparent; + } + .tooltip:hover .tooltiptext { + visibility: visible; + } + +/* Popup text style start */ + +/* Header style start */ + +.header-wrapper +{ + background-color: #002E40; + color: white; + display: flex; + gap: 16px; + align-items: center; + justify-content: space-between; + padding: 20px 32px; +} + +.header-wrapper img { + height:48px; + vertical-align: middle; +} + +@media only screen and (max-width: 600px) { + .header-wrapper { + flex-direction: column; + text-align: center; + padding: 24px 16px; + } +} + +.header-wrapper a { + color: white; +} + +.header-wrapper a:hover { + background: rgba(255, 255, 255, 0.3); +} + +/* Header style end */ + +/* Footer style start */ + +.footer-wrapper +{ + background-color: #002E40; + color: white; + display: flex; + gap: 32px; + align-items: center; + justify-content: space-evenly; + padding: 48px 32px; +} + +.footer-wrapper img { + height:48px; + vertical-align: middle; +} + +@media only screen and (max-width: 600px) { + .footer-wrapper { + flex-direction: column; + text-align: center; + padding: 24px 16px; + } +} + +.footer-wrapper a { + color: white; +} + +.footer-wrapper a:hover { + background: rgba(255, 255, 255, 0.3); +} + +/* Footer style end */ \ No newline at end of file diff --git a/app/static/img/BGS-Logo-White-RGB.svg b/app/static/img/BGS-Logo-White-RGB.svg new file mode 100644 index 00000000..07277834 --- /dev/null +++ b/app/static/img/BGS-Logo-White-RGB.svg @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/static/img/ukri-logo-440x124.png b/app/static/img/ukri-logo-440x124.png new file mode 100644 index 00000000..948ca574 Binary files /dev/null and b/app/static/img/ukri-logo-440x124.png differ diff --git a/app/templates/_base.html b/app/templates/_base.html new file mode 100644 index 00000000..794893b3 --- /dev/null +++ b/app/templates/_base.html @@ -0,0 +1,56 @@ + + + + pyagsapi + + + + + +
+
+ +
+
+ AGS File Utilities Tool +
+
+ API +
+ +
+ Contact +
+
+
+
+
+
+
+ {% block body %} + {% endblock %} +
+
+
+
+ + {% block extrafoot %} + {% endblock %} + + \ No newline at end of file diff --git a/app/templates/index.html b/app/templates/index.html deleted file mode 100644 index dc0fc659..00000000 --- a/app/templates/index.html +++ /dev/null @@ -1,56 +0,0 @@ - - - pyagsapi - - - - BGS Logo -

pyagsapi - AGS File Utilities API

-
-

Validate or convert your AGS files here.

-

-

-

-
-

AGS Validator - Single File

-

Select .ags file for validation (v4.x only). Validation is against the official AGS standard only.

-
- - -
-
-

AGS Validator - Multiple Files

-

Select .ags files for validation (v4.x only). Validation is against the official AGS standard only.

-
- - -
-
-

AGS Converter

-

Convert .ags file(s) to/from .xlsx. Which ever format file is submitted, the opposite will be returned e.g. if 5 .ags files and 3 .xlsx files were submitted the result would be 5 .xlsx files and 3 .ags files

-
- - -
-
-

API Definition

-

- Documentation: Swagger UI ReDoc -

-

- OpenAPI Document -

-
-
-

- pyagsapi was created by and is maintained by the British Geological Survey. It is distributed under the LGPL v3.0 licence. Copyright: © BGS / UKRI 2021 -

-
- - UKRI Logo - - \ No newline at end of file diff --git a/app/templates/landing_page.html b/app/templates/landing_page.html new file mode 100644 index 00000000..2c9cac14 --- /dev/null +++ b/app/templates/landing_page.html @@ -0,0 +1,95 @@ +{% extends "_base.html" %} +{% block title %}Home{% endblock %} +{% block body %} + +
+
+

AGS File Utilities Tool and API

+
+

This tool and asscoated API allow schema validation, data validation and conversion of your AGS files.

+ +
+
+
+

AGS Schema Validator

+

Validation is against the official AGS schema standard only. This is not a data validator

+
+

Select AGS version (default uses dictionary specified in file, if not present, v4.0.4)

+ + + + + +
+

Select .ags file(s) for validation (v4.x only).

+
+

Select reponse format:

+ + + +
+
+ +
+
+
+
+

AGS Data Validator

+

Select .ags file(s) for data validation against the National Geoscience Data Repository requirements

+
+ + +
+

Select reponse format:

+ + + +
+
+ +
+

Validation rules

+

Your files will be validated against the following rules as defined by BGS/NGDC:

+ +
+
+

AGS Converter

+
Convert .ags file(s) to/from .xlsx. + Which ever format file is submitted, the opposite will be returned e.g. if 5 .ags files and 3 .xlsx files were submitted the result would be 5 .xlsx files and 3 .ags files +
+
+
+
+ + +
+
+

API Definition

+

+ Documentation +

+

+ OpenAPI Document +

+

+ ReDoc +

+
+ +{% endblock %} \ No newline at end of file diff --git a/app/templates/schemaresults.html b/app/templates/schemaresults.html new file mode 100644 index 00000000..4e9f6282 --- /dev/null +++ b/app/templates/schemaresults.html @@ -0,0 +1,21 @@ +{% extends "_base.html" %} +{% block title %}Schema_Validation_Results{% endblock %} +{% block body %} + +
+
+

AGS Schema Validation Results

+
+
+
+
+

Metadata

+

Filename: {{ filename }}

+
+
+
+

Errors

+

Item ID: {{ id }}

+
+ +{% endblock %} \ No newline at end of file diff --git a/test/fixtures.py b/test/fixtures.py index aa3dfa4b..dc58be50 100644 --- a/test/fixtures.py +++ b/test/fixtures.py @@ -25,3 +25,9 @@ ('random_binary.ags', ('IndexError: At least one sheet must be visible', 1)), ('real/A3040_03.ags', ("UnboundLocalError: local variable 'group' referenced before assignment", 258)), ] + +DICTIONARIES = { + 'v4_0_3': "Standard_dictionary_v4_0_3.ags", + 'v4_0_4': "Standard_dictionary_v4_0_4.ags", + 'v4_1': "Standard_dictionary_v4_1.ags" +} diff --git a/test/integration/test_api.py b/test/integration/test_api.py index 6a69c479..e5e3286b 100644 --- a/test/integration/test_api.py +++ b/test/integration/test_api.py @@ -8,7 +8,7 @@ from requests_toolbelt.multipart.encoder import MultipartEncoder from app.main import app -from test.fixtures import FROZEN_TIME, ISVALID_RSP_DATA +from test.fixtures import DICTIONARIES, FROZEN_TIME, ISVALID_RSP_DATA from test.fixtures_json import JSON_RESPONSES from test.fixtures_plain_text import PLAIN_TEXT_RESPONSES @@ -48,11 +48,33 @@ async def test_isvalid(async_client, filename, expected): assert body['data'][0] == expected -@pytest.mark.parametrize('fmt,', ['', '?fmt=json']) +@pytest.mark.parametrize('dictionary', DICTIONARIES.keys()) +@pytest.mark.asyncio +async def test_isvalid_custom_dictionary(async_client, dictionary): + # Arrange + filename = TEST_FILE_DIR / 'example1.ags' + mp_encoder = MultipartEncoder( + fields={'file': (filename.name, open(filename, 'rb'), 'text/plain'), + 'std_dictionary': dictionary}) + + # Act + async with async_client as ac: + response = await ac.post( + '/isvalid/', + headers={'Content-Type': mp_encoder.content_type}, + data=mp_encoder.to_string()) + + # Assert + assert response.status_code == 200 + body = response.json() + assert len(body['data']) == 1 + assert body['data'][0] + + @pytest.mark.parametrize('filename, expected', [item for item in JSON_RESPONSES.items()]) @pytest.mark.asyncio -async def test_validate_json(async_client, filename, expected, fmt): +async def test_validate_json(async_client, filename, expected): # Arrange filename = TEST_FILE_DIR / filename mp_encoder = MultipartEncoder( @@ -61,7 +83,7 @@ async def test_validate_json(async_client, filename, expected, fmt): # Act async with async_client as ac: response = await ac.post( - '/validate/' + fmt, + '/validate/', headers={'Content-Type': mp_encoder.content_type}, data=mp_encoder.to_string()) @@ -77,21 +99,22 @@ async def test_validate_json(async_client, filename, expected, fmt): assert body['data'][0]['filename'] == expected['filename'] -@pytest.mark.parametrize('fmt,', ['', '?fmt=json']) @pytest.mark.asyncio -async def test_validatemany_json(async_client, fmt): +async def test_validatemany_json(async_client): # Arrange - files = [] + fields = [] for name in JSON_RESPONSES.keys(): filename = TEST_FILE_DIR / name file = ('files', (filename.name, open(filename, 'rb'), 'text/plain')) - files.append(file) - mp_encoder = MultipartEncoder(fields=files) + fields.append(file) + fields.append(('std_dictionary', 'v4_1')) + fields.append(('fmt', 'json')) + mp_encoder = MultipartEncoder(fields=fields) # Act async with async_client as ac: response = await ac.post( - '/validatemany/' + fmt, + '/validatemany/', headers={'Content-Type': mp_encoder.content_type}, data=mp_encoder.to_string()) @@ -105,6 +128,32 @@ async def test_validatemany_json(async_client, fmt): assert len(body['data']) == len(JSON_RESPONSES) +@pytest.mark.parametrize('dictionary, expected', + [item for item in DICTIONARIES.items()]) +@pytest.mark.asyncio +async def test_validate_custom_dictionary(async_client, dictionary, expected): + # Arrange + filename = TEST_FILE_DIR / 'example1.ags' + mp_encoder = MultipartEncoder( + fields={'file': (filename.name, open(filename, 'rb'), 'text/plain'), + 'std_dictionary': dictionary}) + + # Act + async with async_client as ac: + response = await ac.post( + '/validate/', + headers={'Content-Type': mp_encoder.content_type}, + data=mp_encoder.to_string()) + + # Assert + assert response.status_code == 200 + body = response.json() + assert len(body['data']) == 1 + # Assert + assert body['data'][0]['filename'] == 'example1.ags' + assert body['data'][0]['dictionary'] == expected + + @freeze_time(FROZEN_TIME) @pytest.mark.parametrize('filename, expected', [item for item in PLAIN_TEXT_RESPONSES.items()]) @@ -113,12 +162,13 @@ async def test_validate_text(async_client, filename, expected): # Arrange filename = TEST_FILE_DIR / filename mp_encoder = MultipartEncoder( - fields={'file': (filename.name, open(filename, 'rb'), 'text/plain')}) + fields={'file': (filename.name, open(filename, 'rb'), 'text/plain'), + 'fmt': 'text'}) # Act async with async_client as ac: response = await ac.post( - '/validate/?fmt=text', + '/validate/', headers={'Content-Type': mp_encoder.content_type}, data=mp_encoder.to_string()) @@ -131,17 +181,19 @@ async def test_validate_text(async_client, filename, expected): @pytest.mark.asyncio async def test_validatemany_text(async_client): # Arrange - files = [] - for name in PLAIN_TEXT_RESPONSES.keys(): + fields = [] + for name in JSON_RESPONSES.keys(): filename = TEST_FILE_DIR / name file = ('files', (filename.name, open(filename, 'rb'), 'text/plain')) - files.append(file) - mp_encoder = MultipartEncoder(fields=files) + fields.append(file) + fields.append(('std_dictionary', 'v4_1')) + fields.append(('fmt', 'text')) + mp_encoder = MultipartEncoder(fields=fields) # Act async with async_client as ac: response = await ac.post( - '/validatemany/?fmt=text', + '/validatemany/', headers={'Content-Type': mp_encoder.content_type}, data=mp_encoder.to_string()) diff --git a/test/unit/test_ags.py b/test/unit/test_ags.py index 039ec856..a28733fb 100644 --- a/test/unit/test_ags.py +++ b/test/unit/test_ags.py @@ -6,7 +6,9 @@ import pytest from app import ags -from test.fixtures import BAD_FILE_DATA, FROZEN_TIME, GOOD_FILE_DATA, ISVALID_RSP_DATA +from test.fixtures import (BAD_FILE_DATA, DICTIONARIES, + FROZEN_TIME, GOOD_FILE_DATA, + ISVALID_RSP_DATA) from test.fixtures_json import JSON_RESPONSES from test.fixtures_plain_text import PLAIN_TEXT_RESPONSES @@ -30,6 +32,36 @@ def test_validate(filename, expected): assert response[key] == expected[key] +@pytest.mark.parametrize('dictionary', DICTIONARIES.values()) +def test_validate_custom_dictionary(dictionary): + # Arrange + filename = TEST_FILE_DIR / 'example1.ags' + + # Act + response = ags.validate(filename, + standard_AGS4_dictionary=dictionary) + + # Assert + assert response['filename'] == 'example1.ags' + assert response['dictionary'] == dictionary + + +def test_validate_custom_dictionary_bad_file(): + # Arrange + filename = TEST_FILE_DIR / 'example1.ags' + dictionary = 'bad_file.ags' + + # Act + with pytest.raises(ValueError) as err: + ags.validate(filename, standard_AGS4_dictionary=dictionary) + + # Assert + message = str(err.value) + assert 'dictionary' in message + for key in ags.STANDARD_DICTIONARIES: + assert key in message + + @pytest.mark.parametrize('filename, expected', GOOD_FILE_DATA) def test_convert(tmp_path, filename, expected): # Arrange @@ -78,6 +110,19 @@ def test_is_valid(filename, expected): assert result == expected +@pytest.mark.parametrize('dictionary', DICTIONARIES.values()) +def test_is_valid_custom_dictionary(dictionary): + # Arrange + filename = TEST_FILE_DIR / 'example1.ags' + + # Act + result = ags.is_valid(filename, + standard_AGS4_dictionary=dictionary) + + # Assert + assert result + + @pytest.mark.parametrize('filename', [ 'example1.ags', 'nonsense.ags', 'random_binary.ags', 'real/Blackburn Southern Bypass.ags'])