diff --git a/app/borehole_map.py b/app/borehole_map.py index 1e5d70c..c1bedbd 100644 --- a/app/borehole_map.py +++ b/app/borehole_map.py @@ -22,7 +22,7 @@ def extract_geojson(filepath: Path) -> dict: Read an AGS4 file and extract geojson represenation of LOCA table and metadata. """ - logger.info("Extracting geojson from %s", filepath.name) + logger.info("Extracting geojson from %s", filepath.name) # Read data file tables, load_error, _ = load_tables_reporting_errors(filepath) diff --git a/app/validation.py b/app/validation.py index e4f950a..2f5b73a 100644 --- a/app/validation.py +++ b/app/validation.py @@ -47,6 +47,8 @@ def validate(filename: Path, all_errors = {} all_checkers = [] + bgs_additional_metadata = {} + ags_summary = [] # Don't process if file is not .ags format if filename.suffix.lower() != '.ags': all_errors.update( @@ -57,19 +59,32 @@ def validate(filename: Path, # Run checkers to extract errors and other metadata for checker in checkers: # result is a dictionary with 'errors', 'checker' and other keys - result = checker(filename, standard_AGS4_dictionary=dictionary_file) + result: dict = checker(filename, standard_AGS4_dictionary=dictionary_file) + # Pull 'errors' out to add to running total all_errors.update(result.pop('errors')) all_checkers.append(result.pop('checker')) - # Handle additional metadata + + # Extract checker-dependent additional metadata try: - response['additional_metadata'].update(result.pop('additional_metadata')) + bgs_additional_metadata = result.pop('additional_metadata') except KeyError: - # No additional metadata pass + + try: + ags_summary = result.pop('summary') + except KeyError: + pass + # Add remaining keys to response response.update(result) + # We only want one checker-dependent metadata; use BGS if available. + if bgs_additional_metadata: + response['additional_metadata'] = bgs_additional_metadata + else: + response['summary'] = ags_summary + error_count = len(reduce(lambda total, current: total + current, all_errors.values(), [])) if error_count > 0: message = f'{error_count} error(s) found in file!' diff --git a/test/unit/test_validation.py b/test/unit/test_validation.py index 2716409..43f6e53 100644 --- a/test/unit/test_validation.py +++ b/test/unit/test_validation.py @@ -15,12 +15,32 @@ def mock_check_ags(filename, standard_AGS4_dictionary=None): return dict(checker='ags', dictionary='some_dict', - errors={}) + errors={}, + summary=[{'desc': '7 groups identified in file: PROJ ABBR TRAN TYPE UNIT ' + 'LOCA SAMP', + 'group': '', + 'line': ''}, + {'desc': '1 data row(s) in LOCA group', 'group': '', 'line': ''}, + {'desc': 'Optional DICT group present? False', + 'group': '', + 'line': ''}, + {'desc': 'Optional FILE group present? False', + 'group': '', + 'line': ''}] + ) def mock_check_bgs(filename, **kwargs): return dict(checker='bgs', - errors={'BGS': [{}]}) + errors={'BGS': [{}]}, + additional_metadata={ + 'bgs_all_groups': '7 groups identified in file: ' + 'PROJ ABBR TRAN TYPE UNIT LOCA SAMP', + 'bgs_dict': 'Optional DICT group present: False', + 'bgs_file': 'Optional FILE group present: False', + 'bgs_loca_rows': '1 data rows in LOCA group', + 'bgs_projects': '1 projects found: 121415 (ACME Gas Works Redevelopment)' + }) @freeze_time(FROZEN_TIME) @@ -36,6 +56,17 @@ def test_validate_default_checker(): 'filesize': 0, 'message': 'All checks passed!', 'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc), + 'summary': [{'desc': '7 groups identified in file: PROJ ABBR TRAN TYPE UNIT ' + 'LOCA SAMP', + 'group': '', + 'line': ''}, + {'desc': '1 data row(s) in LOCA group', 'group': '', 'line': ''}, + {'desc': 'Optional DICT group present? False', + 'group': '', + 'line': ''}, + {'desc': 'Optional FILE group present? False', + 'group': '', + 'line': ''}], 'valid': True, 'additional_metadata': {}} @@ -60,7 +91,14 @@ def test_validate_bgs_checker(): 'message': '1 error(s) found in file!', 'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc), 'valid': False, - 'additional_metadata': {}} + 'additional_metadata': {'bgs_all_groups': '7 groups identified in file: PROJ ' + 'ABBR TRAN TYPE UNIT LOCA SAMP', + 'bgs_dict': 'Optional DICT group present: False', + 'bgs_file': 'Optional FILE group present: False', + 'bgs_loca_rows': '1 data rows in LOCA group', + 'bgs_projects': '1 projects found: 121415 (ACME Gas ' + 'Works Redevelopment)'} + } # Act response = validation.validate(filename, checkers=[mock_check_bgs]) @@ -83,7 +121,14 @@ def test_validate_both_checkers(): 'message': '1 error(s) found in file!', 'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc), 'valid': False, - 'additional_metadata': {}} + 'additional_metadata': { + 'bgs_all_groups': '7 groups identified in file: ' + 'PROJ ABBR TRAN TYPE UNIT LOCA SAMP', + 'bgs_dict': 'Optional DICT group present: False', + 'bgs_file': 'Optional FILE group present: False', + 'bgs_loca_rows': '1 data rows in LOCA group', + 'bgs_projects': '1 projects found: 121415 (ACME Gas Works Redevelopment)' + }} # Act response = validation.validate(filename, checkers=[mock_check_bgs, mock_check_ags]) @@ -106,6 +151,7 @@ def test_validate_non_ags(): 'message': '1 error(s) found in file!', 'time': dt.datetime(2021, 8, 23, 14, 25, 43, tzinfo=dt.timezone.utc), 'valid': False, + 'summary': [], 'additional_metadata': {}} # Act