From f27398b38b227b58b26ad9682d07aa10612c9885 Mon Sep 17 00:00:00 2001 From: Matt Reimer Date: Thu, 13 Jun 2024 09:21:54 -0700 Subject: [PATCH] fixing ESLINT on workspace and QRiS metric \validation --- RSXML-RiverscapesXML.code-workspace | 7 +++- RiverscapesXML.code-workspace | 10 +++-- python/test.py | 56 ++++++++++++++++++---------- python/validate.py | 57 +++++++++++++++++++++-------- 4 files changed, 89 insertions(+), 41 deletions(-) diff --git a/RSXML-RiverscapesXML.code-workspace b/RSXML-RiverscapesXML.code-workspace index ad6d6bfb..befe8402 100644 --- a/RSXML-RiverscapesXML.code-workspace +++ b/RSXML-RiverscapesXML.code-workspace @@ -13,11 +13,14 @@ "terminal.integrated.defaultProfile.linux": "zsh", "[python]": { "editor.tabSize": 4, - "editor.formatOnSave": true + "editor.formatOnSave": true, }, + "autopep8.args": [ + "--max-line-length=240", + ], "pylint.args": [ "--extension-pkg-whitelist=pygeoprocessing", - "--ignore=E501", + "--disable=C0301,C0114,C0103,W0719,W0718", "--max-line-length=240" ], "python.terminal.activateEnvironment": true, diff --git a/RiverscapesXML.code-workspace b/RiverscapesXML.code-workspace index b1d42c6a..e5460bd1 100644 --- a/RiverscapesXML.code-workspace +++ b/RiverscapesXML.code-workspace @@ -25,13 +25,15 @@ "terminal.integrated.defaultProfile.linux": "zsh", "[python]": { "editor.tabSize": 4, - "editor.formatOnSave": true + "editor.formatOnSave": true, }, - "python.analysis.extraPaths": ["../../lib/commons"], + "autopep8.args": [ + "--max-line-length=240", + ], "pylint.args": [ "--extension-pkg-whitelist=pygeoprocessing", - "--ignore=E501", - "--max-line-length=240" + "--disable=C0301,C0114,C0103,W0719,W0718", + "--max-line-length=240" ], "python.terminal.activateEnvironment": true, "python.testing.pytestEnabled": true, diff --git a/python/test.py b/python/test.py index d8f6b1cd..6501160f 100644 --- a/python/test.py +++ b/python/test.py @@ -3,16 +3,14 @@ # import sys # import csv import os -from validate import get_xml, collect_files, get_xsd, validate_web_vector_json, validate_xml, validate_qramp +from validate import get_xml, collect_files, get_xsd, validate_web_vector_json, validate_xml, validate_qramp, validate_qris_metric_json # We do this mapping because we want the current version's XML tested against the corrent # version's XSD. XML_DIGEST = [ {'xml': './Programs/**/*.xml', 'xsd': './Program.xsd'}, - {'xml': './RaveBusinessLogic/*.xml', - 'xsd': './RaveBusinessLogic/XSD/project_explorer.xsd'}, - {'xml': './RaveBusinessLogic/V2/*.xml', - 'xsd': './RaveBusinessLogic/XSD/project_explorer.xsd'}, + {'xml': './RaveBusinessLogic/*.xml', 'xsd': './RaveBusinessLogic/XSD/project_explorer.xsd'}, + {'xml': './RaveBusinessLogic/V2/*.xml', 'xsd': './RaveBusinessLogic/XSD/project_explorer.xsd'}, {'xml': './BaseMaps.xml', 'xsd': './BaseMaps.xsd'} ] @@ -48,27 +46,47 @@ def test_xmls(self): errors.append([xml_path, str(e)]) print("Tested XML: {}".format(xml_path)) - self.assertEqual(len(errors), 0, msg='Errors were found: \n{}'.format( - self.err_helper(errors))) + self.assertEqual(len(errors), 0, msg='Errors were found: \n{}'.format(self.err_helper(errors))) def test_validateJSON(self): """We have some JSON in the system that needs validating """ errors = [] - json_paths = collect_files('./Symbology/web/**/*.json') + + print("\nTesting Web Symbologies:\n========================") with open('./Symbology/web/vector.schema.json') as f: + symbology_paths = collect_files('./Symbology/web/**/*.json') schema = json.load(f) - print("\nTesting Web Symbologies:\n========================") - for json_path in json_paths: - try: - with open(json_path, 'r') as f: - json_file = json.load(f) - result, errs = validate_web_vector_json(json_file, schema) - if not result: - errors.append([json_path, str(errs)]) - except Exception as e: - errors.append([json_path, str(e)]) - print("Tested web symbology: {}".format(json_path)) + + for json_path in symbology_paths: + try: + with open(json_path, 'r') as f: + json_file = json.load(f) + result, errs = validate_web_vector_json(json_file, schema) + if not result: + errors.append([json_path, str(errs)]) + except Exception as e: + errors.append([json_path, str(e)]) + print("Tested web symbology: {}".format(json_path)) + + print("\nTesting QRiS Metrics:\n========================") + with open('./QRiS/qris_metrics.schema.json') as f: + metric_paths = [ + *collect_files('./QRiS/metrics/*.json'), + *collect_files('./QRiS/metrics/**/*.json'), + ] + schema = json.load(f) + + for json_path in metric_paths: + try: + with open(json_path, 'r') as f: + json_file = json.load(f) + result, errs = validate_qris_metric_json(json_file, schema) + if not result: + errors.append([json_path, str(errs)]) + except Exception as e: + errors.append([json_path, str(e)]) + print("Tested QRiS Metric: {}".format(json_path)) self.assertEqual(len(errors), 0, msg='Errors were found: \n{}'.format( self.err_helper(errors))) diff --git a/python/validate.py b/python/validate.py index 28545236..1ee0c1c5 100644 --- a/python/validate.py +++ b/python/validate.py @@ -20,8 +20,7 @@ def get_xsd(xsd_path): xsd_encoded = bytes(xsd, encoding='utf-8') errors = validate_xsd(xsd_encoded) if len(errors) > 0: - raise Exception( - 'XSD Failed to validate: {} \n {}'.format(xsd_path, errors)) + raise Exception('XSD Failed to validate: {} \n {}'.format(xsd_path, errors)) return xsd_encoded @@ -55,13 +54,21 @@ def validate_xsd(xsd_str: str): def validate_web_vector_json(json_obj: dict, schema: dict) -> bool: + """ Make sure the Web Vector JSON is valid + + Args: + json_obj (dict): _description_ + schema (dict): _description_ + + Returns: + bool: _description_ + """ result = True errors = [] try: jsonschema.validate(json_obj, schema) except ValidationError as e: - errors.append( - "JSON Did not validate against schema: '{}' in object: {}".format(e.message, e.instance)) + errors.append("JSON Did not validate against schema: '{}' in object: {}".format(e.message, e.instance)) return False, errors if len(json_obj['layerStyles']) < 1: @@ -75,8 +82,7 @@ def validate_web_vector_json(json_obj: dict, schema: dict) -> bool: # Check for mapbox stock layers if 'mapbox://' in o['source']: result = False - errors.append( - 'You cannot use mapbox layers: {}'.format(json.dumps(o))) + errors.append('You cannot use mapbox layers: {}'.format(json.dumps(o))) elif o['type'] == 'raster': result = False errors.append( @@ -86,8 +92,31 @@ def validate_web_vector_json(json_obj: dict, schema: dict) -> bool: [k['source-layer'] for k in json_obj['layerStyles']])) if len(unique_sources) != 1: result = False - errors.append( - 'You cannot consume from multiple sources: {}'.format(unique_sources)) + errors.append('You cannot consume from multiple sources: {}'.format(unique_sources)) + return result, errors + + +def validate_qris_metric_json(json_obj: dict, schema: dict) -> bool: + """ Make sure the QRiS metric JSON is valid + + Args: + json_obj (dict): _description_ + schema (dict): _description_ + + Returns: + bool: _description_ + """ + result = True + errors = [] + try: + jsonschema.validate(json_obj, schema) + except ValidationError as e: + errors.append("JSON Did not validate against schema: '{}' in object: {}".format(e.message, e.instance)) + return False, errors + + # HERE IS WHERE SPECIFIC CHECKS CAN HAPPEN. THESE WOULD BE THINGS THE SCHEMA + # CHECK CAN'T CATCH + return result, errors @@ -106,16 +135,13 @@ def validate_qramp(qramp: str): result = True if lines[0].rstrip() != "# QGIS Generated Color Map Export File": - errors.append( - 'Missing the header line: "# QGIS Generated Color Map Export File". This is probabyl not a QGIS exported color ramp') + errors.append('Missing the header line: "# QGIS Generated Color Map Export File". This is probabyl not a QGIS exported color ramp') result = False if "INTERPOLATION:" not in lines[1]: - errors.append( - 'Missing the intepolation type on line 2: "INTERPOLATED:[DISCRETE|INTERPOLATED|EXACT]". This is probabyl not a QGIS exported color ramp') + errors.append('Missing the intepolation type on line 2: "INTERPOLATED:[DISCRETE|INTERPOLATED|EXACT]". This is probabyl not a QGIS exported color ramp') result = False if lines[1].split(':')[1] not in ['DISCRETE', 'EXACT', 'INTERPOLATED']: - errors.append( - "Interpolation value must be one of: 'DISCRETE', 'EXACT', 'INTERPOLATED'. Got: {}".format(lines[1])) + errors.append("Interpolation value must be one of: 'DISCRETE', 'EXACT', 'INTERPOLATED'. Got: {}".format(lines[1])) result = False pat = "^(.+),([0-9]{1,3},){4}(.+)$" @@ -125,8 +151,7 @@ def validate_qramp(qramp: str): if len(cline.strip()) == 0: continue if not re.match(pat, cline): - errors.append( - 'Value line did not match the pattern: "{}". Got: {}'.format(pat, cline)) + errors.append('Value line did not match the pattern: "{}". Got: {}'.format(pat, cline)) result = False return result, errors