diff --git a/osbenchmark/worker_coordinator/errors.py b/osbenchmark/worker_coordinator/errors.py new file mode 100644 index 000000000..7ce80d85b --- /dev/null +++ b/osbenchmark/worker_coordinator/errors.py @@ -0,0 +1,59 @@ +import re + +def parse_error(error_metadata): + error = error_metadata['error'] + status_code = None + description = "error occured, check logs for details" + operation = UnknownOperationError(description, None) + + if 'status' in error_metadata: + status_code = error_metadata["status"] + + if 'reason' in error: + description = error['reason'] + matches = re.findall(r'\[([^]]*)\]', description) + for match in matches: + if match == "indices:admin/create": + operation = IndexOperationError(description, "index-create", status_code) + elif match == "indices:admin/delete": + operation = IndexOperationError(description, "index-delete", status_code) + elif match == "indices:data/write/bulk": + operation = IndexOperationError(description, "index-append", status_code) + elif match == "indices:admin/refresh": + operation = IndexOperationError(description, "refresh-after-index", status_code) + elif match == "indices:admin/forcemerge": + operation = IndexOperationError(description, "force-merge", status_code) + elif match == "indices:data/read/search": + operation = SearchOperationError(description, "search", status_code) + + return operation + + +class OpenSearchOperationError(): + def __init__(self, description, operation=None, status_code=None): + self.description = description + self.operation = operation + self.status_code = status_code + +class UnknownOperationError(OpenSearchOperationError): + def get_error_message(self): + return self.description + + +class IndexOperationError(OpenSearchOperationError): + def get_error_message(self): + if self.status_code == 403: + return f"permission denied for {self.operation}. check logs for details" + elif self.status_code == 500: + return f"internal server error for {self.operation}. check logs for details" + else: + return self.description + +class SearchOperationError(OpenSearchOperationError): + def get_error_message(self): + if self.status_code == 403: + return f"permission denied for {self.operation}. check logs for details" + elif self.status_code == 500: + return f"internal server error for {self.operation} index. check logs for details" + else: + return self.description diff --git a/osbenchmark/worker_coordinator/worker_coordinator.py b/osbenchmark/worker_coordinator/worker_coordinator.py index 05ac03535..ad8b77dc8 100644 --- a/osbenchmark/worker_coordinator/worker_coordinator.py +++ b/osbenchmark/worker_coordinator/worker_coordinator.py @@ -27,6 +27,7 @@ import concurrent.futures import datetime import itertools +import json import logging import math import multiprocessing @@ -44,7 +45,7 @@ from osbenchmark.worker_coordinator import runner, scheduler from osbenchmark.workload import WorkloadProcessorRegistry, load_workload, load_workload_plugins from osbenchmark.utils import convert, console, net - +from osbenchmark.worker_coordinator.errors import parse_error ################################## # @@ -1690,6 +1691,7 @@ async def execute_single(runner, opensearch, params, on_error): except KeyError as e: logging.getLogger(__name__).exception("Cannot execute runner [%s]; most likely due to missing parameters.", str(runner)) msg = "Cannot execute [%s]. Provided parameters are: %s. Error: [%s]." % (str(runner), list(params.keys()), str(e)) + console.error(msg) raise exceptions.SystemSetupError(msg) if not request_meta_data["success"]: @@ -1698,7 +1700,21 @@ async def execute_single(runner, opensearch, params, on_error): description = request_meta_data.get("error-description") if description: msg += ", Description: %s" % description + console.error(msg) raise exceptions.BenchmarkAssertionError(msg) + + if 'error-description' in request_meta_data: + try: + error_metadata = json.loads(request_meta_data["error-description"]) + # parse error-description metadata + opensearch_operation_error = parse_error(error_metadata) + console.error(opensearch_operation_error.get_error_message()) + except Exception as e: + # error-description is not a valid json so we just print it + console.error(request_meta_data["error-description"]) + + logging.getLogger(__name__).error(request_meta_data["error-description"]) + return total_ops, total_ops_unit, request_meta_data