From 0c4a978173a073227113d109e98e76e1ce5cee6d Mon Sep 17 00:00:00 2001 From: Mustafa Kemal Gilor Date: Fri, 5 May 2023 13:43:53 +0300 Subject: [PATCH] os-exceptions: perform ost warning&exceptions with hyperscan Signed-off-by: Mustafa Kemal Gilor --- hotsos/core/search.py | 2 +- .../core/ycheck/engine/properties/checks.py | 1 + .../openstack/agent/exceptions.py | 98 +++++++++++++------ pyproject.toml | 2 +- requirements.txt | 2 +- 5 files changed, 72 insertions(+), 33 deletions(-) diff --git a/hotsos/core/search.py b/hotsos/core/search.py index 387ddeb98..dea7f85cf 100644 --- a/hotsos/core/search.py +++ b/hotsos/core/search.py @@ -4,6 +4,7 @@ FileSearcher as _FileSearcher, ResultFieldInfo, SearchDef, + HyperscanSearchDef, SequenceSearchDef, ) from searchkit.constraints import ( @@ -51,7 +52,6 @@ def apply_to_line(self, *args, **kwargs): log.info("skipping line constraint since data_root is not a " "sosreport therefore files may be changing") return True - return super().apply_to_line(*args, **kwargs) def apply_to_file(self, *args, **kwargs): diff --git a/hotsos/core/ycheck/engine/properties/checks.py b/hotsos/core/ycheck/engine/properties/checks.py index 093daf8b6..149ddf24f 100644 --- a/hotsos/core/ycheck/engine/properties/checks.py +++ b/hotsos/core/ycheck/engine/properties/checks.py @@ -164,6 +164,7 @@ def initialise(self, vars, input): c = SearchConstraintSearchSince(exprs=COMMON_LOG_DATETIME_EXPRS, hours=hours) + s = FileSearcher(constraint=c) # first load all the search definitions into the searcher for c in self._checks: diff --git a/hotsos/plugin_extensions/openstack/agent/exceptions.py b/hotsos/plugin_extensions/openstack/agent/exceptions.py index 3c66c2dd4..e1ffe2af7 100644 --- a/hotsos/plugin_extensions/openstack/agent/exceptions.py +++ b/hotsos/plugin_extensions/openstack/agent/exceptions.py @@ -11,6 +11,7 @@ from hotsos.core.search import ( FileSearcher, SearchDef, + HyperscanSearchDef, SearchConstraintSearchSince, ) from hotsos.core.utils import cached_property @@ -50,6 +51,7 @@ def _tally_results(self, results): 3: log entry """ exceptions = {} + # log.info(results) for result in results: # strip leading/trailing quotes exc_name = result.get(3).strip("'") @@ -157,29 +159,68 @@ def _add_agent_searches(self, project, agent_name, logs_path, constraints = False tag = "{}.{}".format(project.name, agent_name) - if project.exceptions: - exc_names = "(?:{})".format('|'.join(project.exceptions)) - expr = expr_template.format(exc_names) - self.searchobj.add(SearchDef(expr, tag=tag + '.error', - hint='( ERROR | Traceback)'), - logs_path, - allow_global_constraints=constraints) - - warn_exprs = self._agent_warnings.get(project.name, []) - if warn_exprs: - values = "(?:{})".format('|'.join(warn_exprs)) - expr = expr_template.format(values) - self.searchobj.add(SearchDef(expr, tag=tag + '.warning', - hint='WARNING'), logs_path, - allow_global_constraints=constraints) - - err_exprs = self._agent_errors.get(project.name, []) - if err_exprs: - values = "(?:{})".format('|'.join(err_exprs)) - expr = expr_template.format(values) - sd = SearchDef(expr, tag=tag + '.error', hint='ERROR') - self.searchobj.add(sd, logs_path, - allow_global_constraints=constraints) + + use_hyperscan = True + test_combine_all_expressions = False + + SearchDefType = SearchDef + if use_hyperscan: + SearchDefType = HyperscanSearchDef + + # TODO(mkg): Combine these three to a single hyperscan DB. + # future-mkg: combining these to a single db is not trivial, + # I'll address this later. + + if test_combine_all_expressions: + # This is for testing how combining all expressions + # into a single hyperscan DB would perform. It's not + # functionally correct. + all_exprs = list() + + for el in (project.exceptions, self._agent_warnings.get( + project.name, []), + self._agent_errors.get(project.name, [])): + if not el: + continue + exc_names = "(?:{})".format('|'.join(el)) + expr = expr_template.format( + "CRITICAL|FATAL|ERROR|WARNING|WARN", + exc_names) + all_exprs.append(expr) + + self.searchobj.add( + SearchDefType(all_exprs, + tag=tag + '.error', + hint='( ERROR | Traceback)'), + logs_path, + allow_global_constraints=constraints) + else: + errors = list() + + if project.exceptions: + errors.extend(project.exceptions) + + err_exprs = self._agent_errors.get(project.name, []) + if err_exprs: + errors.extend(err_exprs) + + if len(errors) > 0: + exc_names = "(?:{})".format('|'.join(errors)) + expr = expr_template.format("ERROR", exc_names) + self.searchobj.add( + SearchDefType(expr, tag=tag + '.error', + hint='( ERROR | Traceback)'), + logs_path, + allow_global_constraints=constraints) + + warn_exprs = self._agent_warnings.get(project.name, []) + if warn_exprs: + values = "(?:{})".format('|'.join(warn_exprs)) + expr = expr_template.format("WARNING|WARN", values) + self.searchobj.add( + SearchDefType(expr, tag=tag + '.warning', + hint='WARNING'), logs_path, + allow_global_constraints=constraints) def _load(self): """Register searches for exceptions as well as any other type of issue @@ -217,13 +258,10 @@ def _load(self): prefix_match = (r'(?:{})?'. format(wsgi_prefix or keystone_prefix)) - # Sometimes the exception is printed with just the class name - # and sometimes it is printed with a full import path e.g. - # MyExc or somemod.MyExc so we need to account for both. - exc_obj_full_path_match = r'(?:\S+\.)?' - expr_template = (r"^{}([0-9\-]+) (\S+) .+\S+\s({}{{}})[\s:\.]". - format(prefix_match, exc_obj_full_path_match)) - + expr_template = "^" + prefix_match + \ + r"(\d{{4}}-\d{{2}}-\d{{2}}) "\ + r"(\d{{2}}:\d{{2}}:\d{{2}}\.\d{{3}}) "\ + r"(?:\d+ )?(?:{}) (?:\S+)(?: \[.+\])?.* ((?:\S+\.)?{}):.*" # NOTE: don't check exceptions for deprecated services for agent, log_paths in project.log_paths( include_deprecated_services=False): diff --git a/pyproject.toml b/pyproject.toml index 18a6dc8da..73beb03c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ 'simplejson', 'jinja2', 'cryptography', - 'searchkit >= 0.2.3', + 'searchkit@git+https://github.com/mustafakemalgilor/searchkit.git#egg=enhancement/hyperscan-search-def', 'propertree' ] diff --git a/requirements.txt b/requirements.txt index 26e09755e..752111139 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ jinja2 # More info on issue: https://github.com/canonical/hotsos/issues/326 # [TODO] Bump cryptography to more recent release cryptography==3.4.8 -searchkit >= 0.2.3 +searchkit@git+https://github.com/mustafakemalgilor/searchkit.git@enhancement/hyperscan-search-def propertree fasteners