os-exceptions: perform ost warning&exceptions with hyperscan

Signed-off-by: Mustafa Kemal Gilor <[email protected]>

hotsos/core/search.py

@@ -4,6 +4,7 @@
  FileSearcher as _FileSearcher,
  ResultFieldInfo,
  SearchDef,
+ HyperscanSearchDef,
  SequenceSearchDef,
 )
 from searchkit.constraints import (
@@ -51,7 +52,6 @@ def apply_to_line(self, *args, **kwargs):
  log.info("skipping line constraint since data_root is not a "
  "sosreport therefore files may be changing")
  return True
-
  return super().apply_to_line(*args, **kwargs)
 
  def apply_to_file(self, *args, **kwargs):

hotsos/core/ycheck/engine/properties/checks.py

@@ -164,6 +164,7 @@ def initialise(self, vars, input):
 
  c = SearchConstraintSearchSince(exprs=COMMON_LOG_DATETIME_EXPRS,
  hours=hours)
+
  s = FileSearcher(constraint=c)
  # first load all the search definitions into the searcher
  for c in self._checks:

hotsos/plugin_extensions/openstack/agent/exceptions.py

@@ -10,6 +10,7 @@
 from hotsos.core.search import (
  FileSearcher,
  SearchDef,
+ HyperscanSearchDef,
  SearchConstraintSearchSince,
 )
 
@@ -48,6 +49,7 @@ def _tally_results(self, results):
  3: log entry
  """
  exceptions = {}
+ # log.info(results)
  for result in results:
  # strip leading/trailing quotes
  exc_name = result.get(3).strip("'")
@@ -154,29 +156,68 @@ def _add_agent_searches(self, project, agent_name, logs_path,
  constraints = False
 
  tag = "{}.{}".format(project.name, agent_name)
- if project.exceptions:
- exc_names = "(?:{})".format('|'.join(project.exceptions))
- expr = expr_template.format(exc_names)
- self.searchobj.add(SearchDef(expr, tag=tag + '.error',
- hint='( ERROR | Traceback)'),
- logs_path,
- allow_global_constraints=constraints)
-
- warn_exprs = self._agent_warnings.get(project.name, [])
- if warn_exprs:
- values = "(?:{})".format('|'.join(warn_exprs))
- expr = expr_template.format(values)
- self.searchobj.add(SearchDef(expr, tag=tag + '.warning',
- hint='WARNING'), logs_path,
- allow_global_constraints=constraints)
-
- err_exprs = self._agent_errors.get(project.name, [])
- if err_exprs:
- values = "(?:{})".format('|'.join(err_exprs))
- expr = expr_template.format(values)
- sd = SearchDef(expr, tag=tag + '.error', hint='ERROR')
- self.searchobj.add(sd, logs_path,
- allow_global_constraints=constraints)
+
+ use_hyperscan = True
+ test_combine_all_expressions = False
+
+ SearchDefType = SearchDef
+ if use_hyperscan:
+ SearchDefType = HyperscanSearchDef
+
+ # TODO(mkg): Combine these three to a single hyperscan DB.
+ # future-mkg: combining these to a single db is not trivial,
+ # I'll address this later.
+
+ if test_combine_all_expressions:
+ # This is for testing how combining all expressions
+ # into a single hyperscan DB would perform. It's not
+ # functionally correct.
+ all_exprs = list()
+
+ for el in (project.exceptions, self._agent_warnings.get(
+ project.name, []),
+ self._agent_errors.get(project.name, [])):
+ if not el:
+ continue
+ exc_names = "(?:{})".format('|'.join(el))
+ expr = expr_template.format(
+ "CRITICAL|FATAL|ERROR|WARNING|WARN",
+ exc_names)
+ all_exprs.append(expr)
+
+ self.searchobj.add(
+ SearchDefType(all_exprs,
+ tag=tag + '.error',
+ hint='( ERROR | Traceback)'),
+ logs_path,
+ allow_global_constraints=constraints)
+ else:
+ errors = list()
+
+ if project.exceptions:
+ errors.extend(project.exceptions)
+
+ err_exprs = self._agent_errors.get(project.name, [])
+ if err_exprs:
+ errors.extend(err_exprs)
+
+ if len(errors) > 0:
+ exc_names = "(?:{})".format('|'.join(errors))
+ expr = expr_template.format("ERROR", exc_names)
+ self.searchobj.add(
+ SearchDefType(expr, tag=tag + '.error',
+ hint='( ERROR | Traceback)'),
+ logs_path,
+ allow_global_constraints=constraints)
+
+ warn_exprs = self._agent_warnings.get(project.name, [])
+ if warn_exprs:
+ values = "(?:{})".format('|'.join(warn_exprs))
+ expr = expr_template.format("WARNING|WARN", values)
+ self.searchobj.add(
+ SearchDefType(expr, tag=tag + '.warning',
+ hint='WARNING'), logs_path,
+ allow_global_constraints=constraints)
 
  def _load(self):
  """Register searches for exceptions as well as any other type of issue
@@ -214,13 +255,10 @@ def _load(self):
  prefix_match = (r'(?:{})?'.
  format(wsgi_prefix or keystone_prefix))
 
- # Sometimes the exception is printed with just the class name
- # and sometimes it is printed with a full import path e.g.
- # MyExc or somemod.MyExc so we need to account for both.
- exc_obj_full_path_match = r'(?:\S+\.)?'
- expr_template = (r"^{}([0-9\-]+) (\S+) .+\S+\s({}{{}})[\s:\.]".
- format(prefix_match, exc_obj_full_path_match))
-
+ expr_template = "^" + prefix_match + \
+ r"(\d{{4}}-\d{{2}}-\d{{2}}) "\
+ r"(\d{{2}}:\d{{2}}:\d{{2}}\.\d{{3}}) "\
+ r"(?:\d+ )?(?:{}) (?:\S+)(?: \[.+\])?.* ((?:\S+\.)?{}):.*"
  # NOTE: don't check exceptions for deprecated services
  for agent, log_paths in project.log_paths(
  include_deprecated_services=False):

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
  'simplejson',
  'jinja2',
  'cryptography',
- 'searchkit >= 0.2.3',
+ 'searchkit@git+https://github.com/mustafakemalgilor/searchkit.git#egg=enhancement/hyperscan-search-def',
  'propertree'
 ]
 

requirements.txt

@@ -8,6 +8,6 @@ jinja2
 # More info on issue: https://github.com/canonical/hotsos/issues/326
 # [TODO] Bump cryptography to more recent release
 cryptography==3.4.8
-searchkit >= 0.2.5
+searchkit@git+https://github.com/mustafakemalgilor/searchkit.git@enhancement/hyperscan-search-def
 propertree
 fasteners