openssl versions not parsing correctly; additional logger config

CveXplore/.schema_version

@@ -1,4 +1,4 @@
 {
- "version": "1.7",
+ "version": "1.8",
  "rebuild_needed": true
 }

CveXplore/VERSION

@@ -1 +1 @@
-0.3.20.dev23
+0.3.20.dev23

CveXplore/core/database_maintenance/api_handlers.py

@@ -34,6 +34,98 @@ def process_item(self, item: dict):
  # ).entry
  return item
 
+ @staticmethod
+ def stem(cpe_uri: str):
+ cpe_stem = cpe_uri.split(":")
+ return ":".join(cpe_stem[:5])
+
+ @staticmethod
+ def padded_version(version: str):
+ if version == "-" or version == "":
+ return version
+ else:
+ # normalizing edge cases:
+ version = version.replace("\\(", ".").replace("\\)", ".").rstrip(".")
+
+ ret_list = []
+
+ splitted_version = version.split(".")
+ # perform check if last part of version can be cast to an int
+ try:
+ int(splitted_version[-1])
+ # can be cast to an int, proceed 'normally'
+ for v in splitted_version:
+ try:
+ ret_list.append(f"{int(v):05d}")
+ except ValueError:
+ ret_list.append(v.rjust(5, "0"))
+ except ValueError:
+ # last part of the version cannot be cast to an int, so this means it's either a string or a
+ # string combined with an integer; handle accordingly
+
+ # first handle all version identifiers leading upto the last part
+ if len(splitted_version) > 1:
+ for i in range(len(splitted_version) - 1):
+ try:
+ ret_list.append(f"{int(splitted_version[i]):05d}")
+ except ValueError:
+ ret_list.append(splitted_version[i].rjust(5, "0"))
+
+ # handle the last part
+ # check if the last entry is smaller than 5 characters, if so just use that...
+ if len(splitted_version[-1]) > 5:
+ try:
+ ret_list.append(f"{int(splitted_version[-1]):05d}")
+ except ValueError:
+ ret_list.append(splitted_version[-1].rjust(5, "0"))
+ # check is last entry consists only of alphanumeric characters
+ elif splitted_version[-1].isalpha():
+ ret_list.append(splitted_version[-1].rjust(5, "0"))
+ else:
+ loop_i = 0
+ loop_count = len(splitted_version[-1])
+
+ # int/str combined value; handle accordingly
+ while loop_i < loop_count:
+ current_i = loop_i
+ # probably digit; so check;
+ if splitted_version[-1][loop_i].isdigit():
+ try:
+ ret_list.append(
+ f"{int(splitted_version[-1][loop_i]):05d}"
+ )
+ except ValueError:
+ ret_list.append(
+ splitted_version[-1][loop_i].rjust(5, "0")
+ )
+ finally:
+ # perform check if anything that follows consists only of string characters
+ if splitted_version[-1][loop_i + 1 :].isalpha():
+ ret_list.append(
+ splitted_version[-1][loop_i + 1 :].rjust(5, "0")
+ )
+ # no point proceeding; just break
+ break
+ loop_i += 1
+ else:
+ # ok so probably last part of version identifier is a string; add that with a loop
+ version_string = ""
+ try:
+ while splitted_version[-1][loop_i].isalpha():
+ version_string += splitted_version[-1][loop_i]
+ loop_i += 1
+ except IndexError:
+ # finished splitted_version variable; just pass
+ loop_i += 1
+ pass
+
+ ret_list.append(version_string.rjust(5, "0"))
+
+ if loop_i == current_i:
+ loop_i += 1
+
+ return ".".join(ret_list)
+
  @abstractmethod
  def process_the_item(self, *args):
  raise NotImplementedError

CveXplore/core/database_maintenance/download_handler.py

@@ -15,7 +15,6 @@
 from datetime import timedelta
 from io import BytesIO
 from itertools import islice
-from logging.handlers import RotatingFileHandler
 from shutil import copy
 from typing import Tuple
 
@@ -30,6 +29,8 @@
 from CveXplore.core.general.utils import sanitize
 from CveXplore.core.worker_queue.worker_q import WorkerQueue
 from ..database_indexer.db_indexer import DatabaseIndexer
+from ..logging.handlers.cve_explore_rfh import CveExploreUpdateRfhHandler
+from ..logging.handlers.cve_explore_stream import CveExploreUpdateStreamHandler
 from ..logging.logger_class import AppLogger
 from ...database.connection.database_connection import DatabaseConnection
 
@@ -81,7 +82,8 @@ def __init__(
 
  self.logger = logging.getLogger(logger_name)
 
- self.logger.removeHandler(self.logger.handlers[0])
+ if len(self.logger.handlers) == 1:
+ self.logger.removeHandler(self.logger.handlers[0])
 
  self.logger.propagate = False
 
@@ -91,23 +93,30 @@ def __init__(
 
  crf = None
 
- cli = logging.StreamHandler(stream=sys.stdout)
+ cli = CveExploreUpdateStreamHandler(stream=sys.stdout)
  cli.setFormatter(self.formatter)
  cli.setLevel(logging.INFO)
 
- if self.config.LOGGING_FILE_PATH != "":
- if not os.path.exists(self.config.LOGGING_FILE_PATH):
- os.makedirs(self.config.LOGGING_FILE_PATH)
-
- crf = RotatingFileHandler(
+ if self.config.LOGGING_TO_FILE:
+ crf = CveExploreUpdateRfhHandler(
  filename=f"{self.config.LOGGING_FILE_PATH}/{self.config.LOGGING_UPDATE_FILE_NAME}",
  maxBytes=self.config.LOGGING_MAX_FILE_SIZE,
  backupCount=self.config.LOGGING_BACKLOG,
  )
  crf.setLevel(logging.DEBUG)
  crf.setFormatter(self.formatter)
 
- if not len(self.logger.handlers):
+ if len(self.logger.handlers) > 0:
+ for handler in self.logger.handlers:
+ # add the handlers to the logger
+ # makes sure no duplicate handlers are added
+ if not isinstance(
+ handler, CveExploreUpdateRfhHandler
+ ) and not isinstance(handler, CveExploreUpdateStreamHandler):
+ if crf is not None:
+ self.logger.addHandler(crf)
+ self.logger.addHandler(cli)
+ else:
  if crf is not None:
  self.logger.addHandler(crf)
  self.logger.addHandler(cli)

CveXplore/core/database_maintenance/main_updater.py

@@ -53,6 +53,9 @@ def __init__(self, datasource: DatabaseConnectionBase):
 
  self.do_initialize = False
 
+ def __repr__(self):
+ return f"<<MainUpdater>>"
+
  def validate_schema(self):
  return self.schema_checker.validate_schema()
 

CveXplore/core/database_maintenance/sources_process.py

@@ -47,25 +47,6 @@ def __init__(self):
  def file_to_queue(self, *args):
  pass
 
- @staticmethod
- def stem(cpe_uri: str):
- cpe_stem = cpe_uri.split(":")
- return ":".join(cpe_stem[:5])
-
- @staticmethod
- def padded_version(version: str):
- if version == "-" or version == "":
- return version
- else:
- ret_list = []
- for v in version.split("."):
- try:
- ret_list.append(f"{int(v):05d}")
- except ValueError:
- ret_list.append(v.rjust(5, "0"))
-
- return ".".join(ret_list)
-
  @staticmethod
  def parse_cpe_version(cpename: str):
  cpe_list = cpename.split(":")
@@ -390,25 +371,6 @@ def get_vendor_product(cpeUri: str):
  product = cpeUri.split(":")[4]
  return vendor, product
 
- @staticmethod
- def stem(cpeUri: str):
- cpeArr = cpeUri.split(":")
- return ":".join(cpeArr[:5])
-
- @staticmethod
- def padded_version(version: str):
- if version == "-" or version == "":
- return version
- else:
- ret_list = []
- for v in version.split("."):
- try:
- ret_list.append(f"{int(v):05d}")
- except ValueError:
- ret_list.append(v.rjust(5, "0"))
-
- return ".".join(ret_list)
-
  def file_to_queue(self, *args):
  pass
 

CveXplore/core/database_maintenance/update_base_class.py

@@ -1,17 +1,20 @@
 import logging
-import os
 import sys
-from logging.handlers import RotatingFileHandler
 
 from CveXplore.common.config import Configuration
+from CveXplore.core.logging.handlers.cve_explore_rfh import CveExploreUpdateRfhHandler
+from CveXplore.core.logging.handlers.cve_explore_stream import (
+ CveExploreUpdateStreamHandler,
+)
 
 
 class UpdateBaseClass(object):
  def __init__(self, logger_name: str):
  self.config = Configuration
  self.logger = logging.getLogger(logger_name)
 
- self.logger.removeHandler(self.logger.handlers[0])
+ if len(self.logger.handlers) == 1:
+ self.logger.removeHandler(self.logger.handlers[0])
 
  self.logger.propagate = False
 
@@ -21,23 +24,30 @@ def __init__(self, logger_name: str):
 
  crf = None
 
- cli = logging.StreamHandler(stream=sys.stdout)
+ cli = CveExploreUpdateStreamHandler(stream=sys.stdout)
  cli.setFormatter(self.formatter)
  cli.setLevel(logging.INFO)
 
- if self.config.LOGGING_FILE_PATH != "":
- if not os.path.exists(self.config.LOGGING_FILE_PATH):
- os.makedirs(self.config.LOGGING_FILE_PATH)
-
- crf = RotatingFileHandler(
+ if self.config.LOGGING_TO_FILE:
+ crf = CveExploreUpdateRfhHandler(
  filename=f"{self.config.LOGGING_FILE_PATH}/{self.config.LOGGING_UPDATE_FILE_NAME}",
  maxBytes=self.config.LOGGING_MAX_FILE_SIZE,
  backupCount=self.config.LOGGING_BACKLOG,
  )
  crf.setLevel(logging.DEBUG)
  crf.setFormatter(self.formatter)
 
- if not len(self.logger.handlers):
+ if len(self.logger.handlers) > 0:
+ for handler in self.logger.handlers:
+ # add the handlers to the logger
+ # makes sure no duplicate handlers are added
+ if not isinstance(
+ handler, CveExploreUpdateRfhHandler
+ ) and not isinstance(handler, CveExploreUpdateStreamHandler):
+ if crf is not None:
+ self.logger.addHandler(crf)
+ self.logger.addHandler(cli)
+ else:
  if crf is not None:
  self.logger.addHandler(crf)
  self.logger.addHandler(cli)

CveXplore/core/logging/handlers/cve_explore_rfh.py

@@ -0,0 +1,6 @@
+from logging.handlers import RotatingFileHandler
+
+
+class CveExploreUpdateRfhHandler(RotatingFileHandler):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)

CveXplore/core/logging/handlers/cve_explore_stream.py

@@ -0,0 +1,6 @@
+from logging import StreamHandler
+
+
+class CveExploreUpdateStreamHandler(StreamHandler):
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)