Version 1.3.0 (#16)

* preparing v.1.2.0

* bugfix for version_display_bug

* merged automated testing framework

* updated docs to represenet no-default-filepath

* Added configureable stderr logging, fixed a bug in stderr output handling

* amended changelog

* fixed pip -> pip3 in CLI docs

* re-enable STDERR disabling for stability

* fixed a windows specifc bug

* removed .bat file windows fix

* bumped etp-cli 0.3.6

* amended changelog for etp 3.6

* merged pre v1.3.0

* fixed submodules issue

* v1.3.0 finalized

* modified bug report template

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,29 +10,24 @@ assignees: bitonio, MikeSchiessl
 **Describe the bug**
 A clear and concise description of what the bug is.
 
-**To Reproduce**
+**To Reproduce**  
 Steps to reproduce the behavior:
 1. Go to '...'
 2. Click on '....'
 3. Scroll down to '....'
 4. See error
 
-**Expected behavior**
+**Expected behavior**  
 A clear and concise description of what you expected to happen.
 
-**Screenshots**
+**Screenshots**  
 If applicable, add screenshots to help explain your problem.
 
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
+**ULS Version output**  
+please run the following and attach the output here.
+```bash
+bin/uls.py --version
+```
 
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
+**Additional context**  
 Add any other context about the problem here.

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "test/bats/bats-assert"]
+	path = test/bats/bats-assert
+	url = https://github.com/ztombol/bats-assert.git
+[submodule "test/bats/bats-support"]
+	path = test/bats/bats-support
+	url = https://github.com/ztombol/bats-support.git

Dockerfile

@@ -10,8 +10,8 @@ ARG             HOMEDIR="/opt/akamai-uls"
 ARG             ULS_DIR="$HOMEDIR/uls"
 ARG             EXT_DIR="$ULS_DIR/ext"
 
-ARG             ETP_CLI_VERSION="0.3.5"
-ARG             EAA_CLI_VERSION="0.4.4"
+ARG             ETP_CLI_VERSION="0.3.6"
+ARG             EAA_CLI_VERSION="0.4.5"
 ARG             MFA_CLI_VERSION="0.0.6"
 
 # ENV VARS
@@ -42,6 +42,7 @@ RUN             mkdir -p ${ULS_DIR}
 
 # Install ULS
 COPY            bin/ ${ULS_DIR}/bin
+COPY            var/ ${ULS_DIR}/var
 WORKDIR         ${ULS_DIR}
 RUN             pip3 install -r ${ULS_DIR}/bin/requirements.txt
 

bin/config/global_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.2.0"
+__version__ = "1.3.0"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 
@@ -50,7 +50,8 @@
 input_rerun_retries = 3                         # Number of rerun attempts before giving up
 input_run_delay = 1                             # Time in seconds to wait for the first health check
 input_rerun_delay = 1                           # Time in seconds between rerun attempts
-
+input_disable_stderr = True                     # Enable STDERR output disabling (see value below to specify when this should happen)
+input_disable_stderr_after = 25                 # Disable stderr output after x input_cli cycles --> to prevent buffer overflow
 
 # OUTPUT Configuration
 output_reconnect_retries = 10                   # Number of reconnect attempts before giving up
@@ -62,10 +63,10 @@
 output_tcp_timeout = 10.0                       # TCP SEND / CONNECT Timeout (seconds)
     ## HTTP
 output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'}  # HTTP Additional Headers to send (requests module KV pairs)
+output_http_timeout = 10                        # Timeout after which a request will be considered as failed
     ## FILE
 output_file_encoding = "utf-8"                  # FILE Encoding setting
 output_file_handler_choices = ['SIZE', 'TIME']  # Available Choices for the file handler
-output_file_default_name = 'tmp/uls_file.output'    # Default file name (path + name), only used if not set
 output_file_default_backup_count = 3                # Default number of backup files (after rotation)
 output_file_default_maxbytes = 50* 1024 * 1024      # Default maximum size of a file when rotated by the FILE - handler
 output_file_default_time_use_utc = False            # Use UTC instead of local system time (Default: False)
@@ -83,4 +84,10 @@
 edgerc_eaa_legacy = ["eaa_api_host", "eaa_api_key", "eaa_api_secret"]               # required for EAA - Legacy
 edgerc_mfa = ["mfa_integration_id", "mfa_signing_key"]                              # Required for MFA
 edgerc_documentation_url = "https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md"
-edgerc_mock_file = "ext/edgerc"                 # Required for display the version if no edgercfile was given
+edgerc_mock_file = "ext/edgerc"                 # Required for display the version if no edgercfile was given
+
+# Autoresume Configuration
+autoresume_checkpoint_path = "var/"              # (Default) Path, where the checkpointfiles should be stored to
+autoresume_supported_inputs = ['ETP', 'EAA']           # Internal Var only, to adjust supported inputs
+autoresume_write_after = 1000                    # Write checkpoint only every ${autoresume_write_every} loglines
+

bin/modules/UlsArgsParser.py

@@ -39,6 +39,7 @@ def init():
                         const=True,
                         help=f'Display {uls_config.__tool_name_short__} version and operational information')
 
+
     # ----------------------
     # Input GROUP
     input_group = parser.add_argument_group(title="Input",
@@ -134,7 +135,7 @@ def init():
     output_group.add_argument('--port',
                               action='store',
                               type=int,
-                              default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'),
+                              default=int(os.environ.get('ULS_OUTPUT_PORT') or '0' ),
                               help="Port for TCP/UDP")
 
     # HTTP
@@ -176,7 +177,7 @@ def init():
     output_group.add_argument('--filehandler',
                               action='store',
                               type=str.upper,
-                              default=(os.environ.get('ULS_FILE_HANDLER') or None),
+                              default=(os.environ.get('ULS_FILE_HANDLER') or "SIZE"),
                               choices=uls_config.output_file_handler_choices,
                               help=f"Output file handler - Decides when files are rotated -"
                                    f"Choices: {uls_config.output_file_handler_choices} -"
@@ -186,9 +187,9 @@ def init():
                               action='store',
                               type=str,
                               default=(os.environ.get('ULS_FILE_NAME') or
-                                       uls_config.output_file_default_name),
+                                       None),
                               help=f"Output file destination (path + filename)"
-                                   f" Default: {uls_config.output_file_default_name}")
+                                   f" Default: None")
 
     ## File Backup count
     output_group.add_argument('--filebackupcount',
@@ -227,6 +228,14 @@ def init():
                               help=f"Specifies the file rotation interval based on `--filetime` unit value"
                                    f" Default: {uls_config.output_file_time_interval}")
 
+    ## File Action
+    output_group.add_argument('--fileaction',
+                              action='store',
+                              type=str,
+                              default=(os.environ.get('ULS_FILE_ACTION') or None),
+                              help=f"This enables you to specify your own action upon a file rotation. ('%%s' defines the absolute file_name e.g. /path/to/my_file.log.1)."
+                                   f" Default: <None>")
+
     # ----------------------
     special_group = parser.add_argument_group(title="Transformation",
                                              description="Define Module Settings (Output manipulation)")
@@ -254,6 +263,35 @@ def init():
                               default=(os.environ.get('ULS_TRANSFORMATION_PATTERN') or None),
                               help="Provide a pattern to transform the output (Required for JMESPATH)")
 
+
+
+    #-------------------------
+    resume_group = parser.add_argument_group(title="Autoresume",
+                                             description="Define Autoresume Settings")
+    # Autoresume / Resume Switch
+    resume_group.add_argument('--autoresume', '--resume',
+                        action='store',
+                        type=bool,
+                        dest='autoresume',
+                        default=(os.environ.get('ULS_AUTORESUME') or False),
+                        nargs='?',
+                        const=True,
+                        help=f'Enable automated resume on based on a checkpoint (do not use alongside --starttime)')
+
+    resume_group.add_argument('--autoresumepath',
+                        action='store',
+                        type=str,
+                        dest='autoresumepath',
+                        default=(os.environ.get('ULS_AUTORESUME_PATH') or uls_config.autoresume_checkpoint_path),
+                        help=f'Specify the path where checkpoint files should be written to. (Trailing /) [Default: {uls_config.autoresume_checkpoint_path}]')
+
+    resume_group.add_argument('--autoresumewriteafter',
+                              action='store',
+                              type=int,
+                              dest='autoresumewriteafter',
+                              default=(os.environ.get('ULS_AUTORESUME_WRITEAFTER') or uls_config.autoresume_write_after),
+                              help=f'Specify after how many loglines a checkpoint should be written [Default: {uls_config.autoresume_write_after}]')
+
     return parser.parse_args()
 
 

bin/modules/UlsInputCli.py

@@ -38,7 +38,8 @@ def __init__(self,
                  rawcmd=None,
                  inproxy=None,
                  starttime: int=None,
-                 endtime: int=None):
+                 endtime: int=None,
+                 root_path: str=None):
         """
         Initialzing a new UlsInput handler
         :param product: Input product
@@ -50,6 +51,7 @@ def __init__(self,
         :param inproxy: Proxy config
         :param starttime: Start time in epoch seconds
         :param endtime: End time in epoch seconds
+        :param root_path: Root path of the git repo (to avoid runtime issues)
         """""
 
         # Defaults (may vary later)
@@ -60,6 +62,7 @@ def __init__(self,
         self.rerun_counter = 1
         self.cli_proc = None
         self.run_once = False
+        self.cycle_counter = 0
 
         # Handover Parameters
         self.product = product
@@ -71,12 +74,15 @@ def __init__(self,
         self.inproxy = inproxy
         self.starttime = starttime
         self.endtime = endtime
+        self.root_path = root_path
 
         # Variables (load from uls_config)
         self.run_delay = uls_config.input_run_delay              # Time in seconds to wait for the first health check
         self.rerun_retries = uls_config.input_rerun_retries      # Number of rerun attempts before giving up
         self.rerun_delay = uls_config.input_rerun_delay          # Time in seconds between rerun attempts
         self.bin_python = uls_config.bin_python                  # The python binary
+        self.disable_stderr = uls_config.input_disable_stderr    #Specify if STDERR should be disabled at all after $disable_stderr_after cycles
+        self.disable_stderr_after = uls_config.input_disable_stderr_after   # Disable StdErr Output after # cycles
 
     def _feed_selector(self, feed, product_feeds):
         if feed in product_feeds:
@@ -139,7 +145,7 @@ def proc_create(self):
 
             # EAA config
             if self.product == "EAA":
-                product_path = uls_config.bin_eaa_cli
+                product_path = self.root_path + "/" + uls_config.bin_eaa_cli
                 product_feeds = uls_config.eaa_cli_feeds
                 if not self.rawcmd:
                     my_feed = self._feed_selector(self.feed, product_feeds)
@@ -180,7 +186,7 @@ def proc_create(self):
 
             # ETP config
             elif self.product == "ETP":
-                product_path = uls_config.bin_etp_cli
+                product_path = self.root_path + "/" + uls_config.bin_etp_cli
                 product_feeds = uls_config.etp_cli_feeds
 
                 if not self.cliformat == "JSON":
@@ -210,7 +216,7 @@ def proc_create(self):
 
             # MFA config
             elif self.product == "MFA":
-                product_path = uls_config.bin_mfa_cli
+                product_path = self.root_path + "/" + uls_config.bin_mfa_cli
                 product_feeds = uls_config.mfa_cli_feeds
                 if not self.cliformat == "JSON":
                     aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) "
@@ -243,7 +249,9 @@ def proc_create(self):
                 aka_log.log.critical(f" {self.name} - No valid product selected "
                                      f"(--input={self.product}).")
                 sys.exit(1)
+
             try:
+                self.cycle_counter = 0
                 aka_log.log.info(f'{self.name} - CLI Command:  {" ".join(cli_command)}')
                 os.environ["PYTHONUNBUFFERED"] = "1"
                 self.cli_proc = subprocess.Popen(cli_command,
@@ -253,10 +261,15 @@ def proc_create(self):
                                  f"{' '.join(cli_command)}")
                 self.proc = self.cli_proc
                 self.proc_output = self.cli_proc.stdout
-                os.set_blocking(self.proc_output.fileno(), False)
+
+                # Unblocking on windows causes trouble so we're avoiding it
+                if not os.name == 'nt':
+                    os.set_blocking(self.proc_output.fileno(), False)
+
                 time.sleep(1)
 
                 if not self.check_proc():
+                    #self.rerun_counter += 1
                     raise NameError(f"process [{self.cli_proc.pid}] "
                                     f"exited RC={self.cli_proc.returncode}, REASON: "
                                     f"{self.cli_proc.stderr.read().decode()}")
@@ -267,7 +280,8 @@ def proc_create(self):
                 self.rerun_counter = 1
                 if self.endtime:
                     self.run_once = True
-                self.cli_proc.stderr = subprocess.DEVNULL
+
+                #self.cli_proc.stderr = subprocess.DEVNULL
 
             except Exception as my_error:
                 time.sleep(self.rerun_delay)
@@ -279,18 +293,27 @@ def proc_create(self):
             if self.running is False and self.rerun_counter > self.rerun_retries:
                 aka_log.log.critical(f'{self.name} - Not able to start the CLI for '
                                      f'{self.product}. See above errors. '
-                                     f'Giving up after {self.rerun_counter - 1} retries.')
+                                     f'Giving up after {self.rerun_counter - 2} retries.')
                 sys.exit(1)
 
     def check_proc(self):
         try:
             if self.proc.poll() is None:
 
+                if self.cycle_counter == self.disable_stderr_after and self.disable_stderr:
+                    aka_log.log.info(f"{self.name} - Disabling STDERR output from now on, after {self.cycle_counter} successful cycles")
+                    self.cli_proc.stderr = subprocess.DEVNULL
+                aka_log.log.debug(f'{self.name} - Successful cycles for proc[{self.proc.pid}]: {self.cycle_counter}')
+                self.cycle_counter = self.cycle_counter + 1
                 return True
             else:
                 self.running = False
-                aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]'
-                                  f' was found stale - {self.proc.stderr.read().decode()}')
+                self.rerun_counter += 1
+                if (self.cycle_counter <= self.disable_stderr_after and self.disable_stderr) or not self.disable_stderr:
+                    aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]'
+                                      f' was found stale - Reason:  "{self.proc.stderr.read().decode()}" ')
+                else:
+                    aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}], sadly stderr has been disabled')
                 self.proc_create()
                 return False
 
@@ -299,7 +322,7 @@ def check_proc(self):
                 aka_log.log.critical(f"{self.name} - '--endtime' was specified - so stopping now")
                 sys.exit(0)
             else:
-                aka_log.log.critical(f'{self.name} - Soemthing really '
+                aka_log.log.error(f'{self.name} - Soemthing really '
                                      f'strange happened - message: {my_error}')
 
             self.proc_create()

bin/modules/UlsMonitoring.py

@@ -45,6 +45,7 @@ def __init__(self, stopEvent, product, feed, output):
         # Variables
         self.monitoring_enabled = uls_config.monitoring_enabled                     # Monitoring enable Flag
         self.monitoring_interval = uls_config.monitoring_interval                    # Monitoring interval
+        self._version = uls_config.__version__
 
         # Definitions
         self.name = "UlsMonitoring"                          # Class Human readable name
@@ -79,8 +80,10 @@ def display(self):
                        'uls_product': self._product,
                        'uls_feed': self._feed,
                        'uls_output': self._output,
+                       'uls_version': self._version,
                        'uls_runtime': self._runtime(),
                        'event_count': self.overall_messages_handled,
+                       'event_count_interval': self.window_messages_handled,
                        'event_rate': round(self.window_messages_handled / self.monitoring_interval, 2),
                        'mon_interval': self.monitoring_interval
                     }