diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f258fac..d8bdcc8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,29 +10,24 @@ assignees: bitonio, MikeSchiessl **Describe the bug** A clear and concise description of what the bug is. -**To Reproduce** +**To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behavior** +**Expected behavior** A clear and concise description of what you expected to happen. -**Screenshots** +**Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] +**ULS Version output** +please run the following and attach the output here. +```bash +bin/uls.py --version +``` -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional context** +**Additional context** Add any other context about the problem here. diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..99da997 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "test/bats/bats-assert"] + path = test/bats/bats-assert + url = https://github.com/ztombol/bats-assert.git +[submodule "test/bats/bats-support"] + path = test/bats/bats-support + url = https://github.com/ztombol/bats-support.git diff --git a/Dockerfile b/Dockerfile index 9a1e469..e4973be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,8 @@ ARG HOMEDIR="/opt/akamai-uls" ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" -ARG ETP_CLI_VERSION="0.3.5" -ARG EAA_CLI_VERSION="0.4.4" +ARG ETP_CLI_VERSION="0.3.6" +ARG EAA_CLI_VERSION="0.4.5" ARG MFA_CLI_VERSION="0.0.6" # ENV VARS @@ -42,6 +42,7 @@ RUN mkdir -p ${ULS_DIR} # Install ULS COPY bin/ ${ULS_DIR}/bin +COPY var/ ${ULS_DIR}/var WORKDIR ${ULS_DIR} RUN pip3 install -r ${ULS_DIR}/bin/requirements.txt diff --git a/bin/config/global_config.py b/bin/config/global_config.py index a2f1eb0..e245523 100644 --- a/bin/config/global_config.py +++ b/bin/config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.2.0" +__version__ = "1.3.0" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -50,7 +50,8 @@ input_rerun_retries = 3 # Number of rerun attempts before giving up input_run_delay = 1 # Time in seconds to wait for the first health check input_rerun_delay = 1 # Time in seconds between rerun attempts - +input_disable_stderr = True # Enable STDERR output disabling (see value below to specify when this should happen) +input_disable_stderr_after = 25 # Disable stderr output after x input_cli cycles --> to prevent buffer overflow # OUTPUT Configuration output_reconnect_retries = 10 # Number of reconnect attempts before giving up @@ -62,10 +63,10 @@ output_tcp_timeout = 10.0 # TCP SEND / CONNECT Timeout (seconds) ## HTTP output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'} # HTTP Additional Headers to send (requests module KV pairs) +output_http_timeout = 10 # Timeout after which a request will be considered as failed ## FILE output_file_encoding = "utf-8" # FILE Encoding setting output_file_handler_choices = ['SIZE', 'TIME'] # Available Choices for the file handler -output_file_default_name = 'tmp/uls_file.output' # Default file name (path + name), only used if not set output_file_default_backup_count = 3 # Default number of backup files (after rotation) output_file_default_maxbytes = 50* 1024 * 1024 # Default maximum size of a file when rotated by the FILE - handler output_file_default_time_use_utc = False # Use UTC instead of local system time (Default: False) @@ -83,4 +84,10 @@ edgerc_eaa_legacy = ["eaa_api_host", "eaa_api_key", "eaa_api_secret"] # required for EAA - Legacy edgerc_mfa = ["mfa_integration_id", "mfa_signing_key"] # Required for MFA edgerc_documentation_url = "https://github.com/akamai/uls/blob/main/docs/AKAMAI_API_CREDENTIALS.md" -edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given \ No newline at end of file +edgerc_mock_file = "ext/edgerc" # Required for display the version if no edgercfile was given + +# Autoresume Configuration +autoresume_checkpoint_path = "var/" # (Default) Path, where the checkpointfiles should be stored to +autoresume_supported_inputs = ['ETP', 'EAA'] # Internal Var only, to adjust supported inputs +autoresume_write_after = 1000 # Write checkpoint only every ${autoresume_write_every} loglines + diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py index 0258185..0b044e2 100644 --- a/bin/modules/UlsArgsParser.py +++ b/bin/modules/UlsArgsParser.py @@ -39,6 +39,7 @@ def init(): const=True, help=f'Display {uls_config.__tool_name_short__} version and operational information') + # ---------------------- # Input GROUP input_group = parser.add_argument_group(title="Input", @@ -134,7 +135,7 @@ def init(): output_group.add_argument('--port', action='store', type=int, - default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'), + default=int(os.environ.get('ULS_OUTPUT_PORT') or '0' ), help="Port for TCP/UDP") # HTTP @@ -176,7 +177,7 @@ def init(): output_group.add_argument('--filehandler', action='store', type=str.upper, - default=(os.environ.get('ULS_FILE_HANDLER') or None), + default=(os.environ.get('ULS_FILE_HANDLER') or "SIZE"), choices=uls_config.output_file_handler_choices, help=f"Output file handler - Decides when files are rotated -" f"Choices: {uls_config.output_file_handler_choices} -" @@ -186,9 +187,9 @@ def init(): action='store', type=str, default=(os.environ.get('ULS_FILE_NAME') or - uls_config.output_file_default_name), + None), help=f"Output file destination (path + filename)" - f" Default: {uls_config.output_file_default_name}") + f" Default: None") ## File Backup count output_group.add_argument('--filebackupcount', @@ -227,6 +228,14 @@ def init(): help=f"Specifies the file rotation interval based on `--filetime` unit value" f" Default: {uls_config.output_file_time_interval}") + ## File Action + output_group.add_argument('--fileaction', + action='store', + type=str, + default=(os.environ.get('ULS_FILE_ACTION') or None), + help=f"This enables you to specify your own action upon a file rotation. ('%%s' defines the absolute file_name e.g. /path/to/my_file.log.1)." + f" Default: ") + # ---------------------- special_group = parser.add_argument_group(title="Transformation", description="Define Module Settings (Output manipulation)") @@ -254,6 +263,35 @@ def init(): default=(os.environ.get('ULS_TRANSFORMATION_PATTERN') or None), help="Provide a pattern to transform the output (Required for JMESPATH)") + + + #------------------------- + resume_group = parser.add_argument_group(title="Autoresume", + description="Define Autoresume Settings") + # Autoresume / Resume Switch + resume_group.add_argument('--autoresume', '--resume', + action='store', + type=bool, + dest='autoresume', + default=(os.environ.get('ULS_AUTORESUME') or False), + nargs='?', + const=True, + help=f'Enable automated resume on based on a checkpoint (do not use alongside --starttime)') + + resume_group.add_argument('--autoresumepath', + action='store', + type=str, + dest='autoresumepath', + default=(os.environ.get('ULS_AUTORESUME_PATH') or uls_config.autoresume_checkpoint_path), + help=f'Specify the path where checkpoint files should be written to. (Trailing /) [Default: {uls_config.autoresume_checkpoint_path}]') + + resume_group.add_argument('--autoresumewriteafter', + action='store', + type=int, + dest='autoresumewriteafter', + default=(os.environ.get('ULS_AUTORESUME_WRITEAFTER') or uls_config.autoresume_write_after), + help=f'Specify after how many loglines a checkpoint should be written [Default: {uls_config.autoresume_write_after}]') + return parser.parse_args() diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 9a377a0..1a8249d 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -38,7 +38,8 @@ def __init__(self, rawcmd=None, inproxy=None, starttime: int=None, - endtime: int=None): + endtime: int=None, + root_path: str=None): """ Initialzing a new UlsInput handler :param product: Input product @@ -50,6 +51,7 @@ def __init__(self, :param inproxy: Proxy config :param starttime: Start time in epoch seconds :param endtime: End time in epoch seconds + :param root_path: Root path of the git repo (to avoid runtime issues) """"" # Defaults (may vary later) @@ -60,6 +62,7 @@ def __init__(self, self.rerun_counter = 1 self.cli_proc = None self.run_once = False + self.cycle_counter = 0 # Handover Parameters self.product = product @@ -71,12 +74,15 @@ def __init__(self, self.inproxy = inproxy self.starttime = starttime self.endtime = endtime + self.root_path = root_path # Variables (load from uls_config) self.run_delay = uls_config.input_run_delay # Time in seconds to wait for the first health check self.rerun_retries = uls_config.input_rerun_retries # Number of rerun attempts before giving up self.rerun_delay = uls_config.input_rerun_delay # Time in seconds between rerun attempts self.bin_python = uls_config.bin_python # The python binary + self.disable_stderr = uls_config.input_disable_stderr #Specify if STDERR should be disabled at all after $disable_stderr_after cycles + self.disable_stderr_after = uls_config.input_disable_stderr_after # Disable StdErr Output after # cycles def _feed_selector(self, feed, product_feeds): if feed in product_feeds: @@ -139,7 +145,7 @@ def proc_create(self): # EAA config if self.product == "EAA": - product_path = uls_config.bin_eaa_cli + product_path = self.root_path + "/" + uls_config.bin_eaa_cli product_feeds = uls_config.eaa_cli_feeds if not self.rawcmd: my_feed = self._feed_selector(self.feed, product_feeds) @@ -180,7 +186,7 @@ def proc_create(self): # ETP config elif self.product == "ETP": - product_path = uls_config.bin_etp_cli + product_path = self.root_path + "/" + uls_config.bin_etp_cli product_feeds = uls_config.etp_cli_feeds if not self.cliformat == "JSON": @@ -210,7 +216,7 @@ def proc_create(self): # MFA config elif self.product == "MFA": - product_path = uls_config.bin_mfa_cli + product_path = self.root_path + "/" + uls_config.bin_mfa_cli product_feeds = uls_config.mfa_cli_feeds if not self.cliformat == "JSON": aka_log.log.warning(f"{self.name} - Selected LOG Format ({self.cliformat}) " @@ -243,7 +249,9 @@ def proc_create(self): aka_log.log.critical(f" {self.name} - No valid product selected " f"(--input={self.product}).") sys.exit(1) + try: + self.cycle_counter = 0 aka_log.log.info(f'{self.name} - CLI Command: {" ".join(cli_command)}') os.environ["PYTHONUNBUFFERED"] = "1" self.cli_proc = subprocess.Popen(cli_command, @@ -253,10 +261,15 @@ def proc_create(self): f"{' '.join(cli_command)}") self.proc = self.cli_proc self.proc_output = self.cli_proc.stdout - os.set_blocking(self.proc_output.fileno(), False) + + # Unblocking on windows causes trouble so we're avoiding it + if not os.name == 'nt': + os.set_blocking(self.proc_output.fileno(), False) + time.sleep(1) if not self.check_proc(): + #self.rerun_counter += 1 raise NameError(f"process [{self.cli_proc.pid}] " f"exited RC={self.cli_proc.returncode}, REASON: " f"{self.cli_proc.stderr.read().decode()}") @@ -267,7 +280,8 @@ def proc_create(self): self.rerun_counter = 1 if self.endtime: self.run_once = True - self.cli_proc.stderr = subprocess.DEVNULL + + #self.cli_proc.stderr = subprocess.DEVNULL except Exception as my_error: time.sleep(self.rerun_delay) @@ -279,18 +293,27 @@ def proc_create(self): if self.running is False and self.rerun_counter > self.rerun_retries: aka_log.log.critical(f'{self.name} - Not able to start the CLI for ' f'{self.product}. See above errors. ' - f'Giving up after {self.rerun_counter - 1} retries.') + f'Giving up after {self.rerun_counter - 2} retries.') sys.exit(1) def check_proc(self): try: if self.proc.poll() is None: + if self.cycle_counter == self.disable_stderr_after and self.disable_stderr: + aka_log.log.info(f"{self.name} - Disabling STDERR output from now on, after {self.cycle_counter} successful cycles") + self.cli_proc.stderr = subprocess.DEVNULL + aka_log.log.debug(f'{self.name} - Successful cycles for proc[{self.proc.pid}]: {self.cycle_counter}') + self.cycle_counter = self.cycle_counter + 1 return True else: self.running = False - aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]' - f' was found stale - {self.proc.stderr.read().decode()}') + self.rerun_counter += 1 + if (self.cycle_counter <= self.disable_stderr_after and self.disable_stderr) or not self.disable_stderr: + aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]' + f' was found stale - Reason: "{self.proc.stderr.read().decode()}" ') + else: + aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}], sadly stderr has been disabled') self.proc_create() return False @@ -299,7 +322,7 @@ def check_proc(self): aka_log.log.critical(f"{self.name} - '--endtime' was specified - so stopping now") sys.exit(0) else: - aka_log.log.critical(f'{self.name} - Soemthing really ' + aka_log.log.error(f'{self.name} - Soemthing really ' f'strange happened - message: {my_error}') self.proc_create() diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py index 0945d3c..893b31a 100644 --- a/bin/modules/UlsMonitoring.py +++ b/bin/modules/UlsMonitoring.py @@ -45,6 +45,7 @@ def __init__(self, stopEvent, product, feed, output): # Variables self.monitoring_enabled = uls_config.monitoring_enabled # Monitoring enable Flag self.monitoring_interval = uls_config.monitoring_interval # Monitoring interval + self._version = uls_config.__version__ # Definitions self.name = "UlsMonitoring" # Class Human readable name @@ -79,8 +80,10 @@ def display(self): 'uls_product': self._product, 'uls_feed': self._feed, 'uls_output': self._output, + 'uls_version': self._version, 'uls_runtime': self._runtime(), 'event_count': self.overall_messages_handled, + 'event_count_interval': self.window_messages_handled, 'event_rate': round(self.window_messages_handled / self.monitoring_interval, 2), 'mon_interval': self.monitoring_interval } diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py index a9627b4..ef5f4ef 100644 --- a/bin/modules/UlsOutput.py +++ b/bin/modules/UlsOutput.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import shlex import socket import ast import sys @@ -47,7 +47,8 @@ def __init__(self, output_type: str, filebackupcount=None, filemaxbytes=None, filetime=None, - fileinterval=None): + fileinterval=None, + fileaction=None): """ Initialzing a new UlsOutput handler :param output_type: The desired output format (TCP/ UDP / HTTP) @@ -85,7 +86,7 @@ def __init__(self, output_type: str, if self.output_type in ['TCP', 'UDP'] and host and port: self.host = host self.port = port - elif self.output_type in ['TCP', 'UDP'] and not host and not port: + elif self.output_type in ['TCP', 'UDP'] and (not host or not port): aka_log.log.critical(f"{self.name} - Host or Port has not " f"been set Host: {host} Port: {port} - exiting") sys.exit(1) @@ -97,6 +98,7 @@ def __init__(self, output_type: str, self.http_out_format = http_out_format self.http_out_auth_header = http_out_auth_header self.http_insecure = http_insecure + self.http_timeout = uls_config.output_http_timeout elif self.output_type in ['HTTP'] and not http_url: aka_log.log.critical(f"{self.name} http_out_format http_out_auth_" f"header http_url or http_insecure missing- exiting") @@ -104,12 +106,21 @@ def __init__(self, output_type: str, # File Parameters elif self.output_type in ['FILE']: + if filename == None: + aka_log.log.critical(f"{self.name} file-output was specified, but no file was specified. " + f"Please use --filename to specify a file") + sys.exit(1) self.filehandler = filehandler self.filename = filename self.filebackupcount = filebackupcount self.filemaxbytes = filemaxbytes self.filetime = filetime self.fileinterval = fileinterval + self.fileaction = fileaction + if self.fileaction and not "'%s'" in self.fileaction: + aka_log.log.critical(f"{self.name} file-action was specified, but \'%s\' was not sepcified within the string or %s was not properly escaped with a single quote ('%s') . " + f"Please use --fileaction \"my_script.sh \'%s\'\"") + sys.exit(1) # Variables (load from uls_config) self.reconnect_retries = uls_config.output_reconnect_retries # Number of reconnect attempts before giving up @@ -207,7 +218,7 @@ def connect(self): # Let'S do an options request resp = self.httpSession.options(url=self.http_url, data='{"event":"connection test"}', - verify=self.http_verify_tls) + verify=self.http_verify_tls, timeout=self.http_timeout) if resp.status_code == 200: reconnect_counter = 1 @@ -264,6 +275,32 @@ def connect(self): aka_log.log.critical(f"{self.name} - No valid filehandler has been specified Valid choices: {uls_config.output_file_handler_choices}. Given value: {self.filehandler} - exiting.") sys.exit(1) + ##### Add a hook to trigger file rotation + if self.fileaction and self.filebackupcount == 1: + aka_log.log.debug(f"{self.name} - FileAction has been specified: '{self.fileaction}' - enabling it now") + import subprocess + class UlsRotator: + def __init__(self, fileaction): + self.fileaction = fileaction + self.name = "UlsRotator" + + def __call__(self, source, dest): + # This is exactly what the original filehandler does + if os.path.exists(source): + os.rename(source, dest) + cmd = self.fileaction % (dest) + #print(self.fileaction % (dest)) + aka_log.log.warning(f"{self.name} - Running command {cmd}") + file_proc = subprocess.Popen(shlex.split(cmd)) + + + file_handler.rotator = UlsRotator(self.fileaction) + elif self.fileaction and (self.filebackupcount >= 1 or self.filebackupcount == 0): + aka_log.log.critical( + f"{self.name} - FileAction (--fileaction) has been specifiec but BackoupCount is not 1 (specify --filebackupcount 1) - Exiting") + sys.exit(1) + ##### + self.my_file_writer.addHandler(file_handler) self.my_file_writer.setLevel(logging.INFO) @@ -319,7 +356,8 @@ def send_data(self, data): elif self.output_type == "HTTP": response = self.httpSession.post(url=self.http_url, data=self.http_out_format % (data.decode()), - verify=self.http_verify_tls) + verify=self.http_verify_tls, + timeout=self.http_timeout) aka_log.log.debug(f"{self.name} DATA Send response {response.status_code}," f" {response.text} ") diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 5ac57cf..0f73e22 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -11,19 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import json import subprocess import sys import platform import os.path import configparser +import pathlib +import datetime # ULS modules import modules.aka_log as aka_log import config.global_config as uls_config -def uls_check_sys(): +def uls_check_sys(root_path): """ Collect ULS requirements information and request input if failing """ @@ -44,18 +46,20 @@ def _check_cli_installed(cli_bin): except Exception as my_error: aka_log.log.critical(f"Error checking the cli'tools ") - _check_cli_installed(uls_config.bin_eaa_cli) - _check_cli_installed(uls_config.bin_etp_cli) - _check_cli_installed(uls_config.bin_mfa_cli) + _check_cli_installed(root_path + "/" + uls_config.bin_eaa_cli) + _check_cli_installed(root_path + "/" + uls_config.bin_etp_cli) + _check_cli_installed(root_path + "/" + uls_config.bin_mfa_cli) -def uls_version(): +def uls_version(root_path): """ Collect ULS Version information and display it on STDOUT """ - def _get_cli_version(cli_bin): + + my_edgerc_mock_file = root_path + "/" + uls_config.edgerc_mock_file + def _get_cli_version(cli_bin, edgerc_mock_file): try: - version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "--edgerc", uls_config.edgerc_mock_file, "version"], + version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "--edgerc", edgerc_mock_file, "version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) my_cli_version = version_proc.communicate()[0].decode().strip('\n') @@ -68,26 +72,27 @@ def _get_cli_version(cli_bin): return f"n/a -> ({my_err})" # Create a mocked edgerc file (fix bug no output on missing edgerc) - if os.path.isfile(uls_config.edgerc_mock_file): - os.remove(uls_config.edgerc_mock_file) + if os.path.isfile(my_edgerc_mock_file): + os.remove(my_edgerc_mock_file) - with open(uls_config.edgerc_mock_file, 'x') as mcoked_edgerc_file: - mcoked_edgerc_file.write("[default]\n") + with open(my_edgerc_mock_file, 'x') as mocked_edgerc_file: + mocked_edgerc_file.write("[default]\n") # generate the stdout print(f"{uls_config.__tool_name_long__} Version information\n" f"ULS Version\t\t{uls_config.__version__}\n\n" - f"EAA Version\t\t{_get_cli_version(uls_config.bin_eaa_cli)}\n" - f"ETP Version\t\t{_get_cli_version(uls_config.bin_etp_cli)}\n" - f"MFA Version\t\t{_get_cli_version(uls_config.bin_mfa_cli)}\n\n" + f"EAA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_eaa_cli, my_edgerc_mock_file)}\n" + f"ETP Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_etp_cli, my_edgerc_mock_file)}\n" + f"MFA Version\t\t{_get_cli_version(root_path + '/' + uls_config.bin_mfa_cli, my_edgerc_mock_file)}\n\n" f"OS Plattform\t\t{platform.platform()}\n" f"OS Version\t\t{platform.release()}\n" f"Python Version\t\t{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" f"Docker Status\t\t{check_docker()}\n" + f"RootPath \t\t{root_path}\n" ) # Delete the mocked edgerc file - os.remove(uls_config.edgerc_mock_file) + os.remove(my_edgerc_mock_file) sys.exit(0) @@ -140,4 +145,99 @@ def uls_check_args(input, output): return 0 def check_docker(): - return os.path.isfile('/.dockerenv') \ No newline at end of file + return os.path.isfile('/.dockerenv') + + +def root_path(): + """ + Function to return the root path of the repo + :return: Root path (git root) + """ + return pathlib.Path(__file__).parent.resolve().parent.resolve().parent.resolve() + + +def check_autoresume(input, feed, checkpoint_dir=uls_config.autoresume_checkpoint_path): + # Check if we're in a supported stream / feed + if input not in uls_config.autoresume_supported_inputs or feed == "CONHEALTH": + aka_log.log.critical(f"Input {input} or feed {feed} currently not supported by AUTORESUME - Exiting.") + sys.exit(1) + + checkpoint_file = "uls_" + input.lower() + "_" + feed.lower() + ".ckpt" + checkpoint_full = str(os.path.abspath(checkpoint_dir)) + "/" + checkpoint_file + + if os.path.isfile(checkpoint_full): + aka_log.log.debug(f"Autoresume Checkpoint found: {checkpoint_full}") + if os.stat(checkpoint_full).st_size == 0: + aka_log.log.warning(f"Checkpoint \'{checkpoint_full}\' seems to be empty.") + creation_time = None + checkpoint = None + else: + try: + with open (checkpoint_full, "r") as ckpt_f: + data = json.load(ckpt_f) + if data['creation_time'] and data['checkpoint']: + aka_log.log.debug(f"Autoresume Checkpoint successfully loaded. Checkpoint Time: {data['checkpoint']}, Creation_time: {data['creation_time']}") + creation_time = data['creation_time'] + # Convert the Checkpoint to "epoch Timestamp", depending on the input + if data['input'] == "ETP": + mytime = data['checkpoint'].split("Z")[0] + elif data['input'] == "EAA": + mytime = data['checkpoint'].split("+")[0] + else: + aka_log.log.critical( + f"Unhandeled input data in checkpointfile \'{checkpoint_full}\' --> {input} / {feed} - Exiting.") + sys.exit(1) + checkpoint = int(datetime.datetime(year=int(mytime.split("T")[0].split("-")[0]), + month=int(mytime.split("T")[0].split("-")[1]), + day=int(mytime.split("T")[0].split("-")[2]), + hour=int(mytime.split("T")[1].split(":")[0]), + minute=int(mytime.split("T")[1].split(":")[1]), + second=int(mytime.split("T")[1].split(":")[2]), + ).timestamp()) + aka_log.log.debug(f"Checkpoint timestamp {data['checkpoint']} converted to epoch time {checkpoint}") + else: + aka_log.log.critical(f"Inconsitent data in checkpointfile \'{checkpoint_full}\' --> {data} - Exiting.") + sys.exit(1) + except Exception as readerror: + aka_log.log.critical(f"Error reading data from \'{checkpoint_full}\': {readerror} - Exiting.") + sys.exit(1) + else: + aka_log.log.info(f"No autoresume Checkpoint found - trying to create {checkpoint_full}") + creation_time = None + checkpoint = None + try: + pathlib.Path(checkpoint_full).touch() + except Exception as toucherr: + aka_log.log.critical(f"Error creating {checkpoint_full}: {toucherr} Please check directory / file permissions. - Exiting") + sys.exit(1) + aka_log.log.info( + f"Checkpoint file {checkpoint_full} successfully created") + + return {'filename': checkpoint_full, 'creation_time': creation_time, 'checkpoint': checkpoint} + + +def write_autoresume_ckpt(input, feed, autoresume_file, logline): + aka_log.log.info(f"AUTORESUME - IT's time to write a new checkpoint") + + # Adopt the field to the stream / feed + checkpoint_line = logline.decode() + if input == "ETP" and (feed == "THREAT" or feed =="PROXY" or feed == "AUP"): + checkpoint_timestamp = json.loads(checkpoint_line)['event']['detectionTime'] + if input == "ETP" and feed == "DNS": + checkpoint_timestamp = json.loads(checkpoint_line)['query']['time'] + elif input == "EAA" and feed == "ACCESS": + checkpoint_timestamp = json.loads(checkpoint_line)['datetime'] + else: + aka_log.log.critical( + f"AUTORESUME - Unhandled Inpute / FEed detected {input} / {feed} (this should never happen !!)- Exiting") + sys.exit(1) + + # Write out the file + try: + autoresume_data = {'creation_time': str(datetime.datetime.now()), 'checkpoint': str(checkpoint_timestamp), 'input': input, 'feed': feed} + with open(autoresume_file, "w") as ckpt_fd: + json.dump(autoresume_data, ckpt_fd) + aka_log.log.debug(f"AUTORESUME - Wrote a new checkpoint to {autoresume_file}: {autoresume_data}") + except Exception as write_error: + aka_log.log.critical(f"AUTORESUME - Failure writing data to {autoresume_file} - Data: {autoresume_data} - error: {write_error} - Exiting") + sys.exit(1) diff --git a/bin/uls.py b/bin/uls.py index fa0737f..19f46ad 100755 --- a/bin/uls.py +++ b/bin/uls.py @@ -62,15 +62,29 @@ def main(): # Load the LOG system aka_log.init(uls_args.loglevel, uls_config.__tool_name_short__) + # Determine root directory + root_path = str(UlsTools.root_path()) + # OUTPUT Version Information if uls_args.version: - UlsTools.uls_version() + UlsTools.uls_version(root_path=root_path) # Verify the given core params (at least input and output should be set) UlsTools.uls_check_args(uls_args.input, uls_args.output) + # Avoid confusion alongside "autoresume" (we do this after the input class have initialized to avoid creation of dumb files) + if uls_args.autoresume and uls_args.starttime: + aka_log.log.critical(f"Error using --autoresume alongside --starttime. This is too confusing to me. Exiting.") + sys.exit(1) + elif uls_args.autoresume: + autoresume_file = None + autoresume_lastwrite = 0 + autoresume_data = UlsTools.check_autoresume(input=uls_args.input, feed=uls_args.feed, checkpoint_dir=uls_args.autoresumepath) + uls_args.starttime = autoresume_data['checkpoint'] + autoresume_file = autoresume_data['filename'] + # Check CLI Environment - UlsTools.uls_check_sys() + UlsTools.uls_check_sys(root_path=root_path) # Create & Start monitoring Instance my_monitor = UlsMonitoring.UlsMonitoring(stopEvent=stopEvent, @@ -89,7 +103,8 @@ def main(): inproxy=uls_args.inproxy, rawcmd=uls_args.rawcmd, starttime=uls_args.starttime, - endtime=uls_args.endtime) + endtime=uls_args.endtime, + root_path=root_path) # Connect to the selected input UlsOutput my_output = UlsOutput.UlsOutput(output_type=uls_args.output, @@ -104,7 +119,9 @@ def main(): filebackupcount=uls_args.filebackupcount, filemaxbytes=uls_args.filemaxbytes, filetime=uls_args.filetime, - fileinterval=uls_args.fileinterval) + fileinterval=uls_args.fileinterval, + fileaction=uls_args.fileaction) + # Load a Transformation (if selected) UlsTransformation my_transformer = UlsTransformation.UlsTransformation(transformation=uls_args.transformation, @@ -152,6 +169,12 @@ def main(): aka_log.log.debug(f" {input_data}") for log_line in input_data.splitlines(): + # Write checkpoint to the checkpoint file (if autoresume is enabled) (not after transformation or filter) + if uls_args.autoresume and int(my_monitor.get_message_count()) >= autoresume_lastwrite + uls_args.autoresumewriteafter: + UlsTools.write_autoresume_ckpt(uls_args.input, uls_args.feed, autoresume_file, log_line) + autoresume_lastwrite = int(my_monitor.get_message_count()) + + # Filter Enhancement if uls_args.filter and not filter_pattern.match(log_line): aka_log.log.info(f"SKIPPED LINE due to FILTER rule {uls_args.filter}") @@ -159,7 +182,7 @@ def main(): f"due to FILTER match: {log_line}") continue - # Module Enhancement + # Transformation Enhancement if uls_args.transformation: log_line = my_transformer.transform(log_line) aka_log.log.debug(f"Transformed Logline via " @@ -187,14 +210,14 @@ def main(): uls_config.main_resend_exit_on_fail: aka_log.log.critical(f"MSG[{my_monitor.get_message_count()}] " f"ULS was not able to deliver the log message " - f"{out_data.decode()} - exiting!") + f"{out_data.decode()} after {resend_counter} attempts - Exiting!") sys.exit(1) elif resend_counter == uls_config.main_resend_attempts and \ not uls_config.main_resend_exit_on_fail: aka_log.log.warning( f"MSG[{my_monitor.get_message_count()}] " f"ULS was not able to deliver the log message " - f"{out_data.decode()} - (continuing anyway)") + f"{out_data.decode()} after {resend_counter} attempts - (continuing anyway as my config says)") else: aka_log.log.debug(f"Mainloop, wait {wait} seconds [{my_monitor.get_stats()}]") diff --git a/docs/ADDITIONAL_FEATURES.md b/docs/ADDITIONAL_FEATURES.md index c815003..efd3744 100644 --- a/docs/ADDITIONAL_FEATURES.md +++ b/docs/ADDITIONAL_FEATURES.md @@ -5,6 +5,8 @@ This document handles the usage of features not explained anywhere else. - [FILTER (--filter) Feature](#filter---filter-feature) - [RAWCMD (--rawcmd) Feature](#rawcmd---rawcmd-feature) - [ULS TRANSFORMATIONS](#uls-transformations) +- [AUTORESUME / RESUME](#autoresume--resume) +- ## FILTER (--filter) Feature This feature got introduced in ULS v0.9.0. @@ -37,7 +39,7 @@ Raw commands within ULS can be used to trigger cli calls, that have not been int This allows a more flexible implementation to solve some edge cases. RAWCMD just requires the input to be selected. -Example: +Example: ```bash python3 bin/uls.py -i etp --rawcmd 'event threat -f' -l debug -o raw ``` @@ -47,4 +49,73 @@ Please be aware: Not all output from the cli will be redirected to ULS by defaul # ULS TRANSFORMATIONS Transformations have been introduced to ULS in version `1.2.0` to support additional 3rd party integrations and custom log formats. -Please see the dedicated "[Transformations docs](TRANSFORMATIONS.md)" available. \ No newline at end of file +Please see the dedicated "[Transformations docs](TRANSFORMATIONS.md)" available. + + +# AUTORESUME / RESUME +This feature was introduced in ULS 1.3.0. +Different circumstances (network isse, server maintainence, ...) could lead to an interruption of the log stream or ULS itself. +As this could cause a gap in the continuous log delivery, ULS now offers the option to enable automated resume upon the last recorded checkpoint. + +AUTORESUME will create a checkpoint every 1000 lines of log (configureable) to prevent too many FS operations. +This means, in the worst case, the 1000 lines of log are going to be re-imported. + +Example: +```bash +bin/uls.py -input etp -feed threat --output raw --autoresume +``` +Autoresume can also be set via [environment variables (ENV)](ARGUMENTS_ENV_VARS.md#autoresume). +Attention: Do not use --autoresume alongside the --starttime argument or ENV variable, as ULS would not know where to start from. +For DOCKER based environments, please make sure you're using a volume or mount towards the var (default but configureable) directory within ULS. + +Additional configuration options: +[--autoresumepath](ARGUMENTS_ENV_VARS.md#autoresume) +[--autoresumewriteafter](ARGUMENTS_ENV_VARS.md#autoresume) + + +# POST PROCESSING OF FILES (FileOutput only) +This feature was introduced in ULS 1.3.0. +This allows to take over the control of the file post-processing workflow (asynchronous). +It is highly recommended to specify a binary or script that consumes the absolute file name as input. + +Ensure the following things: +- this will only work with `--output file` +- this will only work with `--filebackupcount 1` +- ensure to escape the `%s` like `'%s'` when specificy on the console + + +## Warnings +- ULS will just fire the process - but not care about "error/fallback handling" +- If the script "messes" up the file, the file is lost! +- If the file is not processed fast enough - the file is lost / overwritten! (size the rotation rules accordingly) +- Output of the "file handler" will be directly written to the console +- ULS will NOT take care of rotating your processed files. Please delete old log files on your own. + + +Example: +/opt/msycript.sh +```bash +#!/bin/bash + +target_dir="/tmp" +file_name="log_$(date +%Y%M%d-%H%m%S).gz" + +if [[ -z $1 ]] ; then + echo "No file given - please specify absolute path" + exit 1 +fi +gzip -cvf $1 > ${target_dir=}/${file_name=} +echo "-> ${target_dir=}/${file_name=}" +rm $1 +``` +You can find the [example script here](examples/scripts/file_handler.sh). + +Run ULS using the fileaction script +```bash +./bin/uls.py -i etp -f threat -o file --filename /tmp/logtest/test1.log --filehandler SIZE --filemaxbytes 10240000 --fileaction --filebackupcount 1 "/opt/msycript.sh '%s'" +``` + +Here's a recommendation on how to use this feature to avoid any "glitches": +Use the --fileaction handler to move the file into an observed queue directory and start a new process from there. +This will ensure a "fast handling" within ULS and provide even more flexibility/stability towards the worklfow and its error handling. + diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index ee98203..2f62147 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -3,47 +3,59 @@ The following tables list all available command line parameters and their corres ## Global -|Parameter|Env - Var|Options|Default|Description| -|---|---|---|---|---| -|-h
--help | n/a | n/a | None | Display help / usage information | -|-l
--loglevel | ULS_LOGLEVEL | 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' | WARNING | Adjust the overall loglevel | -|-v
--version| n/a | n/a | None | Display ULS version information (incl. CLI & OS versions) | +| Parameter | Env - Var | Options | Default | Description | +|--------------------|--------------|-------------------------------------------------|---------|-----------------------------------------------------------| +| -h
--help | n/a | n/a | None | Display help / usage information | +| -l
--loglevel | ULS_LOGLEVEL | 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' | WARNING | Adjust the overall loglevel | +| -v
--version | n/a | n/a | None | Display ULS version information (incl. CLI & OS versions) | + ## INPUT -|Parameter|Env - Var|Options|Default|Description| -|---|---|---|---|---| -|-i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | -|--feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'AUTH','POLICY' | None | Specify the desired INPUT feed | -|--format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | -|--inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT| None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected)| -|--rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature)| -|--edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | -|--section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | -|--starttime| ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection.| -|--endtime| ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | +| Parameter | Env - Var | Options | Default | Description | +|---------------------------|-----------------|-------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | +| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'AUTH','POLICY' | None | Specify the desired INPUT feed | +| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | +| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | +| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | +| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | +| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | +| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | +| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | ## OUTPUT -|Parameter|Output Type|Env - Var|Options|Default|Description| -|---|---|---|---|---|---| -|-o
--output| |ULS_OUTPUT | 'TCP', 'UDP', 'HTTP', 'RAW', 'FILE' | None | Specify the desired OUTPUT target | -|--host |TCP / UDP| ULS_OUTPUT_HOST | xxx.xxx.xxx.xxx | None | Specify the desired OUTPUT target host (TCP/UDP only) | -|--port|TCP / UDP| ULS_OUTPUT_PORT | xxxx | None | Specify the desired OUTPUT target port (TCP/UDP only) | -|--httpurl| HTTP(S) | ULS_HTTP_URL | http(s)://\:\/\ | None | The HTTP target URL. (HTTP only)
Do not use --host / --port for HTTP| -|--httpformat| HTTP(S) | ULS_HTTP_FORMAT| ''|'{"event": %s}'| Specify the expected output format (i.e. json) where %s will be replaced with the event data. -|--httpauthheader| HTTP(S) | ULS_HTTP_AUTH_HEADER | '{"Authorization": "VALUE"}' | None | Specify an Auhtorization header to auth against the HTTP Server (HTTP only)
Example:
'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}' | -|--httpinsecure| HTTP(S) | ULS_HTTP_INSECURE | True | False | Disable TLS CA certificate verification | -|--filehandler| FILE | ULS_FILE_HANDLER | 'SIZE','TIME' | SIZE | Select the handler which decides how the files are rotated if either specific SIZE or TIME has been reached | -|--filename| FILE | ULS_FILE_NAME | '/path/to/file.name' | /tmp/uls_file.output | The PATH + FILENAME where ULS should create the file | -|--filebackupcount | FILE | ULS_FILE_BACKUPCOUNT | '\' | 3 | Select the number of files that should be kept on the file system when rotating the data| -|--filemaxbytes| FILE (SIZE)| ULS_FILE_MAXBYTES | '\' | 50 * 1024 * 1024 = 50 MB | Filesize (in bytes) a file can reach before it will be rotated.
Only on SIZE - Handler (`--filehandler = size`) !!| -|--filetime| FILE (TIME)| ULS_FILE_TIME| ['S','M','H','D','W0'-'W6','midnight'] | 'M' | Specifies the file rotation trigger unit.
S: seconds, M: minutes, H: hours, D: days, 'W0'-'W6' Weekday (W0=Monday), 'midnight': midnight.| -|--fileinterval| FILE (TIME)| ULS_FILE_INTERVAL | \ | 30 | Specifies the file rotation interval based on `--filetime` unit value.
Example: 30 and filetime=M would rotate the file every 30 minutes| +| Parameter | Output Type | Env - Var | Options | Default | Description | +|-------------------|-------------|----------------------|----------------------------------------|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -o
--output | | ULS_OUTPUT | 'TCP', 'UDP', 'HTTP', 'RAW', 'FILE' | None | Specify the desired OUTPUT target | +| --host | TCP / UDP | ULS_OUTPUT_HOST | xxx.xxx.xxx.xxx | None | Specify the desired OUTPUT target host (TCP/UDP only) | +| | | | | | | +| --port | TCP / UDP | ULS_OUTPUT_PORT | xxxx | None | Specify the desired OUTPUT target port (TCP/UDP only) | +| | | | | | | +| --httpurl | HTTP(S) | ULS_HTTP_URL | http(s)://\:\/\ | None | The HTTP target URL. (HTTP only)
Do not use --host / --port for HTTP | +| --httpformat | HTTP(S) | ULS_HTTP_FORMAT | '' | '{"event": %s}' | Specify the expected output format (e.g. json) where %s will be replaced with the event data. /!\ %s can only be used once | +| --httpauthheader | HTTP(S) | ULS_HTTP_AUTH_HEADER | '{"Authorization": "VALUE"}' | None | Specify an Auhtorization header to auth against the HTTP Server (HTTP only)
Example:
'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}' | +| --httpinsecure | HTTP(S) | ULS_HTTP_INSECURE | True | False | Disable TLS CA certificate verification | +| | | | | | | +| --filehandler | FILE | ULS_FILE_HANDLER | 'SIZE','TIME' | SIZE | Select the handler which decides how the files are rotated if either specific SIZE or TIME has been reached | +| --filename | FILE | ULS_FILE_NAME | '/path/to/file.name' | None | The PATH + FILENAME where ULS should create the file | +| --filebackupcount | FILE | ULS_FILE_BACKUPCOUNT | '\' | 3 | Select the number of files that should be kept on the file system when rotating the data | +| --filemaxbytes | FILE (SIZE) | ULS_FILE_MAXBYTES | '\' | 50 * 1024 * 1024 = 50 MB | Filesize (in bytes) a file can reach before it will be rotated.
Only on SIZE - Handler (`--filehandler = size`) !! | +| --filetime | FILE (TIME) | ULS_FILE_TIME | ['S','M','H','D','W0'-'W6','midnight'] | 'M' | Specifies the file rotation trigger unit.
S: seconds, M: minutes, H: hours, D: days, 'W0'-'W6' Weekday (W0=Monday), 'midnight': midnight. | +| --fileinterval | FILE (TIME) | ULS_FILE_INTERVAL | '\' | 30 | Specifies the file rotation interval based on `--filetime` unit value.
Example: 30 and filetime=M would rotate the file every 30 minutes | +| --fileaction | FILE | ULS_FILE_ACTION | \ | None | Specify a file handler script/binary (e.g. bash) where `'%s'` will be replaced with the absolute filename (.e.g. /path/to/myfile.log). /!\ %s can only be used once!
This setting enforces '--filebackupcount' to be set to '1'
[Click here for more information](ADDITIONAL_FEATURES.md#) | ## Special Arguments -|Parameter|Env - Var|Options|Default|Description| -|---|---|---|---|---| -|--filter| ULS_OUTPUT_FILTER | \ | None | Filter (regex) to reduce number of OUTPUT log lines
Only loglines **matching** the `--filter ` argument will bes sent to the output.
[Click here for more information](ADDITIONAL_FEATURES.md#filter---filter-feature)| -|--transformation| ULS_TRANSFORMATION | 'MCAS', 'JMESPATH' | None | OPTIONAL: Specify an optional transformation to manipulate the output format
[Click here for more information](TRANSFORMATIONS.md)| -|--transformationpattern| ULS_TRANSFORMATION_PATTERN | \| None | Specifies the pattern used to transform the log event for the selected transformation. [Click here for more information](TRANSFORMATIONS.md)| \ No newline at end of file +| Parameter | Env - Var | Options | Default | Description | +|-------------------------|----------------------------|------------------------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| --filter | ULS_OUTPUT_FILTER | \ | None | Filter (regex) to reduce number of OUTPUT log lines
Only loglines **matching** the `--filter ` argument will bes sent to the output.
[Click here for more information](ADDITIONAL_FEATURES.md#filter---filter-feature) | +| --transformation | ULS_TRANSFORMATION | 'MCAS', 'JMESPATH' | None | OPTIONAL: Specify an optional transformation to manipulate the output format
[Click here for more information](TRANSFORMATIONS.md) | +| --transformationpattern | ULS_TRANSFORMATION_PATTERN | \ | None | Specifies the pattern used to transform the log event for the selected transformation. [Click here for more information](TRANSFORMATIONS.md) | + +## Autoresume +| Parameter | Env - Var | Options | Default | Description | +|------------------------|---------------------------|-------------------------------|---------|---------------------------------------------------------------------------------------------------------------| +| --autoresume | ULS_AUTORESUME | [True, False] | False | Enable automated resume on based on a checkpoint upon api failure or crash (do not use alongside --starttime) | +| --autoresumepath | ULS_AUTORESUME_PATH | '/path/to/store/checkpoints/' | var/ | Specify the path where checkpoint files should be written to. (Trailing /) | +| --autoresumewriteafter | ULS_AUTORESUME_WRITEAFTER | | 1000 | Specify after how many loglines a checkpoint should be written. | \ No newline at end of file diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index dbba576..b93ec16 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,29 @@ # Version History +## v1.3.0 +||| +|---|---| +|Date|2021-12-17 +|Kind| Feature & Bugfix release +|Author|mschiess@akamai.com +- **Features** + - [internal] Added automated test scripts to improve continuous release quality + - [AUTO-RESUME feature](ADDITIONAL_FEATURES.md#autoresume--resume) enables ULS to automatically continue operation starting from the last saved checkpoint. + - [FileAction support](ADDITIONAL_FEATURES.md#post-processing-of-files-fileoutput-only) to trigger custom scripts upon file rotation event. + + +- **Minor improvements** + - Bumped ETP-CLI to version 0.3.6 in Dockerfile + - Bumped EAA-CLI version to 0.4.5 in Dockerfile + - Added additional fields to the monitoring output ([uls_version, event_count_interval](MONITORING.md)) + + +- **Bugfixes** + - removed hard requirement to run ULS via bin/uls.py - can now be run from everywhere + - introduced HTTP Timeout (for HTTP OUTPUT) to the configuration file (http stream did not issue proper error messages in some cases) + - Fixed an output issue on "CLI failure", added configureable output handling to the config + - replaced pip with pip3 in CLI usage docs + - Fixed a windows bug (bypass blocking on windows) + added a [FAQ entry on how fix a installation specific bug](FAQ.md#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified) + ## v1.2.0 ||| diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md index 402580a..a95f367 100644 --- a/docs/COMMAND_LINE_USAGE.md +++ b/docs/COMMAND_LINE_USAGE.md @@ -44,7 +44,7 @@ pip3 install -r ext/cli-eaa/requirements.txt # Enterprise Threat Protector (ETP) git clone --depth 1 --single-branch https://github.com/akamai/cli-etp.git ext/cli-etp && \ -pip install -r ext/cli-etp/requirements.txt +pip3 install -r ext/cli-etp/requirements.txt # Akamai Phish Proof Multi Factor Authenticator (AKAMAI-MFA) git clone --depth 1 --single-branch https://github.com/akamai/cli-mfa.git ext/cli-mfa && \ diff --git a/docs/FAQ.md b/docs/FAQ.md index 1781efb..10ae86b 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -8,6 +8,8 @@ - [What environmental variables (ENV VARS) are available](#what-environmental-variables-env-vars-are-available-#) - [--inputproxy does not work as expected](#--inputproxy-proxy-does-not-work-as-expected) - [Logs are not showing up in my SIEM](#logs-are-not-showing-up-in-siem) +- [ULS on Windows error: "[WinError 2] The system cannot find the file specified"](#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified) + ---- ## FAQ @@ -62,4 +64,37 @@ Those can also be added to the .evn file when using docker / docker-compose. - Double check for sanity reasons, that no (additional) filters wihtin your SIEM have been applied Some excellent troubleshooting guidance from SPLUNK (but also applies to other SIEM as well) can be found [here](https://docs.splunk.com/Documentation/Splunk/6.4.1/Troubleshooting/Cantfinddata) - \ No newline at end of file +--- +### ULS on Windows error: "[WinError 2] The system cannot find the file specified" +ULS requires the OS to provide a python3 executable. The python installation on Windows somehow (unlike other OS) just installs a "python" executable. +Luckily this is something that can get sorted easily and in multiple different ways (just pick the one that suites you best): + +1) **Copy the binary (recommended)** +Go to your python directory on Windows e.g. `C:\Users\Administrator\AppData\Local\Programs\Python\Python310` or `C:\Program Files\Python\Python310`. +Now copy the `python.exe` executable to `python3.exe` within the same folder. + + +2) **Create a powershell alias (temproary only)** +If you are using powershell, run this before you start ULS. + ```text + Set-Alias -Name python3 -Value python + ``` + +3) **Create a symbolic link (requires [mklink](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/mklink))** + ```text + mklink "C:\path\to\symlink\python3.exe" "C:\path\to\Python3\python.exe" + ``` + +4) **Change ULS config (not recommended)** +You can modify the bin_python variable within the ULS global config file's `bin/config/global_config.py` 'Generic config' section.# +Change + ```text + bin_python = "python3" + ``` + to + ```text + bin_python = "python" + ``` + **WARNING:** This change prevents the global_config.py file to get updated via GIT in the future. You need to manually take care of updating changes within the file. + +--- diff --git a/docs/MONITORING.md b/docs/MONITORING.md index 0e8c591..ebdfe1b 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -3,16 +3,18 @@ This document describes the ULS monitoring output (STDOUT). The output will be sent every 5 minutes to stdout by default. ## Field description -| Field| Example | Description | -|---|---|---| -|dt | "2021-06-09T08:15:35.092889" | Date & Time (OS Timezone)| -|uls_product| "ETP" | Selected ULS product | -|uls_feed| "THREAT" | Selected ULS feed | -|uls_output| "HTTP" | Selected ULS output | -|uls_runtime| "3000" | Time in seconds ULS is already running | -|event_count| "625014" | Number of events handled by ULS (overall) | -|event_rate| "10.97" | Average events per second. Average based on the monitoring interval. (Default 5 minutes)| -|mon_interval| "300" | Monitoring interval in seconds| +| Field | Example | Description | +|----------------------|------------------------------|------------------------------------------------------------------------------------------| +| dt | "2021-06-09T08:15:35.092889" | Date & Time (OS Timezone) | +| uls_product | "ETP" | Selected ULS product | +| uls_feed | "THREAT" | Selected ULS feed | +| uls_output | "HTTP" | Selected ULS output | +| uls_version | "1.3.0" | Version of ULS currently running | +| uls_runtime | "3000" | Time in seconds ULS is already running | +| event_count | "625014" | Number of events handled by ULS (overall) | +| event_count_interval | "32145" | Number of events handlede by ULS (during the last interval) | +| event_rate | "10.97" | Average events per second. Average based on the monitoring interval. (Default 5 minutes) | +| mon_interval | "300" | Monitoring interval in seconds | ## Example Output diff --git a/docs/OUTPUTS.md b/docs/OUTPUTS.md index e2f2419..7f36304 100644 --- a/docs/OUTPUTS.md +++ b/docs/OUTPUTS.md @@ -39,9 +39,9 @@ ULS will resolve domain names using the hosts DNS resolution capabilities. Several SIEM do provide a way to ingest logs via HTTP(S), so ULS is supporting this feature as well. Data is sent towards the HTTP listener via a POST request, containing the data in the payload. -|OUTPUT|Required Parameters|Optional Parameters| -|---|---|---| -|--output http|--httpurl \|--httpformat \ --httpauthheader '{"Authorization": "VALUE"}' --httpinsecure| +|OUTPUT|Required Parameters| Optional Parameters | +|---|---|----------------------------------------------------------------------------------------------------| +|--output http|--httpurl \| --httpformat \
--httpauthheader '{"Authorization": "VALUE"}'
--httpinsecure | More information regarding the parameters can be found [here](ARGUMENTS_ENV_VARS.md#output). **Examples:** @@ -77,9 +77,9 @@ bin/uls.py --input eaa --feed admin --output raw The file output has been introduced in ULS version 1.2.0 to support logging/archiving operations. Data is written to an output file and rotated depending on the giben parameters (see examples below). -|OUTPUT|Required Parameters|Optional Parameters| -|---|---|---| -|--output file|--filehandler \ --filename '/path/to/file.name' --filebackupcount \|--filemaxbytes '\' --filetime ['S','M','H','D','W0'-'W6','midnight'] --fileinterval \| +|OUTPUT| Required Parameters | Optional Parameters | +|---|--------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +|--output file| --filehandler \
--filename '/path/to/file.name'
--filebackupcount \ | --filemaxbytes '\'
--filetime ['S','M','H','D','W0'-'W6','midnight']
--fileinterval \
--fileaction | More information regarding the parameters can be found [here](ARGUMENTS_ENV_VARS.md#output). **Examples:** diff --git a/docs/examples/scripts/file_handler.sh b/docs/examples/scripts/file_handler.sh new file mode 100755 index 0000000..52cb7ab --- /dev/null +++ b/docs/examples/scripts/file_handler.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +target_dir="/tmp" +file_name="log_$(date +%Y%M%d-%H%m%S).gz" + +if [[ -z $1 ]] ; then + echo "No file given - please specify absolute path" + exit 1 +fi +gzip -cvf $1 > ${target_dir=}/${file_name=} +echo "-> ${target_dir=}/${file_name=}" +rm $1 diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..fdd738e --- /dev/null +++ b/test/README.md @@ -0,0 +1,14 @@ +# ULS test scripts +## BATS +We decided to go with [bats](https://bats-core.readthedocs.io/en/stable/) as it provides everything we need for automated cli testing. +The bats job needs to get run from the git root directory: +```bash +bash test/test.sh +``` +For better testing stability, we packed the required dependencies into the test dir as well. + +### Requirements +- [bats](https://bats-core.readthedocs.io/en/stable/) +- timeout command is available +- uls and the cli's are installed +- working (fully fledged .edgerc file - not necessarily providing logs) \ No newline at end of file diff --git a/test/_mocked_edgerc b/test/_mocked_edgerc new file mode 100644 index 0000000..3cb7992 --- /dev/null +++ b/test/_mocked_edgerc @@ -0,0 +1,17 @@ +[testing] +client_secret = testing +host = testing.luna.akamaiapis.net +access_token = testing +client_token = testing + +# EAA MOCK +eaa_api_host = manage.akamai-access.com +eaa_api_key = testing +eaa_api_secret = testing + +# ETP MOCK +etp_config_id = 54321 + +# MFA MOCK +mfa_integration_id = testing +mfa_signing_key = testing diff --git a/test/basic_test.bats b/test/basic_test.bats new file mode 100644 index 0000000..41ef79a --- /dev/null +++ b/test/basic_test.bats @@ -0,0 +1,41 @@ +#!/usr/bin/env bats + + +# Variables + # ULS Binary +uls_bin=bin/uls.py + +uls2_path="bin" +uls2_bin=uls.py + +# Load support libs +load 'test/bats/bats-support/load.bash' +load 'test/bats/bats-assert/load.bash' + +uls_bin=bin/uls.py +uls_edgerc=~/.edgerc +uls_section=default + +# very basic tests +@test "uls.py w/o parametes" { + run $uls_bin + [ "$status" -eq 1 ] +} + +@test "uls.py --version" { + run $uls_bin --version + assert_output --partial "Akamai Unified Log Streamer Version information" + [ "$status" -eq 0 ] +} + +@test "uls.py --help" { + run $uls_bin --help + assert_output --partial "usage: uls.py [-h]" + [ "$status" -eq 0 ] +} + +@test "uls.py --loglevel debug" { + run $uls_bin --loglevel debug + assert_output --partial "ULS D Logging initialized" + [ "$status" -eq 1 ] +} diff --git a/test/bats/bats-assert b/test/bats/bats-assert new file mode 160000 index 0000000..9f88b42 --- /dev/null +++ b/test/bats/bats-assert @@ -0,0 +1 @@ +Subproject commit 9f88b4207da750093baabc4e3f41bf68f0dd3630 diff --git a/test/bats/bats-support b/test/bats/bats-support new file mode 160000 index 0000000..004e707 --- /dev/null +++ b/test/bats/bats-support @@ -0,0 +1 @@ +Subproject commit 004e707638eedd62e0481e8cdc9223ad471f12ee diff --git a/test/negative_test.bats b/test/negative_test.bats new file mode 100644 index 0000000..a40ddb5 --- /dev/null +++ b/test/negative_test.bats @@ -0,0 +1,195 @@ +#!/usr/bin/env bats + + +# Variables + # ULS Binary +uls_bin=bin/uls.py + + +# Load support libs +load 'test/bats/bats-support/load.bash' +load 'test/bats/bats-assert/load.bash' + + + +# Negative tests (failure responses expected) +## Basic negative testes +@test "Checking -i mfa standalone" { + run $uls_bin --input mfa + assert_output --partial "Required argument / ENV var not set: OUTPUT" + [ "$status" -eq 1 ] +} +@test "Checking --input eaa standalone" { + run $uls_bin --input eaa + assert_output --partial "Required argument / ENV var not set: OUTPUT" + [ "$status" -eq 1 ] +} +@test "Input / Feed mismatch" { + run $uls_bin --input eaa --feed auth -o raw + assert_output --partial "Feed (AUTH) not available - Available: ['ACCESS', 'ADMIN', 'CONHEALTH']" + [ "$status" -eq 1 ] +} +@test "Checking --feed and --intput etp" { + run $uls_bin --input etp --feed threat + assert_output --partial "Required argument / ENV var not set: OUTPUT" + [ "$status" -eq 1 ] +} +@test "Nonexistent edgerc" { + run $uls_bin -i eaa -f access -o raw --edgerc /non/existent/edge/rc.rrrrcccc + assert_output --partial "Config file '/non/existent/edge/rc.rrrrcccc' could not be loaded. - Exiting." + [ "$status" -eq 1 ] +} +@test "Nonexistent edgerc section" { + run $uls_bin -i eaa -f access -o raw --section definatelynonexitentsection + assert_output --partial "not found. Available sections:" + [ "$status" -eq 1 ] +} + +## TCP +@test "TCP output failure (no host/no port)" { + run $uls_bin -i eaa -f access -o tcp + assert_output --partial "Host or Port has not been set Host: None Port: 0" + [ "$status" -eq 1 ] +} +@test "TCP output failure (no host)" { + run $uls_bin --input eaa -f access --output tcp --port 2222 + assert_output --partial "Host or Port has not been set Host: None Port: 2222" + [ "$status" -eq 1 ] +} +@test "TCP output failure (no port)" { + run $uls_bin --input eaa -f access --output tcp --host 127.0.0.1 + assert_output --partial "Host or Port has not been set Host: 127.0.0.1 Port: 0" + [ "$status" -eq 1 ] +} +@test "TCP output failure (tcp host / port unrrachable)" { + run $uls_bin --input eaa -f access -o tcp --host 127.0.0.1 --port 7777 + assert_output --partial " UlsOutput not able to connect to 127.0.0.1:7777 - giving up after 10 retries." + [ "$status" -eq 1 ] +} + + +## UDP +@test "UDP output failure (no host/no port)" { + run $uls_bin -i eaa -f access -o udp + assert_output --partial "Host or Port has not been set Host: None Port: 0" + [ "$status" -eq 1 ] +} +@test "UDP output failure (no host)" { + run $uls_bin --input eaa -f access --output udp --port 2222 + assert_output --partial "Host or Port has not been set Host: None Port: 2222" + [ "$status" -eq 1 ] +} +@test "UDP output failure (no port)" { + run $uls_bin --input eaa -f access --output udp --host 127.0.0.1 + assert_output --partial "Host or Port has not been set Host: 127.0.0.1 Port: 0" + [ "$status" -eq 1 ] +} +@test "UDP output failure (tcp host / port unreachable)" { + # UDP won't fail as it is "stateless" + skip "UDP won't fail as it is stateless" + run $uls_bin --input eaa -f access -o udp --host 127.0.0.1 --port 7777 + assert_output --partial " UlsOutput not able to connect to 127.0.0.1:7777 - giving up after 10 retries." + [ "$status" -eq 1 ] +} + + +## HTTP +@test "HTTP output failure (-o http)" { + run $uls_bin -i mfa -f auth --output http + assert_output --partial "http_out_format http_out_auth_header http_url or http_insecure missing- exiting" + [ "$status" -eq 1 ] +} +@test "HTTP output failure (httpurl unreachable)" { + run $uls_bin -i eaa -f access -o http --httpurl https://127.0.0.1:7777 + assert_output --partial " UlsOutput not able to connect to https://127.0.0.1:7777 - giving up after 10 retries." + [ "$status" -eq 1 ] +} + + +## FILE +@test "FILE output failure (-o file)" { + run $uls_bin --input eaa --feed access -o file + assert_output --partial "file-output was specified, but no file was specified. Please use --filename to specify a file" + [ "$status" -eq 1 ] +} +@test "FILE output failure (-o file --filename /non/existent/dir/ec/tory/test.log)" { + rm -fr /non/existent/dir/ec/tory/test.log + run $uls_bin --input eaa --feed access -o file --filename /non/existent/dir/ec/tory/test.log + assert_output --partial " The specified directory /non/existent/dir/ec/tory does not exist or privileges are missing - exiting." + [ "$status" -eq 1 ] + rm -fr /non/existent/dir/ec/tory/test.log +} + +@test "FILE autoresume failure (-o file --filename /tmp/uls_tmplogfile.log --fileaction \"/tmp/test.sh\")" { + run $uls_bin --input eaa --feed access -o file --filename /tmp/uls_tmplogfile.log --fileaction "test.sh" + assert_output --partial "file-action was specified, but '%s' was not sepcified within the string" + [ "$status" -eq 1 ] + rm -fr /tmp/uls_tmplogfile.log +} + +@test "FILE autoresume failure (-o file --filename /tmp/uls_tmplogfile.log --fileaction \"/tmp/test.sh %s\")" { + run $uls_bin --input eaa --feed access -o file --filename /tmp/uls_tmplogfile.log --fileaction "test.sh" + assert_output --partial "or %s was not properly escaped with a single quote ('%s')" + [ "$status" -eq 1 ] + rm -fr /tmp/uls_tmplogfile.log +} + +@test "FILE autoresume failure (-o file --filename /tmp/uls_tmplogfile.log --fileaction \"/tmp/test.sh '%s'\")" { + run $uls_bin --input eaa --feed access -o file --filename /tmp/uls_tmplogfile.log --fileaction "test.sh '%s'" + assert_output --partial "FileAction (--fileaction) has been specifiec but BackoupCount is not 1 (specify --filebackupcount 1)" + [ "$status" -eq 1 ] + rm -fr /tmp/uls_tmplogfile.log +} + + +## TRANSFORMATIONS +@test "TRANSFORM - MCAS wrong input" { + run $uls_bin --input eaa --feed dns --output raw --transformation mcas + assert_output --partial "transformation Microsoft Cloud Applican Security [MCAS] specified, but wrong input/feed or format defined. MCAS only supports Input: ['ETP'] Feed: ['PROXY', 'DNS'] Cliformat: ['JSON'] - (exiting)" + [ "$status" -eq 1 ] +} +@test "TRANSFORM - MCAS wrong feed" { + run $uls_bin --input etp --feed aup --output raw --transformation mcas + assert_output --partial "transformation Microsoft Cloud Applican Security [MCAS] specified, but wrong input/feed or format defined. MCAS only supports Input: ['ETP'] Feed: ['PROXY', 'DNS'] Cliformat: ['JSON'] - (exiting)" + [ "$status" -eq 1 ] +} +@test "TRANSFORM - MCAS wrong format (txt)" { + run $uls_bin --input etp --feed aup --output raw --transformation mcas --format text + assert_output --partial "transformation Microsoft Cloud Applican Security [MCAS] specified, but wrong input/feed or format defined. MCAS only supports Input: ['ETP'] Feed: ['PROXY', 'DNS'] Cliformat: ['JSON'] - (exiting)" + [ "$status" -eq 1 ] +} + +@test "TRANSFORM - JMESPATH empty pattern" { + run $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern "" + assert_output --partial " transformation JMESPath https://jmespath.org/ [JMESPATH] specified, but wrong params given format defined. (Inputformat/pattern)JMESPath only supports Cliformat: ['JSON'] Transformation pattern given: - (exiting)" + [ "$status" -eq 1 ] +} + +@test "TRANSFORM - JMESPATH TEXT format" { + run $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern "asd" --format text + assert_output --partial " transformation JMESPath https://jmespath.org/ [JMESPATH] specified, but wrong params given format defined. (Inputformat/pattern)JMESPath only supports Cliformat: ['JSON'] Transformation pattern given: asd - (exiting)" + [ "$status" -eq 1 ] +} + +### Autoresume +@test "AUTORESUME - inacessible path" { + run $uls_bin --input etp --feed threat --output raw --autoresume --autoresumepath /blabla/blabla + assert_output --partial " [Errno 2] No such file or directory:" + [ "$status" -eq 1 ] +} + +@test "AUTORESUME - corrupt file (wrong data)" { + echo '{"aa": "bb"}' > /tmp/uls_eaa_access.ckpt + run $uls_bin --input eaa --feed access --output raw --autoresume --autoresumepath /tmp/ + assert_output --partial "'creation_time' - Exiting" + [ "$status" -eq 1 ] + rm -fr /tmp/uls_eaa_access.ckpt +} + +@test "AUTORESUME - corrupt file (wrong quoting within file )" { + echo "{'aa': 'bb'}" > /tmp/uls_eaa_access.ckpt + run $uls_bin --input eaa --feed access --output raw --autoresume --autoresumepath /tmp/ + assert_output --partial "Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - Exiting." + [ "$status" -eq 1 ] + rm -fr /tmp/uls_eaa_access.ckpt +} \ No newline at end of file diff --git a/test/positive_test.bats b/test/positive_test.bats new file mode 100644 index 0000000..ac7258b --- /dev/null +++ b/test/positive_test.bats @@ -0,0 +1,148 @@ +#!/usr/bin/env bats + + +## THIS Should be run from ULS ROOT DIR + +# Variables + # ULS Binary +uls_bin=bin/uls.py + + # TESTING EDGERC FILE +#uls_edgerc=~/.edgerc +uls_edgerc=test/_mocked_edgerc + + # TESTING SECTION +uls_section=testing + + # TIMEOUT +uls_test_timeout=30 + + +# Load support libs +load 'test/bats/bats-support/load.bash' +load 'test/bats/bats-assert/load.bash' + + +# POSITIVE tests +## EAA +@test "EAA - ACCESS" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} +@test "EAA - ADMIN" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} +@test "EAA - CONHEALTH" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed admin --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +## ETP +@test "ETP - THREAT" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +@test "ETP - AUP" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed aup --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +@test "ETP - DNS" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +@test "ETP - PROXY" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed proxy --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +## MFA +@test "MFA - AUTH" { + skip "MFA API currently broken" + run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed auth --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +@test "MFA - POLICY" { + skip "MFA API currently broken" + run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed policy --output raw --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +## FILE OUTPUT +@test "FILE: ETP - THREAT" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile.log" --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout + rm -fr /tmp/uls_tmplogfile.log +} + +@test "FILEACTION: ETP - THREAT" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed threat --output file --filename "/tmp/uls_tmplogfile.log" --filebackup 1 --fileaction "/bin/zip '%s'" --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout + rm -fr /tmp/uls_tmplogfile.log +} + +## Transformation +@test "TRANSFORM - MCAS" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input etp --feed dns --output raw --transformation mcas --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +@test "TRANSFORM - JMESPATH" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state]' --edgerc $uls_edgerc --section $uls_section + assert_output "" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout +} + +## AUTORESUME +@test "AUTORESUME - Create File" { + rm -f /tmp/uls_eaa_access.ckpt + run timeout --preserve-status $uls_test_timeout $uls_bin --input eaa --feed access --output raw --edgerc $uls_edgerc --section $uls_section --autoresume --autoresumepath /tmp/ + assert_output "" + #assert_output --partial " seems to be empty" + #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" + #[ "$status" -eq 124 ] #return value from timeout without --preserve status + [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout + rm -f /tmp/uls_eaa_access.ckpt +} \ No newline at end of file diff --git a/test/pre-release.sh b/test/pre-release.sh new file mode 100644 index 0000000..ec75fe0 --- /dev/null +++ b/test/pre-release.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo -ne "CONFIG: \t\t" +cat bin/config/global_config.py | grep "__version__ =" | cut -d " " -f 3 +echo -ne "CHANGELOG: \t\t" +cat docs/CHANGELOG.md | grep "##" | head -n 1 | cut -d "v" -f 2 diff --git a/test/test.sh b/test/test.sh new file mode 100644 index 0000000..056597c --- /dev/null +++ b/test/test.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# test.sh [file - $tests_available] [regex merge] +tests_available="basic positive negative " + +function do_test() { + if [ -f "test/$1_test.bats" ] ; then + echo -e "$1 TESTING\n" + bats test/$1_test.bats $filter + my_exitcode=$? + if [ $my_exitcode -ne 0 ] ; then + exit $my_exitcode + fi + else + echo "file test/$1_test.bats not found" + fi +} + + +if [ "$1" == "all" ] || [ "$1" == "" ] ; then + select="all" +elif [[ $tests_available =~ "$1 " ]] ; then + select=$1 +else + echo "$1 is not a proper selection !!!" + echo "please try: [$tests_available] or all" + exit 1 +fi + +if [ ! -z "$2" ] ; then + filter="-f $2" +else + filter="" +fi + + + +if [ "$select" == "all" ] ; then + for i in $tests_available ; do + echo "> $i" + do_test $i + done +else + do_test $1 +fi + +exit 1 diff --git a/var/.gitkeep b/var/.gitkeep new file mode 100644 index 0000000..e69de29