diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..65b1355 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,63 @@ +FROM python:3.9.5-slim-buster +LABEL maintainer="Mike Schiessl - mike.schiessl@akamai.com" +LABEL APP="Akamai Universal Log Streamer" + +# CONFIGURATION ARGS +ARG HOMEDIR="/opt/akamai-uls" +ARG ULS_DIR="$HOMEDIR/uls" +ARG EXT_DIR="$ULS_DIR/ext" + +ARG ETP_CLI_VERSION="0.3.4" +ARG EAA_CLI_VERSION="0.3.8" +ARG MFA_CLI_VERSION="0.0.4" +ARG ULS_VERSION="0.0.1" + +# ENV VARS +ENV ULS_DIR=$ULS_DIR +ENV EXT_DIR=$EXT_DIR +ENV HOMEDIR=$HOMEDIR + + +# PREPARE ENVIRONMENT +# ENV PREP +RUN apt-get update && \ + apt-get --no-install-recommends -y install \ + curl \ + ca-certificates \ + git && \ + rm -rf /var/lib/apt/lists/ + +# USER & GROUP +RUN groupadd akamai && \ + useradd -g akamai -s /bin/bash -m -d ${HOMEDIR} akamai + +USER akamai +WORKDIR ${HOMEDIR} +RUN mkdir -p ${HOMEDIR}/uls + + +# Install ULS +ENV ULS_VERSION=$ULS_VERSION +RUN git clone --depth 1 -b "${ULS_VERSION}" --single-branch https://github.com/akamai/uls.git ${ULS_DIR} +WORKDIR ${ULS_DIR} + +# Install external CLI'S +## ETP CLI +ENV ETP_CLI_VERSION=$ETP_CLI_VERSION +RUN git clone --depth 1 -b "${ETP_CLI_VERSION}" --single-branch https://github.com/akamai/cli-etp.git ${EXT_DIR}/cli-etp && \ + pip install -r ${EXT_DIR}/cli-etp/requirements.txt + +## EAA CLI +ENV EAA-CLI_VERSION=$EAA_CLI_VERSION +RUN git clone --depth 1 -b "${EAA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-eaa.git ${EXT_DIR}/cli-eaa && \ + pip install -r ${EXT_DIR}/cli-eaa/requirements.txt +## MFA CLI +ENV MFA-CLI_VERSION=$MFA_CLI_VERSION +RUN git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch https://github.com/akamai/cli-mfa.git ${EXT_DIR}/cli-mfa && \ + pip install -r ${EXT_DIR}/cli-mfa/requirements.txt + +# ENTRYPOINTS / CMD +#CMD /usr/local/bin/python3 ${ULS_DIR}/bin/uls.py +ENTRYPOINT ["/usr/local/bin/python3","bin/uls.py"] + +# EOF diff --git a/README.md b/README.md index 5c780c7..75d61bc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,86 @@ -# uls -Unified Log Streamer (ULS) +# Akamai Unified Log Streamer (ULS) + +## Introduction +The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Akamai Secure Enterprise Access Products +- [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp) +- [Enterprise Threat Protector (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) +- [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp) + +Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box. +It can be run directly as Python code, as a provided Docker container or through `docker compose` scripts. + + +![ULS docker compose usage](docs/images/ula_docker-compose_complex_example.png) + + +## Table of contents +- [Akamai Unified Log Streamer (ULS)](#akamai-unified-log-streamer-uls) + - [Introduction](#introduction) + - [Table of contents](#table-of-contents) + - [Key Features](#key-features) + - [Documentation](#documentation) + - [Command Line Usage](#command-line-usage) + - [Docker](#docker) + - [Docker-compose](#docker-compose) + - [Development](#development) + - [Support](#support) + +## Key Features + +- Supported Inputs (Secure Enterprise Access Products) + - [Enterprise Application Access (EAA)](https://www.akamai.com/us/en/products/security/enterprise-application-access.jsp) + - [Enterprise Threat Protectors (ETP)](https://www.akamai.com/us/en/products/security/enterprise-threat-protector.jsp) + - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp) + + +- Supported data outputs + - TCP Socket (tcp://host:port) + - UDP Socket (udp://host:port) + - HTTP(S) URL (http(s)://host:port/path) (supporting Authentication) + + +- Operation types + - [python (command line)](./docs/COMMAND_LINE_USAGE.md) + - [docker](./docs/DOCKER_USAGE.md) + - [docker-compose](./docs/DOCKER-COMPOSE_USAGE.md) + + +- Additional Features + - [Monitoring output](./docs/MONITORING.md) + - Debug information (log level adjustment) + - HTTP CA CERT verification skipping + - Adoptable HTTP - POST format + +## Documentation +ULS can be operated in many ways. +Before setting up ULS, please understand your SIEM ingestion capabilities and configure an ingest method on your SIEM. +More information for specific SIEM solutions can be found in [this directory](./docs/SIEM/SIEM_OVERVIEW.md) and in your SIEM documentation. + +### Command Line Usage +![ULS command line usage](docs/images/uls_cli_help_example.png) +For more information, please visit [this document](./docs/COMMAND_LINE_USAGE.md) + +### Docker +![ULS docker usage](docs/images/uls_docker_etp_threat_example.png) +For more information, please visit [this document](./docs/DOCKER_USAGE.md) + +### Docker-compose +![ULS docker compose usage](docs/images/ula_docker-compose_complex_example.png) +For more information, please visit [this document](./docs/DOCKER-COMPOSE_USAGE.md) + + +## Development + +For the latest stable version of this software, please check the [release section](https://github.com/akamai/uls/releases) of the repo. The `main` [branch](https://github.com/akamai/uls) will retain the stable versions. +To ensure a continuous development of this tool, all new updates will go into the `development` [branch](https://github.com/akamai/uls/tree/development) of this repo. +The `development` branch can be subject to change and could also represent a broken version of this software. +In parallel, all new versions within the "main" branch will also be available on the [ULS docker hub space](https://hub.docker.com/repository/docker/akamai/uls). + +Contributions to this software can be provided via [Pull Requests](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests) and will get merged after successful review. + +## Support + +Akamai ULS is provided "as-is". It is not supported by Akamai Support. Akamai is neither liable for the function nor for any caused problems that come along with the usage or caused by this tool. To report an issue, feature request or bug, please open a new issue into the [GitHub Issues page](https://github.com/akamai/uls/issues). +This software is released under the "Apache License". Please refer to the [LICENSE](./LICENSE) document for more information. + +[Pull requests](#development) to improve the code or enhance the functionality are welcome. diff --git a/bin/config/global_config.py b/bin/config/global_config.py new file mode 100644 index 0000000..f0b6316 --- /dev/null +++ b/bin/config/global_config.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +# Common global variables / constants +__version__ = "0.0.1" +__tool_name_long__ = "Akamai Unified Log Streamer" +__tool_name_short__ = "ULS" + + +# Generic config +bin_python = "python3" # Python binary to use (use OS standard when not using path) + # EAA +bin_eaa_cli = "ext/cli-eaa/bin/akamai-eaa" # Path to the EAA CLI Executable +eaa_cli_feeds = ['ACCESS', 'ADMIN'] # Available EAA CLI feeds + # ETP +bin_etp_cli = "ext/cli-etp/bin/akamai-etp" # Path to the ETP CLI Executable +etp_cli_feeds = ['THREAT', 'AUP'] # Available ETP CLI feeds + # MFA +bin_mfa_cli = "ext/cli-mfa/bin/akamai-mfa" # Path to the MFA CLI Executable +mfa_cli_feeds = ['POLICY', 'AUTH'] # Available MFA CLI feeds + + # INPUT Choices +input_choices = ['EAA', 'ETP', 'MFA'] # Available input types +input_format_choices = ['JSON', 'TEXT'] # Available input format choices (need to be supported by cli) + + # OUTPUT Choices +output_choices = ['TCP', 'HTTP', 'UDP'] # Definition of OUTPUT Choices + + # LogLevels +log_levels_available = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] +log_level_default = 'WARNING' + + +# INPUT Configuration +input_rerun_retries = 3 # Number of rerun attempts before giving up +input_run_delay = 1 # Time in seconds to wait for the first health check +input_rerun_delay = 1 # Time in seconds between rerun attempts + + +# OUTPUT Configuration +output_reconnect_retries = 10 # Number of reconnect attempts before giving up +output_reconnect_delay = 1 # Time in seconds between reconnect attempts +output_udp_send_buffer = 262144 # UDP Send buffer in bytes +output_udp_timeout = 10.0 # UDP SEND / CONNECT Timeout (seconds) +output_tcp_send_buffer = 262144 # TCP Send buffer in bytes +output_tcp_timeout = 10.0 # TCP SEND / CONNECT Timeout (seconds) + # Additional Headers to send (requests module KV pairs) +output_http_header = {'User-Agent': f'{__tool_name_long__}/{__version__}'} + + +# Monitoring Configuration +monitoring_enabled = True # Set to false to disable monitoring outputs +monitoring_interval = 5 * 60 # Monitoring output interval (seconds) diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py new file mode 100644 index 0000000..8495a7f --- /dev/null +++ b/bin/modules/UlsArgsParser.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +import argparse +import os +import config.global_config as uls_config + + +def init(): + # Argument Parsing + parser = argparse.ArgumentParser(description=f"{uls_config.__tool_name_long__}", + formatter_class=argparse.RawTextHelpFormatter) + # Common params + parser.add_argument('-l', '--loglevel', + action='store', + type=str.upper, + default=(os.environ.get('ULS_LOGLEVEL') or uls_config.log_level_default), + choices=uls_config.log_levels_available, + help=f'Adjust the loglevel Default: {uls_config.log_level_default}') + + # Version Information + parser.add_argument('-v', '--version', + action='store', + type=bool, + default=False, + nargs='?', + const=True, + help=f'Display {uls_config.__tool_name_short__} version and operational information') + + # ---------------------- + # Input GROUP + input_group = parser.add_argument_group(title="Input", + description="Define INPUT Settings (AKAMAI API)") + + # INPUT_SELECTOR + input_group.add_argument('-i', '--input', + action='store', + type=str.upper, + default=(os.environ.get('ULS_INPUT') or None), + choices=uls_config.input_choices, + help="Select the Input Source. Default: None", ) + # INPUT_FEED + input_group.add_argument('--feed', + action='store', + type=str.upper, + default=(os.environ.get('ULS_FEED') or 'DEFAULT'), + help="Select data feed [CLI-DEFAULT]") + # INPUT FORMAT + input_group.add_argument('--format', + action='store', + dest="cliformat", + type=str.upper, + default=(os.environ.get('ULS_FORMAT') or "JSON"), + choices=uls_config.input_format_choices, + help="Select log output format Default: JSON") + # INPUT PROXY + input_group.add_argument('--inproxy', '--inputproxy', + dest='inproxy', + type=str, + default=(os.environ.get('ULS_INPUT_PROXY') or None), + help="Use a proxy Server for the INPUT requests (fetching data from AKAMAI API'S)") + # RAWCMD + input_group.add_argument('--rawcmd', + action='store', + type=str, + default=(os.environ.get('ULS_RAWCMD') or None), + help="Overwrite the cli command with your parameters. (python3 akamai-cli $rawcmd)") + # EDGERC + input_group.add_argument('--edgerc', + action='store', + type=str, + dest="credentials_file", + default=(os.environ.get('ULS_EDGERC') or '~/.edgerc'), + help="Location of the credentials file (default is ~/.edgerc)") + # EDGERC-SECTION + input_group.add_argument('--section', + action='store', + type=str, + dest="credentials_file_section", + default=(os.environ.get('ULS_SECTION') or 'default'), + help="Credentials file Section's name to use ('default' if not specified).") + + # ---------------------- + # Output GROUP + output_group = parser.add_argument_group(title="Output", + description="Define OUTPUT Settings (SIEM)") + + # OUTPUT Selector + output_group.add_argument('-o', '--output', + action='store', + type=str.upper, + default=(os.environ.get('ULS_OUTPUT') or None), + choices=uls_config.output_choices, + help="Select the Output Destination Default: None") + + # Output HOST + output_group.add_argument('--host', + action='store', + type=str, + default=(os.environ.get('ULS_OUTPUT_HOST') or None), + help="Host for TCP/UDP") + + # OUTPUT PORT + output_group.add_argument('--port', + action='store', + type=int, + default=int(os.environ.get('ULS_OUTPUT_PORT') or '0'), + help="Port for TCP/UDP") + + # HTTP URL + output_group.add_argument('--httpurl', + action='store', + type=str, + default=(os.environ.get('ULS_HTTP_URL') or None), + help=f'Full http(s) target url i.e. ' + f'https://my.splunk.host:9091/services/collector/event"') + + # HTTP AUTH HEADER + output_group.add_argument('--httpauthheader', + action='store', + type=str, + default=(os.environ.get('ULS_HTTP_AUTH_HEADER') or None), + help='HTTP Header for authorization. Example: ' + '\'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}\'') + + # Disable HTTP TLS verification + output_group.add_argument('--httpinsecure', + action='store', + type=bool, + default=(os.environ.get('ULS_HTTP_NO_VERIFY_TLS') or False), + nargs='?', + const=True, + help=f'Disable TLS CA Certificate verification. Default: False') + + # HTTP FORMAT DEFINITION + output_group.add_argument('--httpformat', + action='store', + type=str, + default=(os.environ.get('ULS_HTTP_FORMAT') or '{"event": %s}'), + help='HTTP Message format expected by http receiver ' + '(%%s defines the data string). Default \'{\"event\": %%s}\'') + + return parser.parse_args() + +# EOF diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py new file mode 100644 index 0000000..9055d4d --- /dev/null +++ b/bin/modules/UlsInputCli.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import time +import shlex +import modules.aka_log as aka_log +import config.global_config as uls_config +import platform + + +def uls_version(): + """ + Collect ULS Version information and display it on STDOUT + """ + def _get_cli_version(cli_bin): + try: + version_proc = subprocess.Popen([uls_config.bin_python, cli_bin, "version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + my_cli_version = version_proc.communicate()[0].decode().strip('\n') + version_proc.terminate() + if my_cli_version: + return my_cli_version + else: + return "n/a" + except Exception as my_err: + return f"n/a -> ({my_err})" + + # generate the stdout + print(f"{uls_config.__tool_name_long__} Version information\n" + f"ULS Version\t\t{uls_config.__version__}\n\n" + f"EAA Version\t\t{_get_cli_version(uls_config.bin_eaa_cli)}\n" + f"ETP Version\t\t{_get_cli_version(uls_config.bin_etp_cli)}\n" + f"MFA Version\t\t{_get_cli_version(uls_config.bin_mfa_cli)}\n\n" + f"OS Plattform\t\t{platform.platform()}\n" + f"OS Version\t\t{platform.release()}\n" + f"Python Version\t\t{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" + ) + sys.exit(0) + + +class UlsInputCli: + def __init__(self): + + self.run_delay = uls_config.input_run_delay # Time in seconds to wait for the first health check + self.rerun_retries = uls_config.input_rerun_retries # Number of rerun attempts before giving up + self.rerun_delay = uls_config.input_rerun_delay # Time in seconds between rerun attempts + self.bin_python = uls_config.bin_python # The python binary + + # Defaults (may vary later) + self.name = "UlsInputCli" # Class Human readable name + self.running = False # Internal Running tracker - do not touch + self.proc = None + self.proc_output = None + + def _feed_selector(self, feed, product_feeds): + if feed in product_feeds: + # feed matches the given list + aka_log.log.debug(f'{self.name} - selected feed: {feed}') + elif not feed: + # Set default (first of feeds) + feed = product_feeds[0] + aka_log.log.debug(f'{self.name} - using default feed: {feed}') + else: + aka_log.log.critical( + f"{self.name} - Feed ({feed}) not available - Available: {product_feeds}") + sys.exit(1) + return feed + + def _format_selector(self, cliformat): + if cliformat in uls_config.input_format_choices: + + return cliformat + elif not cliformat: + cliformat = 'JSON' + aka_log.log.debug(f'{self.name} - using default format: {cliformat}') + else: + aka_log.log.critical( + f"{self.name} - FORMAT ({cliformat}) not available") + sys.exit(1) + return cliformat + + def _prep_proxy(self, proxy): + if proxy: + return ['--proxy', proxy] + else: + return "" + + def _prep_edgegridauth(self, credentials_file, credentials_file_section): + edgegrid_auth = ['--edgerc', credentials_file, '--section', credentials_file_section] + return edgegrid_auth + + def _uls_useragent(self, product, feed): + my_useragent = f'ULS/{uls_config.__version__}_{product}-{feed}' + return ["--user-agent-prefix", my_useragent] + + def proc_create(self, product=None, + feed=None, + cliformat=None, + credentials_file="~/.edgerc", + credentials_file_section="default", + rawcmd=None, + inproxy=None): + + rerun_counter = 1 + + while self.running is False and rerun_counter <= self.rerun_retries: + edgegrid_auth = self._prep_edgegridauth(credentials_file, credentials_file_section) + aka_log.log.debug(f'{self.name} - selected product: {product}') + + # EAA config + if product == "EAA": + product_path = uls_config.bin_eaa_cli + product_feeds = uls_config.eaa_cli_feeds + if not rawcmd: + feed = self._feed_selector(feed, product_feeds) + cli_command = [self.bin_python, product_path, 'log', feed.lower(), '-f'] + cli_command[2:2] = self._uls_useragent(product_path, product, feed) + cli_command[2:2] = edgegrid_auth + cli_command[2:2] = self._prep_proxy(inproxy) + if self._format_selector(cliformat) == "JSON": + cli_command.append('-j') + else: + cli_command = [self.bin_python, product_path] + \ + self._uls_useragent(product, feed) +\ + shlex.split(rawcmd) + + # ETP config + elif product == "ETP": + product_path = uls_config.bin_etp_cli + product_feeds = uls_config.etp_cli_feeds + if not rawcmd: + feed = self._feed_selector(feed, product_feeds) + cli_command = [self.bin_python, product_path, 'event', feed.lower(), '-f'] + cli_command[2:2] = self._uls_useragent(product, feed) + cli_command[2:2] = edgegrid_auth + cli_command[2:2] = self._prep_proxy(inproxy) + else: + cli_command = [self.bin_python, product_path] +\ + self._uls_useragent(product, feed) +\ + shlex.split(rawcmd) + + # MFA config + elif product == "MFA": + product_path = uls_config.bin_mfa_cli + product_feeds = uls_config.mfa_cli_feeds + if not rawcmd: + feed = self._feed_selector(feed, product_feeds) + cli_command = [self.bin_python, product_path, 'event', feed.lower(), '-f'] + cli_command[2:2] = self._uls_useragent(product, feed) + cli_command[2:2] = edgegrid_auth + cli_command[2:2] = self._prep_proxy(inproxy) + else: + cli_command = [self.bin_python, product_path] +\ + self._uls_useragent(product, feed) +\ + shlex.split(rawcmd) + + # Everything else (undefined) + else: + aka_log.log.critical(f" {self.name} - No valid product selected (--input={product}).") + sys.exit(1) + try: + aka_log.log.debug(f'{self.name} - CLI Command: {cli_command}') + cli_proc = subprocess.Popen(cli_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + aka_log.log.debug(f"{self.name} - started PID[{cli_proc.pid}]: {cli_command}") + self.proc = cli_proc + self.proc_output = cli_proc.stdout + time.sleep(1) + + if not self.check_proc(): + raise NameError(f"process [{cli_proc.pid}] " + f"exited rc={cli_proc.returncode}: {cli_proc.stderr.read()}") + self.running = True + + except Exception as my_error: + time.sleep(self.rerun_delay) + self.running = False + rerun_counter += 1 + aka_log.log.error(f'{self.name} - {my_error} -> {self.proc.stderr.read()}') + + if self.running is False and rerun_counter > self.rerun_retries: + aka_log.log.critical(f'Not able to start the CLI for {product}. See above errors' + f'giving up after {rerun_counter - 1} retries.') + sys.exit(1) + + def check_proc(self): + try: + if self.proc.poll() is None: + return True + else: + self.running = False + aka_log.log.error(f'{self.name} - CLI process [{self.proc.pid}]' + f' was found stale -> {self.proc.stderr.read()}') + return False + except: + return False + +# EOF diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py new file mode 100644 index 0000000..cfcafe0 --- /dev/null +++ b/bin/modules/UlsMonitoring.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +import time +import threading +import json +import datetime + +import modules.aka_log as aka_log +import config.global_config as uls_config + + +class UlsMonitoring: + + def __init__(self, stopEvent, product, feed, output): + """ + Hanlde ULS self monitoring, spills out performance counter on stdout. + + Args: + stopEvent (threading.Event): Event from the controlling thread to tell the monitoring to stop + product (string): Akamai Product name/acronym + feed (string): specific data feed being consumed by ULS + """ + + self._stopEvent = stopEvent + self._product = product + self._feed = feed + self._output = output + + # Prevent other thread interact with the performance counters + self._metricLock = threading.Lock() + + # Variables + self.monitoring_enabled = uls_config.monitoring_enabled # Monitoring enable Flag + self.monitoring_interval = uls_config.monitoring_interval # Monitoring interval + + # Definitions + self.name = "UlsMonitoring" # Class Human readable name + self.overall_messages_handled = 0 # Define overall number of messages handled + self.window_messages_handled = 0 # Define mon_window number of messages handled + self.init_time = time.time() # Define the init time + + # Define the working thread, daemon allows us to offload + # of the main program termination to python + self.mon_thread = threading.Thread(target=self.display, daemon=True) + + def start(self): + if self.monitoring_enabled: + aka_log.log.debug(f"{self.name} monitoring thread started...") + # Start the background thread + self.mon_thread.start() + else: + aka_log.log.debug(f"{self.name} monitoring was disabled - not starting.") + + def display(self): + """ + Entry point for the monitoring thread + """ + try: # Exception handling is crucial once on the thread + while not self._stopEvent.is_set(): + aka_log.log.debug(f"{self.name} sleeping {self.monitoring_interval} sec...") + # Wait return True unless the timer expired, which is when + # ULS is still active and we safely report the activity + if not self._stopEvent.wait(self.monitoring_interval): + mon_msg = { + 'dt': datetime.datetime.utcnow().isoformat(), + 'uls_product': self._product, + 'uls_feed': self._feed, + 'uls_outpout': self._output, + 'uls_runtime': self._runtime(), + 'event_count': self.overall_messages_handled, + 'event_rate': round(self.window_messages_handled / self.monitoring_interval, 2), + 'mon_interval': self.monitoring_interval + } + print(json.dumps(mon_msg)) + # Reset window based vars + with self._metricLock: + self.window_messages_handled = 0 + except Exception as e: + aka_log.log.exception(e) + + def increase_message_count(self): + with self._metricLock: + self.overall_messages_handled = self.overall_messages_handled + 1 + self.window_messages_handled = self.window_messages_handled + 1 + + def _runtime(self): + return int(time.time() - self.init_time) + +# EOF + diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py new file mode 100644 index 0000000..dcdac00 --- /dev/null +++ b/bin/modules/UlsOutput.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +import socket +import requests +import ast +import sys +import time +import threading + +# ULS specific modules +import modules.aka_log as aka_log +import config.global_config as uls_config + +stopEvent = threading.Event() + + +class UlsOutput: + + def __init__(self): + # Variables (load from uls_config) + self.reconnect_retries = uls_config.output_reconnect_retries # Number of reconnect attempts before giving up + self.udp_send_buffer = uls_config.output_udp_send_buffer # UDP Send buffer in bytes + self.udp_timeout = uls_config.output_udp_timeout # UDP SEND / CONNECT Timeout (seconds) + self.tcp_send_buffer = uls_config.output_tcp_send_buffer # TCP Send buffer in bytes + self.tcp_timeout = uls_config.output_tcp_timeout # TCP SEND / CONNECT Timeout (seconds) + self.http_header = uls_config.output_http_header # Additional Headers + + # Defaults (may vary later) + self.name = "UlsOutput" # Class Human readable name + self.http_verify_tls = False # whether to verify the Certificate CA (True) or not (False) + self.connected = False # Internal Connection tracker - do not touch + self.output_type = None + self.http_out_format = None + self.http_url = None + self.httpSession = None + self.port = None + self.host = None + self.clientSocket = None + + def connect(self, output_type: str, host: str, port: int, + http_out_format=None, + http_out_auth_header=None, + http_url=None, + http_insecure=False): + """ + Connecting the tcp output socket. in addition we've added some error/reconnection handling + :param output_type: The desired output format (TCP/ UDP / HTTP) + :param host: hostname or ip address (TCP/UDP) + :param port: tcp port number (TCP/UDP) + :param http_url: URL (scheme://host:port/path) (HTTP) + :param http_out_format: HTTP Output format ((HTTP) + :param http_out_auth_header: HTTP Authentication header (HTTP) + :param http_insecure: (bool) Disable TLS verification (HTTP) + :return: + """ + + reconnect_counter = 1 + while not stopEvent.is_set() and self.connected is False and reconnect_counter <= self.reconnect_retries: + # Check & set output type + if output_type in ['TCP', 'HTTP', 'UDP']: + self.output_type = output_type + aka_log.log.debug(f"{self.name} Selected Output Type: {self.output_type} ") + else: + aka_log.log.critical(f"{self.name} target was not defined {output_type} ") + sys.exit(1) + try: + # TCP Connector + if self.output_type == "TCP": + # add a check if required vars are set + aka_log.log.debug(f"{self.name} attempting to connect via TCP to {host}:{port} ") + self.clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # check + self.clientSocket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.tcp_send_buffer) + self.clientSocket.connect((host, port)) + self.clientSocket.settimeout(self.tcp_timeout) + reconnect_counter = 1 + self.connected = True + aka_log.log.info(f"{self.name} successful connected to tcp://{host}:{port} ") + + # UDP Connector + if self.output_type == "UDP": + aka_log.log.debug(f"{self.name} attempting to connect via UDP to {host}:{port} ") + self.clientSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.host = host + self.port = port + self.clientSocket.settimeout(self.udp_timeout) + self.clientSocket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.udp_send_buffer) + reconnect_counter = 1 + self.connected = True + aka_log.log.info(f"{self.name} successful connected to udp://{host}:{port} ") + + # HTTP Connector + if self.output_type == "HTTP": + self.httpSession = requests.session() + # Prepare & set the headers + if http_out_auth_header: + headers = self.http_header | ast.literal_eval(http_out_auth_header) + else: + headers = self.http_header + aka_log.log.debug(f"{self.name} adding http headers: {headers}") + self.httpSession.headers.update(headers) + # Output Format + self.http_out_format = http_out_format + aka_log.log.debug(f"{self.name} setting http output format: {self.http_out_format}") + # TLS Verification + if http_insecure: + # DISABLE insecure warnings (if verify=FALSE) + requests.packages.urllib3.disable_warnings() + self.http_verify_tls = False # Use the inverted boolean expression ;) + aka_log.log.debug( + f"{self.name} TLS CA Certificate verification has been disabled - this is insecure !!") + elif not http_insecure: + self.http_verify_tls = True + aka_log.log.debug( + f"{self.name} TLS CA Certificate verification is turned on.") + else: + aka_log.log.critical(f'{self.name} HTTP insecure was not set to a boolean value (True|False) ' + f'- we got "{http_insecure}" instead') + sys.exit(1) + # Check the URL + if not http_url: + aka_log.log.critical(f'{self.name} HTTP output selected but no URL given. ' + f'Use --httpurl instead of --host / --port') + sys.exit(1) + else: + aka_log.log.debug(f"{self.name} attempting to connect via HTTP to {http_url} ") + + # Let'S do an options request + self.http_url = http_url + resp = self.httpSession.options(url=self.http_url, data='{"event":"connection test"}', + verify=self.http_verify_tls) + + if resp.status_code == 200: + reconnect_counter = 1 + self.connected = True + aka_log.log.info(f"{self.name} successful connected to {self.http_url} ") + else: + aka_log.log.error(f"{self.name} error connecting to {self.http_url}. " + f"StatusCode: {resp.status_code} Reason: {resp.text} [{reconnect_counter}]") + time.sleep(uls_config.output_reconnect_delay) + self.connected = False + reconnect_counter = 1 + + except Exception as con_error: + aka_log.log.debug(f"{self.name} issue: {con_error}") + if not self.output_type == 'HTTP': + aka_log.log.error(f"{self.name} error connecting to {host}:{port} [{reconnect_counter}]") + else: + aka_log.log.error(f"{self.name} error connecting to {self.http_url} [{reconnect_counter}]") + reconnect_counter += 1 + self.connected = False + time.sleep(uls_config.output_reconnect_delay) + + if self.connected is False and reconnect_counter > self.reconnect_retries: + if not self.output_type == 'HTTP': + aka_log.log.critical(f"{self.name} not able to connect to {host}:{port} - " + f"giving up after {reconnect_counter - 1} retries.") + else: + aka_log.log.critical(f"{self.name} not able to connect to {self.http_url} - " + f"giving up after {reconnect_counter - 1} retries.") + sys.exit(1) + + def send_data(self, data): + """ + Transfer binary data towards the established TCP socket. + We also try to handle issues across the connection (potential data loss?) + :param data: binary + :return: + """ + try: + if self.output_type == "TCP": + self.clientSocket.sendall(data) + + elif self.output_type == "UDP": + self.clientSocket.sendto(data, (self.host, self.port)) + + elif self.output_type == "HTTP": + response = self.httpSession.post(url=self.http_url, + data=self.http_out_format % (data.decode()), + verify=self.http_verify_tls) + aka_log.log.debug(f"{self.name} DATA Send response {response.status_code}," + f" {response.text} ") + else: + aka_log.log.critical(f"{self.name} target was not defined {self.output_type} ") + sys.exit(1) + except Exception as my_error: + aka_log.log.error(f"{self.name} Issue sending data {my_error}") + self.connected = False + + def tear_down(self): + """ + Tear down all resources + """ + if self.output_type == "TCP" or self.output_type == "UDP": + aka_log.log.debug(f"{self.name} closing socket {self.clientSocket}") + if self.clientSocket: + self.clientSocket.close() + if self.output_type == "HTTP": + aka_log.log.debug(f"{self.name} closing HTTP Session {self.httpSession}") + if self.httpSession: + self.httpSession.close() +# EOF diff --git a/bin/modules/aka_log.py b/bin/modules/aka_log.py new file mode 100644 index 0000000..63d22e4 --- /dev/null +++ b/bin/modules/aka_log.py @@ -0,0 +1,11 @@ +import logging + +def init(loglevel='WARNING', loggername=None): + global log + log = logging.getLogger(loggername) + logging.basicConfig(format='%(asctime)s %(name)s %(levelname).1s %(message)s') + log.setLevel(loglevel) + log.debug("Logging initialized") + return log + +# EOF diff --git a/bin/uls.py b/bin/uls.py new file mode 100644 index 0000000..f7a4b23 --- /dev/null +++ b/bin/uls.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import select +import signal +import threading + +# ULS specific modules +import modules.aka_log as aka_log +import modules.UlsArgsParser as aka_parser +import modules.UlsOutput as UlsOutput +import modules.UlsInputCli as UlsInputCli +import config.global_config as uls_config +import modules.UlsMonitoring as UlsMonitoring + +stopEvent = threading.Event() + + +def sigterm_handler(signum, frame): + """ + Upon SIGTERM, we signal any other pending activities + to stop right away + """ + aka_log.log.debug(f"SIGTERM ({signum}) detected, setting stopEvent") + stopEvent.set() + + +def control_break_handler(): + """ + Upon CTRL + C, we signal any other pending activities + to stop right away + """ + aka_log.log.debug("Control+C detected, setting stopEvent") + stopEvent.set() + + +def main(): + + signal.signal(signal.SIGTERM, sigterm_handler) + + # Load the Argument / ENV Var handler + uls_args = aka_parser.init() + if uls_args.version: + UlsInputCli.uls_version() + + # Load the LOG system + aka_log.init(uls_args.loglevel, uls_config.__tool_name_short__) + + # Create instances for Input and Output stream handler + my_monitor = UlsMonitoring.UlsMonitoring(stopEvent, uls_args.input, uls_args.feed, uls_args.output) + my_monitor.start() + my_output = UlsOutput.UlsOutput() + my_input = UlsInputCli.UlsInputCli() + + # Now let's handle the data and send input to output + while not stopEvent.is_set(): + try: + # (Re)Connect the input handler + my_input.proc_create(product=uls_args.input, + feed=uls_args.feed, + cliformat=uls_args.cliformat, + credentials_file=uls_args.credentials_file, + credentials_file_section=uls_args.credentials_file_section, + rawcmd=uls_args.rawcmd) + + input_poll = select.poll() + input_poll.register(my_input.proc_output) + my_input.check_proc() + + # (RE)Connect the output handler + my_output.connect(output_type=uls_args.output, + host=uls_args.host, + port=uls_args.port, + http_out_format=uls_args.httpformat, + http_out_auth_header=uls_args.httpauthheader, + http_url=uls_args.httpurl, + http_insecure=uls_args.httpinsecure) + + if input_poll.poll(10): + data = my_input.proc_output.readline() + if data: + aka_log.log.debug(f"DATA: {data}") + my_monitor.increase_message_count() + my_output.send_data(data) + except KeyboardInterrupt: + control_break_handler() + + my_output.tear_down() + + if stopEvent.is_set(): + sys.exit(100) + + +if __name__ == "__main__": + main() + +# EOF diff --git a/docker-compose/complex/docker-compose.yml b/docker-compose/complex/docker-compose.yml new file mode 100644 index 0000000..642d598 --- /dev/null +++ b/docker-compose/complex/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.0" +services: + etp-threat: + image: akamai/uls:latest + restart: always + env_file: etp-threat.env + volumes: + - type: bind + source: /Users/mschiess/.edgerc + target: /opt/akamai-uls/.edgerc + eaa-access: + image: akamai/uls:latest + restart: always + env_file: eaa-access.env + volumes: + - type: bind + source: /Users/mschiess/.edgerc + target: /opt/akamai-uls/.edgerc + eaa-admin: + image: akamai/uls:latest + restart: always + env_file: eaa-admin.env + volumes: + - type: bind + source: /Users/mschiess/.edgerc + target: /opt/akamai-uls/.edgerc \ No newline at end of file diff --git a/docker-compose/complex/eaa-access.env b/docker-compose/complex/eaa-access.env new file mode 100644 index 0000000..91de9a2 --- /dev/null +++ b/docker-compose/complex/eaa-access.env @@ -0,0 +1,28 @@ +# This is a sample ENV file for EAA - Threat logs (via ULS) + +# GENERIC Config +ULS_LOGLEVEL=DEBUG + +# INPUT CONFIGURATION + # THE INPUT PRODUCT [EAA | ETP | MFA] + ULS_INPUT=EAA + # THE INPUT FEED + # EAA: [ ADMIN | ACCESS] + # ETP: [ THREAT | AUP ] + # MFA: [ POLICY | AUTH ] + ULS_FEED=ACCESS + # INPUT FORRMAT + ULS_FORMAT=JSON + # LOCATION OF THE AKAMAI .EDGERC FILE + ULS_EDGERC='~/.edgerc' + # RELEVANT SECTION WITHIN THE EDGERC FILE + ULS_SECTION=akamaidemo + + +# OUTPUT CONFIGURATION + # OUTPUT PATH [ TCP / UDP / HTTP ] + ULS_OUTPUT=UDP + # OUTPUT TARGET HOST + ULS_OUTPUT_HOST=192.168.86.34 + # OUTPUT TARGET PORT + ULS_OUTPUT_PORT=9091 \ No newline at end of file diff --git a/docker-compose/complex/eaa-admin.env b/docker-compose/complex/eaa-admin.env new file mode 100644 index 0000000..8276b1c --- /dev/null +++ b/docker-compose/complex/eaa-admin.env @@ -0,0 +1,28 @@ +# This is a sample ENV file for EAA - Threat logs (via ULS) + +# GENERIC Config +ULS_LOGLEVEL=DEBUG + +# INPUT CONFIGURATION + # THE INPUT PRODUCT [EAA | ETP | MFA] + ULS_INPUT=EAA + # THE INPUT FEED + # EAA: [ ADMIN | ACCESS] + # ETP: [ THREAT | AUP ] + # MFA: [ POLICY | AUTH ] + ULS_FEED=ADMIN + # INPUT FORRMAT + ULS_FORMAT=JSON + # LOCATION OF THE AKAMAI .EDGERC FILE + ULS_EDGERC='~/.edgerc' + # RELEVANT SECTION WITHIN THE EDGERC FILE + ULS_SECTION=akamaidemo + + +# OUTPUT CONFIGURATION + # OUTPUT PATH [ TCP / UDP / HTTP ] + ULS_OUTPUT=TCP + # OUTPUT TARGET HOST + ULS_OUTPUT_HOST=192.168.86.34 + # OUTPUT TARGET PORT + ULS_OUTPUT_PORT=9091 \ No newline at end of file diff --git a/docker-compose/complex/etp-threat.env b/docker-compose/complex/etp-threat.env new file mode 100644 index 0000000..c1331af --- /dev/null +++ b/docker-compose/complex/etp-threat.env @@ -0,0 +1,28 @@ +# This is a sample ENV file for EAA - Threat logs (via ULS) + +# GENERIC Config +ULS_LOGLEVEL=DEBUG + +# INPUT CONFIGURATION + # THE INPUT PRODUCT [EAA | ETP | MFA] + ULS_INPUT=ETP + # THE INPUT FEED + # EAA: [ ADMIN | ACCESS] + # ETP: [ THREAT | AUP ] + # MFA: [ POLICY | AUTH ] + ULS_FEED=THREAT + # INPUT FORRMAT + ULS_FORMAT=JSON + # LOCATION OF THE AKAMAI .EDGERC FILE + ULS_EDGERC='~/.edgerc' + # RELEVANT SECTION WITHIN THE EDGERC FILE + ULS_SECTION=akamaidemo + + +# OUTPUT CONFIGURATION + # OUTPUT PATH [ TCP / UDP / HTTP ] + ULS_OUTPUT=TCP + # OUTPUT TARGET HOST + ULS_OUTPUT_HOST=192.168.86.34 + # OUTPUT TARGET PORT + ULS_OUTPUT_PORT=9091 \ No newline at end of file diff --git a/docker-compose/simple/docker-compose.yml b/docker-compose/simple/docker-compose.yml new file mode 100644 index 0000000..f0a5383 --- /dev/null +++ b/docker-compose/simple/docker-compose.yml @@ -0,0 +1,10 @@ +version: "3.0" +services: + etp-threat: + image: uls:latest + restart: always + env_file: etp-threat.env + volumes: + - type: bind + source: /Users/mschiess/.edgerc + target: /opt/akamai-uls/.edgerc \ No newline at end of file diff --git a/docker-compose/simple/etp-threat.env b/docker-compose/simple/etp-threat.env new file mode 100644 index 0000000..c1331af --- /dev/null +++ b/docker-compose/simple/etp-threat.env @@ -0,0 +1,28 @@ +# This is a sample ENV file for EAA - Threat logs (via ULS) + +# GENERIC Config +ULS_LOGLEVEL=DEBUG + +# INPUT CONFIGURATION + # THE INPUT PRODUCT [EAA | ETP | MFA] + ULS_INPUT=ETP + # THE INPUT FEED + # EAA: [ ADMIN | ACCESS] + # ETP: [ THREAT | AUP ] + # MFA: [ POLICY | AUTH ] + ULS_FEED=THREAT + # INPUT FORRMAT + ULS_FORMAT=JSON + # LOCATION OF THE AKAMAI .EDGERC FILE + ULS_EDGERC='~/.edgerc' + # RELEVANT SECTION WITHIN THE EDGERC FILE + ULS_SECTION=akamaidemo + + +# OUTPUT CONFIGURATION + # OUTPUT PATH [ TCP / UDP / HTTP ] + ULS_OUTPUT=TCP + # OUTPUT TARGET HOST + ULS_OUTPUT_HOST=192.168.86.34 + # OUTPUT TARGET PORT + ULS_OUTPUT_PORT=9091 \ No newline at end of file diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md new file mode 100644 index 0000000..992cc73 --- /dev/null +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -0,0 +1,33 @@ +# List of parameters / Environmental variables +The following tables list all available command line parameters and their corresponding environmental variables (for advanced usage). + + +## Global +|Parameter|Env - Var|Options|Default|Description| +|---|---|---|---|---| +|-h
--help | n/a | n/a | None | Display help / usage information | +|-l
--loglevel | ULS_LOGLEVEL | 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' | WARNING | Adjust the overall loglevel | +|-v
--version| n/a | n/a | None | Display ULS version information (incl. CLI & OS versions) | + + +## INPUT +|Parameter|Env - Var|Options|Default|Description| +|---|---|---|---|---| +|-i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | +|--feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN'
ETP: 'THREAT', 'AUP'
MFA: 'AUTH' | None | Specify the desired INPUT feed | +|--format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | +|--inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT| None | Adjust proxy usage for INPUT data collection (cli) | +|--rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI| +|--edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | +|--section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | + +## OUTPUT +|Parameter|Env - Var|Options|Default|Description| +|---|---|---|---|---| +|-o
--output| ULS_OUTPUT | 'TCP' | None | Specify the desired OUTPUT target | +|--host | ULS_OUTPUT_HOST | xxx.xxx.xxx.xxx | None | Specify the desired OUTPUT target host (TCP/UDP only) | +|--port| ULS_OUTPUT_PORT | xxxx | None | Specify the desired OUTPUT target port (TCP/UDP only) | +|--httpurl| ULS_HTTP_URL | http(s)://\:\/\ | None | The HTTP target URL. (HTTP only)
Do not use --host / --port for HTTP| +|--httpformat| ULS_HTTP_FORMAT| ''|'{"event": %s}'| Specify the expected output format (i.e. json) where %s will be replaced with the event data. +|--httpauthheader| ULS_HTTP_AUTH_HEADER | '{"Authorization": "VALUE"}' | None | Specify an Auhtorization header to auth against the HTTP Server (HTTP only)
Example:
'{"Authorization": "Splunk xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxx"}' | +|--httpinsecure| ULS_HTTP_INSECURE | True | False | Disable TLS CA certificate verification | diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md new file mode 100644 index 0000000..672e619 --- /dev/null +++ b/docs/CHANGELOG.md @@ -0,0 +1,17 @@ +# Version History + + +## v0.0.1 (Initial Commit) +|version|v0.0.1| +|---|---| +|Date|2021-06-09 +|Kind|Initial Commit +|Author|mschiess@akamai.com
adrocho@akamai.com +- INPUT: EAA, ETP, MFA (based on CLI's) +- OUTPUT: HTTP, TCP, UDP +- Docker & docker-compose examples +- Error & Reconnection handling +- Monitoring hook introduced Example: +- Kill Signal handling +- Configuration file `bin/config/global_config.py` +- Documentation & usage examples \ No newline at end of file diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md new file mode 100644 index 0000000..8142524 --- /dev/null +++ b/docs/COMMAND_LINE_USAGE.md @@ -0,0 +1,72 @@ +# ULS Command Line Usage +This document describes the "command line usage" of the ULS software. +All commands referenced in this document are run from the repositories root level. + + +### Overview +- [Requirements](#requirements) +- [Installation](#installation) +- [Usage](#usage) + +## Requirements +To run the operations within the following documentation, you need to have the following tools installed: +- git +- python >= 3.9 (including pip) +- Akamai EDGEGRID credentials file (`.edgerc`) +- Understanding of available [ULS CLI PARAMETERS](ARGUMENTS_ENV_VARS.md) + +## Installation +### Enterprise Access CLI's +The Secure Enterprise Access Products CLI Tools need to be installed into the `ext` directory within this repo. +Please run the following commands to download the CLI tools and install the requirements. +```bash +# Enterprise Application Access (EAA) +git clone --depth 1 --single-branch https://github.com/akamai/cli-etp.git ext/cli-etp && \ +pip install -r ext/cli-eaa/requirements.txt + +# Enterprise Threat Protector (ETP) +git clone --depth 1 --single-branch https://github.com/akamai/cli-etp.git ext/cli-etp && \ +pip install ext/cli-etp/requirements.txt + +# Akamai Phish Proof Multi Factor Authenticator (AKAMAI-MFA) +git clone --depth 1 --single-branch https://github.com/akamai/cli-mfa.git ext/cli-mfa && \ +pip install -r ext/cli-mfa/requirements.txt +``` + +## Usage +The command line interface is split into 3 different sections: +- Global commands (i.e. --loglevel debug) +- Input configuration (i.e. --input eaa) +- Output configuration (i.e. --output tcp) + +A full list of options/parameters can be printed by typing +```bash +python3 bin/uls.py --help +``` + +Starting ULS on the command line, ULS will run in foreground and literally run forever (unless terminated). +As a docker/container usage is recommended, ULS does not bring any threading/daemon support right now. +All log output will be directed to STDOUT by default. + +### Usage examples +- EAA ADMIN LOG ==> TCP LISTENER + ```bash + python3 bin/uls.py --input eaa --feed admin --output tcp --host 10.10.10.200 --port 9090 + ``` + +- ETP THREAT LOG ==> UDP LISTENER + ```bash + python3 bin/uls.py --input etp --feed threat --output udp --host 10.10.10.200 --port 9090 + ``` +- MFA AUTH LOG ==> HTTP LISTENER (SPLUNK) + disabled TLS verification + ```bash + python3 bin/uls.py --input=MFA --feed auth --output HTTP --httpformat '{"event": %s}' --httpauthheader '{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}' --httpurl "https://127.0.0.1:9091/services/collector/event" --httpinsecure + ``` + +- Logging to a file and sending process to the background + ```bash + python3 bin/uls.py --input etp --feed threat --output udp --host 10.10.10.200 --port 9090 &> /path/to/my/logfile & + ``` + Rather consider [docker usage](./DOCKER_USAGE.md) instead of this + \ No newline at end of file diff --git a/docs/DEBUGGING.md b/docs/DEBUGGING.md new file mode 100644 index 0000000..f5d05b2 --- /dev/null +++ b/docs/DEBUGGING.md @@ -0,0 +1,52 @@ +# ULS DEBUGGING +This document describes the debugging of the ULS software. +Please make sure you follow the steps before filing an issue on the [GitHub Issues Page](https://github.com/akamai/uls/issues). +Follow the steps to collect "supportive" data you should also provide when filing an issue. + +## Table of contents +- [Version Information](#version-information) +- [Debug Output](#debug-output) + +## Version information +Providing information about relevant module & software versions can help identify issues. +###Commands to trigger version output +####Command Line: +```bash +python3 bin/uls.py --version +``` +####Docker +```bash +docker run -ti --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly --rm akamai/uls -v +``` + +###Example Output +```text +Akamai Unified Log Streamer Version information +ULS Version 0.0.1 + +EAA Version 0.3.8 +ETP Version 0.3.4 +MFA Version 0.0.4 + +OS Plattform Linux-5.10.25-linuxkit-x86_64-with-glibc2.28 +OS Version 5.10.25-linuxkit +Python Version 3.9.5 + +``` + + +## Debug Output +To debug problems into depth, ULS provides an extremely verbose output about every step processed within ULS. +###Commands to trigger debug output +####Command Line: +```bash +python3 bin/uls.py --loglevel debug +``` +####Docker +```bash +docker run -ti --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly \ + --rm akamai/uls --loglevel debug \ + +``` + +Instead of adding it to the command line, you can also set the `ULS_LOGLEVEL` ENV VAR to "DEBUG" diff --git a/docs/DOCKER-COMPOSE_USAGE.md b/docs/DOCKER-COMPOSE_USAGE.md new file mode 100644 index 0000000..62557e9 --- /dev/null +++ b/docs/DOCKER-COMPOSE_USAGE.md @@ -0,0 +1,59 @@ +# ULS Docker-Compose Usage +This document describes the "docker compose" of the ULS software. +All commands referenced in this document are run from the repositories root level. +The `docker compose` command used in this doc, was recently integrated into the docker cli (currently beta). Nevertheless, all of the `docker-compose`commands will work as well. + +![ULS docker compose usage](./images/ula_docker-compose_complex_example.png) +### Overview +- [Requirements](#requirements) +- [Usage](#usage) + +## Requirements +- [Docker](https://www.docker.com/) needs to be installed on an **linux** OS (Windows not supported) +- [docker-compose](https://docs.docker.com/compose/install/) needs to be installed alongside docker + As alternative to the above, you can now use the latest docker cli with the command `docker compose` +- Access to the docker image (see Installation section within the [DOCKER documentation](./DOCKER_USAGE.md#installation)) +- Akamai EDGEGRID credentials file (`.edgerc`) +- Understanding of available [ULS Environmental Variables](ARGUMENTS_ENV_VARS.md) + +## Usage +Docker compose is the recommended way to run AKAMAI ULS in a production environment. +Docker compose enables the definition of multiple parallel uls instances providing different data to multiple ingestion points. + +To start the docker-compose script please run the following: +```bash +cd docker-compose/simple +docker compose up +``` + + +In order to run the docker-compose as DAEMON in the background, use the following command + ```bash +cd docker-compose/simple +docker compose up -d + ``` + +### Usage Examples +- Simple docker-compose setup that will ship ETP-THREAT events via TCP + ```bash + cd docker-compose/simple + docker compose up + ``` + This will run the "simple" use case in foreground. + The `docker-compose.yml` file will reference the `etp-threat.env` and provide the configuration from that file. + **Files:** + - [docker-compose.yml](../docker-compose/simple/docker-compose.yml) + - [etp-threat.env](../docker-compose/simple/etp-threat.env) + + +- Complex docker-compose setup delivering different streams to different endpoints + ```bash + cd docker-compose/complex + docker compose up + ``` + This triggers a more complex setup consisting out of 3 different data feeds. + **Files:** + - [docker-compose.yml](../docker-compose/complex/docker-compose.yml) + - [etp-threat.env](../docker-compose/complex/etp-threat.env) + - [eaa-admin.env](../docker-compose/complex/eaa-admin.env) + - [eaa-access.env](../docker-compose/complex/eaa-access.env) \ No newline at end of file diff --git a/docs/DOCKER_USAGE.md b/docs/DOCKER_USAGE.md new file mode 100644 index 0000000..b954ed8 --- /dev/null +++ b/docs/DOCKER_USAGE.md @@ -0,0 +1,77 @@ +# ULS Docker Usage +This document describes the "docker" of the ULS software. +All commands referenced in this document are run from the repositories root level. +![img.png](images/uls_docker_etp_threat_example.png) + +### Overview +- [ULS Docker Usage](#uls-docker-usage) + - [Overview](#overview) + - [Requirements](#requirements) + - [Installation](#installation) + - [Usage](#usage) + +## Requirements +- [Docker](https://www.docker.com/) needs to be installed on an **GNU/Linux** OS + - Note: Windows is not supported, please use HyperV with a Linux VM +- Access to the docker image (see [installation](#installation)) +- Akamai EDGEGRID credentials file (`.edgerc`) +- Understanding of available [ULS Environmental Variables and CLI PARAMETERS](ARGUMENTS_ENV_VARS.md) + +## Installation +There are two options to retrieve the docker image: +- DockerHub + Pull the latest image from DockerHubs online repository + ```bash + docker pull akamai/uls:latest + ``` +- Build using Dockerfile + Locally build the container using the `Dockerfile` provided with this repo. + ```bash + docker build --force-rm -t akamai/uls:latest . + ``` +For both of the above options the image can be verified with the following command: +```bash +docker image ls | grep uls +``` +should return something like (where size, fingerprint and time will differ) +```text +akamai/uls latest 2a822d4ab406 16 hours ago 929MB +``` + +## Usage +Using the dockerized approach, you have two different options to set up the options and parameters: + +- Docker Command Line Arguments + ```bash + docker run -d --name uls_etp-threat -ti \ + --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly \ + akamai/uls \ + --input etp --feed threat --output tcp --host 10.10.10.10 --port 9091 + ``` + +- Docker Environmental Variables´ + ```bash + docker run -d --name uls_etp-threat -ti \ + --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly \ + --env ULS_INPUT=ETP \ + --env ULS_FEED=THREAT \ + --env ULS_OUTPUT=TCP \ + --env ULS_OUTPUT_HOST=10.10.10.10 \ + --env ULS_OUTPUT_PORT=9091 \ + akamai/uls + ``` + +Both of the above examples would do the exact same thing. +You can find a full set of command line parameters along with the according ENV variables in this document. + +Right now, mounting the `.edgerc` file into the container is the only way applying the authentication. This might get fixed in some later version. +Please change the `source=` according to your needs within the mount lines. +```bash +--mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly +``` + +- Display version information + ```bash + docker run -ti --mount type=bind,source="/path/to/your/.edgerc",target="/opt/akamai-uls/.edgerc",readonly --rm akamai/uls -v + ``` + ![img.png](images/uls_docker_version_example.png) \ No newline at end of file diff --git a/docs/MONITORING.md b/docs/MONITORING.md new file mode 100644 index 0000000..11f5b6c --- /dev/null +++ b/docs/MONITORING.md @@ -0,0 +1,23 @@ +# ULS Monitoring +This document describes the ULS monitoring output (STDOUT). +The output will be sent every 5 minutes to stdout by default. + +## Field description +| Field| Example | Description | +|---|---|---| +|dt | "2021-06-09T08:15:35.092889" | Date & Time (OS Timezone)| +|uls_product| "ETP" | Selected ULS product | +|uls_feed| "THREAT" | Selected ULS feed | +|uls_output| "HTTP" | Selected ULS output | +|uls_runtime| "3000" | Time in seconds ULS is already running | +|event_count| "625014" | Number of events handled by ULS (overall) | +|event_rate| "10.97" | Average events per second. Average based on the monitoring interval. (Default 5 minutes)| +|mon_interval| "300" | Monitoring interval in seconds| + + +## Example Output +The output is delivered in JSON format +```json +{"dt": "2021-06-09T08:15:35.092889", "uls_product": "ETP", "uls_feed": "THREAT", "uls_outpout": "HTTP", "uls_runtime": 300, "event_count": 504, "event_rate": 1.68, "mon_interval": 300} +``` + diff --git a/docs/SIEM/GRAYLOG/README.md b/docs/SIEM/GRAYLOG/README.md new file mode 100644 index 0000000..493baa0 --- /dev/null +++ b/docs/SIEM/GRAYLOG/README.md @@ -0,0 +1,20 @@ +# GRAYLOG +This document describes how to configure [Graylog](https://www.graylog.org/) in order to receive data from ULS. +The recommended way is the Raw/Plaintext TCP Input. + +## INPUT CONFIG +### Raw/Plaintext TCP +Go to System -> Inputs and create a new Input: +![img.png](img.png) + + +## FIELD EXTRACTION +As Extractor, you can use the "JSON" extractor using the standard settings. +Eventually it is required to add additional JSON extractors for fields like "event_aupCategories", "query_resolved", ... +Please see the [extractor documentation](https://docs.graylog.org/en/4.0/pages/extractors.html) for more help. + + +## KNOWN ISSUES + +### Some fields are not getting extracted properly +See [this bug report](https://community.graylog.org/t/search-at-json-object-field/14735/6) within the graylog universe. \ No newline at end of file diff --git a/docs/SIEM/GRAYLOG/img.png b/docs/SIEM/GRAYLOG/img.png new file mode 100644 index 0000000..f5248bc Binary files /dev/null and b/docs/SIEM/GRAYLOG/img.png differ diff --git a/docs/SIEM/QRADAR/README.md b/docs/SIEM/QRADAR/README.md new file mode 100644 index 0000000..1962332 --- /dev/null +++ b/docs/SIEM/QRADAR/README.md @@ -0,0 +1,8 @@ +# QRADAR +This document describes how to configure [QRADAR](https://www.ibm.com/security/security-intelligence/qradar) in order to receive data from ULS. +The recommended way (in order to minimize network/encryption overhead) is the TCP (Syslog) connector. + +## INPUTS +### HTTP OUTPUT +Please follow the [QRADAR documentation](https://www.ibm.com/docs/en/dsm?topic=options-http-receiver-protocol-configuration). + diff --git a/docs/SIEM/SIEM_OVERVIEW.md b/docs/SIEM/SIEM_OVERVIEW.md new file mode 100644 index 0000000..5d3bbe5 --- /dev/null +++ b/docs/SIEM/SIEM_OVERVIEW.md @@ -0,0 +1,10 @@ +# SIEM OVERVIEW +Following are some references on how ULS can be implemented alng some common SIEM solutions. +If a SIEM is not listed, does not mean it will not work. It just has not (yet) been documented. + +All contributions in terms of documentation a welcome. + +## SIEM Integration guides (alphabetical) +- [GRAYLOG](./GRAYLOG/README.md) +- [QRADAR](./QRADAR/README.md) +- [SPLUNK](./SPLUNK/README.md) diff --git a/docs/SIEM/SPLUNK/README.md b/docs/SIEM/SPLUNK/README.md new file mode 100644 index 0000000..da105da --- /dev/null +++ b/docs/SIEM/SPLUNK/README.md @@ -0,0 +1,26 @@ +# SPLUNK +This document describes how to configure [Splunk](https://www.splunk.com/) in order to receive data from ULS. +The recommended way (in order to minimize network/encryption overhead) is the TCP connector. +Nevertheless, ULS has been tested with TCP & HTTP output module towards splunk. + +## INPUTS +### TCP INPUT +Please follow the [SPLUNK DOCUMENTATION](https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/Monitornetworkports). + +### HTTP EVENT COLLECTOR +Please follow the [SPLUNK DOCUMENTATION](https://docs.splunk.com/Documentation/Splunk/8.2.0/Data/UsetheHTTPEventCollector). + + +## KNOWN ISSUES +### Line breaking with tcp (streaming) input fails +Depending on your configured settings, SPLUNK could fail determining the line breaks correctly. +Many messages might appear as "one" event within splunk. +To fix this, please follow the instructions below: + +Add the following to the file `$SPLUNK_HOME/etc/system/local/props.conf`: +```text +[akamai_etp] +SHOULD_LINEMERGE = false +``` +The default linebreaker `LINE_BREAKER = ([\r\n]+)` configuration should perfectly match. +More information on props can be found [here](https://docs.splunk.com/Documentation/Splunk/Latest/Admin/Propsconf) \ No newline at end of file diff --git a/docs/images/ula_docker-compose_complex_example.png b/docs/images/ula_docker-compose_complex_example.png new file mode 100644 index 0000000..7d5bcec Binary files /dev/null and b/docs/images/ula_docker-compose_complex_example.png differ diff --git a/docs/images/uls_cli_help_example.png b/docs/images/uls_cli_help_example.png new file mode 100644 index 0000000..8e2acb0 Binary files /dev/null and b/docs/images/uls_cli_help_example.png differ diff --git a/docs/images/uls_docker_etp_threat_example.png b/docs/images/uls_docker_etp_threat_example.png new file mode 100644 index 0000000..f99b39b Binary files /dev/null and b/docs/images/uls_docker_etp_threat_example.png differ diff --git a/docs/images/uls_docker_version_example.png b/docs/images/uls_docker_version_example.png new file mode 100644 index 0000000..877eb2c Binary files /dev/null and b/docs/images/uls_docker_version_example.png differ