From e7cd15c578a9c2c93c34de7f0d37369148d69df6 Mon Sep 17 00:00:00 2001 From: Mike Schiessl <77062930+MikeSchiessl@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:49:26 +0200 Subject: [PATCH] v1.8.2 (#70) --- Dockerfile | 2 +- README.md | 3 +- bin/modules/UlsArgsParser.py | 2 +- bin/modules/UlsTools.py | 2 +- bin/uls_config/global_config.py | 6 +- docs/AKAMAI_API_CREDENTIALS.md | 2 +- docs/CHANGELOG.md | 343 ++++++++++-------- docs/COMMAND_LINE_USAGE.md | 8 +- docs/FAQ.md | 84 +++-- docs/LOG_OVERVIEW.md | 12 + .../kubernetes/helm/akamai-uls/Chart.yaml | 2 +- test/basic_test.bats | 23 ++ test/positive_test.bats | 2 +- 13 files changed, 284 insertions(+), 207 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e5d055..7c732be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ARG HOMEDIR="/opt/akamai-uls" ARG ULS_DIR="$HOMEDIR/uls" ARG EXT_DIR="$ULS_DIR/ext" -ARG ETP_CLI_VERSION="0.4.7" +ARG ETP_CLI_VERSION="0.4.8" ARG EAA_CLI_VERSION="0.6.10" ARG MFA_CLI_VERSION="0.1.1" ARG GC_CLI_VERSION="v0.0.5" diff --git a/README.md b/README.md index b663906..82a273a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Aka Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box. ULS can send data into any SIEM that supports either file, TCP, UDP or HTTP ingestion. -It can be run directly as Python code, as a provided Docker container, through `docker compose` scripts or through helm within kubernetes. +ULS can be run directly as Python code, as a Docker container, through `docker compose` scripts or through helm within kubernetes. +Running ULS on Windows (python) is also supported (but not extensively tested, yet). ![ULS docker compose usage](docs/images/uls_docker-compose_complex_example.png) diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py index 2847d46..722155f 100644 --- a/bin/modules/UlsArgsParser.py +++ b/bin/modules/UlsArgsParser.py @@ -52,7 +52,7 @@ def init(): dest='logdatefmt', type=str, default=(os.environ.get('ULS_LOG_DATEFORMAT') or uls_config.log_datefmt), - help=f"Adjust the logging date/time format to your needs, (Default: {uls_config.log_datefmt.replace("%", "%%")})") + help=f"Adjust the logging date/time format to your needs, (Default: {uls_config.log_datefmt.replace('%', '%%')})") # Added double %% to have argsparser display proper string as it tries do to % replacement :D # put loglines into debug log diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 3556378..3a65c5c 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -344,7 +344,7 @@ def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"): def callhome(nocallhome_state: bool, input: str = "n/a", feed: str = "n/a", output: str = "n/a", position: str = "n/a"): if not nocallhome_state: try: - url = f"/{position}?version={uls_config.__version__}&input={input}&feed={feed}&output={output}&install_id={get_install_id()['install_id']}&os_platform={platform.platform()}&pyhton={sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}&container={check_container()}" + url = f"/{position}?version={uls_config.__version__}&input={input}&feed={feed}&output={output}&install_id={get_install_id()['install_id']}&os_platform={platform.platform()}&python={sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}&container={check_container()}" aka_log.log.debug(f"Sending a CallHome request containing the following data: {url}") result = requests.get(uls_config.callhome_url + url, timeout=int(uls_config.callhome_timeout)) aka_log.log.debug(f"Callhome response code: {result.status_code}") diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py index 77d8a8e..74dd87c 100644 --- a/bin/uls_config/global_config.py +++ b/bin/uls_config/global_config.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 - +import sys # Common global variables / constants -__version__ = "1.8.1" +__version__ = "1.8.2" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" # Generic config -bin_python = "python3" # Python binary to use (use OS standard when not using path) +bin_python = sys.executable # Python binary to use (use OS standard when not using path) output_line_breaker = '\r\n' # Line breaking type (to split messages when streaming data) main_wait_default = 0.01 # Default wait time within the main loop main_wait_max = 60 # Maximum wait time for the main loop diff --git a/docs/AKAMAI_API_CREDENTIALS.md b/docs/AKAMAI_API_CREDENTIALS.md index 59a356f..04b6572 100644 --- a/docs/AKAMAI_API_CREDENTIALS.md +++ b/docs/AKAMAI_API_CREDENTIALS.md @@ -145,7 +145,7 @@ Guardicore is using the portal users for API access. Therefore it is recommended - Go to Administration - Select "Users" in the left navigation tree - Click the "Create User" button -- Enter a username and a password, select "Guest" as permission scheme +- Enter a username and a password, select "Support" as permission scheme - Confirm by clicking the SAVE button - Now logout and login with the newly created user and follow tha password change procedure - Note down your guardicore Adminsitration (=API) url without https diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d26b441..f436a32 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,21 +1,44 @@ # Version History +## v1.8.2 +| | | +|--------|---------------------| +| Date | 2024-XX-XX | +| Kind | MINOR release | +| Author | mschiess@akamai.com | + +- **Features** + - Enabled Windows Support (natively run in python 3.12+ on Windows) + +- **Minor improvements** + - [docker] bumped CLI-ETP version to 0.4.8 (future api support fix) + +- **Bugfixes** + - Fixed a bug that caused an incompatibility with python versions < 3.12 + +- **Housekeeping** + - improved python version testing (sampling py3.9 to 3.12) + +--- + ## v1.8.1 -||| -|---|---| -|Date|2024-08-28 -|Kind| MINOR release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2024-08-28 | +| Kind | MINOR release | +| Author | mschiess@akamai.com | - **Bugfixes** - Merged a missing fix from the development branch +--- + ## v1.8.0 -||| -|---|---| -|Date|2024-08-27 -|Kind| MAJOR release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2024-08-27 | +| Kind | MAJOR release | +| Author | mschiess@akamai.com | - **Features** - Prometheus monitoring support added to allow smoother monitoring into third party (prometheus compatible) monitoring sytems. More information [here](MONITORING.md#prometheus) @@ -36,11 +59,11 @@ --- ## v1.7.5 -||| -|---|---| -|Date|2024-07-16 -|Kind| MINOR release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2024-07-16 | +| Kind | MINOR release | +| Author | mschiess@akamai.com | - **Minor improvements** - Updated Command Line usage docs (ACC logs installation) @@ -51,11 +74,11 @@ --- ## v1.7.4 -||| -|---|---| -|Date|2024-06-17 -|Kind| MINOR release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2024-06-17 | +| Kind | MINOR release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Features** - New Input & Feed available - Akamai Control Center Events @@ -70,11 +93,11 @@ --- ## v1.7.3 -||| -|---|---| -|Date|2024-04-02 -|Kind| MINOR release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2024-04-02 | +| Kind | MINOR release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Features** - introduced "audit logs" for Guardicore @@ -91,11 +114,11 @@ --- ## v1.7.2 -||| -|---|---| -|Date|2024-02-08 -|Kind| MINOR release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2024-02-08 | +| Kind | MINOR release | +| Author | mschiess@akamai.com | - **Minor improvements** - Introduced **Secure Internet Access** (formerly ETP) as INPUT specification (as an alias to ETP) - added "ETP NETCON" to the autoresume feature @@ -114,11 +137,11 @@ --- ## v1.7.1 -||| -|---|---| -|Date|2023-10-11 -|Kind| BUGFIX release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2023-10-11 | +| Kind | BUGFIX release | +| Author | mschiess@akamai.com | - **BUGFIX** - Fixed a bug in the ETP & EAA CLI that prevented ULS to run properly in docker environment - [docker] bumped CLI-EAA to "0.6.3" @@ -128,11 +151,11 @@ --- ## v1.7.0 -||| -|---|---| -|Date|2023-10-10 -|Kind| FEATURE release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2023-10-10 | +| Kind | FEATURE release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Features** - Allowing the configuration of the HTTPFORMATTYPE, which controls the building of payloads for aggregated HTTP requests (click [here](FAQ.md#what-is-http-formattype) for additional information) - Allow adjustment of the "INPUT QUEUE SIZE" threshold (--inputqueuesize) in order to handle huge API pages and fast API output @@ -150,11 +173,11 @@ --- ## v1.6.6 -||| -|---|---| -|Date|2023-08-23 -|Kind| Minor release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2023-08-23 | +| Kind | Minor release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Features** - Added '--httpliveness' to disable HTTP(S) OPTIONS request for liveness checking - Added new feed for ETP: Network traffic connections details (netcon) [Requires CLI-ETP >= 0.4.2] @@ -168,22 +191,22 @@ --- ## v1.6.5 -||| -|---|---| -|Date|2023-07-28 -|Kind| Minor release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2023-07-28 | +| Kind | Minor release | +| Author | mschiess@akamai.com | - **Minor improvements** - Allow manipulation of the [TCP & UDP output format](ARGUMENTS_ENV_VARS.md#list-of-parameters--environmental-variables) (--tcpudpformat / ULS_TCPUDP_FORMAT). - [docker] bumped source image to 3.11.4-slim-bookworm (new debian release) --- ## v1.6.4 -||| -|---|---| -|Date|2023-05-02 -|Kind| Minor release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2023-05-02 | +| Kind | Minor release | +| Author | mschiess@akamai.com | - **Minor improvements** - Updated docs to clarify the required timestamp format (undefined --> epoch time in seconds) - [docker] bumped CLI-EAA version to "0.5.7" @@ -197,11 +220,11 @@ ## v1.6.3 -||| -|---|---| -|Date|2022-11-29 -|Kind| Minor release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2022-11-29 | +| Kind | Minor release | +| Author | mschiess@akamai.com | - **Minor improvements** - Introduced '--httpaggregate' / 'ULS_HTTP_AGGREGATE' option to allow easier management of the HTTP(S) aggregation function @@ -216,11 +239,11 @@ ## v1.6.2 -||| -|---|---| -|Date|2022-10 -|Kind| Minor release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2022-10 | +| Kind | Minor release | +| Author | mschiess@akamai.com | - **Minor improvements** - Bumped EAA CLI to version 0.5.1 (additional SIEM fields - EAA release 2022.02) @@ -233,11 +256,11 @@ ## v1.6.1 -||| -|---|---| -|Date|2022-10 -|Kind| BUGFIX release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2022-10 | +| Kind | BUGFIX release | +| Author | mschiess@akamai.com | - **Minor improvements** - Dropped CLI installation verification for CLI's not used by ULS @@ -248,11 +271,11 @@ ## v1.6.0 -||| -|---|---| -|Date|2022-09 -|Kind| FEATURE release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2022-09 | +| Kind | FEATURE release | +| Author | mschiess@akamai.com | - **Features** - Support for Akamai Guardicore Segmentation (experimental) - Available feeds: netlog, incident, agent, system @@ -273,11 +296,11 @@ ## v1.5.1 -||| -|---|---| -|Date|2022-08 -|Kind| BUGFIX release -|Author|androcho@akamai.com +| | | +|--------|---------------------| +| Date | 2022-08 | +| Kind | BUGFIX release | +| Author | androcho@akamai.com | - **Bugfix** - Use cli-eaa version 0.5.0.2 fixing a timezone issue on the EAA ADMIN feed @@ -286,11 +309,11 @@ ## v1.5.0 -||| -|---|---| -|Date|2022-07 -|Kind| FEATURE release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2022-07 | +| Kind | FEATURE release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Performance improvements** - Rework to handle large number of events (100k+ per minute) and fail safe @@ -310,11 +333,11 @@ --- ## v1.4.0 -||| -|---|---| -|Date|2022-05-20 -|Kind| FEATURE release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2022-05-20 | +| Kind | FEATURE release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Features** - Device Inventory (DEVINV) feed added for EAA (requires eaa-cli >= 0.4.9.1) @@ -329,22 +352,22 @@ --- ## v1.3.5 -||| -|---|---| -|Date|2022-04-05 -|Kind| Bugfix release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2022-04-05 | +| Kind | Bugfix release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Minor improvements** - More QRADAR log source type definitions (thx to bitonio) - Added docker-compose ETP - Tenant example --- ## v1.3.4 -||| -|---|---| -|Date|2022-03-08 -|Kind| Bugfix release -|Author|mschiess@akamai.comm androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2022-03-08 | +| Kind | Bugfix release | +| Author | mschiess@akamai.comm androcho@akamai.com | - **Minor improvements** - Added QRADAR log source type definitions (thx to bitonio) - Added SUMO Logic (thx to huskar20 for the contribution) @@ -354,11 +377,11 @@ ## v1.3.3 -||| -|---|---| -|Date|2022-02-28 -|Kind| Bugfix release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2022-02-28 | +| Kind | Bugfix release | +| Author | mschiess@akamai.com | - **Bugfix** - Adopted to new MFA CLI Version (only single feed "EVENT" available anymore) - Amended new dates to the file headers @@ -366,11 +389,11 @@ --- ## v1.3.2 -||| -|---|---| -|Date|2022-02-10 -|Kind| Bugfix release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2022-02-10 | +| Kind | Bugfix release | +| Author | mschiess@akamai.com | - **Features** - Kubernetes deployment example / Helm charts added ([start here](KUBERNETES_USAGE.md)) @@ -391,22 +414,22 @@ ## v1.3.1 -||| -|---|---| -|Date|2021-12-20 -|Kind| Bugfix release -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2021-12-20 | +| Kind | Bugfix release | +| Author | mschiess@akamai.com | - **Bugfixes** - fixed a checkpoint issue when using ETP / THREAD - some doc fixes --- ## v1.3.0 -||| -|---|---| -|Date|2021-12-17 -|Kind| Feature & Bugfix release -|Author|mschiess@akamai.com +| | | +|--------|--------------------------| +| Date | 2021-12-17 | +| Kind | Feature & Bugfix release | +| Author | mschiess@akamai.com | - **Features** - [internal] Added automated test scripts to improve continuous release quality - [AUTO-RESUME feature](ADDITIONAL_FEATURES.md#autoresume--resume) enables ULS to automatically continue operation starting from the last saved checkpoint. @@ -428,11 +451,11 @@ --- ## v1.2.0 -||| -|---|---| -|Date|2021-11-02 -|Kind| Feature & Bugfix release -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2021-11-02 | +| Kind | Feature & Bugfix release | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Features** - [Transformation Support for output format transformation ](TRANSFORMATIONS.md)(additional log formats and integrations) introduced - [MCAS transformation](TRANSFORMATIONS.md#microsoft-cloud-application-security-mcas): Microsoft Cloud Application Security @@ -457,11 +480,11 @@ --- ## v1.1.0 -||| -|---|---| -|Date|2021-08-18 -|Kind|Bugfix / Feature -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2021-08-18 | +| Kind | Bugfix / Feature | +| Author | mschiess@akamai.com | - **Features** - Added **DNS** and **PROXY** feeds to ETP Input (<3 Sara) - **Minor improvements** @@ -475,11 +498,11 @@ --- ## v1.0.0 -||| -|---|---| -|Date|2021-08-10 -|Kind|Bugfix / Feature -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2021-08-10 | +| Kind | Bugfix / Feature | +| Author | mschiess@akamai.com, androcho@akamai.com | - **Minor improvements** - EdgeRC file check (preflight) and "~" expansion to solve some common issues - fixed some typos in the "docker-compose" file @@ -490,11 +513,11 @@ --- ## v0.9.0 -||| -|---|---| -|Date|2021-07-26-2021 -|Kind|Bugfix / Feature -|Author|mschiess@akamai.com, androcho@akamai.com +| | | +|--------|------------------------------------------| +| Date | 2021-07-26-2021 | +| Kind | Bugfix / Feature | +| Author | mschiess@akamai.com, androcho@akamai.com | - Minor improvements - fixed some typos / instructions - bumped EAA version to 0.4.1 @@ -509,11 +532,11 @@ --- ## v0.0.4 -||| -|---|---| -|Date|2021-06-17 -|Kind|Bugfix / Feature -|Author|mschiess@akamai.com +| | | +|--------|---------------------| +| Date | 2021-06-17 | +| Kind | Bugfix / Feature | +| Author | mschiess@akamai.com | - Minor improvements - Wait_time and wait_max shifted to config - added -f flag as alternative to --flag @@ -526,11 +549,11 @@ --- ## v0.0.3 -||| -|---|---| -|Date|2021-06-15 -|Kind|Bugfix / Feature -|Author|mschiess@akamai.com
androcho@akamai.com +| | | +|--------|----------------------------------------------| +| Date | 2021-06-15 | +| Kind | Bugfix / Feature | +| Author | mschiess@akamai.com
androcho@akamai.com | - introduced line breaker variable for output - fixed a bug in the "poll" handling - fixed a bug that caused Popen PIPE to hang in certain circumstances @@ -539,22 +562,22 @@ --- ## v0.0.2 -||| -|---|---| -|Date|2021-06-10 -|Kind|Bugfix -|Author|mschiess@akamai.com
androcho@akamai.com +| | | +|--------|----------------------------------------------| +| Date | 2021-06-10 | +| Kind | Bugfix | +| Author | mschiess@akamai.com
androcho@akamai.com | - fixed monitoring output bug in docker-compose - fixed bug in Dockerfile that prevented development builds - fixed a bug in EAA CLI handler --- ## v0.0.1 (Initial Commit) -|version|v0.0.1| -|---|---| -|Date|2021-06-09 -|Kind|Initial Commit -|Author|mschiess@akamai.com
androcho@akamai.com +| version | v0.0.1 | +|---------|----------------------------------------------| +| Date | 2021-06-09 | +| Kind | Initial Commit | +| Author | mschiess@akamai.com
androcho@akamai.com | - INPUT: EAA, ETP, MFA (based on CLI's) - OUTPUT: HTTP, TCP, UDP - Docker & docker-compose examples diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md index 89ee11a..27cb6ac 100644 --- a/docs/COMMAND_LINE_USAGE.md +++ b/docs/COMMAND_LINE_USAGE.md @@ -28,14 +28,17 @@ To run the operations in this documentation, make sure you have the following to - python >= `3.9` (including pip >= `22.2`) - Akamai API credentials file - `.edgerc` (see [API Credentials](AKAMAI_API_CREDENTIALS.md) for creation instructions) - Understanding of available [ULS CLI PARAMETERS](ARGUMENTS_ENV_VARS.md) -- Access to `github.com`, `pypi.org`, `pythonhosted.org` and `pypi.python.org` within your firewall +- Access to `github.com`, `uls-beacon.akamaized.net`, `pypi.org`, `pythonhosted.org`, `pypi.python.org` as well as your Akamai API hostname (see edgerc) within your firewall + +The above apploes as well for Linux and Windows OS. ## Installation To install ULS, you can choose 2 different ways: automated or manual. ### Automated Installation -The automated installation actually does everything, the described below in the manual installation but saves you from the copying the blocks. +The automated installation actually does everything, the described below in the manual installation but saves you from the copying the blocks. +It will likely only work on "Linux" based operating systems. ```bash # Got to your preferred installation folder (it will install to a subdirectory ./uls @@ -45,6 +48,7 @@ bash get-uls.sh ``` ### Manual Installation +Those steps are the same on windows and linux based operating systems. Please make sure you have properly installed the above [requirements](#pre-requisites) according to your OS in order to follow the next steps. #### Clone ULS repository diff --git a/docs/FAQ.md b/docs/FAQ.md index a3d1287..e055bdd 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -8,7 +8,6 @@ - [What environmental variables (ENV VARS) are available](#what-environmental-variables-env-vars-are-available-#) - [--inputproxy does not work as expected](#--inputproxy-proxy-does-not-work-as-expected) - [Logs are not showing up in my SIEM](#logs-are-not-showing-up-in-siem) -- [ULS on Windows error: "[WinError 2] The system cannot find the file specified"](#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified) - [ULS does not start due to missing field in config](#uls-does-not-start-due-to-missing-field-in-config) - [ULS throws TLS an error when connecting towards Guardicore API (--input GC)](#uls-throws-tls-an-error-when-connecting-towards-guardicore-api---input-gc) - [WHY JMESPATH and not JSONPATH](#why-jmespath-and-not-jsonpath) @@ -16,6 +15,8 @@ - [Error: "Capacity exceeded, too many incoming data vs. slow output"](#error-capacity-exceeded-too-many-incoming-data-vs-slow-output) - [Error: "Invalid timestamp" on API call](#error-invalid-timestamp-on-api-call) - [I do not want to send any data to Akamai](#i-do-not-want-to-send-any-data-to-akamai) +- [Can i run ULS on Windows Operating Systems](#can-i-run-uls-on-windows-operating-systems) +- [ULS on Windows error: "[WinError 2] The system cannot find the file specified"](#uls-on-windows-error-winerror-2-the-system-cannot-find-the-file-specified) ---- ## FAQ @@ -70,39 +71,6 @@ Those can also be added to the .evn file when using docker / docker-compose. - Double check for sanity reasons, that no (additional) filters wihtin your SIEM have been applied Some excellent troubleshooting guidance from SPLUNK (but also applies to other SIEM as well) can be found [here](https://docs.splunk.com/Documentation/Splunk/6.4.1/Troubleshooting/Cantfinddata) ---- -### ULS on Windows error: "[WinError 2] The system cannot find the file specified" -ULS requires the OS to provide a python3 executable. The python installation on Windows somehow (unlike other OS) just installs a "python" executable. -Luckily this is something that can get sorted easily and in multiple different ways (just pick the one that suites you best): - -1) **Copy the binary (recommended)** -Go to your python directory on Windows e.g. `C:\Users\Administrator\AppData\Local\Programs\Python\Python310` or `C:\Program Files\Python\Python310`. -Now copy the `python.exe` executable to `python3.exe` within the same folder. - - -2) **Create a powershell alias (temproary only)** -If you are using powershell, run this before you start ULS. - ```text - Set-Alias -Name python3 -Value python - ``` - -3) **Create a symbolic link (requires [mklink](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/mklink))** - ```text - mklink "C:\path\to\symlink\python3.exe" "C:\path\to\Python3\python.exe" - ``` - -4) **Change ULS config (not recommended)** -You can modify the bin_python variable within the ULS global config file's `bin/config/global_config.py` 'Generic config' section.# -Change - ```text - bin_python = "python3" - ``` - to - ```text - bin_python = "python" - ``` - **WARNING:** This change prevents the global_config.py file to get updated via GIT in the future. You need to manually take care of updating changes within the file. - --- ### ULS does not start due to missing field in config If you try to start ULS but it exits with an error similar to @@ -194,6 +162,7 @@ With ULS Version 1.8.0 we introduced a call home functionality that once sends d This data helps us continue the ULS development in the future. So if possible, please allow ULS to send this data. We are not sending any sensitive or PII data. The Debug logs show the exact data that has been sent. + The data includes: - current ULS version - ULS input @@ -209,5 +178,50 @@ Example data: /uls_start?version=1.8.0-alpha&input=EAA&feed=ACCESS&output=RAW&install_id=OU5UR0RHLTIwMjIxMTI4LTEuNi4y&os_platform=macOS-14.5-arm64-arm-64bit&pyhton=3.12.4&container=False ``` +The domain the data will be sent to: `uls-beacon.akamaized.net` + If you still want to disable the CallHome functionality within ULS, -you can do so by setting the `--nocallhome` command line parameter OR by using the ENV VAR: `export ULS_NOCALLHOME=TRUE` \ No newline at end of file +you can do so by setting the `--nocallhome` command line parameter OR by using the ENV VAR: `export ULS_NOCALLHOME=TRUE` + +--- +### Can I run ULS on Windows Operating Systems +Generically speaking: Yes - we fixed a couple of bugs and since ULS v1.8.2 you should be able to run the ULS code natively under windows. +You need to install the requirements ([see here](COMMAND_LINE_USAGE.md#pre-requisites)) and ULS should run without any issue. +An alternative to running it natively with python on windows, is to use the docker version which also runs under windows but might be come with additional docker [license cost](https://docs.docker.com/subscription/desktop-license/). + +Please be aware that we're currently unable to test every new ULS version directly on windows, so there might be bugs we're unable to spot at build time of the release. +Please verify functionality before an update on your staging system. + +--- +### ULS on Windows error: "[WinError 2] The system cannot find the file specified" +ULS requires the OS to provide a python3 executable. The python installation on Windows somehow (unlike other OS) just installs a "python" executable. +Luckily this is something that can get sorted easily and in multiple different ways (just pick the one that suites you best): + +1) **Copy the binary (recommended)** +Go to your python directory on Windows e.g. `C:\Users\Administrator\AppData\Local\Programs\Python\Python310` or `C:\Program Files\Python\Python310`. +Now copy the `python.exe` executable to `python3.exe` within the same folder. + + +2) **Create a powershell alias (temproary only)** +If you are using powershell, run this before you start ULS. + ```text + Set-Alias -Name python3 -Value python + ``` + +3) **Create a symbolic link (requires [mklink](https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/mklink))** + ```text + mklink "C:\path\to\symlink\python3.exe" "C:\path\to\Python3\python.exe" + ``` + +4) **Change ULS config (not recommended)** +You can modify the bin_python variable within the ULS global config file's `bin/config/global_config.py` 'Generic config' section.# +Change + ```text + bin_python = "python3" + ``` + to + ```text + bin_python = "python" + ``` + **WARNING:** This change prevents the global_config.py file to get updated via GIT in the future. You need to manually take care of updating changes within the file. + diff --git a/docs/LOG_OVERVIEW.md b/docs/LOG_OVERVIEW.md index 7191c27..edbe3a4 100644 --- a/docs/LOG_OVERVIEW.md +++ b/docs/LOG_OVERVIEW.md @@ -26,6 +26,7 @@ Here are some examples (per product) and links to additional information. - [Guardicore](#guardicore) - [NETLOG](#netlog) - [INCIDENT](#incident) + - [AUDIT](#gc-audit) - [Linode](#linode) - [AUDIT Logs](#audit-logs) - [AKAMAI ACC](#akamai-acc) @@ -2659,6 +2660,17 @@ Additional information regarding the MFA log fields can be found on [here](https +### GC Audit +
+ Guardicore AUDIT log example (JSON) + +```json +{"_id": "fedeabc1230123", "doc_version": 91, "time": 1725952232281, "username": "username", "title": "User authentication", "description": "User description", "path": "/api/v3.0/authenticate", "method": "POST", "remote_addr": "11.22.33.44", "user_agent": "python-requests/2.31.0", "status_code": 200, "expire_at": 1757488232281, "id": "fedeabc1230123"} +``` + +
+ + --- ## Linode ### AUDIT Logs diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 7b45dd6..02fcbad 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -4,4 +4,4 @@ description: Akamai Universal Log Streamer Helm installation type: application version: 2.0.0 -appVersion: "1.8.1" +appVersion: "1.8.2" diff --git a/test/basic_test.bats b/test/basic_test.bats index b15ded0..ac2cbd0 100644 --- a/test/basic_test.bats +++ b/test/basic_test.bats @@ -49,6 +49,29 @@ current_version=$(cat docs/CHANGELOG.md | grep "##" | head -n 1 | sed 's/.* v//' assert_output --partial $current_version [ "$status" -eq 0 ] } +@test "python3.9 uls.py --version" { + run /opt/homebrew/bin/python3.9 $uls_bin --version + assert_output --partial "Akamai Unified Log Streamer Version information" + [ "$status" -eq 0 ] +} + +@test "python3.10 uls.py --version" { + run /opt/homebrew/bin/python3.10 $uls_bin --version + assert_output --partial "Akamai Unified Log Streamer Version information" + [ "$status" -eq 0 ] +} + +@test "python3.11 uls.py --version" { + run /opt/homebrew/bin/python3.11 $uls_bin --version + assert_output --partial "Akamai Unified Log Streamer Version information" + [ "$status" -eq 0 ] +} + +@test "python3.12 uls.py --version" { + run /opt/homebrew/bin/python3.12 $uls_bin --version + assert_output --partial "Akamai Unified Log Streamer Version information" + [ "$status" -eq 0 ] +} @test "cat bin/config/global - Version output should be ($current_version) according to CHANGELOG" { run echo $(cat bin/uls_config/global_config.py | grep "__version__ =" | cut -d " " -f 3) diff --git a/test/positive_test.bats b/test/positive_test.bats index 84a05e7..740ab0a 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -31,7 +31,7 @@ if [ "$mocked_edgerc"=="FALSE" ] ; then etp_assert="configId" gc_assert="flow_id" linode_assert="" - jmespath_assert="['" + jmespath_assert='["' else # TESTING EDGERC FILE & section uls_edgerc=test/_mocked_edgerc