v1.3.5 (#25)

# Version History
## v1.3.5
|||
|---|---|
|Date|2022-04-06
|Kind| Bugfix release
|Author|[email protected], [email protected]
- **Minor improvements**
  - More QRADAR log source type definitions (thx to bitonio)
  - Added docker-compose ETP - Tenant example

bin/config/global_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "1.3.4"
+__version__ = "1.3.5"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 

docs/CHANGELOG.md

@@ -1,14 +1,13 @@
 # Version History
-## v1.3.3
+## v1.3.5
 |||
 |---|---|
-|Date|2022-02-28
+|Date|2022-04-06
 |Kind| Bugfix release
-|Author|[email protected]
-- **Bugfix**
-  - Adopted to new MFA CLI Version (only single feed "EVENT" available anymore)
-  - Amended new dates to the file headers
-  - Added volume to dockerfile as data storage for "autoresume"
+|Author|[email protected], [email protected]
+- **Minor improvements**
+  - More QRADAR log source type definitions (thx to bitonio)
+  - Added docker-compose ETP - Tenant example
 
 ## v1.3.4
 |||

docs/SIEM/QRADAR/build-sample-event.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Description
+# -----------
+# Creates a sample file to test against QRadar DSM Editor.
+#
+# Requirements
+# ------------
+# - Akamai CLI with eaa modules
+# - Credentials in the default section of the ~/.edgerc file
+
+# Adjust how far back you want the log to go
+# It will vary based on your EAA account activity
+START=$(( $(date +%s) - 7 * 24 * 60 * 60 ))
+
+shuf() { awk 'BEGIN {srand(); OFMT="%.17f"} {print rand(), $0}' "$@" |
+               sort -k1,1n | cut -d ' ' -f2-; }
+
+tmp_dir=$(mktemp -d -t ak-uls-qradar-XXXXXXXXXX)
+echo "Working in temporary directory $tmp_dir"
+
+function print_usage() {
+    echo "Usage:"
+    echo "$0 [eaa|etp]"
+}
+
+case "$1" in
+
+    "eaa")
+        echo "Fetching access events..."
+        akamai eaa log admin --start $START --json --output $tmp_dir/eaa_admin.json
+        echo "Fetching admin audit events..."
+        akamai eaa log access --start $START --json --output $tmp_dir/eaa_access.json
+        echo "Fetching connector health events..."
+        akamai eaa connector list --perf --json > $tmp_dir/eaa_conhealth.json
+
+        cat $tmp_dir/eaa_admin.json | shuf | head -n 50 > $tmp_dir/eaa_admin_min.json
+        cat $tmp_dir/eaa_access.json | shuf | head -n 50 > $tmp_dir/eaa_access_min.json
+        cat $tmp_dir/eaa_conhealth.json | shuf > $tmp_dir/eaa_conhealth_min.json
+
+        cat $tmp_dir/eaa_admin_min.json $tmp_dir/eaa_access_min.json $tmp_dir/eaa_conhealth_min.json | shuf > eaa_feeds_combined_sample.json
+
+        stat eaa_feeds_combined_sample.json
+        echo "File 'eaa_feeds_combined_sample.json' created in the current directory ($(pwd))."
+        ;;
+
+    "etp")
+        # Window for ETP is 1 hour since we have a lot of logs in the lab
+        START=$(( $(date +%s) - 1 * 60 * 60 ))
+        event_types=( "dns" "proxy" "aup" "threat" )
+        for event_type in "${event_types[@]}"
+        do
+            echo "Fetching ${event_type} events..."
+            akamai etp event --start $START --output "$tmp_dir/etp_${event_type}.json" ${event_type}
+            cat "$tmp_dir/etp_${event_type}.json" | shuf | head -n 50 > "$tmp_dir/etp_min_${event_type}.json"
+            if [ "$2" == "preview" ]; then
+                echo "# START PREVIEW ${event_type}"
+                head -n1 "$tmp_dir/etp_min_${event_type}.json"
+                echo "# END PREVIEW ${event_type}"
+            fi
+        done
+        for event_type in "${event_types[@]}"
+        do
+            cat $tmp_dir/etp_min_*.json | shuf > etp_feeds_combined_sample.json
+        done
+        ;;
+
+    *)
+        print_usage
+        exit 1
+        ;;
+
+esac
+
+rm -v -rf $tmp_dir

docs/examples/docker-compose/README.md

@@ -4,6 +4,11 @@ Within this directory, we provide some `docker compose` examples including examp
 The docker-compose.yml controls the docker - relevant attributes like mounting the `.edgerc` file into the container.
 The `.env` files control the ULS via dedicated [ENVIRONMENTAL VARIABLES](../../ARGUMENTS_ENV_VARS.md).
 
-The [simple](simple/README.md) directory provides a simple example running ULS via `docker compose`  
+The [simple](simple/README.md) directory provides a simple example running ULS via `docker compose`   
 The [complex](complex/README.md) directory provides a more "real world" example combining multiple feeds and different outputs.  
-The [example](examples/README.md) directory provides different configuration snippets.
+The [example](examples/README.md) directory provides different configuration snippets.  
+<<<<<<< HEAD
+The [etp-multi-tenant](etp-tenants/README.md) directory shows how logs from different ETP tenants can be collected.  
+=======
+The [etp-multi-tenant](etp-tenants/README.md) directory shows how logs from different ETP tenants can be collected.
+>>>>>>> v.1.3.5

docs/examples/docker-compose/etp-tenants/.edgerc-example

@@ -0,0 +1,14 @@
+[etp_tenant1]
+client_secret = …
+host = akab-….luna.akamaiapis.net
+access_token = akab-…
+client_token = akab-…
+etp_config_id = 123
+
+[etp_tenant2]
+; same first 4 credentials (client_secret, host, access_token, client_token) as etp_tenant1
+client_secret = …
+host = akab-….luna.akamaiapis.net
+access_token = akab-…
+client_token = akab-…
+etp_config_id = 456

docs/examples/docker-compose/etp-tenants/README.md

@@ -0,0 +1,22 @@
+# ULS DOCKER ETP-TENANT EXAMPLES
+
+Enterprise Threat Protector (ETP) allows customers and partners to manage multiple ETP tenants each coming with separated data feeds.
+
+See [Akamai ETP multi-tenancy documentation](https://techdocs.akamai.com/etp/docs/delegated-tenant-access#multi-tenancy) for more details.
+
+ULS can be configured to be a data hub to fetch and distribute these feeds from multiple tenants into one or multiple destinations.
+
+This directory contains configuration examples (for simple copy & paste usage) that illustrate ETP multi-tenant feature.
+
+## docker-compose.yml
+
+[This file](docker-compose.yml) contains examples for 2 different ETP tenants collecting the same "threat" feed.
+
+## ENV files
+The files contains all available ENV VARS explained in a single file.  
+Tenant 1 [etp-threat-tenant-1.env](./etp-threat-tenant-1.env)  
+Tenant 2 [etp-threat-tenant-2.env](./etp-threat-tenant-2.env)  
+
+## EDGERC example
+
+This is a sample `.edgerc` file explaining the ["ETP Multi Tenant support"](./.edgerc-example)

docs/examples/docker-compose/etp-tenants/docker-compose.yml

@@ -0,0 +1,20 @@
+version: "3.0"
+services:
+  etp-threat-tenant-1:
+    image: akamai/uls:latest
+    restart: always
+    env_file: etp-threat-tenant-1.env
+    volumes:
+      - type: bind
+        source: /path/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+        read_only: true
+  etp-threat-tenant-2:
+    image: akamai/uls:latest
+    restart: always
+    env_file: etp-threat-tenant-2.env
+    volumes:
+      - type: bind
+        source: /path/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+        read_only: true

docs/examples/docker-compose/etp-tenants/etp-threat-tenant-1.env

@@ -0,0 +1,20 @@
+# This is a sample ENV file for ULS ETP - Threat logs TENANT 1(output via HTTP to SPLUNK)
+
+# GENERIC Config
+ULS_LOGLEVEL=INFO
+
+# TENANT SELECTION
+    ULS_SECTION="etp_tenant1"
+
+# INPUT CONFIGURATION
+    ULS_INPUT=ETP
+    ULS_FEED=THREAT
+    ULS_FORMAT=JSON
+    ULS_EDGERC=/opt/akamai-uls/.edgerc
+
+# OUTPUT CONFIGURATION
+    ULS_OUTPUT=HTTP
+    ULS_HTTP_URL=https://127.0.0.1:8088/services/collector/event
+    ULS_HTTP_AUTH_HEADER={"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}
+    ULS_HTTP_INSECURE="True"
+    ULS_HTTP_FORMAT={"event": %s}

docs/examples/docker-compose/etp-tenants/etp-threat-tenant-2.env

@@ -0,0 +1,20 @@
+# This is a sample ENV file for ULS ETP - Threat logs TENANT 2(output via HTTP to SPLUNK)
+
+# GENERIC Config
+ULS_LOGLEVEL=INFO
+
+# TENANT SELECTION
+    ULS_SECTION="etp_tenant2"
+
+# INPUT CONFIGURATION
+    ULS_INPUT=ETP
+    ULS_FEED=THREAT
+    ULS_FORMAT=JSON
+    ULS_EDGERC=/opt/akamai-uls/.edgerc
+
+# OUTPUT CONFIGURATION
+    ULS_OUTPUT=HTTP
+    ULS_HTTP_URL=https://127.0.0.1:8088/services/collector/event
+    ULS_HTTP_AUTH_HEADER={"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}
+    ULS_HTTP_INSECURE="True"
+    ULS_HTTP_FORMAT={"event": %s}

docs/examples/kubernetes/helm/akamai-uls/Chart.yaml

@@ -3,5 +3,5 @@ name: akamai-uls
 description: Akamai Universal Log Streamer Helm installation
 
 type: application
-version: 1.3.4
-appVersion: "1.3.4"
+version: 1.3.5
+appVersion: "1.3.5"

test/basic_test.bats

@@ -3,14 +3,14 @@
 
 # Variables
     # ULS Binary
-uls_bin=bin/uls.py
+uls_bin="bin/uls.py"
 
 uls2_path="bin"
-uls2_bin=uls.py
+uls2_bin="uls.py"
 
 # Load support libs
-load 'test/bats/bats-support/load.bash'
-load 'test/bats/bats-assert/load.bash'
+load 'bats/bats-support/load.bash'
+load 'bats/bats-assert/load.bash'
 
 uls_bin=bin/uls.py
 uls_edgerc=~/.edgerc

test/negative_test.bats

@@ -7,8 +7,8 @@ uls_bin=bin/uls.py
 
 
 # Load support libs
-load 'test/bats/bats-support/load.bash'
-load 'test/bats/bats-assert/load.bash'
+load 'bats/bats-support/load.bash'
+load 'bats/bats-assert/load.bash'
 
 
 

test/positive_test.bats

@@ -19,8 +19,8 @@ uls_test_timeout=30
 
 
 # Load support libs
-load 'test/bats/bats-support/load.bash'
-load 'test/bats/bats-assert/load.bash'
+load 'bats/bats-support/load.bash'
+load 'bats/bats-assert/load.bash'
 
 
 #  POSITIVE tests