diff --git a/Dockerfile b/Dockerfile
index a4e0c526..77574e6d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,8 +23,6 @@
# use as builder image to pull in required deps
FROM alpine@sha256:6457d53fb065d6f250e1504b9bc42d5b6c65941d57532c072d929dd0628977d0 AS builder
-LABEL org.opencontainers.image.source="https://github.com/alpinelinux/docker-alpine"
-
ENV PYTHONUNBUFFERED=1
COPY requirements-docker.txt /tmp/requirements-docker.txt
@@ -48,6 +46,17 @@ FROM alpine@sha256:6457d53fb065d6f250e1504b9bc42d5b6c65941d57532c072d929dd062897
COPY --from=builder /usr /usr
+LABEL \
+ org.opencontainers.image.title="ElectricEye" \
+ org.opencontainers.image.description="ElectricEye is a multi-cloud, multi-SaaS Python CLI tool for Asset Management, Security Posture Management & Attack Surface Monitoring supporting 100s of services and evaluations to harden your CSP & SaaS environments with controls mapped to over 20 industry, regulatory, and best practice controls frameworks." \
+ org.opencontainers.image.version="3.0" \
+ org.opencontainers.image.created="2024-02-02T00:00:00Z" \
+ org.opencontainers.image.documentation="https://github.com/jonrau1/ElectricEye" \
+ org.opencontainers.image.revision="sha256:6457d53fb065d6f250e1504b9bc42d5b6c65941d57532c072d929dd0628977d0" \
+ org.opencontainers.image.source="https://github.com/alpinelinux/docker-alpine" \
+ org.opencontainers.image.licenses="Apache-2.0" \
+ org.opencontainers.image.authors="opensource@electriceye.cloud"
+
# NOTE: This will copy all application files and auditors to the container
# IMPORTANT: ADD YOUR TOML CONFIGURATIONS BEFORE YOU BUILD THIS! - or use docker run -v options to override
diff --git a/README.md b/README.md
index 5f7e7e0e..ea851a3f 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ ElectricEye is a multi-cloud, multi-SaaS Python CLI tool for Asset Management, S
- Multi-faceted Attack Surface Monitoring uses tools such as VirusTotal, Nmap, Shodan.io, Detect-Secrets, and CISA's KEV to locate assets indexed on the internet, find exposed services, locate exploitable vulnerabilities, and malicious packages in artifact repositories, respectively.
-- Outputs to [AWS Security Hub](https://aws.amazon.com/security-hub/), [AWS DocumentDB](https://aws.amazon.com/documentdb/), JSON, CSV, HTML Reports, [MongoDB](https://www.mongodb.com/), [Amazon SQS](https://aws.amazon.com/sqs/), [PostgreSQL](https://www.postgresql.org/), [Slack](https://slack.com/) (via Slack App Bots), and [FireMon Cloud Defense](https://www.firemon.com/introducing-disruptops/).
+- Outputs to [AWS Security Hub](https://aws.amazon.com/security-hub/), the [Open Cyber Security Framework (OCSF)](https://github.com/ocsf/) [V1.1.0](https://schema.ocsf.io/1.1.0/?extensions=) in JSON, [AWS DocumentDB](https://aws.amazon.com/documentdb/), JSON, CSV, HTML Reports, [MongoDB](https://www.mongodb.com/), [Amazon SQS](https://aws.amazon.com/sqs/), [PostgreSQL](https://www.postgresql.org/), [Slack](https://slack.com/) (via Slack App Bots), and [FireMon Cloud Defense](https://www.firemon.com/introducing-disruptops/).
ElectricEye's core concept is the **Auditor** which are sets of Python scripts that run **Checks** per Service dedicated to a specific SaaS vendor or public cloud service provider called an **Assessment Target**. You can run an entire Assessment Target, a specific Auditor, or a specific Check within an Auditor. After ElectricEye is done with evaluations, it supports over a dozen types of **Outputs** ranging from an HTML executive report to AWS DocumentDB clusters - you can run multiple Outputs as you see fit.
diff --git a/docs/outputs/OUTPUTS.md b/docs/outputs/OUTPUTS.md
index 35fb3cd7..a84aac4f 100644
--- a/docs/outputs/OUTPUTS.md
+++ b/docs/outputs/OUTPUTS.md
@@ -492,6 +492,20 @@ To use this Output include the following arguments in your ElectricEye CLI: `pyt
}
```
+## Open Cyber Security Format (OCSF) V1.1.0 Output
+
+The OCSF V1.1.0 Output selection will convert all ElectricEye findings into the OCSF format (in JSON) which is a normalized and standardized security-centric data model, well-suited to ingestion in Data Lakes and Data Lake Houses built upon Amazon Security Lake, AWS Glue Data Catalog, Snowflake, Apache Iceberg, Google BigQuery, and more. The Event Class used for this finding is [`compliance_finding [2003]`](https://schema.ocsf.io/1.1.0/classes/compliance_finding?extensions=)
+
+This Output will provide the `ProductFields.AssetDetails` information, it is mapped within `resource.data`.
+
+To use this Output include the following arguments in your ElectricEye CLI: `python3 eeauditor/controller.py {..args..} -o ocsf_v1_1_0`
+
+### Example Open Cyber Security Format (OCSF) V1.1.0 Output
+
+```json
+{}
+```
+
## MongoDB & AWS DocumentDB Output
The MongoDB Output selection will write all ElectricEye findings to a MongoDB database or to an AWS DocumentDB Instance/Cluster along with the `ProductFields.AssetDetails` using `pymongo`. To facilitate mutable records being written to a Collection, ElectricEye will duplicate the ASFF `Id` (the finding's GUID) into the MongoDB `_id` field and write all records sequentially using the `update_one(upsert=True)` method within `pymongo`. This is written with a filter to replace the entire record where and existing `_id` is located.
diff --git a/eeauditor/cloud_utils.py b/eeauditor/cloud_utils.py
index 7e683708..204f6f1f 100644
--- a/eeauditor/cloud_utils.py
+++ b/eeauditor/cloud_utils.py
@@ -18,14 +18,17 @@
#specific language governing permissions and limitations
#under the License.
+import logging
import boto3
from tomli import load as tomload
-from sys import exit
-from os import environ, path
+import sys
+from os import environ, path, chmod
from re import compile
import json
from botocore.exceptions import ClientError
+logger = logging.getLogger(__name__)
+
# Boto3 Clients
sts = boto3.client("sts")
ssm = boto3.client("ssm")
@@ -51,13 +54,16 @@ def __init__(self, assessmentTarget):
# From TOML [global]
if data["global"]["aws_multi_account_target_type"] not in AWS_MULTI_ACCOUNT_TARGET_TYPE_CHOICES:
- print("Invalid option for [global.aws_multi_account_target_type].")
- exit(2)
+ logger.error("Invalid option for [global.aws_multi_account_target_type].")
+ sys.exit(2)
self.awsMultiAccountTargetType = data["global"]["aws_multi_account_target_type"]
if data["global"]["credentials_location"] not in CREDENTIALS_LOCATION_CHOICES:
- print(f"Invalid option for [global.credentials_location]. Must be one of {str(CREDENTIALS_LOCATION_CHOICES)}.")
- exit(2)
+ logger.error(
+ "Invalid option for [global.credentials_location]. Must be one of %s.",
+ CREDENTIALS_LOCATION_CHOICES
+ )
+ sys.exit(2)
self.credentialsLocation = data["global"]["credentials_location"]
##################################
@@ -75,14 +81,14 @@ def __init__(self, assessmentTarget):
self.awsAccountTargets = awsAccountTargets
elif self.awsMultiAccountTargetType == "OU":
if not awsAccountTargets:
- print("OU was specified but targets were not specified.")
- exit(2)
+ logger.error("OU was specified but targets were not specified.")
+ sys.exit(2)
# Regex to check for Valid OUs
ouIdRegex = compile(r"^ou-[0-9a-z]{4,32}-[a-z0-9]{8,32}$")
for ou in awsAccountTargets:
if not ouIdRegex.match(ou):
- print(f"Invalid Organizational Unit ID {ou}.")
- exit(2)
+ logger.error(f"Invalid Organizational Unit ID {ou}.")
+ sys.exit(2)
self.awsAccountTargets = self.get_aws_accounts_from_organizational_units(awsAccountTargets)
elif self.awsMultiAccountTargetType == "Organization":
self.awsAccountTargets = self.get_aws_accounts_from_organization()
@@ -102,8 +108,8 @@ def __init__(self, assessmentTarget):
# Process ["aws_electric_eye_iam_role_name"]
electricEyeRoleName = data["regions_and_accounts"]["aws"]["aws_electric_eye_iam_role_name"]
if electricEyeRoleName == (None or ""):
- print(f"A value for ['aws_electric_eye_iam_role_name'] was not provided. Fix the TOML file and run ElectricEye again.")
- exit(2)
+ logger.error(f"A value for ['aws_electric_eye_iam_role_name'] was not provided. Fix the TOML file and run ElectricEye again.")
+ sys.exit(2)
self.electricEyeRoleName = electricEyeRoleName
# GCP
@@ -111,8 +117,8 @@ def __init__(self, assessmentTarget):
# Process ["gcp_project_ids"]
gcpProjects = data["regions_and_accounts"]["gcp"]["gcp_project_ids"]
if not gcpProjects:
- print("No GCP Projects were provided in [regions_and_accounts.gcp.gcp_project_ids].")
- exit(2)
+ logger.error("No GCP Projects were provided in [regions_and_accounts.gcp.gcp_project_ids].")
+ sys.exit(2)
else:
self.gcpProjectIds = gcpProjects
@@ -151,8 +157,8 @@ def __init__(self, assessmentTarget):
ociTenancyId, ociUserId, ociRegionName, ociCompartments, ociUserApiKeyFingerprint, ociUserApiKeyPemValue
]
):
- print(f"One of your Oracle Cloud TOML entries in [regions_and_accounts.oci] or [credentials.oci] is empty!")
- exit(2)
+ logger.error(f"One of your Oracle Cloud TOML entries in [regions_and_accounts.oci] or [credentials.oci] is empty!")
+ sys.exit(2)
# Assign ["regions_and_accounts"]["oci"] values to `self`
self.ociTenancyId = ociTenancyId
@@ -197,15 +203,15 @@ def __init__(self, assessmentTarget):
# Azure
elif assessmentTarget == "Azure":
- print("Coming soon!")
+ logger.info("Coming soon!")
# Alibaba Cloud
elif assessmentTarget == "Alibaba":
- print("Coming soon!")
+ logger.info("Coming soon!")
# VMWare Cloud on AWS
elif assessmentTarget == "VMC":
- print("Coming soon!")
+ logger.info("Coming soon!")
###################################
# SOFTWARE-AS-A-SERVICE PROVIDERS #
@@ -227,8 +233,8 @@ def __init__(self, assessmentTarget):
snowInstanceName, snowInstanceRegion, snowUserName, snowUserLoginBreachRate
]
):
- print(f"One of your ServiceNow TOML entries in [credentials.servicenow] is empty!")
- exit(2)
+ logger.error(f"One of your ServiceNow TOML entries in [credentials.servicenow] is empty!")
+ sys.exit(2)
# Retrieve ServiceNow ElectricEye user password
serviceNowPwVal = serviceNowValues["servicenow_sspm_password_value"]
@@ -267,8 +273,8 @@ def __init__(self, assessmentTarget):
m365ClientId, m365SecretId, m365TenantId, m365TenantLocation
]
):
- print(f"One of your M365 TOML entries in [credentials.m365] is empty!")
- exit(2)
+ logger.error(f"One of your M365 TOML entries in [credentials.m365] is empty!")
+ sys.exit(2)
# This value (tenant location) will always be in plaintext
self.m365TenantLocation = m365TenantLocation
@@ -333,8 +339,8 @@ def __init__(self, assessmentTarget):
salesforceAppClientId, salesforceAppClientSecret, salesforceApiUsername, salesforceApiPassword, salesforceUserSecurityToken, salesforceInstanceLocation, salesforceFailedLoginBreachingRate, salesforceApiVersion
]
):
- print(f"One of your Salesforce TOML entries in [credentials.salesforce] is empty!")
- exit(2)
+ logger.error(f"One of your Salesforce TOML entries in [credentials.salesforce] is empty!")
+ sys.exit(2)
# The failed login breaching rate and API Version will be in plaintext/env vars
environ["SALESFORCE_FAILED_LOGIN_BREACHING_RATE"] = salesforceFailedLoginBreachingRate
@@ -426,10 +432,12 @@ def get_credential_from_aws_ssm(self, value, configurationName):
Retrieves a TOML variable from AWS Systems Manager Parameter Store and returns it
"""
- # Check that a value was provided
- if value == (None or ""):
- print(f"A value for {configurationName} was not provided. Fix the TOML file and run ElectricEye again.")
- exit(2)
+ if value is None or value == "":
+ logger.error(
+ "A value for %s was not provided. Fix the TOML file and run ElectricEye again.",
+ configurationName
+ )
+ sys.exit(2)
# Retrieve the credential from SSM Parameter Store
try:
@@ -438,6 +446,10 @@ def get_credential_from_aws_ssm(self, value, configurationName):
WithDecryption=True
)["Parameter"]["Value"]
except ClientError as e:
+ logger.error(
+ "Failed to retrieve the credential for %s from SSM Parameter Store: %s",
+ configurationName, e
+ )
raise e
return credential
@@ -446,20 +458,22 @@ def get_credential_from_aws_secrets_manager(self, value, configurationName):
"""
Retrieves a TOML variable from AWS Secrets Manager and returns it
"""
+ if value is None or value == "":
+ logger.error(
+ "A value for %s was not provided. Fix the TOML file and run ElectricEye again.",
+ configurationName
+ )
+ sys.exit(2)
- # Check that a value was provided
- if value == (None or ""):
- print(f"A value for {configurationName} was not provided. Fix the TOML file and run ElectricEye again.")
- exit(2)
-
- # Retrieve the credential from AWS Secrets Manager
try:
- credential = asm.get_secret_value(
- SecretId=value,
- )["SecretString"]
+ credential = asm.get_secret_value(SecretId=value)["SecretString"]
except ClientError as e:
+ logger.error(
+ "Failed to retrieve the credential for %s from AWS Secrets Manager: %s",
+ configurationName, e
+ )
raise e
-
+
return credential
def get_aws_accounts_from_organization(self):
@@ -469,38 +483,38 @@ def get_aws_accounts_from_organization(self):
try:
accounts = [account["Id"] for account in org.list_accounts()["Accounts"] if account["Status"] == "ACTIVE"]
except ClientError as e:
+ logger.error(
+ "Failed to retrieve accounts from AWS Organizations: %s", e
+ )
raise e
-
+
return accounts
def get_aws_accounts_from_organizational_units(self, targets):
"""
Uses Organizations ListAccountsForParent API to get a list of "ACTIVE" AWS Accounts for specified OUs
"""
- accounts = []
- # Sometimes the caller Account may not be in the OU, add them in
- accounts.append(sts.get_caller_identity()["Account"])
+ accounts = [sts.get_caller_identity()["Account"]] # Caller account is added directly.
for parent in targets:
- print(f"Processing accounts for Organizational Unit {parent}.")
+ logger.info("Processing accounts for Organizational Unit %s.", parent)
try:
- for account in org.list_accounts_for_parent(ParentId=parent)["Accounts"]:
- if account["Status"] == "ACTIVE":
- if account["Id"] not in accounts:
- accounts.append(account["Id"])
- else:
- continue
+ active_accounts = [account["Id"] for account in org.list_accounts_for_parent(ParentId=parent)["Accounts"] if account["Status"] == "ACTIVE"]
+ accounts.extend(account for account in active_accounts if account not in accounts)
except ClientError as e:
+ logger.error(
+ "Failed to retrieve accounts for Organizational Unit %s: %s",
+ parent, e
+ )
raise e
-
+
return accounts
-
+
# This function is called outside of this Class
- def create_aws_session(account, partition, region, roleName):
+ def create_aws_session(account: str, partition: str, region: str, roleName: str) -> boto3.Session:
"""
- Uses STS AssumeRole to create a temporary Boto3 Session with a specified Account, Partition, and Region
+ Creates a Boto3 Session by assuming a given AWS IAM Role
"""
-
crossAccountRoleArn = f"arn:{partition}:iam::{account}:role/{roleName}"
try:
@@ -508,7 +522,12 @@ def create_aws_session(account, partition, region, roleName):
RoleArn=crossAccountRoleArn,
RoleSessionName="ElectricEye"
)
+ logger.info("Assumed role: %s successfully", crossAccountRoleArn)
except ClientError as e:
+ logger.error(
+ "Failed to assume role %s: %s",
+ crossAccountRoleArn, e
+ )
raise e
session = boto3.Session(
@@ -521,10 +540,11 @@ def create_aws_session(account, partition, region, roleName):
return session
# This function is called outside of this Class and from create_aws_session()
- def check_aws_partition(region):
+ def check_aws_partition(region: str):
"""
Returns the AWS Partition based on the current Region of a Session
"""
+
# GovCloud partition override
if region in ["us-gov-east-1", "us-gov-west-1"]:
partition = "aws-us-gov"
@@ -543,31 +563,37 @@ def check_aws_partition(region):
return partition
# This function is called outside of this Class
- def get_aws_support_eligiblity(session):
+ def get_aws_support_eligibility(session):
support = session.client("support")
try:
support.describe_trusted_advisor_checks(language='en')
supportEligible = True
+ logger.info("AWS Support is eligible.")
except ClientError as e:
- if str(e) == "An error occurred (SubscriptionRequiredException) when calling the DescribeTrustedAdvisorChecks operation: Amazon Web Services Premium Support Subscription is required to use this service.":
+ if "SubscriptionRequiredException" in str(e):
supportEligible = False
+ logger.warning("AWS Support is not eligible: %s", e)
else:
+ logger.error("Error checking AWS Support eligibility: %s", e)
raise e
return supportEligible
# This function is called outside of this Class
- def get_aws_shield_advanced_eligiblity(session):
+ def get_aws_shield_advanced_eligibility(session):
shield = session.client("shield")
try:
shield.describe_subscription()
shieldEligible = True
+ logger.info("AWS Shield Advanced is eligible.")
except ClientError as e:
- if str(e) == "An error occurred (ResourceNotFoundException) when calling the DescribeSubscription operation: The subscription does not exist.":
+ if "ResourceNotFoundException" in str(e):
shieldEligible = False
+ logger.warning("AWS Shield Advanced is not eligible: %s", e)
else:
+ logger.error("Error checking AWS Shield Advanced eligibility: %s", e)
raise e
return shieldEligible
@@ -583,18 +609,22 @@ def setup_gcp_credentials(self, credentialValue):
by this overall Class (CloudConfig), writes it to a JSON file, and specifies that location as the environment variable "GOOGLE_APPLICATION_CREDENTIALS"
"""
here = path.abspath(path.dirname(__file__))
- # Write the result of ["gcp_service_account_json_payload_value"] to file
- with open(f"{here}/gcp_cred.json", 'w') as jsonfile:
- json.dump(
- json.loads(
- credentialValue
- ),
- jsonfile,
- indent=2
+ credentials_file_path = path.join(here, 'gcp_cred.json')
+
+ # Attempt to parse the credential value and write it to a file
+ try:
+ credentials = json.loads(credentialValue)
+ with open(credentials_file_path, 'w') as jsonfile:
+ json.dump(credentials, jsonfile, indent=2)
+ chmod(credentials_file_path, 0o600) # Set file to be readable and writable only by the owner
+ except json.JSONDecodeError as e:
+ logger.error(
+ "Failed to parse GCP credentials JSON: %s", e
)
- # Set Cred global path
- print(f"{here}/gcp_cred.json saved to environment variable")
- environ["GOOGLE_APPLICATION_CREDENTIALS"] = f"{here}/gcp_cred.json"
+ raise e
+
+ logger.info("%s saved to environment variable", credentials_file_path)
+ environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_file_path
def setup_oci_credentials(self, credentialValue):
"""
@@ -602,12 +632,14 @@ def setup_oci_credentials(self, credentialValue):
contents to a file and save the location to an environment variable to be used
"""
here = path.abspath(path.dirname(__file__))
- # Write the result of ["oci_user_api_key_private_key_pem_contents_value"] to file
- with open(f"{here}/oci_api_key.pem", "w") as f:
+ credentials_file_path = path.join(here, 'oci_api_key.pem')
+
+ # Write the PEM contents to a file
+ with open(credentials_file_path, "w") as f:
f.write(credentialValue)
+ chmod(credentials_file_path, 0o600) # Set file to be readable and writable only by the owner
- # Set the location
- print(f"{here}/oci_api_key.pem saved to environment variable")
- environ["OCI_PEM_FILE_PATH"] = f"{here}/oci_api_key.pem"
+ logger.info("%s saved to environment variable", credentials_file_path)
+ environ["OCI_PEM_FILE_PATH"] = credentials_file_path
## EOF
\ No newline at end of file
diff --git a/requirements-docker.txt b/requirements-docker.txt
index 1aca9046..f106407d 100644
--- a/requirements-docker.txt
+++ b/requirements-docker.txt
@@ -6,7 +6,7 @@ google-api-python-client>=2.88.0
oci>=2.104.0
pluginbase==1.0.1
psycopg2-binary==2.9.6
-pymongo==4.5.0
+pymongo==4.6.1
pysnow<=0.7.17
python3-nmap==1.6.0
tomli==2.0.1
diff --git a/requirements.txt b/requirements.txt
index bd6ef7a0..75759742 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,12 +3,12 @@ boto3==1.34.34
click==8.1.7
detect-secrets==1.4.0
google-api-python-client>=2.88.0
-matplotlib==3.8.0
+matplotlib==3.8.2
oci>=2.104.0
pandas==2.0.3
pluginbase==1.0.1
psycopg2-binary==2.9.6
-pymongo==4.5.0
+pymongo==4.6.1
pysnow<=0.7.17
python3-nmap==1.6.0
tomli==2.0.1
diff --git a/screenshots/ElectricEye2023Architecture.svg b/screenshots/ElectricEye2023Architecture.svg
deleted file mode 100644
index 4edcb0bc..00000000
--- a/screenshots/ElectricEye2023Architecture.svg
+++ /dev/null
@@ -1 +0,0 @@
-
\ No newline at end of file
diff --git a/screenshots/ElectricEye2024Architecture.svg b/screenshots/ElectricEye2024Architecture.svg
new file mode 100644
index 00000000..180cf7e9
--- /dev/null
+++ b/screenshots/ElectricEye2024Architecture.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/screenshots/ElectricEyeAnimated.gif b/screenshots/ElectricEyeAnimated.gif
index b935f8c8..c9c9aadf 100644
Binary files a/screenshots/ElectricEyeAnimated.gif and b/screenshots/ElectricEyeAnimated.gif differ
diff --git a/screenshots/architecture-for-github-thumbnail.jpg b/screenshots/architecture-for-github-thumbnail.jpg
index 840eade0..d5b8090a 100644
Binary files a/screenshots/architecture-for-github-thumbnail.jpg and b/screenshots/architecture-for-github-thumbnail.jpg differ
diff --git a/screenshots/extras/ElectricEye.pptx b/screenshots/extras/ElectricEye.pptx
index c704d091..ff95da69 100644
Binary files a/screenshots/extras/ElectricEye.pptx and b/screenshots/extras/ElectricEye.pptx differ