Securityhub events

data-collection/README.md

@@ -34,6 +34,7 @@ List of modules and objects collected:
 | `backup`                     |  AWS Backup           | Management Account  | Collects Backup Restore and Copy Jobs. Requires [activation of cross-account](https://docs.aws.amazon.com/aws-backup/latest/devguide/manage-cross-account.html#enable-cross-account) |
 | `health-evetns`      |  AWS Health | Management Accounts  | Collect AWS Health notificaitons via AWS Organizational view  |
 | `licence-manager`      |  AWS License Manager | Management Accounts  | Collect Licences and Grants |
+| `SecurityHub-events`      |  AWS Security Hub | Management Account  | Collect AWS Security Hub Events via Event Bus from a single AWS organization |
 
 
 

data-collection/deploy/deploy-data-collection.yaml

@@ -33,6 +33,7 @@ Metadata:
           - IncludeTransitGatewayModule
           - IncludeAWSFeedsModule
           - IncludeLicenseManagerModule
+          - IncludeSecurityHubModule
     ParameterLabels:
       DestinationBucket:
         default: 'Destination S3 bucket prefix'
@@ -82,6 +83,8 @@ Metadata:
         default: 'Include AWS Health Events Module'
       IncludeLicenseManagerModule:
         default: 'Include Marketplace Licensing Collection'
+      IncludeSecurityHubModule:
+        default: 'Include AWS Security Hub Module'
 
 Mappings:
   RegionMap:
@@ -221,6 +224,11 @@ Parameters:
     Description: Collects Marketplace Licenses and Grants
     AllowedValues: ['yes', 'no']
     Default: 'no'
+  IncludeSecurityHubModule:
+    Type: String
+    Description: Collects AWS Security Hub Events
+    AllowedValues: ['yes', 'no']
+    Default: 'no'
 
 
 Conditions:
@@ -238,6 +246,7 @@ Conditions:
   DeployAWSFeedsModule: !Equals [ !Ref IncludeAWSFeedsModule, "yes"]
   DeployHealthEventsModule: !Equals [ !Ref IncludeHealthEventsModule, "yes"]
   DeployLicenseManagerModule: !Equals [ !Ref IncludeLicenseManagerModule, "yes"]
+  DeploySecurityHubModule: !Equals [!Ref IncludeSecurityHubModule, "yes"]
   DeployPricingModule: !Or
     - !Condition DeployInventoryCollectorModule
     - !Condition DeployRDSUtilizationModule
@@ -257,6 +266,7 @@ Conditions:
       - !Condition DeployTransitGatewayModule
       - !Condition DeployHealthEventsModule
       - !Condition DeployLicenseManagerModule
+      - !Condition DeploySecurityHubModule
   RegionsInScopeIsEmpty: !Equals
     - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
     - ""
@@ -903,6 +913,17 @@ Resources:
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
 
+  SecurityHubModule:
+    Type: AWS::CloudFormation::Stack
+    Condition: DeploySecurityHubModule
+    Properties:
+      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.amazonaws.com/cfn/data-collection/module-securityhub.yaml"
+      Parameters:
+        DatabaseName: !Ref DatabaseName
+        DestinationBucket: !Ref S3Bucket
+        ResourcePrefix: !Ref ResourcePrefix
+        LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn
+
   BackupModule:
     Type: AWS::CloudFormation::Stack
     Condition: DeployBackupModule

data-collection/deploy/module-securityhub.yaml

@@ -0,0 +1,318 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: Retrieves AWS Security Hub details across AWS organization
+Parameters:
+  DatabaseName:
+    Type: String
+    Description: Name of the Athena database to be created to hold lambda information
+    Default: optimization_data
+  DestinationBucket:
+    Type: String
+    Description: Name of the S3 Bucket that exists or needs to be created to hold costanomaly information
+    AllowedPattern: (?=^.{3,63}$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$)
+  CFDataName:
+    Type: String
+    Description: The name of what this cf is doing.
+    Default: securityhub
+  ResourcePrefix:
+    Type: String
+    Description: This prefix will be placed in front of all roles created. Note you may wish to add a dash at the end to make more readable
+  LambdaAnalyticsARN:
+    Type: String
+    Description: Arn of lambda for Analytics
+
+Resources:
+  LambdaRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub "${ResourcePrefix}${CFDataName}-LambdaRole"
+      AssumeRolePolicyDocument:
+        Statement:
+          - Action:
+              - sts:AssumeRole
+            Effect: Allow
+            Principal:
+              Service:
+                - lambda.amazonaws.com
+        Version: 2012-10-17
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      Path: /
+      Policies:
+        - PolicyName: "S3-Access"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: "Allow"
+                Action:
+                  - "s3:PutObject"
+                  - "s3:GetObject"
+                  - "s3:PutObjectAcl"
+                Resource:
+                  - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}/*"
+              - Effect: "Allow"
+                Action:
+                  - "s3:ListBucket"
+                Resource:
+                  - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}"
+
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W28 # Resource found with an explicit name, this disallows updates that require replacement of this resource
+            reason: "Need explicit name to identify role actions"
+
+  EventBridgeInvokeFirehoseRole:
+    UpdateReplacePolicy: "Delete"
+    Type: "AWS::IAM::Role"
+    DeletionPolicy: "Delete"
+    Properties:
+      Path: "/service-role/"
+      MaxSessionDuration: 3600
+      RoleName: !Sub "${ResourcePrefix}${CFDataName}-EventBridgeInvokeFirehose"
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+        - Action: "sts:AssumeRole"
+          Effect: "Allow"
+          Principal:
+            Service: "events.amazonaws.com"
+      Policies:
+        - PolicyName: "Firehose-WriteAccess"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: "Allow"
+                Action:
+                  - "firehose:PutRecord"
+                  - "firehose:PutRecordBatch"
+                Resource:
+                  - !GetAtt SecHubEventsFirehoseDeliveryStream.Arn
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W28 # Resource found with an explicit name, this disallows updates that require replacement of this resource
+            reason: "Need explicit name to identify role actions"
+
+  KinesisFirehoseRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub "${ResourcePrefix}${CFDataName}-KinesisFireHoseRole"
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Action: sts:AssumeRole
+            Effect: Allow
+            Principal:
+              Service: firehose.amazonaws.com
+            Condition:
+              StringEquals:
+                'sts:ExternalId': !Ref 'AWS::AccountId'
+      Path: /
+      Policies:
+        - PolicyName: "SecurityHub-S3-Access"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: "Allow"
+                Action:
+                - "s3:PutObject"
+                - "s3:GetObject"
+                - "s3:ListBucketMultipartUploads"
+                - "s3:AbortMultipartUpload"
+                Resource:
+                - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}/*"
+              - Effect: "Allow"
+                Action:
+                  - "s3:ListBucket"
+                  - "s3:GetBucketLocation"
+                Resource:
+                - !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}/*"
+              - Effect: "Allow"
+                Action:
+                  - "lambda:InvokeFunction"
+                  - "lambda:GetFunctionConfiguration"
+                Resource:
+                - !GetAtt LambdaFunction.Arn
+              - Effect: "Allow"
+                Action:
+                  - "logs:CreateLogStream"
+                  - "logs:PutLogEvents"
+                Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/*"
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W28 # Resource found with an explicit name, this disallows updates that require replacement of this resource
+            reason: "Need explicit name to identify role actions"
+
+  EventRuleSecurityHubKinesisIntegration:
+    Type: "AWS::Events::Rule"
+    Properties:
+      EventPattern:
+        detail-type:
+        - "Security Hub Findings - Imported"
+        source:
+        - "aws.securityhub" # only AWS can send events with source aws.*
+        - "cid.test" # uncomment for testing
+      Targets:
+      - Arn: !GetAtt SecHubEventsFirehoseDeliveryStream.Arn
+        RoleArn: !GetAtt EventBridgeInvokeFirehoseRole.Arn
+        Id: "Firehose"
+      State: "ENABLED"
+      Name: !Sub "${ResourcePrefix}${CFDataName}-SecurityHubToFirehose"
+
+  LambdaFunction:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: !Sub '${ResourcePrefix}${CFDataName}-transformation-Lambda'
+      Description: "Lambda function to transform Security Hub events"
+      Runtime: python3.12
+      Architectures: [x86_64]
+      Code:
+        ZipFile: |
+          import os
+          import re
+          import json
+          import logging
+          import base64
+
+          def rename_keys(payload):
+              """Recursively rename all special characters in keys to '_'. """
+              if isinstance(payload, dict):
+                  renamed_payload = {}
+                  for key, value in payload.items():
+                      renamed_key = re.sub(r'\W+', '_', key)
+                      renamed_value = rename_keys(value)
+                      renamed_payload[renamed_key] = renamed_value
+                  return renamed_payload
+              elif isinstance(payload, list):
+                  renamed_list = []
+                  for item in payload:
+                      renamed_item = rename_keys(item)
+                      renamed_list.append(renamed_item)
+                  return renamed_list
+              elif isinstance(payload, (str, int, float, bool, type(None))):
+                  return payload
+              else:
+                  if hasattr(payload, "__dict__"):
+                      renamed_payload = {}
+                      for key, value in payload.__dict__.items():
+                          renamed_key = re.sub(r'\W+', '_', key)
+                          renamed_value = rename_keys(value)
+                          renamed_payload[renamed_key] = renamed_value
+                      return renamed_payload
+                  else:
+                      return payload
+
+          def lambda_handler(event, context):
+              logger = logging.getLogger()
+              payload = event['records']
+              transformed_records = []
+
+              for record in payload:
+                  data = record['data']
+                  decoded_data = base64.b64decode(data).decode('utf-8')
+                  print(f"Input record JSON: {decoded_data}")
+                  renamed_payload = rename_keys(json.loads(decoded_data))
+                  flattened_data = json.dumps(renamed_payload, separators=(',', ':'))
+                  transformed_record = {
+                      'recordId': record['recordId'],
+                      'result': 'Ok',
+                      'data': base64.b64encode(flattened_data.encode('utf-8')).decode('utf-8')
+                  }
+                  print(f"Transformed record: {transformed_record}")
+                  transformed_records.append(transformed_record)
+              return {'records': transformed_records}
+      Handler: "index.lambda_handler"
+      MemorySize: 2688
+      Timeout: 900
+      Role: !GetAtt LambdaRole.Arn
+    Metadata:
+      cfn_nag:
+        rules_to_suppress:
+          - id: W89 # Lambda functions should be deployed inside a VPC
+            reason: "No need for VPC in this case"
+          - id: W92 #  Lambda functions should define ReservedConcurrentExecutions to reserve simultaneous executions
+            reason: "No need for simultaneous execution"
+
+  FirehoseLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/kinesis/${ResourcePrefix}${CFDataName}-detail-stream"
+      RetentionInDays: 60
+
+  SecHubEventsFirehoseDeliveryStream:
+      Type: AWS::KinesisFirehose::DeliveryStream
+      Properties:
+        DeliveryStreamName: !Sub '${ResourcePrefix}${CFDataName}-detail-stream'
+        DeliveryStreamType: DirectPut
+        DeliveryStreamEncryptionConfigurationInput:
+          KeyType: AWS_OWNED_CMK
+        ExtendedS3DestinationConfiguration:
+          BucketARN: !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}"
+          Prefix: !Sub "${CFDataName}/securityhub_events/!{timestamp:yyyy}/!{timestamp:MM}/!{timestamp:dd}/"
+          ErrorOutputPrefix: !Sub "${CFDataName}/securityhub_errors/!{timestamp:yyyy/MM/}/!{firehose:error-output-type}"
+          RoleARN: !GetAtt KinesisFirehoseRole.Arn
+          CloudWatchLoggingOptions:
+            Enabled: true
+            LogGroupName: !Ref FirehoseLogGroup
+            LogStreamName: SecHubEventsFirehoseDeliveryStreamLog
+          BufferingHints:
+            IntervalInSeconds: 900
+            SizeInMBs: 30
+          CompressionFormat: "GZIP"
+          ProcessingConfiguration:
+            Enabled: true
+            Processors:
+            - Type: Lambda
+              Parameters:
+              - ParameterName: LambdaArn
+                ParameterValue: !GetAtt LambdaFunction.Arn
+              - ParameterName: BufferIntervalInSeconds
+                ParameterValue: 600
+              - ParameterName: BufferSizeInMBs
+                ParameterValue: 3
+
+  TableSecurityHubEvents:
+    Type: AWS::Glue::Table
+    Properties:
+      CatalogId: !Ref "AWS::AccountId"
+      DatabaseName: !Ref DatabaseName
+      TableInput:
+        Name: securityhub_events
+        TableType: EXTERNAL_TABLE
+        PartitionKeys:
+        - { Name: datehour,    Type: string }
+        StorageDescriptor:
+          Columns:
+          - { Name: version,     Type: string }
+          - { Name: id,          Type: string }
+          - { Name: detail_type, Type: string }
+          - { Name: source,      Type: string }
+          - { Name: account,     Type: string }
+          - { Name: time,        Type: string }
+          - { Name: region,      Type: string }
+          - { Name: resources,   Type: array<string> }
+          - { Name: detail,      Type: string }
+          InputFormat: org.apache.hadoop.mapred.TextInputFormat
+          OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+          Location: !Sub "s3://${DestinationBucket}/${CFDataName}/securityhub_events/"
+          Parameters: {}
+          SerdeInfo:
+            SerializationLibrary: org.openx.data.jsonserde.JsonSerDe
+            Parameters:
+              serialization.format: '1'
+        Parameters:
+          EXTERNAL: 'TRUE'
+          projection.datehour.format: yyyy/MM/dd
+          projection.datehour.interval: '1'
+          projection.datehour.interval.unit: DAYS
+          projection.datehour.range: 2024/07/01,NOW
+          projection.datehour.type: date
+          projection.enabled: 'true'
+          storage.location.template: !Sub "s3://${DestinationBucket}/${CFDataName}/securityhub_events/${!datehour}"
+
+  AnalyticsExecutor:
+    Type: Custom::LambdaAnalyticsExecutor
+    Properties:
+      ServiceToken: !Ref LambdaAnalyticsARN
+      Name: !Ref CFDataName

data-collection/test/utils.py

@@ -220,6 +220,7 @@ def initial_deploy_stacks(cloudformation, account_id, org_unit_id, bucket):
             {'ParameterKey': 'IncludeBackupModule',             'ParameterValue': "yes"},
             {'ParameterKey': 'IncludeAWSFeedsModule',           'ParameterValue': "yes"},
             {'ParameterKey': 'IncludeHealthEventsModule',       'ParameterValue': "yes"},
+            {'ParameterKey': 'IncludeSecurityHubModule',        'ParameterValue': "yes"},
             {'ParameterKey': 'ManagementAccountID',             'ParameterValue': account_id},
             {'ParameterKey': 'ManagementAccountRole',           'ParameterValue': "Lambda-Assume-Role-Management-Account"},
             {'ParameterKey': 'MultiAccountRoleName',            'ParameterValue': "Optimization-Data-Multi-Account-Role"},