add component test for filtering and fix lint issues

Signed-off-by: Kavindu Dodanduwa <[email protected]> # Conflicts: # x-pack/filebeat/input/awss3/s3_test.go
elastic · Dec 31, 2024 · e81eb64 · e81eb64
1 parent cbdb70f
commit e81eb64
Show file tree

Hide file tree

Showing 6 changed files with 428 additions and 182 deletions.
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -747,7 +747,7 @@ filebeat.modules:
 #------------------------------- Coredns Module -------------------------------
 - module: coredns
   # Fileset for native deployment
-  log:
+  log: 
     enabled: false
 
     # Set custom paths for the log files. If left empty,
@@ -756,7 +756,7 @@ filebeat.modules:
 
 #----------------------------- Crowdstrike Module -----------------------------
 - module: crowdstrike
-
+  
   falcon:
     enabled: false
 
@@ -827,7 +827,7 @@ filebeat.modules:
 #------------------------------ Envoyproxy Module ------------------------------
 - module: envoyproxy
   # Fileset for native deployment
-  log:
+  log: 
     enabled: false
 
     # Set custom paths for the log files. If left empty,
@@ -1488,147 +1488,147 @@ filebeat.modules:
     #var.password:
 
 #------------------------------ Salesforce Module ------------------------------
-# Configuration file for Salesforce module in Filebeat
-
-# Common Configurations:
-# - enabled: Set to true to enable ingestion of Salesforce module fileset
-# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s
-# - api_version: API version for Salesforce, version should be greater than 46.0
-
-# Authentication Configurations:
-# User-Password Authentication:
-# - enabled: Set to true to enable user-password authentication
-# - client.id: Client ID for user-password authentication
-# - client.secret: Client secret for user-password authentication
-# - token_url: Token URL for user-password authentication
-# - username: Username for user-password authentication
-# - password: Password for user-password authentication
-
-# JWT Authentication:
-# - enabled: Set to true to enable JWT authentication
-# - client.id: Client ID for JWT authentication
-# - client.username: Username for JWT authentication
-# - client.key_path: Path to client key for JWT authentication
-# - url: Audience URL for JWT authentication
-
-# Event Monitoring:
-# - real_time: Set to true to enable real-time logging using object type data collection
-# - real_time_interval: Interval for real-time logging
-
-# Event Log File:
-# - event_log_file: Set to true to enable event log file type data collection
-# - elf_interval: Interval for event log file
-# - log_file_interval: Interval type for log file collection, either Hourly or Daily
-
-- module: salesforce
-
-  apex:
-    enabled: false
-    var.initial_interval: 1d
-    var.api_version: 56
-
-    var.authentication:
-      user_password_flow:
-        enabled: true
-        client.id: "<YourClientIdHere>"
-        client.secret: "<YourClientSecretHere>"
-        token_url: "<YourTokenURLHere>"
-        username: "<YourUsernameHere>"
-        password: "<YourPasswordHere>"
-      jwt_bearer_flow:
-        enabled: false
-        client.id: "<YourClientIdHere>"
-        client.username: "<YourClientUsernameHere>"
-        client.key_path: "<YourClientKeyPathHere>"
-        url: "https://login.salesforce.com"
-
-    var.url: "https://instance_id.my.salesforce.com"
-
-    var.event_log_file: true
-    var.elf_interval: 1h
-    var.log_file_interval: "Hourly"
-
-  login:
-    enabled: false
-    var.initial_interval: 1d
-    var.api_version: 56
-
-    var.authentication:
-      user_password_flow:
-        enabled: true
-        client.id: "<YourClientIdHere>"
-        client.secret: "client-secret"
-        token_url: "<YourTokenURLHere>"
-        username: "<YourUsernameHere>"
-        password: "<YourPasswordHere>"
-      jwt_bearer_flow:
-        enabled: false
-        client.id: "<YourClientIdHere>"
-        client.username: "<YourClientUsernameHere>"
-        client.key_path: "<YourClientKeyPathHere>"
-        url: "https://login.salesforce.com"
-
-    var.url: "https://instance_id.my.salesforce.com"
-
-    var.event_log_file: true
-    var.elf_interval: 1h
-    var.log_file_interval: "Hourly"
-
-    var.real_time: true
-    var.real_time_interval: 5m
-
-  logout:
-    enabled: false
-    var.initial_interval: 1d
-    var.api_version: 56
-
-    var.authentication:
-      user_password_flow:
-        enabled: true
-        client.id: "<YourClientIdHere>"
-        client.secret: "client-secret"
-        token_url: "<YourTokenURLHere>"
-        username: "<YourUsernameHere>"
-        password: "<YourPasswordHere>"
-      jwt_bearer_flow:
-        enabled: false
-        client.id: "<YourClientIdHere>"
-        client.username: "<YourClientUsernameHere>"
-        client.key_path: "<YourClientKeyPathHere>"
-        url: "https://login.salesforce.com"
-
-    var.url: "https://instance_id.my.salesforce.com"
-
-    var.event_log_file: true
-    var.elf_interval: 1h
-    var.log_file_interval: "Hourly"
-
-    var.real_time: true
-    var.real_time_interval: 5m
-
-  setupaudittrail:
-    enabled: false
-    var.initial_interval: 1d
-    var.api_version: 56
-
-    var.authentication:
-      user_password_flow:
-        enabled: true
-        client.id: "<YourClientIdHere>"
-        client.secret: "client-secret"
-        token_url: "<YourTokenURLHere>"
-        username: "<YourUsernameHere>"
-        password: "<YourPasswordHere>"
-      jwt_bearer_flow:
-        enabled: false
-        client.id: "<YourClientIdHere>"
-        client.username: "<YourClientUsernameHere>"
-        client.key_path: "<YourClientKeyPathHere>"
-        url: "https://login.salesforce.com"
-
-    var.url: "https://instance_id.my.salesforce.com"
-
-    var.real_time: true
+# Configuration file for Salesforce module in Filebeat
+
+# Common Configurations:
+# - enabled: Set to true to enable ingestion of Salesforce module fileset
+# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s
+# - api_version: API version for Salesforce, version should be greater than 46.0
+
+# Authentication Configurations:
+# User-Password Authentication:
+# - enabled: Set to true to enable user-password authentication
+# - client.id: Client ID for user-password authentication
+# - client.secret: Client secret for user-password authentication
+# - token_url: Token URL for user-password authentication
+# - username: Username for user-password authentication
+# - password: Password for user-password authentication
+
+# JWT Authentication:
+# - enabled: Set to true to enable JWT authentication
+# - client.id: Client ID for JWT authentication
+# - client.username: Username for JWT authentication
+# - client.key_path: Path to client key for JWT authentication
+# - url: Audience URL for JWT authentication
+
+# Event Monitoring:
+# - real_time: Set to true to enable real-time logging using object type data collection
+# - real_time_interval: Interval for real-time logging
+
+# Event Log File:
+# - event_log_file: Set to true to enable event log file type data collection
+# - elf_interval: Interval for event log file
+# - log_file_interval: Interval type for log file collection, either Hourly or Daily
+
+- module: salesforce
+
+  apex:
+    enabled: false
+    var.initial_interval: 1d
+    var.api_version: 56
+
+    var.authentication:
+      user_password_flow:
+        enabled: true
+        client.id: "<YourClientIdHere>"
+        client.secret: "<YourClientSecretHere>"
+        token_url: "<YourTokenURLHere>"
+        username: "<YourUsernameHere>"
+        password: "<YourPasswordHere>"
+      jwt_bearer_flow:
+        enabled: false
+        client.id: "<YourClientIdHere>"
+        client.username: "<YourClientUsernameHere>"
+        client.key_path: "<YourClientKeyPathHere>"
+        url: "https://login.salesforce.com"
+
+    var.url: "https://instance_id.my.salesforce.com"
+
+    var.event_log_file: true
+    var.elf_interval: 1h
+    var.log_file_interval: "Hourly"
+
+  login:
+    enabled: false
+    var.initial_interval: 1d
+    var.api_version: 56
+
+    var.authentication:
+      user_password_flow:
+        enabled: true
+        client.id: "<YourClientIdHere>"
+        client.secret: "client-secret"
+        token_url: "<YourTokenURLHere>"
+        username: "<YourUsernameHere>"
+        password: "<YourPasswordHere>"
+      jwt_bearer_flow:
+        enabled: false
+        client.id: "<YourClientIdHere>"
+        client.username: "<YourClientUsernameHere>"
+        client.key_path: "<YourClientKeyPathHere>"
+        url: "https://login.salesforce.com"
+
+    var.url: "https://instance_id.my.salesforce.com"
+
+    var.event_log_file: true
+    var.elf_interval: 1h
+    var.log_file_interval: "Hourly"
+
+    var.real_time: true
+    var.real_time_interval: 5m
+
+  logout:
+    enabled: false
+    var.initial_interval: 1d
+    var.api_version: 56
+
+    var.authentication:
+      user_password_flow:
+        enabled: true
+        client.id: "<YourClientIdHere>"
+        client.secret: "client-secret"
+        token_url: "<YourTokenURLHere>"
+        username: "<YourUsernameHere>"
+        password: "<YourPasswordHere>"
+      jwt_bearer_flow:
+        enabled: false
+        client.id: "<YourClientIdHere>"
+        client.username: "<YourClientUsernameHere>"
+        client.key_path: "<YourClientKeyPathHere>"
+        url: "https://login.salesforce.com"
+
+    var.url: "https://instance_id.my.salesforce.com"
+
+    var.event_log_file: true
+    var.elf_interval: 1h
+    var.log_file_interval: "Hourly"
+
+    var.real_time: true
+    var.real_time_interval: 5m
+
+  setupaudittrail:
+    enabled: false
+    var.initial_interval: 1d
+    var.api_version: 56
+
+    var.authentication:
+      user_password_flow:
+        enabled: true
+        client.id: "<YourClientIdHere>"
+        client.secret: "client-secret"
+        token_url: "<YourTokenURLHere>"
+        username: "<YourUsernameHere>"
+        password: "<YourPasswordHere>"
+      jwt_bearer_flow:
+        enabled: false
+        client.id: "<YourClientIdHere>"
+        client.username: "<YourClientUsernameHere>"
+        client.key_path: "<YourClientKeyPathHere>"
+        url: "https://login.salesforce.com"
+
+    var.url: "https://instance_id.my.salesforce.com"
+
+    var.real_time: true
     var.real_time_interval: 5m
 #----------------------------- Google Santa Module -----------------------------
 - module: santa
@@ -4273,7 +4273,7 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-
+  
   # Configure automatic file rotation on every startup. The default is true.
   #rotate_on_startup: true
 
@@ -4450,15 +4450,15 @@ setup.template.settings:
 
 # ======================== Data Stream Lifecycle (DSL) =========================
 
-# Configure Data Stream Lifecycle to manage data streams while connected to Serverless elasticsearch.
+# Configure Data Stream Lifecycle to manage data streams while connected to Serverless elasticsearch. 
 # These settings are mutually exclusive with ILM settings which are not supported in Serverless projects.
 
 # Enable DSL support. Valid values are true, or false.
 #setup.dsl.enabled: true
 
 # Set the lifecycle policy name or pattern. For DSL, this name must match the data stream that the lifecycle is for.
 # The default data stream pattern is filebeat-%{[agent.version]}"
-# The template string `%{[agent.version]}` will resolve to the current stack version.
+# The template string `%{[agent.version]}` will resolve to the current stack version. 
 # The other possible template value is `%{[beat.name]}`.
 #setup.dsl.data_stream_pattern: "filebeat-%{[agent.version]}"
 

diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go
@@ -147,7 +147,7 @@ func (c *config) Validate() error {
 	if c.StartTimestamp != "" {
 		_, err := time.Parse(time.RFC3339, c.StartTimestamp)
 		if err != nil {
-			return fmt.Errorf("invalid input for start_timestamp: %v", err)
+			return fmt.Errorf("invalid input for start_timestamp: %w", err)
 		}
 	}
 
@@ -304,6 +304,7 @@ func (c config) sqsConfigModifier(o *sqs.Options) {
 		o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled
 	}
 	if c.AWSConfig.Endpoint != "" {
+		//nolint:staticcheck // not changing through this PR
 		o.EndpointResolver = sqs.EndpointResolverFromURL(c.AWSConfig.Endpoint)
 	}
 }

diff --git a/x-pack/filebeat/input/awss3/s3_filters.go b/x-pack/filebeat/input/awss3/s3_filters.go
@@ -94,11 +94,7 @@ func newStartTimestampFilter(start time.Time) *startTimestampFilter {
 }
 
 func (s startTimestampFilter) isValid(objState state) bool {
-	if s.timeStart.Before(objState.LastModified) {
-		return true
-	}
-
-	return false
+	return s.timeStart.Before(objState.LastModified)
 }
 
 func (s startTimestampFilter) getID() string {
@@ -120,11 +116,7 @@ func newOldestTimeFilter(timespan time.Duration) *oldestTimeFilter {
 }
 
 func (s oldestTimeFilter) isValid(objState state) bool {
-	if s.timeOldest.Before(objState.LastModified) {
-		return true
-	}
-
-	return false
+	return s.timeOldest.Before(objState.LastModified)
 }
 
 func (s oldestTimeFilter) getID() string {