diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 7958303ac95..eeb3770e8fe 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -747,7 +747,7 @@ filebeat.modules: #------------------------------- Coredns Module ------------------------------- - module: coredns # Fileset for native deployment - log: + log: enabled: false # Set custom paths for the log files. If left empty, @@ -756,7 +756,7 @@ filebeat.modules: #----------------------------- Crowdstrike Module ----------------------------- - module: crowdstrike - + falcon: enabled: false @@ -827,7 +827,7 @@ filebeat.modules: #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy # Fileset for native deployment - log: + log: enabled: false # Set custom paths for the log files. If left empty, @@ -1488,147 +1488,147 @@ filebeat.modules: #var.password: #------------------------------ Salesforce Module ------------------------------ -# Configuration file for Salesforce module in Filebeat - -# Common Configurations: -# - enabled: Set to true to enable ingestion of Salesforce module fileset -# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s -# - api_version: API version for Salesforce, version should be greater than 46.0 - -# Authentication Configurations: -# User-Password Authentication: -# - enabled: Set to true to enable user-password authentication -# - client.id: Client ID for user-password authentication -# - client.secret: Client secret for user-password authentication -# - token_url: Token URL for user-password authentication -# - username: Username for user-password authentication -# - password: Password for user-password authentication - -# JWT Authentication: -# - enabled: Set to true to enable JWT authentication -# - client.id: Client ID for JWT authentication -# - client.username: Username for JWT authentication -# - client.key_path: Path to client key for JWT authentication -# - url: Audience URL for JWT authentication - -# Event Monitoring: -# - real_time: Set to true to enable real-time logging using object type data collection -# - real_time_interval: Interval for real-time logging - -# Event Log File: -# - event_log_file: Set to true to enable event log file type data collection -# - elf_interval: Interval for event log file -# - log_file_interval: Interval type for log file collection, either Hourly or Daily - -- module: salesforce - - apex: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - login: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - var.real_time: true - var.real_time_interval: 5m - - logout: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - var.real_time: true - var.real_time_interval: 5m - - setupaudittrail: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.real_time: true +# Configuration file for Salesforce module in Filebeat + +# Common Configurations: +# - enabled: Set to true to enable ingestion of Salesforce module fileset +# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s +# - api_version: API version for Salesforce, version should be greater than 46.0 + +# Authentication Configurations: +# User-Password Authentication: +# - enabled: Set to true to enable user-password authentication +# - client.id: Client ID for user-password authentication +# - client.secret: Client secret for user-password authentication +# - token_url: Token URL for user-password authentication +# - username: Username for user-password authentication +# - password: Password for user-password authentication + +# JWT Authentication: +# - enabled: Set to true to enable JWT authentication +# - client.id: Client ID for JWT authentication +# - client.username: Username for JWT authentication +# - client.key_path: Path to client key for JWT authentication +# - url: Audience URL for JWT authentication + +# Event Monitoring: +# - real_time: Set to true to enable real-time logging using object type data collection +# - real_time_interval: Interval for real-time logging + +# Event Log File: +# - event_log_file: Set to true to enable event log file type data collection +# - elf_interval: Interval for event log file +# - log_file_interval: Interval type for log file collection, either Hourly or Daily + +- module: salesforce + + apex: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + login: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + var.real_time: true + var.real_time_interval: 5m + + logout: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + var.real_time: true + var.real_time_interval: 5m + + setupaudittrail: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.real_time: true var.real_time_interval: 5m #----------------------------- Google Santa Module ----------------------------- - module: santa @@ -4273,7 +4273,7 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + # Configure automatic file rotation on every startup. The default is true. #rotate_on_startup: true @@ -4450,7 +4450,7 @@ setup.template.settings: # ======================== Data Stream Lifecycle (DSL) ========================= -# Configure Data Stream Lifecycle to manage data streams while connected to Serverless elasticsearch. +# Configure Data Stream Lifecycle to manage data streams while connected to Serverless elasticsearch. # These settings are mutually exclusive with ILM settings which are not supported in Serverless projects. # Enable DSL support. Valid values are true, or false. @@ -4458,7 +4458,7 @@ setup.template.settings: # Set the lifecycle policy name or pattern. For DSL, this name must match the data stream that the lifecycle is for. # The default data stream pattern is filebeat-%{[agent.version]}" -# The template string `%{[agent.version]}` will resolve to the current stack version. +# The template string `%{[agent.version]}` will resolve to the current stack version. # The other possible template value is `%{[beat.name]}`. #setup.dsl.data_stream_pattern: "filebeat-%{[agent.version]}" diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go index 0216cf1505f..bc62ed9f871 100644 --- a/x-pack/filebeat/input/awss3/config.go +++ b/x-pack/filebeat/input/awss3/config.go @@ -147,7 +147,7 @@ func (c *config) Validate() error { if c.StartTimestamp != "" { _, err := time.Parse(time.RFC3339, c.StartTimestamp) if err != nil { - return fmt.Errorf("invalid input for start_timestamp: %v", err) + return fmt.Errorf("invalid input for start_timestamp: %w", err) } } @@ -304,6 +304,7 @@ func (c config) sqsConfigModifier(o *sqs.Options) { o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled } if c.AWSConfig.Endpoint != "" { + //nolint:staticcheck // not changing through this PR o.EndpointResolver = sqs.EndpointResolverFromURL(c.AWSConfig.Endpoint) } } diff --git a/x-pack/filebeat/input/awss3/s3_filters.go b/x-pack/filebeat/input/awss3/s3_filters.go index 3647a28c800..df9fbfedd80 100644 --- a/x-pack/filebeat/input/awss3/s3_filters.go +++ b/x-pack/filebeat/input/awss3/s3_filters.go @@ -94,11 +94,7 @@ func newStartTimestampFilter(start time.Time) *startTimestampFilter { } func (s startTimestampFilter) isValid(objState state) bool { - if s.timeStart.Before(objState.LastModified) { - return true - } - - return false + return s.timeStart.Before(objState.LastModified) } func (s startTimestampFilter) getID() string { @@ -120,11 +116,7 @@ func newOldestTimeFilter(timespan time.Duration) *oldestTimeFilter { } func (s oldestTimeFilter) isValid(objState state) bool { - if s.timeOldest.Before(objState.LastModified) { - return true - } - - return false + return s.timeOldest.Before(objState.LastModified) } func (s oldestTimeFilter) getID() string { diff --git a/x-pack/filebeat/input/awss3/s3_filters_test.go b/x-pack/filebeat/input/awss3/s3_filters_test.go index 710e06170ee..096acea6f77 100644 --- a/x-pack/filebeat/input/awss3/s3_filters_test.go +++ b/x-pack/filebeat/input/awss3/s3_filters_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/elastic/elastic-agent-libs/logp" + "github.com/stretchr/testify/assert" ) @@ -29,14 +30,14 @@ func Test_filterProvider(t *testing.T) { tests := []struct { name string - cfg config + cfg *config inputState state runFilterForCount int expectFilterResults []bool }{ { name: "Simple run - all valid result", - cfg: config{ + cfg: &config{ StartTimestamp: "2024-11-26T21:00:00Z", IgnoreOlder: 10 * time.Minute, }, @@ -46,7 +47,7 @@ func Test_filterProvider(t *testing.T) { }, { name: "Simple run - all invalid result", - cfg: config{ + cfg: &config{ StartTimestamp: "2024-11-26T21:00:00Z", }, inputState: newState("bucket", "key", "eTag", time.Unix(0, 0)), @@ -55,14 +56,14 @@ func Test_filterProvider(t *testing.T) { }, { name: "Simple run - no filters hence valid result", - cfg: config{}, + cfg: &config{}, inputState: newState("bucket", "key", "eTag", time.Now()), runFilterForCount: 1, expectFilterResults: []bool{true}, }, { name: "Single filter - ignore old invalid result", - cfg: config{ + cfg: &config{ IgnoreOlder: 1 * time.Minute, }, inputState: newState("bucket", "key", "eTag", time.Unix(time.Now().Add(-2*time.Minute).Unix(), 0)), @@ -71,7 +72,7 @@ func Test_filterProvider(t *testing.T) { }, { name: "Combined filters - ignore old won't affect first run if timestamp is given but will affect thereafter", - cfg: config{ + cfg: &config{ StartTimestamp: "2024-11-26T21:00:00Z", IgnoreOlder: 10 * time.Minute, }, @@ -83,7 +84,7 @@ func Test_filterProvider(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - provider := newFilterProvider(&test.cfg) + provider := newFilterProvider(test.cfg) results := make([]bool, 0, test.runFilterForCount) for i := 0; i < test.runFilterForCount; i++ { diff --git a/x-pack/filebeat/input/awss3/s3_input.go b/x-pack/filebeat/input/awss3/s3_input.go index 9ec5398797d..31715c4d08f 100644 --- a/x-pack/filebeat/input/awss3/s3_input.go +++ b/x-pack/filebeat/input/awss3/s3_input.go @@ -8,9 +8,10 @@ import ( "context" "errors" "fmt" + "sync" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/ratelimit" - "sync" "github.com/elastic/beats/v7/filebeat/beater" v2 "github.com/elastic/beats/v7/filebeat/input/v2" diff --git a/x-pack/filebeat/input/awss3/s3_test.go b/x-pack/filebeat/input/awss3/s3_test.go index 2f79cb44a48..5518a1808e1 100644 --- a/x-pack/filebeat/input/awss3/s3_test.go +++ b/x-pack/filebeat/input/awss3/s3_test.go @@ -9,13 +9,14 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - - "github.com/elastic/elastic-agent-libs/logp" ) func TestS3Poller(t *testing.T) { @@ -130,21 +131,24 @@ func TestS3Poller(t *testing.T) { s3ObjProc := newS3ObjectProcessorFactory(nil, mockAPI, nil, backupConfig{}) states, err := newStates(nil, store, listPrefix) require.NoError(t, err, "states creation must succeed") + + cfg := config{ + NumberOfWorkers: numberOfWorkers, + BucketListInterval: pollInterval, + BucketARN: bucket, + BucketListPrefix: listPrefix, + RegionName: "region", + } poller := &s3PollerInput{ - log: logp.NewLogger(inputName), - config: config{ - NumberOfWorkers: numberOfWorkers, - BucketListInterval: pollInterval, - BucketARN: bucket, - BucketListPrefix: listPrefix, - RegionName: "region", - }, + log: logp.NewLogger(inputName), + config: cfg, s3: mockAPI, pipeline: pipeline, s3ObjectHandler: s3ObjProc, states: states, provider: "provider", metrics: newInputMetrics("", nil, 0), + filterProvider: newFilterProvider(&cfg), } poller.runPoll(ctx) }) @@ -268,6 +272,14 @@ func TestS3Poller(t *testing.T) { s3ObjProc := newS3ObjectProcessorFactory(nil, mockS3, nil, backupConfig{}) states, err := newStates(nil, store, listPrefix) require.NoError(t, err, "states creation must succeed") + + cfg := config{ + NumberOfWorkers: numberOfWorkers, + BucketListInterval: pollInterval, + BucketARN: bucket, + BucketListPrefix: "key", + RegionName: "region", + } poller := &s3PollerInput{ log: logp.NewLogger(inputName), config: config{ @@ -283,15 +295,254 @@ func TestS3Poller(t *testing.T) { states: states, provider: "provider", metrics: newInputMetrics("", nil, 0), + filterProvider: newFilterProvider(&cfg), } poller.run(ctx) }) } -func TestS3ReaderLoop(t *testing.T) { - -} - -func TestS3WorkerLoop(t *testing.T) { - +func Test_S3StateHandling(t *testing.T) { + bucket := "bucket" + logger := logp.NewLogger(inputName) + fixedTimeNow := time.Now() + + tests := []struct { + name string + s3Objects []types.Object + config *config + initStates []state + runPollFor int + expectStateIDs []string + }{ + { + name: "State unchanged - registry backed state", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(time.Unix(1732622400, 0)), // 2024-11-26T12:00:00Z + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + }, + initStates: []state{newState(bucket, "obj-A", "etag-A", time.Unix(1732622400, 0))}, // 2024-11-26T12:00:00Z + runPollFor: 1, + expectStateIDs: []string{stateID(bucket, "obj-A", "etag-A", time.Unix(1732622400, 0))}, // 2024-11-26T12:00:00Z + }, + { + name: "State cleanup - remove existing registry entry based on ignore older filter", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(time.Unix(1732622400, 0)), // 2024-11-26T12:00:00Z + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + IgnoreOlder: 1 * time.Second, + }, + initStates: []state{newState(bucket, "obj-A", "etag-A", time.Unix(1732622400, 0))}, // 2024-11-26T12:00:00Z + runPollFor: 1, + expectStateIDs: []string{}, + }, + { + name: "State cleanup - remove existing registry entry based on timestamp filter", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(time.Unix(1732622400, 0)), // 2024-11-26T12:00:00Z + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + StartTimestamp: "2024-11-27T12:00:00Z", + }, + initStates: []state{newState(bucket, "obj-A", "etag-A", time.Unix(1732622400, 0))}, // 2024-11-26T12:00:00Z + runPollFor: 1, + expectStateIDs: []string{}, + }, + { + name: "State updated - no filters", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(time.Unix(1732622400, 0)), // 2024-11-26T12:00:00Z + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + }, + runPollFor: 1, + expectStateIDs: []string{stateID(bucket, "obj-A", "etag-A", time.Unix(1732622400, 0))}, // 2024-11-26T12:00:00Z + }, + { + name: "State updated - ignore old filter", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(fixedTimeNow), + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + IgnoreOlder: 1 * time.Hour, + }, + runPollFor: 1, + expectStateIDs: []string{stateID(bucket, "obj-A", "etag-A", fixedTimeNow)}, + }, + { + name: "State updated - timestamp filter", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(fixedTimeNow), + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + StartTimestamp: "2024-11-26T12:00:00Z", + }, + runPollFor: 1, + expectStateIDs: []string{stateID(bucket, "obj-A", "etag-A", fixedTimeNow)}, + }, + { + name: "State updated - combined filters of ignore old and timestamp entry exist after first run", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(time.Unix(1732622400, 0)), // 2024-11-26T12:00:00Z + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + IgnoreOlder: 1 * time.Hour, + StartTimestamp: "2024-11-20T12:00:00Z", + }, + // run once to validate initial coverage of entries till start timestamp + runPollFor: 1, + expectStateIDs: []string{stateID(bucket, "obj-A", "etag-A", time.Unix(1732622400, 0))}, // 2024-11-26T12:00:00Z + }, + { + name: "State updated - combined filters of ignore old and timestamp remove entry after second run", + s3Objects: []types.Object{ + { + Key: aws.String("obj-A"), + ETag: aws.String("etag-A"), + LastModified: aws.Time(time.Unix(1732622400, 0)), // 2024-11-26T12:00:00Z + }, + }, + config: &config{ + NumberOfWorkers: 1, + BucketListInterval: 1 * time.Second, + BucketARN: bucket, + IgnoreOlder: 1 * time.Hour, + StartTimestamp: "2024-11-20T12:00:00Z", + }, + // run twice to validate removal by ignore old filter + runPollFor: 2, + expectStateIDs: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // given - setup and mocks + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + ctrl, ctx := gomock.WithContext(ctx, t) + defer ctrl.Finish() + + mockS3API := NewMockS3API(ctrl) + mockS3Pager := NewMockS3Pager(ctrl) + mockObjHandler := NewMockS3ObjectHandlerFactory(ctrl) + mockS3ObjectHandler := NewMockS3ObjectHandler(ctrl) + + gomock.InOrder( + mockS3API.EXPECT(). + ListObjectsPaginator(gomock.Eq(bucket), ""). + AnyTimes(). + DoAndReturn(func(_, _ string) s3Pager { + return mockS3Pager + }), + ) + + for i := 0; i < test.runPollFor; i++ { + mockS3Pager.EXPECT().HasMorePages().Times(1).DoAndReturn(func() bool { return true }) + mockS3Pager.EXPECT().HasMorePages().Times(1).DoAndReturn(func() bool { return false }) + } + + mockS3Pager.EXPECT(). + NextPage(gomock.Any()). + Times(test.runPollFor). + DoAndReturn(func(_ context.Context, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + return &s3.ListObjectsV2Output{Contents: test.s3Objects}, nil + }) + + mockObjHandler.EXPECT().Create(gomock.Any(), gomock.Any()).AnyTimes().Return(mockS3ObjectHandler) + mockS3ObjectHandler.EXPECT().ProcessS3Object(gomock.Any(), gomock.Any()).AnyTimes(). + DoAndReturn(func(log *logp.Logger, eventCallback func(e beat.Event)) error { + eventCallback(beat.Event{}) + return nil + }) + + store := openTestStatestore() + s3States, err := newStates(logger, store, "") + require.NoError(t, err, "States creation must succeed") + + // Note - add init states as if we are deriving them from registry + for _, st := range test.initStates { + err := s3States.AddState(st) + require.NoError(t, err, "State add should not error") + } + + poller := &s3PollerInput{ + log: logger, + config: *test.config, + s3: mockS3API, + pipeline: newFakePipeline(), + s3ObjectHandler: mockObjHandler, + states: s3States, + metrics: newInputMetrics("state-test: "+test.name, nil, 0), + filterProvider: newFilterProvider(test.config), + } + + // when - run polling for desired time + for i := 0; i < test.runPollFor; i++ { + poller.runPoll(ctx) + <-time.After(500 * time.Millisecond) + } + + // then - desired state entries + + // state must only contain expected state IDs + require.Equal(t, len(test.expectStateIDs), len(s3States.states)) + for _, id := range test.expectStateIDs { + if s3States.states[id] == nil { + t.Errorf("state with ID %s should exist", id) + } + } + }) + } }