diff --git a/.gitattributes b/.gitattributes index 525ad2883d2cdc..05a7c47f6c7a69 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ website/pages/docs/plugins/sources/**/tables.md linguist-generated +**/schema.json linguist-generated diff --git a/plugins/source/aws/Makefile b/plugins/source/aws/Makefile index 7a75bc6bbe22bb..d392bc530821d5 100644 --- a/plugins/source/aws/Makefile +++ b/plugins/source/aws/Makefile @@ -46,6 +46,10 @@ gen-docs: build sed -i.bak -e 's_(\(.*\).md)_(\1)_' ../../../website/tables/aws/*.md rm -rf ../../../website/tables/aws/*.bak +.PHONY: gen-spec-schema +gen-spec-schema: + go run client/spec/gen/main.go + # All gen targets .PHONY: gen -gen: gen-mocks gen-docs +gen: gen-spec-schema gen-mocks gen-docs diff --git a/plugins/source/aws/client/account.go b/plugins/source/aws/client/account.go index 9cd7ccf4db89a9..bd45cdfa433cee 100644 --- a/plugins/source/aws/client/account.go +++ b/plugins/source/aws/client/account.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/smithy-go" "github.com/cloudquery/cloudquery/plugins/source/aws/client/services" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/rs/zerolog" ) @@ -21,7 +22,7 @@ type svcsDetail struct { svcs Services } -func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec, adminAccountSts AssumeRoleAPIClient, account Account) (*svcsDetail, error) { +func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, adminAccountSts AssumeRoleAPIClient, account spec.Account) (*svcsDetail, error) { if account.AccountName == "" { account.AccountName = account.ID } @@ -46,7 +47,7 @@ func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, aws awsCfg, err := ConfigureAwsSDK(ctx, logger, awsPluginSpec, account, adminAccountSts) if err != nil { warningMsg := logger.Warn().Str("account", account.AccountName).Err(err) - if account.source == "org" { + if account.Source == spec.AccountSourceOrg { warningMsg.Msg("Unable to assume role in account") return nil, nil } diff --git a/plugins/source/aws/client/aws_sdk.go b/plugins/source/aws/client/aws_sdk.go index a44d5d6b6af254..1aaaf5b640697e 100644 --- a/plugins/source/aws/client/aws_sdk.go +++ b/plugins/source/aws/client/aws_sdk.go @@ -9,29 +9,24 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/rs/zerolog" ) -func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec, account Account, stsClient AssumeRoleAPIClient) (aws.Config, error) { +func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, account spec.Account, stsClient AssumeRoleAPIClient) (aws.Config, error) { var err error var awsCfg aws.Config - maxAttempts := 10 - if awsPluginSpec.MaxRetries != nil { - maxAttempts = *awsPluginSpec.MaxRetries - } - maxBackoff := 30 - if awsPluginSpec.MaxBackoff != nil { - maxBackoff = *awsPluginSpec.MaxBackoff - } + // This sets MaxRetries & MaxBackoff, too + awsPluginSpec.SetDefaults() configFns := []func(*config.LoadOptions) error{ config.WithDefaultRegion(defaultRegion), // https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/retries-timeouts/ config.WithRetryer(func() aws.Retryer { return retry.NewStandard(func(so *retry.StandardOptions) { - so.MaxAttempts = maxAttempts - so.MaxBackoff = time.Duration(maxBackoff) * time.Second + so.MaxAttempts = *awsPluginSpec.MaxRetries + so.MaxBackoff = time.Duration(*awsPluginSpec.MaxBackoff) * time.Second so.RateLimiter = &NoRateLimiter{} }) }), diff --git a/plugins/source/aws/client/client.go b/plugins/source/aws/client/client.go index 83831c50a69b13..8e405bf5d55869 100644 --- a/plugins/source/aws/client/client.go +++ b/plugins/source/aws/client/client.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types" "github.com/aws/smithy-go/logging" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/state" "github.com/rs/zerolog" @@ -32,7 +33,7 @@ type Client struct { LanguageCode string Backend state.Client specificRegions bool - Spec *Spec + Spec *spec.Spec accountMutex map[string]*sync.Mutex } @@ -88,13 +89,13 @@ func (s *ServicesManager) InitServicesForPartitionAccount(partition, accountId s s.services[partition][accountId].Regions = funk.UniqString(append(s.services[partition][accountId].Regions, svcs.Regions...)) } -func NewAwsClient(logger zerolog.Logger, spec *Spec) Client { +func NewAwsClient(logger zerolog.Logger, s *spec.Spec) Client { return Client{ ServicesManager: &ServicesManager{ services: ServicesPartitionAccountMap{}, }, logger: logger, - Spec: spec, + Spec: s, accountMutex: map[string]*sync.Mutex{}, } } @@ -200,14 +201,14 @@ func (c *Client) withLanguageCode(code string) *Client { } // Configure is the entrypoint into configuring the AWS plugin. It is called by the plugin initialization in resources/plugin/aws.go -func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.ClientMeta, error) { - if err := spec.Validate(); err != nil { +func Configure(ctx context.Context, logger zerolog.Logger, s spec.Spec) (schema.ClientMeta, error) { + if err := s.Validate(); err != nil { return nil, fmt.Errorf("spec validation failed: %w", err) } - spec.SetDefaults() + s.SetDefaults() - if spec.TableOptions != nil { - structVal := reflect.ValueOf(*spec.TableOptions) + if s.TableOptions != nil { + structVal := reflect.ValueOf(*s.TableOptions) fieldNum := structVal.NumField() for i := 0; i < fieldNum; i++ { field := structVal.Field(i) @@ -218,7 +219,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.Cl } } - client := NewAwsClient(logger, &spec) + client := NewAwsClient(logger, &s) var adminAccountSts AssumeRoleAPIClient @@ -231,11 +232,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.Cl } } if len(client.Spec.Accounts) == 0 { - client.Spec.Accounts = []Account{ - { - ID: defaultVar, - }, - } + client.Spec.Accounts = []spec.Account{{ID: defaultVar}} } initLock := sync.Mutex{} diff --git a/plugins/source/aws/client/organizations.go b/plugins/source/aws/client/organizations.go index d4c2f4e8dbebd6..7d3e7082833833 100644 --- a/plugins/source/aws/client/organizations.go +++ b/plugins/source/aws/client/organizations.go @@ -4,6 +4,7 @@ import ( "context" "github.com/cloudquery/cloudquery/plugins/source/aws/client/services" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/thoas/go-funk" "github.com/aws/aws-sdk-go-v2/aws" @@ -15,10 +16,10 @@ import ( ) // Parses org configuration and grabs the appropriate accounts -func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec) ([]Account, AssumeRoleAPIClient, error) { +func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec) ([]spec.Account, AssumeRoleAPIClient, error) { // If user doesn't specify any configs for admin account instantiate default values if awsPluginSpec.Organization.AdminAccount == nil { - awsPluginSpec.Organization.AdminAccount = &Account{ + awsPluginSpec.Organization.AdminAccount = &spec.Account{ AccountName: "Default-Admin-Account", LocalProfile: "", } @@ -46,7 +47,7 @@ func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec * } // Load accounts from the appropriate endpoint as well as normalizing response -func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services.OrganizationsClient, region string) ([]Account, error) { +func loadAccounts(ctx context.Context, awsPluginSpec *spec.Spec, accountsApi services.OrganizationsClient, region string) ([]spec.Account, error) { var rawAccounts []orgTypes.Account var err error if len(awsPluginSpec.Organization.OrganizationUnits) > 0 { @@ -56,10 +57,10 @@ func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services } if err != nil { - return []Account{}, err + return []spec.Account{}, err } seen := map[string]struct{}{} - accounts := make([]Account, 0) + accounts := make([]spec.Account, 0) for _, account := range rawAccounts { // Only load Active accounts if account.Status != orgTypes.AccountStatusActive || account.Id == nil { @@ -83,21 +84,21 @@ func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services roleArn.Partition = parsed.Partition } - accounts = append(accounts, Account{ + accounts = append(accounts, spec.Account{ ID: *account.Id, RoleARN: roleArn.String(), RoleSessionName: awsPluginSpec.Organization.ChildAccountRoleSessionName, ExternalID: awsPluginSpec.Organization.ChildAccountExternalID, LocalProfile: awsPluginSpec.Organization.AdminAccount.LocalProfile, Regions: awsPluginSpec.Organization.ChildAccountRegions, - source: "org", + Source: spec.AccountSourceOrg, }) } return accounts, err } // Get Accounts for specific Organizational Units -func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *AwsOrg, region string) ([]orgTypes.Account, error) { +func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *spec.Org, region string) ([]orgTypes.Account, error) { q := awsOrg.OrganizationUnits var ou string var rawAccounts []orgTypes.Account @@ -158,7 +159,7 @@ func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient } // Get All accounts in a specific organization -func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *AwsOrg, region string) ([]orgTypes.Account, error) { +func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *spec.Org, region string) ([]orgTypes.Account, error) { var rawAccounts []orgTypes.Account accountsPaginator := organizations.NewListAccountsPaginator(accountsApi, &organizations.ListAccountsInput{}) for accountsPaginator.HasMorePages() { diff --git a/plugins/source/aws/client/organizations_test.go b/plugins/source/aws/client/organizations_test.go index df160a958075d6..97ad492a193eb9 100644 --- a/plugins/source/aws/client/organizations_test.go +++ b/plugins/source/aws/client/organizations_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/golang/mock/gomock" "github.com/aws/aws-sdk-go-v2/aws" @@ -111,24 +112,24 @@ func Test_loadAccounts(t *testing.T) { ctx := context.Background() tests := []struct { name string - spec *Spec + spec *spec.Spec want []string wantErr error }{ { name: "all_accounts", - spec: &Spec{ - Organization: &AwsOrg{ - AdminAccount: &Account{}, + spec: &spec.Spec{ + Organization: &spec.Org{ + AdminAccount: &spec.Account{}, }, }, want: []string{"id-child1-account", "id-child2-account", "id-parent1-account", "id-parent2-account", "id-top-level-account"}, }, { name: "all_accounts_with_skip_member_accounts", - spec: &Spec{ - Organization: &AwsOrg{ - AdminAccount: &Account{}, + spec: &spec.Spec{ + Organization: &spec.Org{ + AdminAccount: &spec.Account{}, SkipMemberAccounts: []string{"id-child2-account", "id-parent1-account", "id-parent2-account", "id-top-level-account"}, }, }, @@ -136,74 +137,74 @@ func Test_loadAccounts(t *testing.T) { }, { name: "org_root", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"root"}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-top-level-account", "id-child1-account", "id-parent1-account", "id-child2-account", "id-parent2-account"}, }, { name: "ou_parent1", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"ou-parent1"}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-parent1-account", "id-child1-account"}, }, { name: "ou_parent1_and_parent2", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"ou-parent1", "ou-parent2"}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-parent1-account", "id-child1-account", "id-parent2-account", "id-child2-account"}, }, { name: "ou_parent1_skip_child1", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"ou-parent1"}, SkipMemberAccounts: []string{"id-child1-account"}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-parent1-account"}, }, { name: "ou_root_skip_parent1", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"root"}, SkipOrganizationalUnits: []string{"ou-parent1"}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-top-level-account", "id-parent2-account", "id-child2-account"}, }, { name: "ou_root_skip_parent1", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"root"}, SkipOrganizationalUnits: []string{"ou-parent1"}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-top-level-account", "id-parent2-account", "id-child2-account"}, }, { name: "ou_root_and_parent1", - spec: &Spec{ - Organization: &AwsOrg{ + spec: &spec.Spec{ + Organization: &spec.Org{ OrganizationUnits: []string{"root", "ou-parent1"}, SkipOrganizationalUnits: []string{}, - AdminAccount: &Account{}, + AdminAccount: &spec.Account{}, }, }, want: []string{"id-top-level-account", "id-parent1-account", "id-child1-account", "id-parent2-account", "id-child2-account"}, diff --git a/plugins/source/aws/client/spec.go b/plugins/source/aws/client/spec.go deleted file mode 100644 index 1f86108c8d4473..00000000000000 --- a/plugins/source/aws/client/spec.go +++ /dev/null @@ -1,133 +0,0 @@ -package client - -import ( - "errors" - "fmt" - "regexp" - "time" - - "github.com/aws/aws-sdk-go-v2/aws/arn" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" - "github.com/cloudquery/plugin-sdk/v4/scheduler" -) - -const ( - defaultMaxConcurrency = 50000 -) - -type Account struct { - ID string `json:"id"` - AccountName string `json:"account_name,omitempty"` - LocalProfile string `json:"local_profile,omitempty"` - RoleARN string `json:"role_arn,omitempty"` - RoleSessionName string `json:"role_session_name,omitempty"` - ExternalID string `json:"external_id,omitempty"` - DefaultRegion string `json:"default_region,omitempty"` - Regions []string `json:"regions,omitempty"` - source string -} - -type AwsOrg struct { - OrganizationUnits []string `json:"organization_units,omitempty"` - SkipMemberAccounts []string `json:"skip_member_accounts,omitempty"` - SkipOrganizationalUnits []string `json:"skip_organization_units,omitempty"` - AdminAccount *Account `json:"admin_account"` - MemberCredentials *Account `json:"member_trusted_principal"` - ChildAccountRoleName string `json:"member_role_name,omitempty"` - ChildAccountRoleSessionName string `json:"member_role_session_name,omitempty"` - ChildAccountExternalID string `json:"member_external_id,omitempty"` - ChildAccountRegions []string `json:"member_regions,omitempty"` -} - -type EventBasedSync struct { - FullSync *bool `json:"full_sync,omitempty"` - Account Account `json:"account"` - KinesisStreamARN string `json:"kinesis_stream_arn"` - StartTime *time.Time `json:"start_time,omitempty"` -} -type Spec struct { - Regions []string `json:"regions,omitempty"` - Accounts []Account `json:"accounts"` - Organization *AwsOrg `json:"org"` - AWSDebug bool `json:"aws_debug,omitempty"` - MaxRetries *int `json:"max_retries,omitempty"` - MaxBackoff *int `json:"max_backoff,omitempty"` - EndpointURL string `json:"custom_endpoint_url,omitempty"` - HostnameImmutable *bool `json:"custom_endpoint_hostname_immutable,omitempty"` - PartitionID string `json:"custom_endpoint_partition_id,omitempty"` - SigningRegion string `json:"custom_endpoint_signing_region,omitempty"` - InitializationConcurrency int `json:"initialization_concurrency"` - UsePaidAPIs bool `json:"use_paid_apis"` - TableOptions *tableoptions.TableOptions `json:"table_options,omitempty"` - Concurrency int `json:"concurrency"` - EventBasedSync *EventBasedSync `json:"event_based_sync,omitempty"` - Scheduler scheduler.Strategy `json:"scheduler,omitempty"` -} - -func (s *Spec) Validate() error { - if s.EndpointURL != "" { - if s.PartitionID == "" { - return fmt.Errorf("custom_endpoint_partition_id is required when custom_endpoint_url is set") - } - if s.SigningRegion == "" { - return fmt.Errorf("custom_endpoint_signing_region is required when custom_endpoint_url is set") - } - if s.HostnameImmutable == nil { - return fmt.Errorf("custom_endpoint_hostname_immutable is required when custom_endpoint_url is set") - } - } - - if s.Organization != nil && len(s.Accounts) > 0 { - return errors.New("specifying accounts via both the Accounts and Org properties is not supported. To achieve both, use multiple source configurations") - } - if s.Organization != nil { - if s.Organization.ChildAccountRoleName == "" { - return fmt.Errorf("member_role_name is required when using org configuration") - } - if err := validateOUs(s.Organization.OrganizationUnits); err != nil { - return fmt.Errorf("invalid organization_units: %w", err) - } - if err := validateOUs(s.Organization.SkipOrganizationalUnits); err != nil { - return fmt.Errorf("invalid skip_organization_units: %w", err) - } - } - if s.TableOptions != nil { - if err := s.TableOptions.Validate(); err != nil { - return fmt.Errorf("invalid table_options: %w", err) - } - } - - if s.EventBasedSync != nil { - _, err := arn.Parse(s.EventBasedSync.KinesisStreamARN) - if err != nil { - return fmt.Errorf("failed to parse kinesis arn (%s): %w", s.EventBasedSync.KinesisStreamARN, err) - } - } - return nil -} - -func validateOUs(ous []string) error { - r := regexp.MustCompile(`^((ou\-[0-9a-z]{4,32}\-[a-z0-9]{8,32})|(r\-[0-9a-z]{4,32}))$`) - for _, ou := range ous { - if !r.MatchString(ou) { - return fmt.Errorf(`invalid OU: %s (should match "ou-*-*" or "r-*" with lowercase letters or digits)`, ou) - } - } - return nil -} - -func (s *Spec) SetDefaults() { - if s.InitializationConcurrency <= 0 { - s.InitializationConcurrency = 4 - } - if s.TableOptions == nil { - s.TableOptions = &tableoptions.TableOptions{} - } - if s.Concurrency == 0 { - s.Concurrency = defaultMaxConcurrency - } - if s.EventBasedSync != nil && s.EventBasedSync.FullSync == nil { - fullSync := true - s.EventBasedSync.FullSync = &fullSync - } -} diff --git a/plugins/source/aws/client/spec/account.go b/plugins/source/aws/client/spec/account.go new file mode 100644 index 00000000000000..22a39d7cbb13fe --- /dev/null +++ b/plugins/source/aws/client/spec/account.go @@ -0,0 +1,21 @@ +package spec + +type Account struct { + ID string `json:"id" jsonschema:"required,minLength=1"` + AccountName string `json:"account_name,omitempty"` + LocalProfile string `json:"local_profile,omitempty"` + RoleARN string `json:"role_arn,omitempty" jsonschema:"pattern=^arn(:[^:\n]*){5}([:/].*)?$"` + RoleSessionName string `json:"role_session_name,omitempty"` + ExternalID string `json:"external_id,omitempty"` + DefaultRegion string `json:"default_region,omitempty"` + Regions []string `json:"regions,omitempty" jsonschema:"minLength=1"` + + // explicitly ignore in JSON parsing, as this is filled in later + Source AccountSource `json:"-"` +} + +type AccountSource string + +const ( + AccountSourceOrg = "org" +) diff --git a/plugins/source/aws/client/spec/account_test.go b/plugins/source/aws/client/spec/account_test.go new file mode 100644 index 00000000000000..057240c1fa9b95 --- /dev/null +++ b/plugins/source/aws/client/spec/account_test.go @@ -0,0 +1,107 @@ +package spec + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/require" +) + +func TestAccountJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty accounts", + Spec: `{"accounts":[]}`, + }, + { + Name: "null accounts", + Spec: `{"accounts":null}`, + }, + { + Name: "bad accounts", + Err: true, + Spec: `{"accounts":[123]}`, + }, + { + Name: "empty account", + Err: true, + Spec: `{"accounts":[{}]}`, + }, + { + Name: "null account", + Err: true, + Spec: `{"accounts":[null]}`, + }, + { + Name: "bad account", + Err: true, + Spec: `{"accounts":[123]}`, + }, + { + Name: "proper account", + Spec: func() string { + var input Account + require.NoError(t, faker.FakeObject(&input)) + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.RoleARN = randomARN.String() + + return `{"accounts":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + { + Name: "bad account.role_arn", + Err: true, + Spec: func() string { + var input Account + require.NoError(t, faker.FakeObject(&input)) + return `{"accounts":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + { + Name: "missing account.id", + Err: true, + Spec: func() string { + var input Account + require.NoError(t, faker.FakeObject(&input)) + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.RoleARN = randomARN.String() + + return `{"accounts":[` + jsonschema.WithRemovedKeys(t, &input, "id") + `]}` + }(), + }, + { + Name: "empty account.region", + Err: true, + Spec: func() string { + var input Account + require.NoError(t, faker.FakeObject(&input)) + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.RoleARN = randomARN.String() + + input.Regions = []string{""} + return `{"accounts":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + { + Name: "filled in accounts with null org", + Spec: func() string { + var account Account + require.NoError(t, faker.FakeObject(&account)) + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + account.RoleARN = randomARN.String() + + return `{"org":null,"accounts":[` + jsonschema.WithRemovedKeys(t, &account) + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/spec/event_based.go b/plugins/source/aws/client/spec/event_based.go new file mode 100644 index 00000000000000..3809c3306325d4 --- /dev/null +++ b/plugins/source/aws/client/spec/event_based.go @@ -0,0 +1,22 @@ +package spec + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws/arn" +) + +type EventBasedSync struct { + FullSync *bool `json:"full_sync,omitempty" jsonschema:"default=true"` + Account Account `json:"account"` + KinesisStreamARN string `json:"kinesis_stream_arn" jsonschema:"required,pattern=^arn(:[^:\n]*){5}([:/].*)?$"` + StartTime *time.Time `json:"start_time,omitempty" jsonschema:"default=now"` +} + +func (e *EventBasedSync) Validate() error { + if !arn.IsARN(e.KinesisStreamARN) { + return fmt.Errorf("kinesis_stream_arn %q is not a valid ARN", e.KinesisStreamARN) + } + return nil +} diff --git a/plugins/source/aws/client/spec/event_based_test.go b/plugins/source/aws/client/spec/event_based_test.go new file mode 100644 index 00000000000000..801796bd2a12cb --- /dev/null +++ b/plugins/source/aws/client/spec/event_based_test.go @@ -0,0 +1,62 @@ +package spec + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/require" +) + +func TestEventBasedSyncJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty", + Err: true, + Spec: `{"event_based_sync":{}}`, + }, + { + Name: "null", + Spec: `{"event_based_sync":null}`, + }, + { + Name: "bad", + Err: true, + Spec: `{"event_based_sync":123}`, + }, + { + Name: "proper", + Spec: func() string { + var input EventBasedSync + require.NoError(t, faker.FakeObject(&input)) + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.KinesisStreamARN = randomARN.String() + input.Account.RoleARN = randomARN.String() + return `{"event_based_sync":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "bad kinesis_stream_arn", + Err: true, + Spec: func() string { + var input EventBasedSync + require.NoError(t, faker.FakeObject(&input)) + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.Account.RoleARN = randomARN.String() + return `{"event_based_sync":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "missing kinesis_stream_arn", + Err: true, + Spec: func() string { + var input EventBasedSync + require.NoError(t, faker.FakeObject(&input)) + return `{"event_based_sync":` + jsonschema.WithRemovedKeys(t, &input, "kinesis_stream_arn") + `}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/spec/gen/main.go b/plugins/source/aws/client/spec/gen/main.go new file mode 100644 index 00000000000000..5dee419780c321 --- /dev/null +++ b/plugins/source/aws/client/spec/gen/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" + "github.com/cloudquery/codegen/jsonschema" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + jsonschema.GenerateIntoFile(new(spec.Spec), path.Join(currDir(), "..", "schema.json")) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/plugins/source/aws/client/spec/org.go b/plugins/source/aws/client/spec/org.go new file mode 100644 index 00000000000000..cf17c56fb286f8 --- /dev/null +++ b/plugins/source/aws/client/spec/org.go @@ -0,0 +1,41 @@ +package spec + +import ( + "fmt" + "regexp" +) + +type Org struct { + AdminAccount *Account `json:"admin_account"` + MemberCredentials *Account `json:"member_trusted_principal"` + ChildAccountRoleName string `json:"member_role_name,omitempty" jsonschema:"required,minLength=1"` + ChildAccountRoleSessionName string `json:"member_role_session_name,omitempty"` + ChildAccountExternalID string `json:"member_external_id,omitempty"` + ChildAccountRegions []string `json:"member_regions,omitempty"` + OrganizationUnits []string `json:"organization_units,omitempty" jsonschema:"pattern=^((ou-[0-9a-z]{4\\,32}-[a-z0-9]{8\\,32})|(r-[0-9a-z]{4\\,32}))$"` + SkipOrganizationalUnits []string `json:"skip_organization_units,omitempty" jsonschema:"pattern=^((ou-[0-9a-z]{4\\,32}-[a-z0-9]{8\\,32})|(r-[0-9a-z]{4\\,32}))$"` + SkipMemberAccounts []string `json:"skip_member_accounts,omitempty"` +} + +func (o *Org) Validate() error { + if o.ChildAccountRoleName == "" { + return fmt.Errorf("member_role_name is required when using org configuration") + } + if err := validateOUs(o.OrganizationUnits); err != nil { + return fmt.Errorf("invalid organization_units: %w", err) + } + if err := validateOUs(o.SkipOrganizationalUnits); err != nil { + return fmt.Errorf("invalid skip_organization_units: %w", err) + } + return nil +} + +func validateOUs(ous []string) error { + r := regexp.MustCompile(`^((ou-[0-9a-z]{4,32}-[a-z0-9]{8,32})|(r-[0-9a-z]{4,32}))$`) + for _, ou := range ous { + if !r.MatchString(ou) { + return fmt.Errorf(`invalid OU: %s (should match "ou-*-*" or "r-*" with lowercase letters or digits)`, ou) + } + } + return nil +} diff --git a/plugins/source/aws/client/spec/org_test.go b/plugins/source/aws/client/spec/org_test.go new file mode 100644 index 00000000000000..58cf8bc14d1b17 --- /dev/null +++ b/plugins/source/aws/client/spec/org_test.go @@ -0,0 +1,231 @@ +package spec + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/require" +) + +func TestOrgJSONSchema(t *testing.T) { + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty", + Err: true, + Spec: `{"org":{}}`, + }, + { + Name: "null", + Spec: `{"org":null}`, + }, + { + Name: "bad", + Err: true, + Spec: `{"org":123}`, + }, + { + Name: "proper", + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = ou + input.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "missing member_role_name", + Err: true, + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = ou + input.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input, "member_role_name") + `}` + }(), + }, + { + Name: "empty member_role_name", + Err: true, + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = ou + input.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + input.ChildAccountRoleName = "" + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "null organization_units", + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = nil + input.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "empty organization_units", + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = []string{} + input.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "bad organization_units", + Err: true, + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "null skip_organization_units", + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = ou + input.SkipOrganizationalUnits = nil + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "empty skip_organization_units", + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = ou + input.SkipOrganizationalUnits = []string{} + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "bad skip_organization_units", + Err: true, + Spec: func() string { + var input Org + require.NoError(t, faker.FakeObject(&input)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + input.OrganizationUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + input.AdminAccount.RoleARN = randomARN.String() + input.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &input) + `}` + }(), + }, + { + Name: "org with null accounts", + Spec: func() string { + var org Org + require.NoError(t, faker.FakeObject(&org)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + org.OrganizationUnits = ou + org.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + org.AdminAccount.RoleARN = randomARN.String() + org.MemberCredentials.RoleARN = randomARN.String() + + return `{"accounts":null,"org":` + jsonschema.WithRemovedKeys(t, &org) + `}` + }(), + }, + { + Name: "org with empty accounts", + Spec: func() string { + var org Org + require.NoError(t, faker.FakeObject(&org)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + org.OrganizationUnits = ou + org.SkipOrganizationalUnits = ou + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + org.AdminAccount.RoleARN = randomARN.String() + org.MemberCredentials.RoleARN = randomARN.String() + + return `{"accounts":[],"org":` + jsonschema.WithRemovedKeys(t, &org) + `}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/spec/schema.json b/plugins/source/aws/client/spec/schema.json new file mode 100644 index 00000000000000..b7b0e15ae9b8a3 --- /dev/null +++ b/plugins/source/aws/client/spec/schema.json @@ -0,0 +1,3556 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/spec", + "$ref": "#/$defs/Spec", + "$defs": { + "AccessanalyzerFindings": { + "properties": { + "list_findings": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CustomAccessAnalyzerListFindingsInput" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "Account": { + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "account_name": { + "type": "string" + }, + "local_profile": { + "type": "string" + }, + "role_arn": { + "type": "string", + "pattern": "^arn(:[^:\n]*){5}([:/].*)?$" + }, + "role_session_name": { + "type": "string" + }, + "external_id": { + "type": "string" + }, + "default_region": { + "type": "string" + }, + "regions": { + "oneOf": [ + { + "items": { + "type": "string", + "minLength": 1 + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "id" + ] + }, + "AwsSecurityFindingFilters": { + "properties": { + "AwsAccountId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "CompanyName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ComplianceAssociatedStandardsId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ComplianceSecurityControlId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ComplianceStatus": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Confidence": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "CreatedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Criticality": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Description": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsConfidence": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsCriticality": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsRelatedFindingsId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsRelatedFindingsProductArn": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsSeverityLabel": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsSeverityOriginal": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingProviderFieldsTypes": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FirstObservedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "GeneratorId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Id": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Keyword": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/KeywordFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LastObservedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "MalwareName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "MalwarePath": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "MalwareState": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "MalwareType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkDestinationDomain": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkDestinationIpV4": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/IpFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkDestinationIpV6": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/IpFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkDestinationPort": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkDirection": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkProtocol": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkSourceDomain": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkSourceIpV4": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/IpFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkSourceIpV6": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/IpFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkSourceMac": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkSourcePort": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NoteText": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NoteUpdatedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NoteUpdatedBy": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProcessLaunchedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProcessName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProcessParentPid": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProcessPath": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProcessPid": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProcessTerminatedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProductArn": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProductFields": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/MapFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ProductName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "RecommendationText": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "RecordState": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Region": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "RelatedFindingsId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "RelatedFindingsProductArn": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceIamInstanceProfileArn": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceImageId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceIpV4Addresses": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/IpFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceIpV6Addresses": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/IpFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceKeyName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceLaunchedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceSubnetId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsEc2InstanceVpcId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsIamAccessKeyCreatedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsIamAccessKeyPrincipalName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsIamAccessKeyStatus": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsIamAccessKeyUserName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsIamUserUserName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsS3BucketOwnerId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceAwsS3BucketOwnerName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceContainerImageId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceContainerImageName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceContainerLaunchedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceContainerName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceDetailsOther": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/MapFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourcePartition": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceRegion": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceTags": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/MapFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Sample": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/BooleanFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "SeverityLabel": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "SeverityNormalized": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "SeverityProduct": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "SourceUrl": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ThreatIntelIndicatorCategory": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ThreatIntelIndicatorLastObservedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ThreatIntelIndicatorSource": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ThreatIntelIndicatorSourceUrl": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ThreatIntelIndicatorType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ThreatIntelIndicatorValue": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Title": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Type": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "UpdatedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "UserDefinedFields": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/MapFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "VerificationState": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "WorkflowState": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "WorkflowStatus": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter_1" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "BooleanFilter": { + "properties": { + "Value": { + "type": "boolean" + } + }, + "additionalProperties": false, + "type": "object" + }, + "CloudtrailAPIs": { + "properties": { + "lookup_events": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CustomLookupEventsOpts" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CloudwatchGetMetricStatisticsInput": { + "properties": { + "EndTime": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "Period": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "StartTime": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "ExtendedStatistics": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Statistics": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Unit": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "CloudwatchListMetricsInput": { + "properties": { + "Dimensions": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DimensionFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "IncludeLinkedAccounts": { + "type": "boolean" + }, + "MetricName": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Namespace": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "OwningAccount": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "RecentlyActive": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "CloudwatchMetric": { + "properties": { + "list_metrics": { + "$ref": "#/$defs/CloudwatchListMetricsInput" + }, + "get_metric_statistics": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CloudwatchGetMetricStatisticsInput" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CloudwatchMetrics": { + "items": { + "$ref": "#/$defs/CloudwatchMetric" + }, + "type": "array" + }, + "CostCategoryValues": { + "properties": { + "Key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "MatchOptions": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Values": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CostExplorerAPIs": { + "properties": { + "get_cost_and_usage": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CustomGetCostAndUsageInput" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "Criterion": { + "properties": { + "Contains": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Eq": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Exists": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "Neq": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CustomAccessAnalyzerListFindingsInput": { + "properties": { + "Filter": { + "oneOf": [ + { + "additionalProperties": { + "$ref": "#/$defs/Criterion" + }, + "type": "object" + }, + { + "type": "null" + } + ] + }, + "MaxResults": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "Sort": { + "oneOf": [ + { + "$ref": "#/$defs/SortCriteria" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CustomGetCostAndUsageInput": { + "properties": { + "Granularity": { + "type": "string" + }, + "Metrics": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "TimePeriod": { + "oneOf": [ + { + "$ref": "#/$defs/DateInterval" + }, + { + "type": "null" + } + ] + }, + "Filter": { + "oneOf": [ + { + "$ref": "#/$defs/Expression" + }, + { + "type": "null" + } + ] + }, + "GroupBy": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/GroupDefinition" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CustomGetFindingsOpts": { + "properties": { + "Filters": { + "oneOf": [ + { + "$ref": "#/$defs/AwsSecurityFindingFilters" + }, + { + "type": "null" + } + ] + }, + "MaxResults": { + "type": "integer", + "maximum": 100, + "minimum": 1 + }, + "SortCriteria": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/SortCriterion" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CustomInspector2ListFindingsInput": { + "properties": { + "FilterCriteria": { + "oneOf": [ + { + "$ref": "#/$defs/FilterCriteria" + }, + { + "type": "null" + } + ] + }, + "MaxResults": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "SortCriteria": { + "oneOf": [ + { + "$ref": "#/$defs/SortCriteria_1" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CustomListTasksOpts": { + "properties": { + "ContainerInstance": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "DesiredStatus": { + "type": "string" + }, + "Family": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "LaunchType": { + "type": "string" + }, + "MaxResults": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 100 + }, + "ServiceName": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "StartedBy": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "CustomLookupEventsOpts": { + "properties": { + "EndTime": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "EventCategory": { + "type": "string" + }, + "LookupAttributes": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/LookupAttribute" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "MaxResults": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "StartTime": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "DateFilter": { + "properties": { + "EndInclusive": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + }, + "StartInclusive": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "DateFilter_1": { + "properties": { + "DateRange": { + "oneOf": [ + { + "$ref": "#/$defs/DateRange" + }, + { + "type": "null" + } + ] + }, + "End": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Start": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "DateInterval": { + "properties": { + "End": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Start": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "DateRange": { + "properties": { + "Unit": { + "type": "string" + }, + "Value": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object" + }, + "DimensionFilter": { + "properties": { + "Name": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "DimensionValues": { + "properties": { + "Key": { + "type": "string" + }, + "MatchOptions": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Values": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "ECSTaskAPIs": { + "properties": { + "list_tasks": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CustomListTasksOpts" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "EventBasedSync": { + "properties": { + "full_sync": { + "oneOf": [ + { + "type": "boolean", + "default": true + }, + { + "type": "null" + } + ] + }, + "account": { + "$ref": "#/$defs/Account" + }, + "kinesis_stream_arn": { + "type": "string", + "pattern": "^arn(:[^:\n]*){5}([:/].*)?$" + }, + "start_time": { + "oneOf": [ + { + "type": "string", + "format": "date-time", + "default": "now" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "kinesis_stream_arn" + ] + }, + "Expression": { + "properties": { + "And": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/Expression" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "CostCategories": { + "oneOf": [ + { + "$ref": "#/$defs/CostCategoryValues" + }, + { + "type": "null" + } + ] + }, + "Dimensions": { + "oneOf": [ + { + "$ref": "#/$defs/DimensionValues" + }, + { + "type": "null" + } + ] + }, + "Not": { + "oneOf": [ + { + "$ref": "#/$defs/Expression" + }, + { + "type": "null" + } + ] + }, + "Or": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/Expression" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Tags": { + "oneOf": [ + { + "$ref": "#/$defs/TagValues" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "FilterCriteria": { + "properties": { + "AwsAccountId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "CodeVulnerabilityDetectorName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "CodeVulnerabilityDetectorTags": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "CodeVulnerabilityFilePath": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ComponentId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ComponentType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Ec2InstanceImageId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Ec2InstanceSubnetId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Ec2InstanceVpcId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EcrImageArchitecture": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EcrImageHash": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EcrImagePushedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EcrImageRegistry": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EcrImageRepositoryName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EcrImageTags": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "EpssScore": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ExploitAvailable": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingArn": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingStatus": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FindingType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FirstObservedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "FixAvailable": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "InspectorScore": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/NumberFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LambdaFunctionExecutionRoleArn": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LambdaFunctionLastModifiedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LambdaFunctionLayers": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LambdaFunctionName": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LambdaFunctionRuntime": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "LastObservedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "NetworkProtocol": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "PortRange": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/PortRangeFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "RelatedVulnerabilities": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceTags": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/MapFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "ResourceType": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Severity": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Title": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "UpdatedAt": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/DateFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "VendorSeverity": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "VulnerabilityId": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "VulnerabilitySource": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/StringFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "VulnerablePackages": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/PackageFilter" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "GroupDefinition": { + "properties": { + "Key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Type": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Inspector2APIs": { + "properties": { + "list_findings": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CustomInspector2ListFindingsInput" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "IpFilter": { + "properties": { + "Cidr": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "KeywordFilter": { + "properties": { + "Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "LookupAttribute": { + "properties": { + "AttributeKey": { + "type": "string" + }, + "AttributeValue": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "MapFilter": { + "properties": { + "Comparison": { + "type": "string" + }, + "Key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "MapFilter_1": { + "properties": { + "Comparison": { + "type": "string" + }, + "Key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "NumberFilter": { + "properties": { + "LowerInclusive": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "UpperInclusive": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "NumberFilter_1": { + "properties": { + "Eq": { + "type": "number" + }, + "Gte": { + "type": "number" + }, + "Lte": { + "type": "number" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Org": { + "properties": { + "admin_account": { + "oneOf": [ + { + "$ref": "#/$defs/Account" + }, + { + "type": "null" + } + ] + }, + "member_trusted_principal": { + "oneOf": [ + { + "$ref": "#/$defs/Account" + }, + { + "type": "null" + } + ] + }, + "member_role_name": { + "type": "string", + "minLength": 1 + }, + "member_role_session_name": { + "type": "string" + }, + "member_external_id": { + "type": "string" + }, + "member_regions": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "organization_units": { + "oneOf": [ + { + "items": { + "type": "string", + "pattern": "^((ou-[0-9a-z]{4,32}-[a-z0-9]{8,32})|(r-[0-9a-z]{4,32}))$" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "skip_organization_units": { + "oneOf": [ + { + "items": { + "type": "string", + "pattern": "^((ou-[0-9a-z]{4,32}-[a-z0-9]{8,32})|(r-[0-9a-z]{4,32}))$" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "skip_member_accounts": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "member_role_name" + ] + }, + "PackageFilter": { + "properties": { + "Architecture": { + "oneOf": [ + { + "$ref": "#/$defs/StringFilter" + }, + { + "type": "null" + } + ] + }, + "Epoch": { + "oneOf": [ + { + "$ref": "#/$defs/NumberFilter" + }, + { + "type": "null" + } + ] + }, + "Name": { + "oneOf": [ + { + "$ref": "#/$defs/StringFilter" + }, + { + "type": "null" + } + ] + }, + "Release": { + "oneOf": [ + { + "$ref": "#/$defs/StringFilter" + }, + { + "type": "null" + } + ] + }, + "SourceLambdaLayerArn": { + "oneOf": [ + { + "$ref": "#/$defs/StringFilter" + }, + { + "type": "null" + } + ] + }, + "SourceLayerHash": { + "oneOf": [ + { + "$ref": "#/$defs/StringFilter" + }, + { + "type": "null" + } + ] + }, + "Version": { + "oneOf": [ + { + "$ref": "#/$defs/StringFilter" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "PortRangeFilter": { + "properties": { + "BeginInclusive": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "EndInclusive": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "SecurityHubAPIs": { + "properties": { + "get_findings": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/CustomGetFindingsOpts" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "SortCriteria": { + "properties": { + "AttributeName": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "OrderBy": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "SortCriteria_1": { + "properties": { + "Field": { + "type": "string" + }, + "SortOrder": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "SortCriterion": { + "properties": { + "Field": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "SortOrder": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Spec": { + "allOf": [ + { + "if": { + "properties": { + "custom_endpoint_url": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "custom_endpoint_url" + ] + }, + "then": { + "properties": { + "custom_endpoint_partition_id": { + "type": "string", + "minLength": 1 + }, + "custom_endpoint_signing_region": { + "type": "string", + "minLength": 1 + }, + "custom_endpoint_hostname_immutable": { + "type": "boolean" + } + }, + "required": [ + "custom_endpoint_partition_id", + "custom_endpoint_signing_region", + "custom_endpoint_hostname_immutable" + ] + } + }, + { + "not": { + "properties": { + "org": { + "$ref": "#/$defs/Org" + }, + "accounts": { + "items": { + "$ref": "#/$defs/Account" + }, + "type": "array", + "minItems": 1 + } + }, + "required": [ + "org", + "accounts" + ] + } + } + ], + "properties": { + "regions": { + "oneOf": [ + { + "items": { + "type": "string", + "minLength": 1 + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "accounts": { + "oneOf": [ + { + "items": { + "$ref": "#/$defs/Account" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "org": { + "oneOf": [ + { + "$ref": "#/$defs/Org" + }, + { + "type": "null" + } + ] + }, + "aws_debug": { + "type": "boolean" + }, + "max_retries": { + "oneOf": [ + { + "type": "integer", + "default": 10 + }, + { + "type": "null" + } + ] + }, + "max_backoff": { + "oneOf": [ + { + "type": "integer", + "default": 30 + }, + { + "type": "null" + } + ] + }, + "custom_endpoint_url": { + "type": "string" + }, + "custom_endpoint_hostname_immutable": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "custom_endpoint_partition_id": { + "type": "string" + }, + "custom_endpoint_signing_region": { + "type": "string" + }, + "initialization_concurrency": { + "type": "integer", + "minimum": 1, + "default": 4 + }, + "concurrency": { + "type": "integer", + "minimum": 1, + "default": 50000 + }, + "use_paid_apis": { + "type": "boolean", + "default": false + }, + "table_options": { + "oneOf": [ + { + "$ref": "#/$defs/TableOptions" + }, + { + "type": "null" + } + ] + }, + "event_based_sync": { + "oneOf": [ + { + "$ref": "#/$defs/EventBasedSync" + }, + { + "type": "null" + } + ] + }, + "scheduler": { + "$ref": "#/$defs/Strategy" + } + }, + "additionalProperties": false, + "type": "object" + }, + "Strategy": { + "type": "string", + "enum": [ + "dfs", + "round-robin", + "shuffle" + ], + "title": "CloudQuery scheduling strategy", + "default": "dfs" + }, + "StringFilter": { + "properties": { + "Comparison": { + "type": "string" + }, + "Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "StringFilter_1": { + "properties": { + "Comparison": { + "type": "string" + }, + "Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "TableOptions": { + "properties": { + "aws_alpha_cloudwatch_metrics": { + "oneOf": [ + { + "$ref": "#/$defs/CloudwatchMetrics" + }, + { + "type": "null" + } + ] + }, + "aws_cloudtrail_events": { + "oneOf": [ + { + "$ref": "#/$defs/CloudtrailAPIs" + }, + { + "type": "null" + } + ] + }, + "aws_accessanalyzer_analyzer_findings": { + "oneOf": [ + { + "$ref": "#/$defs/AccessanalyzerFindings" + }, + { + "type": "null" + } + ] + }, + "aws_inspector2_findings": { + "oneOf": [ + { + "$ref": "#/$defs/Inspector2APIs" + }, + { + "type": "null" + } + ] + }, + "aws_alpha_costexplorer_cost_custom": { + "oneOf": [ + { + "$ref": "#/$defs/CostExplorerAPIs" + }, + { + "type": "null" + } + ] + }, + "aws_securityhub_findings": { + "oneOf": [ + { + "$ref": "#/$defs/SecurityHubAPIs" + }, + { + "type": "null" + } + ] + }, + "aws_ecs_cluster_tasks": { + "oneOf": [ + { + "$ref": "#/$defs/ECSTaskAPIs" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + }, + "TagValues": { + "properties": { + "Key": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "MatchOptions": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "Values": { + "oneOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "type": "object" + } + } +} diff --git a/plugins/source/aws/client/spec/spec.go b/plugins/source/aws/client/spec/spec.go new file mode 100644 index 00000000000000..6456a6a1954ca4 --- /dev/null +++ b/plugins/source/aws/client/spec/spec.go @@ -0,0 +1,165 @@ +package spec + +import ( + _ "embed" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" + "github.com/cloudquery/plugin-sdk/v4/scheduler" + "github.com/invopop/jsonschema" + orderedmap "github.com/wk8/go-ordered-map/v2" +) + +type Spec struct { + Regions []string `json:"regions,omitempty" jsonschema:"minLength=1"` + Accounts []Account `json:"accounts"` + Organization *Org `json:"org"` + AWSDebug bool `json:"aws_debug,omitempty"` + MaxRetries *int `json:"max_retries,omitempty" jsonschema:"default=10"` + MaxBackoff *int `json:"max_backoff,omitempty" jsonschema:"default=30"` + EndpointURL string `json:"custom_endpoint_url,omitempty"` + HostnameImmutable *bool `json:"custom_endpoint_hostname_immutable,omitempty"` + PartitionID string `json:"custom_endpoint_partition_id,omitempty"` + SigningRegion string `json:"custom_endpoint_signing_region,omitempty"` + InitializationConcurrency int `json:"initialization_concurrency" jsonschema:"minimum=1,default=4"` + Concurrency int `json:"concurrency" jsonschema:"minimum=1,default=50000"` + UsePaidAPIs bool `json:"use_paid_apis" jsonschema:"default=false"` + TableOptions *tableoptions.TableOptions `json:"table_options,omitempty"` + EventBasedSync *EventBasedSync `json:"event_based_sync,omitempty"` + Scheduler scheduler.Strategy `json:"scheduler,omitempty"` +} + +// JSONSchemaExtend is required to verify: +// 1.if `custom_endpoint_url` is present then the following fields are required: +// * `custom_endpoint_partition_id` +// * `custom_endpoint_signing_region` +// * `custom_endpoint_hostname_immutable` +// 2. Make `org` & `accounts` mutually exclusive +func (Spec) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.AllOf = []*jsonschema.Schema{ + { + // custom_endpoint_url => custom_endpoint_partition_id, custom_endpoint_signing_region, custom_endpoint_hostname_immutable + If: &jsonschema.Schema{ + // We also need to make sure that `custom_endpoint_url` isn't "" + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + url := *sc.Properties.Value("custom_endpoint_url") + url.MinLength = aws.Uint64(1) + properties.Set("custom_endpoint_url", &url) + return properties + }(), + Required: []string{"custom_endpoint_url"}, + }, + Then: &jsonschema.Schema{ + // require properties not to be empty or null + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + + partitionID := *sc.Properties.Value("custom_endpoint_partition_id") + partitionID.MinLength = aws.Uint64(1) + properties.Set("custom_endpoint_partition_id", &partitionID) + + signingRegion := *sc.Properties.Value("custom_endpoint_signing_region") + signingRegion.MinLength = aws.Uint64(1) + properties.Set("custom_endpoint_signing_region", &signingRegion) + + hostnameImmutable := *sc.Properties.Value("custom_endpoint_hostname_immutable").OneOf[0] // spec is 0, null is 1st + properties.Set("custom_endpoint_hostname_immutable", &hostnameImmutable) + + return properties + }(), + Required: []string{"custom_endpoint_partition_id", "custom_endpoint_signing_region", "custom_endpoint_hostname_immutable"}, + }, + }, + { + Not: &jsonschema.Schema{ + // org & accounts are mutually exclusive + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("org", sc.Properties.Value("org").OneOf[0]) // spec is 0, null is 1st + + // we take a value because we'll be updating the items spec + accounts := *sc.Properties.Value("accounts").OneOf[0] // spec is 0, null is 1st + accounts.MinItems = aws.Uint64(1) + properties.Set("accounts", &accounts) + return properties + }(), + Required: []string{"org", "accounts"}, + }, + }, + } +} + +func (s *Spec) Validate() error { + if s.EndpointURL != "" { + if s.PartitionID == "" { + return fmt.Errorf("custom_endpoint_partition_id is required when custom_endpoint_url is set") + } + if s.SigningRegion == "" { + return fmt.Errorf("custom_endpoint_signing_region is required when custom_endpoint_url is set") + } + if s.HostnameImmutable == nil { + return fmt.Errorf("custom_endpoint_hostname_immutable is required when custom_endpoint_url is set") + } + } + + if s.Organization != nil && len(s.Accounts) > 0 { + return errors.New("specifying accounts via both the Accounts and Org properties is not supported. To achieve both, use multiple source configurations") + } + if s.Organization != nil { + if err := s.Organization.Validate(); err != nil { + return fmt.Errorf("invalid org: %w", err) + } + } + + if s.TableOptions != nil { + if err := s.TableOptions.Validate(); err != nil { + return fmt.Errorf("invalid table_options: %w", err) + } + } + + if s.EventBasedSync != nil { + if err := s.EventBasedSync.Validate(); err != nil { + return fmt.Errorf("invalid event_based_sync: %w", err) + } + } + return nil +} + +func (s *Spec) SetDefaults() { + if s.TableOptions == nil { + s.TableOptions = &tableoptions.TableOptions{} + } + // also call set defaults + s.TableOptions.SetDefaults() + + if s.InitializationConcurrency <= 0 { + const defaultInitializationConcurrency = 4 + s.InitializationConcurrency = defaultInitializationConcurrency + } + + if s.Concurrency <= 0 { + const defaultMaxConcurrency = 50000 + s.Concurrency = defaultMaxConcurrency + } + + if s.EventBasedSync != nil && s.EventBasedSync.FullSync == nil { + fullSync := true + s.EventBasedSync.FullSync = &fullSync + } + + if s.MaxRetries == nil { + maxRetries := 10 + s.MaxRetries = &maxRetries + } + + if s.MaxBackoff == nil { + maxBackoff := 30 + s.MaxBackoff = &maxBackoff + } +} + +//go:embed schema.json +var JSONSchema string diff --git a/plugins/source/aws/client/spec/spec_test.go b/plugins/source/aws/client/spec/spec_test.go new file mode 100644 index 00000000000000..880ed460b07766 --- /dev/null +++ b/plugins/source/aws/client/spec/spec_test.go @@ -0,0 +1,542 @@ +package spec + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/require" +) + +func TestSpecValidate(t *testing.T) { + tests := []struct { + name string + spec *Spec + wantErr bool + }{ + { + name: "valid accounts", + spec: &Spec{ + Accounts: []Account{ + {ID: "123456789012"}, + {ID: "cq-playground"}, + }, + }, + wantErr: false, + }, + { + name: "valid org", + spec: &Spec{ + Organization: &Org{ + ChildAccountRoleName: "test", + OrganizationUnits: []string{"ou-1234-12345678"}, + }, + }, + wantErr: false, + }, + { + name: "invalid org", + spec: &Spec{ + Organization: &Org{ + ChildAccountRoleName: "test", + OrganizationUnits: []string{"123"}, + }, + }, + wantErr: true, + }, + { + name: "missing member account role name", + spec: &Spec{ + Organization: &Org{}, + }, + wantErr: true, + }, + { + name: "valid skip ou", + spec: &Spec{ + Organization: &Org{ + ChildAccountRoleName: "test", + OrganizationUnits: []string{"ou-1234-12345678"}, + SkipOrganizationalUnits: []string{"ou-1234-45678901"}, + }, + }, + wantErr: false, + }, + { + name: "invalid skip ou", + spec: &Spec{ + Organization: &Org{ + ChildAccountRoleName: "test", + OrganizationUnits: []string{"ou-1234-12345678"}, + SkipOrganizationalUnits: []string{"456"}, + }, + }, + wantErr: true, + }, + { + name: "both account and org", + spec: &Spec{ + Accounts: []Account{ + {ID: "123456789012"}, + }, + Organization: &Org{ + ChildAccountRoleName: "test", + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.spec.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Spec.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestJSONSchema(t *testing.T) { + // Accounts, Org, TableOptions & EventBasedSync are tested separately + jsonschema.TestJSONSchema(t, JSONSchema, []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + // We check that accounts aren't present together with org, though + { + Name: "accounts with org", + Err: true, + Spec: func() string { + var account Account + require.NoError(t, faker.FakeObject(&account)) + + var randomARN arn.ARN + require.NoError(t, faker.FakeObject(&randomARN)) + account.RoleARN = randomARN.String() + + var org Org + require.NoError(t, faker.FakeObject(&org)) + + ou := []string{"ou-abcdefg123-qwerty789", "r-qwerty789"} + org.OrganizationUnits = ou + org.SkipOrganizationalUnits = ou + + org.AdminAccount.RoleARN = randomARN.String() + org.MemberCredentials.RoleARN = randomARN.String() + + return `{"org":` + jsonschema.WithRemovedKeys(t, &org) + + `,"accounts":[` + jsonschema.WithRemovedKeys(t, &account) + `]}` + }(), + }, + { + Name: "null regions", + Spec: `{"regions":null}`, + }, + { + Name: "empty regions", + Spec: `{"regions":[]}`, + }, + { + Name: "bad regions type", + Err: true, + Spec: `{"regions":123}`, + }, + { + Name: "bad region type", + Err: true, + Spec: `{"regions":[1,2,3]}`, + }, + { + Name: "empty region", + Err: true, + Spec: `{"regions":["a","b",""]}`, + }, + { + Name: "proper regions", + Spec: `{"regions":["a","b","c"]}`, + }, + { + Name: "proper aws_debug", + Spec: `{"aws_debug":true}`, + }, + { + Name: "bad aws_debug", + Err: true, + Spec: `{"aws_debug":123}`, + }, + { + Name: "null aws_debug", + Err: true, + Spec: `{"aws_debug":null}`, + }, + { + Name: "proper max_retries", + Spec: `{"max_retries":123}`, + }, + { + Name: "bad max_retries", + Err: true, + Spec: `{"max_retries":true}`, + }, + { + Name: "null max_retries", + Spec: `{"max_retries":null}`, + }, + { + Name: "proper max_backoff", + Spec: `{"max_backoff":123}`, + }, + { + Name: "bad max_backoff", + Err: true, + Spec: `{"max_backoff":true}`, + }, + { + Name: "null max_backoff", + Spec: `{"max_backoff":null}`, + }, + { + Name: "null custom_endpoint_url", + Err: true, + Spec: `{"custom_endpoint_url":null}`, + }, + { + Name: "bad custom_endpoint_url", + Err: true, + Spec: `{"custom_endpoint_url":123}`, + }, + { + Name: "proper custom_endpoint_url & dependent", + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url without custom_endpoint_partition_id", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url with empty custom_endpoint_partition_id", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url with null custom_endpoint_partition_id", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": null, + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url with bad custom_endpoint_partition_id", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": 123, + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url without custom_endpoint_signing_region", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url with empty custom_endpoint_signing_region", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url with null custom_endpoint_signing_region", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": null, + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url with bad custom_endpoint_signing_region", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": 123, + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "custom_endpoint_url without custom_endpoint_hostname_immutable", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region" +} +`, + }, + { + Name: "custom_endpoint_url with null custom_endpoint_hostname_immutable", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": null +} +`, + }, + { + Name: "custom_endpoint_url with bad custom_endpoint_hostname_immutable", + Err: true, + Spec: ` +{ + "custom_endpoint_url": "url", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": 123 +} +`, + }, + { + Name: "empty custom_endpoint_url", + Spec: ` +{ + "custom_endpoint_url": "", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "empty custom_endpoint_url without custom_endpoint_partition_id", + Spec: ` +{ + "custom_endpoint_url": "", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "empty custom_endpoint_url without custom_endpoint_signing_region", + Spec: ` +{ + "custom_endpoint_url": "", + "custom_endpoint_partition_id": "id", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "empty custom_endpoint_url without custom_endpoint_hostname_immutable", + Spec: ` +{ + "custom_endpoint_url": "", + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region" +} +`, + }, + { + Name: "no custom_endpoint_url", + Spec: ` +{ + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "no custom_endpoint_url without custom_endpoint_partition_id", + Spec: ` +{ + "custom_endpoint_signing_region": "region", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "no custom_endpoint_url without custom_endpoint_signing_region", + Spec: ` +{ + "custom_endpoint_partition_id": "id", + "custom_endpoint_hostname_immutable": true +} +`, + }, + { + Name: "no custom_endpoint_url without custom_endpoint_hostname_immutable", + Spec: ` +{ + "custom_endpoint_partition_id": "id", + "custom_endpoint_signing_region": "region" +} +`, + }, + { + Name: "proper initialization_concurrency", + Spec: `{"initialization_concurrency":123}`, + }, + { + Name: "zero initialization_concurrency", + Err: true, + Spec: `{"initialization_concurrency":0}`, + }, + { + Name: "bad initialization_concurrency", + Err: true, + Spec: `{"initialization_concurrency":-1}`, + }, + { + Name: "float initialization_concurrency", + Err: true, + Spec: `{"initialization_concurrency":4.5}`, + }, + { + Name: "null initialization_concurrency", + Err: true, + Spec: `{"initialization_concurrency":null}`, + }, + { + Name: "proper concurrency", + Spec: `{"concurrency":123}`, + }, + { + Name: "zero concurrency", + Err: true, + Spec: `{"concurrency":0}`, + }, + { + Name: "bad concurrency", + Err: true, + Spec: `{"concurrency":-1}`, + }, + { + Name: "float concurrency", + Err: true, + Spec: `{"concurrency":4.5}`, + }, + { + Name: "null concurrency", + Err: true, + Spec: `{"concurrency":null}`, + }, + { + Name: "false use_paid_apis", + Spec: `{"use_paid_apis":false}`, + }, + { + Name: "true use_paid_apis", + Spec: `{"use_paid_apis":true}`, + }, + { + Name: "null use_paid_apis", + Err: true, + Spec: `{"use_paid_apis":null}`, + }, + { + Name: "bad use_paid_apis type", + Err: true, + Spec: `{"use_paid_apis":123}`, + }, + // Scheduler tests are included for completeness’s sake, but should be done in scheduler package instead + { + Name: "dfs scheduler", + Spec: `{"scheduler":"dfs"}`, + }, + { + Name: "round-robin scheduler", + Spec: `{"scheduler":"round-robin"}`, + }, + { + Name: "shuffle scheduler", + Spec: `{"scheduler":"shuffle"}`, + }, + { + Name: "empty scheduler", + Err: true, + Spec: `{"scheduler":""}`, + }, + { + Name: "bad scheduler", + Err: true, + Spec: `{"scheduler":"bad"}`, + }, + { + Name: "bad scheduler type", + Err: true, + Spec: `{"scheduler":123}`, + }, + { + Name: "null scheduler type", + Err: true, + Spec: `{"scheduler":null}`, + }, + // detailed table_options cases are tested separately + { + Name: "null table_options", + Spec: `{"table_options":null}`, + }, + { + Name: "empty table_options", + Spec: `{"table_options":null}`, + }, + }) +} + +func TestEnsureJSONSchema(t *testing.T) { + data, err := jsonschema.Generate(new(Spec)) + require.NoError(t, err) + require.JSONEqf(t, string(data), JSONSchema, "new schema should be:\n%s\n", string(data)) +} diff --git a/plugins/source/aws/client/tableoptions/accessanalyzer.go b/plugins/source/aws/client/spec/tableoptions/accessanalyzer.go similarity index 84% rename from plugins/source/aws/client/tableoptions/accessanalyzer.go rename to plugins/source/aws/client/spec/tableoptions/accessanalyzer.go index 66f39465cf1d50..ce99fa7dcdd023 100644 --- a/plugins/source/aws/client/tableoptions/accessanalyzer.go +++ b/plugins/source/aws/client/spec/tableoptions/accessanalyzer.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) type AccessanalyzerFindings struct { @@ -32,6 +33,12 @@ func (c *CustomAccessAnalyzerListFindingsInput) UnmarshalJSON(data []byte) error return json.Unmarshal(b, &c.ListFindingsInput) } +// JSONSchemaExtend is required to remove `AnalyzerArn` & `NextToken`. +func (CustomAccessAnalyzerListFindingsInput) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("AnalyzerArn") + sc.Properties.Delete("NextToken") +} + func (c *AccessanalyzerFindings) validateListFindings() error { for _, opt := range c.ListFindingOpts { if aws.ToString(opt.NextToken) != "" { diff --git a/plugins/source/aws/client/spec/tableoptions/accessanalyzer_test.go b/plugins/source/aws/client/spec/tableoptions/accessanalyzer_test.go new file mode 100644 index 00000000000000..e64f3081af2c46 --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/accessanalyzer_test.go @@ -0,0 +1,96 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAAListFindings(t *testing.T) { + u := CustomAccessAnalyzerListFindingsInput{} + require.NoError(t, faker.FakeObject(&u)) + + api := AccessanalyzerFindings{ + ListFindingOpts: []CustomAccessAnalyzerListFindingsInput{u}, + } + // Ensure that the validation works as expected + err := api.Validate() + assert.EqualError(t, err, "invalid input: cannot set NextToken in ListFindings") + api.ListFindingOpts[0].NextToken = nil + + err = api.Validate() + assert.EqualError(t, err, "invalid input: cannot set AnalyzerARN in ListFindings") + api.ListFindingOpts[0].AnalyzerArn = nil + + // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields + err = api.Validate() + assert.Nil(t, err) +} + +func TestCustomAccessAnalyzerListFindingsInput_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(AccessanalyzerFindings{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "empty list_findings", + Spec: `{"list_findings":[]}`, + }, + { + Name: "null list_findings", + Spec: `{"list_findings":null}`, + }, + { + Name: "bad list_findings", + Err: true, + Spec: `{"list_findings":123}`, + }, + { + Name: "empty list_findings entry", + Spec: `{"list_findings":[{}]}`, + }, + { + Name: "null list_findings", + Err: true, + Spec: `{"list_findings":[null]}`, + }, + { + Name: "bad list_findings entry", + Err: true, + Spec: `{"list_findings":[123]}`, + }, + { + Name: "proper list_findings", + Spec: func() string { + var input CustomAccessAnalyzerListFindingsInput + require.NoError(t, faker.FakeObject(&input)) + return `{"list_findings":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken", "AnalyzerArn") + `]}` + }(), + }, + { + Name: "list_findings.AnalyzerArn is present", + Err: true, + Spec: func() string { + var input CustomAccessAnalyzerListFindingsInput + require.NoError(t, faker.FakeObject(&input)) + return `{"list_findings":[` + jsonschema.WithRemovedKeys(t, &input, "AnalyzerArn") + `]}` + }(), + }, + { + Name: "list_findings.NextToken is present", + Err: true, + Spec: func() string { + var input CustomAccessAnalyzerListFindingsInput + require.NoError(t, faker.FakeObject(&input)) + return `{"list_findings":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/change_case.go b/plugins/source/aws/client/spec/tableoptions/change_case.go similarity index 100% rename from plugins/source/aws/client/tableoptions/change_case.go rename to plugins/source/aws/client/spec/tableoptions/change_case.go diff --git a/plugins/source/aws/client/tableoptions/cloudtrail.go b/plugins/source/aws/client/spec/tableoptions/cloudtrail.go similarity index 85% rename from plugins/source/aws/client/tableoptions/cloudtrail.go rename to plugins/source/aws/client/spec/tableoptions/cloudtrail.go index 7347819cd1342e..0b14c26b03e552 100644 --- a/plugins/source/aws/client/tableoptions/cloudtrail.go +++ b/plugins/source/aws/client/spec/tableoptions/cloudtrail.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) type CloudtrailAPIs struct { @@ -31,6 +32,11 @@ func (c *CustomLookupEventsOpts) UnmarshalJSON(data []byte) error { return json.Unmarshal(b, &c.LookupEventsInput) } +// JSONSchemaExtend is required to remove `NextToken`. +func (CustomLookupEventsOpts) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("NextToken") +} + func (c *CloudtrailAPIs) validateLookupEvents() error { for _, opt := range c.LookupEventsOpts { if aws.ToString(opt.NextToken) != "" { diff --git a/plugins/source/aws/client/spec/tableoptions/cloudtrail_test.go b/plugins/source/aws/client/spec/tableoptions/cloudtrail_test.go new file mode 100644 index 00000000000000..38dfcc301f54cc --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/cloudtrail_test.go @@ -0,0 +1,83 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLookupEvents(t *testing.T) { + u := CustomLookupEventsOpts{} + require.NoError(t, faker.FakeObject(&u)) + + api := CloudtrailAPIs{ + LookupEventsOpts: []CustomLookupEventsOpts{u}, + } + // Ensure that the validation works as expected + err := api.Validate() + assert.EqualError(t, err, "invalid input: cannot set NextToken in LookupEvents") + + // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields + api.LookupEventsOpts[0].NextToken = nil + err = api.Validate() + assert.Nil(t, err) +} + +func TestCustomLookupEventsOpts_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(CloudtrailAPIs{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "empty lookup_events", + Spec: `{"lookup_events":[]}`, + }, + { + Name: "null lookup_events", + Spec: `{"lookup_events":null}`, + }, + { + Name: "bad lookup_events", + Err: true, + Spec: `{"lookup_events":123}`, + }, + { + Name: "empty lookup_events entry", + Spec: `{"lookup_events":[{}]}`, + }, + { + Name: "null lookup_events entry", + Err: true, + Spec: `{"lookup_events":[null]}`, + }, + { + Name: "bad lookup_events entry", + Err: true, + Spec: `{"lookup_events":[123]}`, + }, + { + Name: "proper lookup_events", + Spec: func() string { + var input CustomLookupEventsOpts + require.NoError(t, faker.FakeObject(&input)) + return `{"lookup_events":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + { + Name: "lookup_events.NextToken is present", + Err: true, + Spec: func() string { + var input CustomLookupEventsOpts + require.NoError(t, faker.FakeObject(&input)) + return `{"lookup_events":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/cloudwatch.go b/plugins/source/aws/client/spec/tableoptions/cloudwatch.go similarity index 72% rename from plugins/source/aws/client/tableoptions/cloudwatch.go rename to plugins/source/aws/client/spec/tableoptions/cloudwatch.go index d23a2e6fecb840..c32b36f1666345 100644 --- a/plugins/source/aws/client/tableoptions/cloudwatch.go +++ b/plugins/source/aws/client/spec/tableoptions/cloudwatch.go @@ -7,12 +7,63 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) +type ( + CloudwatchMetrics []CloudwatchMetric + CloudwatchMetric struct { + ListMetricsOpts CloudwatchListMetricsInput `json:"list_metrics,omitempty"` + GetMetricStatisticsOpts []CloudwatchGetMetricStatisticsInput `json:"get_metric_statistics,omitempty"` + } +) + +func (c CloudwatchMetrics) Validate() error { + for _, m := range c { + if err := m.Validate(); err != nil { + return err + } + } + return nil +} + +func (c *CloudwatchMetric) Validate() error { + return errors.Join(c.validateListMetricsOpts(), c.validateGetMetricStatisticsOpts()) +} + +func (c *CloudwatchMetric) validateListMetricsOpts() error { + if aws.ToString(c.ListMetricsOpts.NextToken) != "" { + return errors.New("invalid input: cannot set NextToken in CloudwatchMetrics.ListMetricsOpts") + } + return nil +} + +func (c CloudwatchMetric) validateGetMetricStatisticsOpts() error { + for _, opt := range c.GetMetricStatisticsOpts { + if aws.ToString(opt.Namespace) != "" { + return errors.New("invalid input: cannot set Namespace in CloudwatchMetrics.GetMetricStatisticsOpts") + } + if aws.ToString(opt.MetricName) != "" { + return errors.New("invalid input: cannot set MetricName in CloudwatchMetrics.GetMetricStatisticsOpts") + } + if len(opt.Dimensions) > 0 { + return errors.New("invalid input: cannot set Dimensions in CloudwatchMetrics.GetMetricStatisticsOpts") + } + } + return nil +} + type CloudwatchGetMetricStatisticsInput struct { cloudwatch.GetMetricStatisticsInput } +// JSONSchemaExtend is required to remove `Namespace`, `MetricName` & `Dimensions`. +func (CloudwatchGetMetricStatisticsInput) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("Namespace") + sc.Properties.Delete("MetricName") + sc.Properties.Delete("Dimensions") +} + // UnmarshalJSON implements the json.Unmarshaler interface for the CloudwatchGetMetricStatisticsInput type. // It is the same as default, but allows the use of underscore in the JSON field names. func (c *CloudwatchGetMetricStatisticsInput) UnmarshalJSON(data []byte) error { @@ -26,13 +77,6 @@ func (c *CloudwatchGetMetricStatisticsInput) UnmarshalJSON(data []byte) error { return json.Unmarshal(b, &c.GetMetricStatisticsInput) } -type CloudwatchMetric struct { - ListMetricsOpts CloudwatchListMetricsInput `json:"list_metrics,omitempty"` - GetMetricStatisticsOpts []CloudwatchGetMetricStatisticsInput `json:"get_metric_statistics,omitempty"` -} - -type CloudwatchMetrics []CloudwatchMetric - type CloudwatchListMetricsInput struct { cloudwatch.ListMetricsInput } @@ -50,43 +94,7 @@ func (c *CloudwatchListMetricsInput) UnmarshalJSON(data []byte) error { return json.Unmarshal(b, &c.ListMetricsInput) } -func (c *CloudwatchMetric) validateListMetricsOpts() error { - if aws.ToString(c.ListMetricsOpts.NextToken) != "" { - return errors.New("invalid input: cannot set NextToken in CloudwatchMetrics.ListMetricsOpts") - } - return nil -} - -func (c CloudwatchMetric) validateGetMetricStatisticsOpts() error { - for _, opt := range c.GetMetricStatisticsOpts { - if aws.ToString(opt.Namespace) != "" { - return errors.New("invalid input: cannot set Namespace in CloudwatchMetrics.GetMetricStatisticsOpts") - } - if aws.ToString(opt.MetricName) != "" { - return errors.New("invalid input: cannot set MetricName in CloudwatchMetrics.GetMetricStatisticsOpts") - } - if len(opt.Dimensions) > 0 { - return errors.New("invalid input: cannot set Dimensions in CloudwatchMetrics.GetMetricStatisticsOpts") - } - } - return nil -} - -func (c *CloudwatchMetric) Validate() error { - if err := c.validateListMetricsOpts(); err != nil { - return err - } - return c.validateGetMetricStatisticsOpts() -} - -func (c CloudwatchMetrics) Validate() error { - for _, m := range c { - if err := m.validateListMetricsOpts(); err != nil { - return err - } - if err := m.validateGetMetricStatisticsOpts(); err != nil { - return err - } - } - return nil +// JSONSchemaExtend is required to remove `NextToken`. +func (CloudwatchListMetricsInput) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("NextToken") } diff --git a/plugins/source/aws/client/spec/tableoptions/cloudwatch_test.go b/plugins/source/aws/client/spec/tableoptions/cloudwatch_test.go new file mode 100644 index 00000000000000..c43e2c75a7dba1 --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/cloudwatch_test.go @@ -0,0 +1,126 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/require" +) + +func TestCloudwatchListMetricsInput_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(CloudwatchMetrics{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `[]`, + }, + { + Name: "empty list_metrics", + Spec: `[{"list_metrics":{}}]`, + }, + { + Name: "null list_metrics", + Err: true, + Spec: `[{"list_metrics":null}]`, + }, + { + Name: "bad list_metrics", + Err: true, + Spec: `[{"list_metrics":123}]`, + }, + { + Name: "proper list_metrics", + Spec: func() string { + var listInput CloudwatchListMetricsInput + require.NoError(t, faker.FakeObject(&listInput)) + return `[{"list_metrics":` + jsonschema.WithRemovedKeys(t, &listInput, "NextToken") + `}]` + }(), + }, + { + Name: "list_metrics.NextToken present", + Err: true, + Spec: func() string { + var listInput CloudwatchListMetricsInput + require.NoError(t, faker.FakeObject(&listInput)) + return `[{"list_metrics":` + jsonschema.WithRemovedKeys(t, &listInput) + `}]` + }(), + }, + }) +} + +func TestCloudwatchGetMetricStatisticsInput_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(CloudwatchMetrics{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `[]`, + }, + { + Name: "empty get_metric_statistics", + Spec: `[{"get_metric_statistics":[]}]`, + }, + { + Name: "null get_metric_statistics", + Spec: `[{"get_metric_statistics":null}]`, + }, + { + Name: "bad get_metric_statistics", + Err: true, + Spec: `[{"get_metric_statistics":123}]`, + }, + { + Name: "empty get_metric_statistics entry", + Spec: `[{"get_metric_statistics":[{}]}]`, + }, + { + Name: "null get_metric_statistics entry", + Err: true, + Spec: `[{"get_metric_statistics":[null]}]`, + }, + { + Name: "bad get_metric_statistics entry", + Err: true, + Spec: `[{"get_metric_statistics":[123]}]`, + }, + { + Name: "proper get_metric_statistics", + Spec: func() string { + var getInput CloudwatchGetMetricStatisticsInput + require.NoError(t, faker.FakeObject(&getInput)) + return `[{"get_metric_statistics":[` + jsonschema.WithRemovedKeys(t, &getInput, "Dimensions", "MetricName", "Namespace") + `]}]` + }(), + }, + { + Name: "get_metric_statistics.Dimensions present", + Err: true, + Spec: func() string { + var getInput CloudwatchGetMetricStatisticsInput + require.NoError(t, faker.FakeObject(&getInput)) + return `[{"get_metric_statistics":[` + jsonschema.WithRemovedKeys(t, &getInput, "MetricName", "Namespace") + `]}]` + }(), + }, + { + Name: "get_metric_statistics.MetricName present", + Err: true, + Spec: func() string { + var getInput CloudwatchGetMetricStatisticsInput + require.NoError(t, faker.FakeObject(&getInput)) + return `[{"get_metric_statistics":[` + jsonschema.WithRemovedKeys(t, &getInput, "Dimensions", "Namespace") + `]}]` + }(), + }, + { + Name: "get_metric_statistics.Namespace present", + Err: true, + Spec: func() string { + var getInput CloudwatchGetMetricStatisticsInput + require.NoError(t, faker.FakeObject(&getInput)) + return `[{"get_metric_statistics":[` + jsonschema.WithRemovedKeys(t, &getInput, "Dimensions", "MetricName") + `]}]` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/costexplorer.go b/plugins/source/aws/client/spec/tableoptions/costexplorer.go similarity index 85% rename from plugins/source/aws/client/tableoptions/costexplorer.go rename to plugins/source/aws/client/spec/tableoptions/costexplorer.go index 3a7ef999f67379..b9dd0b89e74aa0 100644 --- a/plugins/source/aws/client/tableoptions/costexplorer.go +++ b/plugins/source/aws/client/spec/tableoptions/costexplorer.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/costexplorer" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) type CostExplorerAPIs struct { @@ -17,6 +18,11 @@ type CustomGetCostAndUsageInput struct { costexplorer.GetCostAndUsageInput } +// JSONSchemaExtend is required to remove `NextPageToken`. +func (CustomGetCostAndUsageInput) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("NextPageToken") +} + // UnmarshalJSON implements the json.Unmarshaler interface for the CustomLookupEventsOpts type. // It is the same as default, but allows the use of underscore in the JSON field names. func (c *CustomGetCostAndUsageInput) UnmarshalJSON(data []byte) error { diff --git a/plugins/source/aws/client/spec/tableoptions/costexplorer_test.go b/plugins/source/aws/client/spec/tableoptions/costexplorer_test.go new file mode 100644 index 00000000000000..7ae3aca5b80f63 --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/costexplorer_test.go @@ -0,0 +1,65 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/require" +) + +func TestCustomGetCostAndUsageInput_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(CostExplorerAPIs{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "empty get_cost_and_usage", + Spec: `{"get_cost_and_usage":[]}`, + }, + { + Name: "null get_cost_and_usage", + Spec: `{"get_cost_and_usage":null}`, + }, + { + Name: "bad get_cost_and_usage", + Err: true, + Spec: `{"get_cost_and_usage":123}`, + }, + { + Name: "empty get_cost_and_usage entry", + Spec: `{"get_cost_and_usage":[{}]}`, + }, + { + Name: "null get_cost_and_usage entry", + Err: true, + Spec: `{"get_cost_and_usage":[null]}`, + }, + { + Name: "bad get_cost_and_usage entry", + Err: true, + Spec: `{"get_cost_and_usage":[123]}`, + }, + { + Name: "proper get_cost_and_usage", + Spec: func() string { + var input CustomGetCostAndUsageInput + require.NoError(t, faker.FakeObject(&input)) + return `{"get_cost_and_usage":[` + jsonschema.WithRemovedKeys(t, &input, "NextPageToken") + `]}` + }(), + }, + { + Name: "get_cost_and_usage.NextPageToken is present", + Err: true, + Spec: func() string { + var input CustomGetCostAndUsageInput + require.NoError(t, faker.FakeObject(&input)) + return `{"get_cost_and_usage":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/ecs_cluster_tasks.go b/plugins/source/aws/client/spec/tableoptions/ecs_cluster_tasks.go similarity index 78% rename from plugins/source/aws/client/tableoptions/ecs_cluster_tasks.go rename to plugins/source/aws/client/spec/tableoptions/ecs_cluster_tasks.go index b336b0b65ab281..51ea8cbddffe28 100644 --- a/plugins/source/aws/client/tableoptions/ecs_cluster_tasks.go +++ b/plugins/source/aws/client/spec/tableoptions/ecs_cluster_tasks.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) type ECSTaskAPIs struct { @@ -31,6 +32,14 @@ func (s *CustomListTasksOpts) UnmarshalJSON(data []byte) error { return json.Unmarshal(b, &s.ListTasksInput) } +// JSONSchemaExtend is required to remove `NextToken` & `Cluster`, as well as add default for `MaxResults`. +func (CustomListTasksOpts) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("NextToken") + sc.Properties.Delete("Cluster") + + sc.Properties.Value("MaxResults").Default = 100 +} + func (s *ECSTaskAPIs) validateListTasks() error { for _, opt := range s.ListTasksOpts { if aws.ToString(opt.NextToken) != "" { @@ -44,7 +53,7 @@ func (s *ECSTaskAPIs) validateListTasks() error { return nil } -func (s *ECSTaskAPIs) setDefaults() { +func (s *ECSTaskAPIs) SetDefaults() { for i := 0; i < len(s.ListTasksOpts); i++ { if aws.ToInt32(s.ListTasksOpts[i].MaxResults) == 0 { s.ListTasksOpts[i].MaxResults = aws.Int32(100) @@ -53,6 +62,5 @@ func (s *ECSTaskAPIs) setDefaults() { } func (s *ECSTaskAPIs) Validate() error { - s.setDefaults() return s.validateListTasks() } diff --git a/plugins/source/aws/client/spec/tableoptions/ecs_cluster_tasks_test.go b/plugins/source/aws/client/spec/tableoptions/ecs_cluster_tasks_test.go new file mode 100644 index 00000000000000..f8e15296274d79 --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/ecs_cluster_tasks_test.go @@ -0,0 +1,91 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestListTasks(t *testing.T) { + u := CustomListTasksOpts{} + require.NoError(t, faker.FakeObject(&u)) + api := ECSTaskAPIs{ + ListTasksOpts: []CustomListTasksOpts{u}, + } + // Ensure that the validation works as expected + err := api.Validate() + assert.EqualError(t, err, "invalid input: cannot set NextToken in ListTasks") + + // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields + api.ListTasksOpts[0].NextToken = nil + err = api.Validate() + assert.EqualError(t, err, "invalid input: cannot set Cluster in ListTasks") +} + +func TestCustomListTasksOpts_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(ECSTaskAPIs{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "empty list_tasks", + Spec: `{"list_tasks":[]}`, + }, + { + Name: "null list_tasks", + Spec: `{"list_tasks":null}`, + }, + { + Name: "bad list_tasks", + Err: true, + Spec: `{"list_tasks":123}`, + }, + { + Name: "empty list_tasks entry", + Spec: `{"list_tasks":[{}]}`, + }, + { + Name: "null list_tasks entry", + Err: true, + Spec: `{"list_tasks":[null]}`, + }, + { + Name: "bad list_tasks entry", + Err: true, + Spec: `{"list_tasks":123}`, + }, + { + Name: "proper list_tasks", + Spec: func() string { + var input CustomListTasksOpts + require.NoError(t, faker.FakeObject(&input)) + return `{"list_tasks":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken", "Cluster") + `]}` + }(), + }, + { + Name: "list_tasks.NextToken is present", + Err: true, + Spec: func() string { + var input CustomListTasksOpts + require.NoError(t, faker.FakeObject(&input)) + return `{"list_tasks":[` + jsonschema.WithRemovedKeys(t, &input, "Cluster") + `]}` + }(), + }, + { + Name: "list_tasks.Cluster is present", + Err: true, + Spec: func() string { + var input CustomListTasksOpts + require.NoError(t, faker.FakeObject(&input)) + return `{"list_tasks":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/inspector2.go b/plugins/source/aws/client/spec/tableoptions/inspector2.go similarity index 85% rename from plugins/source/aws/client/tableoptions/inspector2.go rename to plugins/source/aws/client/spec/tableoptions/inspector2.go index 72b91b42c9461d..6f5c75efc71c05 100644 --- a/plugins/source/aws/client/tableoptions/inspector2.go +++ b/plugins/source/aws/client/spec/tableoptions/inspector2.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/inspector2" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) type Inspector2APIs struct { @@ -17,6 +18,11 @@ type CustomInspector2ListFindingsInput struct { inspector2.ListFindingsInput } +// JSONSchemaExtend is required to remove `NextToken`. +func (CustomInspector2ListFindingsInput) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("NextToken") +} + // UnmarshalJSON implements the json.Unmarshaler interface for the CustomLookupEventsOpts type. // It is the same as default, but allows the use of underscore in the JSON field names. func (c *CustomInspector2ListFindingsInput) UnmarshalJSON(data []byte) error { diff --git a/plugins/source/aws/client/spec/tableoptions/inspector2_test.go b/plugins/source/aws/client/spec/tableoptions/inspector2_test.go new file mode 100644 index 00000000000000..7ced7a7c3b8495 --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/inspector2_test.go @@ -0,0 +1,84 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInspector2ListFindings(t *testing.T) { + u := CustomInspector2ListFindingsInput{} + require.NoError(t, faker.FakeObject(&u)) + + api := Inspector2APIs{ + ListFindingsOpts: []CustomInspector2ListFindingsInput{u}, + } + // Ensure that the validation works as expected + err := api.Validate() + assert.EqualError(t, err, "invalid input: cannot set NextToken in ListFindings") + + // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields + api.ListFindingsOpts[0].NextToken = nil + err = api.Validate() + + assert.Nil(t, err) +} + +func TestCustomInspector2ListFindingsInput_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(Inspector2APIs{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "empty list_findings", + Spec: `{"list_findings":[]}`, + }, + { + Name: "null list_findings", + Spec: `{"list_findings":null}`, + }, + { + Name: "bad list_findings", + Err: true, + Spec: `{"list_findings":123}`, + }, + { + Name: "empty list_findings entry", + Spec: `{"list_findings":[{}]}`, + }, + { + Name: "null list_findings entry", + Err: true, + Spec: `{"list_findings":[null]}`, + }, + { + Name: "bad list_findings entry", + Err: true, + Spec: `{"list_findings":[123]}`, + }, + { + Name: "proper list_findings", + Spec: func() string { + var input CustomInspector2ListFindingsInput + require.NoError(t, faker.FakeObject(&input)) + return `{"list_findings":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + { + Name: "list_findings.NextToken is present", + Err: true, + Spec: func() string { + var input CustomInspector2ListFindingsInput + require.NoError(t, faker.FakeObject(&input)) + return `{"list_findings":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/securityhub.go b/plugins/source/aws/client/spec/tableoptions/securityhub.go similarity index 79% rename from plugins/source/aws/client/tableoptions/securityhub.go rename to plugins/source/aws/client/spec/tableoptions/securityhub.go index 636fcea76505ac..fde7680b66638f 100644 --- a/plugins/source/aws/client/tableoptions/securityhub.go +++ b/plugins/source/aws/client/spec/tableoptions/securityhub.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/securityhub" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/invopop/jsonschema" ) type SecurityHubAPIs struct { @@ -31,6 +32,15 @@ func (s *CustomGetFindingsOpts) UnmarshalJSON(data []byte) error { return json.Unmarshal(b, &s.GetFindingsInput) } +// JSONSchemaExtend is required to remove `NextToken` as well as add min & max for `MaxResults`. +func (CustomGetFindingsOpts) JSONSchemaExtend(sc *jsonschema.Schema) { + sc.Properties.Delete("NextToken") + + maxResults := sc.Properties.Value("MaxResults") + maxResults.Minimum = json.Number("1") + maxResults.Maximum = json.Number("100") +} + func (s *SecurityHubAPIs) validateGetFindingEvent() error { for _, opt := range s.GetFindingsOpts { if aws.ToString(opt.NextToken) != "" { @@ -45,7 +55,7 @@ func (s *SecurityHubAPIs) validateGetFindingEvent() error { return nil } -func (s *SecurityHubAPIs) setDefaults() { +func (s *SecurityHubAPIs) SetDefaults() { for i := 0; i < len(s.GetFindingsOpts); i++ { if s.GetFindingsOpts[i].MaxResults == 0 { s.GetFindingsOpts[i].MaxResults = 100 @@ -54,6 +64,5 @@ func (s *SecurityHubAPIs) setDefaults() { } func (s *SecurityHubAPIs) Validate() error { - s.setDefaults() return s.validateGetFindingEvent() } diff --git a/plugins/source/aws/client/spec/tableoptions/securityhub_test.go b/plugins/source/aws/client/spec/tableoptions/securityhub_test.go new file mode 100644 index 00000000000000..1a613992f366c3 --- /dev/null +++ b/plugins/source/aws/client/spec/tableoptions/securityhub_test.go @@ -0,0 +1,104 @@ +package tableoptions + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/plugin-sdk/v4/faker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetFindings(t *testing.T) { + u := CustomGetFindingsOpts{} + require.NoError(t, faker.FakeObject(&u)) + api := SecurityHubAPIs{ + GetFindingsOpts: []CustomGetFindingsOpts{u}, + } + // Ensure that the validation works as expected + err := api.Validate() + assert.EqualError(t, err, "invalid input: cannot set NextToken in GetFindings") + + // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields + api.GetFindingsOpts[0].NextToken = nil + err = api.Validate() + assert.EqualError(t, err, "invalid range: MaxResults must be within range [1-100]") +} + +func TestCustomGetFindingsOpts_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(SecurityHubAPIs{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "empty get_findings", + Spec: `{"get_findings":[]}`, + }, + { + Name: "null get_findings", + Spec: `{"get_findings":null}`, + }, + { + Name: "bad get_findings", + Err: true, + Spec: `{"get_findings":123}`, + }, + { + Name: "empty get_findings entry", + Spec: `{"get_findings":[{}]}`, + }, + { + Name: "null get_findings entry", + Err: true, + Spec: `{"get_findings":[null]}`, + }, + { + Name: "bad get_findings entry", + Err: true, + Spec: `{"get_findings":[123]}`, + }, + { + Name: "proper get_findings", + Spec: func() string { + var input CustomGetFindingsOpts + require.NoError(t, faker.FakeObject(&input)) + input.MaxResults = 10 // range 1-100 + return `{"get_findings":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + { + Name: "get_findings.NextToken is present", + Err: true, + Spec: func() string { + var input CustomGetFindingsOpts + require.NoError(t, faker.FakeObject(&input)) + input.MaxResults = 10 // range 1-100 + return `{"get_findings":[` + jsonschema.WithRemovedKeys(t, &input) + `]}` + }(), + }, + { + Name: "get_findings.MaxResults > 100", + Err: true, + Spec: func() string { + var input CustomGetFindingsOpts + require.NoError(t, faker.FakeObject(&input)) + input.MaxResults = 1000 // range 1-100 + return `{"get_findings":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + { + Name: "get_findings.MaxResults < 1", + Err: true, + Spec: func() string { + var input CustomGetFindingsOpts + require.NoError(t, faker.FakeObject(&input)) + input.MaxResults = 0 // range 1-100 + return `{"get_findings":[` + jsonschema.WithRemovedKeys(t, &input, "NextToken") + `]}` + }(), + }, + }) +} diff --git a/plugins/source/aws/client/tableoptions/table_options.go b/plugins/source/aws/client/spec/tableoptions/table_options.go similarity index 67% rename from plugins/source/aws/client/tableoptions/table_options.go rename to plugins/source/aws/client/spec/tableoptions/table_options.go index 4acc14311ffbb6..a649c8206d238e 100644 --- a/plugins/source/aws/client/tableoptions/table_options.go +++ b/plugins/source/aws/client/spec/tableoptions/table_options.go @@ -1,10 +1,17 @@ package tableoptions -import "reflect" +import ( + "reflect" +) -type customInputValidation interface { - Validate() error -} +type ( + customInputValidation interface { + Validate() error + } + defaultsSetter interface { + SetDefaults() + } +) type TableOptions struct { CloudwatchMetrics CloudwatchMetrics `json:"aws_alpha_cloudwatch_metrics,omitempty"` @@ -16,8 +23,21 @@ type TableOptions struct { ECSTasks *ECSTaskAPIs `json:"aws_ecs_cluster_tasks,omitempty"` } -func (t TableOptions) Validate() error { - v := reflect.ValueOf(t) +func (t *TableOptions) SetDefaults() { + v := reflect.ValueOf(*t) + for i := 0; i < v.NumField(); i++ { + table := v.Field(i).Interface() + if !reflect.ValueOf(table).IsNil() { + tableInput, ok := table.(defaultsSetter) + if ok { + tableInput.SetDefaults() + } + } + } +} + +func (t *TableOptions) Validate() error { + v := reflect.ValueOf(*t) for i := 0; i < v.NumField(); i++ { table := v.Field(i).Interface() if !reflect.ValueOf(table).IsNil() { diff --git a/plugins/source/aws/client/tableoptions/table_options_test.go b/plugins/source/aws/client/spec/tableoptions/table_options_test.go similarity index 77% rename from plugins/source/aws/client/tableoptions/table_options_test.go rename to plugins/source/aws/client/spec/tableoptions/table_options_test.go index 390987f1258eb5..0c88e50ae28066 100644 --- a/plugins/source/aws/client/tableoptions/table_options_test.go +++ b/plugins/source/aws/client/spec/tableoptions/table_options_test.go @@ -19,6 +19,7 @@ import ( inspector2types "github.com/aws/aws-sdk-go-v2/service/inspector2/types" "github.com/aws/aws-sdk-go-v2/service/securityhub" securityhubtypes "github.com/aws/aws-sdk-go-v2/service/securityhub/types" + "github.com/cloudquery/codegen/jsonschema" "github.com/stretchr/testify/require" "github.com/cloudquery/plugin-sdk/v4/faker" @@ -26,7 +27,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" ) -func TestTableOptionsValidate(t *testing.T) { +func TestTableOptions_Validate(t *testing.T) { tOpts := TableOptions{} err := tOpts.Validate() if err != nil { @@ -60,6 +61,26 @@ func TestTableOptionsValidate(t *testing.T) { } } +func TestTableOptions_SetDefaults(t *testing.T) { + opts := &TableOptions{ + SecurityHubFindings: &SecurityHubAPIs{ + GetFindingsOpts: []CustomGetFindingsOpts{{}}, + }, + ECSTasks: &ECSTaskAPIs{ + ListTasksOpts: []CustomListTasksOpts{{}}, + }, + } + data, err := json.Marshal(opts) + require.NoError(t, err) + + require.NotPanics(t, opts.SetDefaults) + + // check something did change + dataWithDefaults, err := json.Marshal(opts) + require.NoError(t, err) + require.NotEqual(t, string(data), string(dataWithDefaults)) +} + // TestTableOptionsUnmarshal tests that the TableOptions struct can be unmarshaled from JSON using // snake_case keys. func TestTableOptionsUnmarshal(t *testing.T) { @@ -125,3 +146,27 @@ func TestTableOptionsUnmarshal(t *testing.T) { t.Fatalf("mismatch between objects after loading from snake case json: %v", diff) } } + +func TestJSONSchema(t *testing.T) { + schema, err := jsonschema.Generate(TableOptions{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "all null", + Spec: `{ + "aws_alpha_cloudwatch_metrics": null, + "aws_cloudtrail_events": null, + "aws_accessanalyzer_analyzer_findings": null, + "aws_inspector2_findings": null, + "aws_alpha_costexplorer_cost_custom": null, + "aws_securityhub_findings": null, + "aws_ecs_cluster_tasks": null +}`, + }, + }) +} diff --git a/plugins/source/aws/client/spec_test.go b/plugins/source/aws/client/spec_test.go deleted file mode 100644 index ed264efb05a894..00000000000000 --- a/plugins/source/aws/client/spec_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package client - -import "testing" - -func TestSpecValidate(t *testing.T) { - tests := []struct { - name string - spec *Spec - wantErr bool - }{ - { - name: "valid accounts", - spec: &Spec{ - Accounts: []Account{ - {ID: "123456789012"}, - {ID: "cq-playground"}, - }, - }, - wantErr: false, - }, - { - name: "valid org", - spec: &Spec{ - Organization: &AwsOrg{ - ChildAccountRoleName: "test", - OrganizationUnits: []string{"ou-1234-12345678"}, - }, - }, - wantErr: false, - }, - { - name: "invalid org", - spec: &Spec{ - Organization: &AwsOrg{ - ChildAccountRoleName: "test", - OrganizationUnits: []string{"123"}, - }, - }, - wantErr: true, - }, - { - name: "missing member account role name", - spec: &Spec{ - Organization: &AwsOrg{}, - }, - wantErr: true, - }, - { - name: "valid skip ou", - spec: &Spec{ - Organization: &AwsOrg{ - ChildAccountRoleName: "test", - OrganizationUnits: []string{"ou-1234-12345678"}, - SkipOrganizationalUnits: []string{"ou-1234-45678901"}, - }, - }, - wantErr: false, - }, - { - name: "invalid skip ou", - spec: &Spec{ - Organization: &AwsOrg{ - ChildAccountRoleName: "test", - OrganizationUnits: []string{"ou-1234-12345678"}, - SkipOrganizationalUnits: []string{"456"}, - }, - }, - wantErr: true, - }, - { - name: "both account and org", - spec: &Spec{ - Accounts: []Account{ - {ID: "123456789012"}, - }, - Organization: &AwsOrg{ - ChildAccountRoleName: "test", - }, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := tt.spec.Validate(); (err != nil) != tt.wantErr { - t.Errorf("Spec.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/plugins/source/aws/client/tableoptions/accessanalyzer_test.go b/plugins/source/aws/client/tableoptions/accessanalyzer_test.go deleted file mode 100644 index 1a3492059d6313..00000000000000 --- a/plugins/source/aws/client/tableoptions/accessanalyzer_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package tableoptions - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cloudquery/plugin-sdk/faker" -) - -func TestAAListFindings(t *testing.T) { - u := CustomAccessAnalyzerListFindingsInput{} - require.NoError(t, faker.FakeObject(&u)) - - api := AccessanalyzerFindings{ - ListFindingOpts: []CustomAccessAnalyzerListFindingsInput{u}, - } - // Ensure that the validation works as expected - err := api.Validate() - assert.EqualError(t, err, "invalid input: cannot set NextToken in ListFindings") - api.ListFindingOpts[0].NextToken = nil - - err = api.Validate() - assert.EqualError(t, err, "invalid input: cannot set AnalyzerARN in ListFindings") - api.ListFindingOpts[0].AnalyzerArn = nil - - // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields - err = api.Validate() - assert.Nil(t, err) -} diff --git a/plugins/source/aws/client/tableoptions/cloudtrail_test.go b/plugins/source/aws/client/tableoptions/cloudtrail_test.go deleted file mode 100644 index eb51ec8a9412e0..00000000000000 --- a/plugins/source/aws/client/tableoptions/cloudtrail_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package tableoptions - -import ( - "testing" - - "github.com/cloudquery/plugin-sdk/v4/faker" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLookupEvents(t *testing.T) { - u := CustomLookupEventsOpts{} - require.NoError(t, faker.FakeObject(&u)) - - api := CloudtrailAPIs{ - LookupEventsOpts: []CustomLookupEventsOpts{u}, - } - // Ensure that the validation works as expected - err := api.Validate() - assert.EqualError(t, err, "invalid input: cannot set NextToken in LookupEvents") - - // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields - api.LookupEventsOpts[0].NextToken = nil - err = api.Validate() - assert.Nil(t, err) -} diff --git a/plugins/source/aws/client/tableoptions/ecs_cluster_tasks_test.go b/plugins/source/aws/client/tableoptions/ecs_cluster_tasks_test.go deleted file mode 100644 index 2b2ed30bafa7a7..00000000000000 --- a/plugins/source/aws/client/tableoptions/ecs_cluster_tasks_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package tableoptions - -import ( - "testing" - - "github.com/cloudquery/plugin-sdk/v4/faker" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestListTasks(t *testing.T) { - u := CustomListTasksOpts{} - require.NoError(t, faker.FakeObject(&u)) - api := ECSTaskAPIs{ - ListTasksOpts: []CustomListTasksOpts{u}, - } - // Ensure that the validation works as expected - err := api.Validate() - assert.EqualError(t, err, "invalid input: cannot set NextToken in ListTasks") - - // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields - api.ListTasksOpts[0].NextToken = nil - err = api.Validate() - assert.EqualError(t, err, "invalid input: cannot set Cluster in ListTasks") -} diff --git a/plugins/source/aws/client/tableoptions/inspector2_test.go b/plugins/source/aws/client/tableoptions/inspector2_test.go deleted file mode 100644 index 084c5ce1613d80..00000000000000 --- a/plugins/source/aws/client/tableoptions/inspector2_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package tableoptions - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/cloudquery/plugin-sdk/faker" -) - -func TestInspector2ListFindings(t *testing.T) { - u := CustomInspector2ListFindingsInput{} - require.NoError(t, faker.FakeObject(&u)) - - api := Inspector2APIs{ - ListFindingsOpts: []CustomInspector2ListFindingsInput{u}, - } - // Ensure that the validation works as expected - err := api.Validate() - assert.EqualError(t, err, "invalid input: cannot set NextToken in ListFindings") - - // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields - api.ListFindingsOpts[0].NextToken = nil - err = api.Validate() - - assert.Nil(t, err) -} diff --git a/plugins/source/aws/client/tableoptions/securityhub_test.go b/plugins/source/aws/client/tableoptions/securityhub_test.go deleted file mode 100644 index 016adc454083a9..00000000000000 --- a/plugins/source/aws/client/tableoptions/securityhub_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package tableoptions - -import ( - "testing" - - "github.com/cloudquery/plugin-sdk/v4/faker" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGetFindings(t *testing.T) { - u := CustomGetFindingsOpts{} - require.NoError(t, faker.FakeObject(&u)) - api := SecurityHubAPIs{ - GetFindingsOpts: []CustomGetFindingsOpts{u}, - } - // Ensure that the validation works as expected - err := api.Validate() - assert.EqualError(t, err, "invalid input: cannot set NextToken in GetFindings") - - // Ensure that as soon as the validation passes that there are no unexpected empty or nil fields - api.GetFindingsOpts[0].NextToken = nil - err = api.Validate() - assert.EqualError(t, err, "invalid range: MaxResults must be within range [1-100]") -} diff --git a/plugins/source/aws/client/testing.go b/plugins/source/aws/client/testing.go index a266499609f4f8..cc1cc8d365a380 100644 --- a/plugins/source/aws/client/testing.go +++ b/plugins/source/aws/client/testing.go @@ -7,13 +7,13 @@ import ( "testing" "time" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/plugin" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" - - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" "github.com/cloudquery/plugin-sdk/v4/scheduler" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" + sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/golang/mock/gomock" "github.com/rs/zerolog" ) @@ -34,7 +34,7 @@ func AwsMockTestHelper(t *testing.T, parentTable *schema.Table, builder func(*te zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro}, ).Level(zerolog.DebugLevel).With().Timestamp().Logger() - var awsSpec Spec + var awsSpec spec.Spec awsSpec.SetDefaults() awsSpec.UsePaidAPIs = true awsSpec.TableOptions = &testOpts.TableOptions @@ -71,7 +71,7 @@ func AwsCreateMockClient(t *testing.T, ctrl *gomock.Controller, builder func(*te zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro}, ).Level(zerolog.DebugLevel).With().Timestamp().Logger() - var awsSpec Spec + var awsSpec spec.Spec awsSpec.SetDefaults() awsSpec.UsePaidAPIs = true awsSpec.TableOptions = &testOpts.TableOptions diff --git a/plugins/source/aws/go.mod b/plugins/source/aws/go.mod index 7252fd9f953c71..5f3a70c9e33f91 100644 --- a/plugins/source/aws/go.mod +++ b/plugins/source/aws/go.mod @@ -130,13 +130,15 @@ require ( github.com/aws/aws-sdk-go-v2/service/xray v1.17.2 github.com/aws/smithy-go v1.14.1 github.com/basgys/goxml2json v1.1.0 - github.com/cloudquery/codegen v0.3.4 + github.com/cloudquery/codegen v0.3.5 github.com/cloudquery/plugin-sdk/v4 v4.12.3 github.com/cockroachdb/cockroachdb-parser v0.0.0-20230705064001-302c9ad52e1a github.com/gertd/go-pluralize v0.2.1 + github.com/ghodss/yaml v1.0.0 github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 + github.com/invopop/jsonschema v0.11.0 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/mitchellh/mapstructure v1.5.0 github.com/mjibson/sqlfmt v0.5.0 @@ -144,6 +146,7 @@ require ( github.com/rs/zerolog v1.29.1 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.9.3 + github.com/wk8/go-ordered-map/v2 v2.1.8 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.3.0 google.golang.org/grpc v1.57.0 @@ -152,7 +155,18 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231002001222-7ded38b478cd +// https://github.com/cloudquery/jsonschema@cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231006094144-b42fc04b92bb + require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect + github.com/CloudyKit/jet/v6 v6.2.0 // indirect + github.com/Joker/jade v1.1.3 // indirect + github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect @@ -166,43 +180,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect - github.com/bitly/go-simplejson v0.5.0 // indirect - github.com/cloudquery/plugin-sdk v1.45.0 - github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/getsentry/sentry-go v0.24.1 // indirect - github.com/ghodss/yaml v1.0.0 - github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/cobra v1.6.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect -) - -require ( - github.com/BurntSushi/toml v1.3.2 // indirect - github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect - github.com/CloudyKit/jet/v6 v6.2.0 // indirect - github.com/Joker/jade v1.1.3 // indirect - github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/apache/arrow/go/v13 v13.0.0-20230731205701-112f94971882 // indirect - github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect + github.com/bitly/go-simplejson v0.5.0 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.10.1 // indirect @@ -211,16 +192,19 @@ require ( github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/cloudquery/cloudquery-api-go v1.2.4 // indirect github.com/cloudquery/plugin-pb-go v1.12.1 // indirect + github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect github.com/cockroachdb/apd/v3 v3.1.0 // indirect github.com/cockroachdb/errors v1.9.0 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/dave/dst v0.27.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.15.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/getsentry/sentry-go v0.24.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -228,15 +212,21 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect - github.com/invopop/jsonschema v0.11.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -246,6 +236,8 @@ require ( github.com/kataras/pio v0.0.12 // indirect github.com/kataras/sitemap v0.0.6 // indirect github.com/kataras/tunnel v0.0.4 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/labstack/echo/v4 v4.11.1 // indirect @@ -254,19 +246,24 @@ require ( github.com/lib/pq v1.10.6 // indirect github.com/mailgun/raymond/v2 v2.0.48 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20230904192822-1876fd5063bc // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pierrre/geohash v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/tdewolff/minify/v2 v2.12.9 // indirect github.com/tdewolff/parse/v2 v2.6.8 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -277,8 +274,8 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosssi/ace v0.0.5 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect @@ -295,6 +292,7 @@ require ( golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.13.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect diff --git a/plugins/source/aws/go.sum b/plugins/source/aws/go.sum index 8dba27bf8054d9..7c2e9c9b824603 100644 --- a/plugins/source/aws/go.sum +++ b/plugins/source/aws/go.sum @@ -408,12 +408,12 @@ github.com/cloudquery/arrow/go/v14 v14.0.0-20231002001222-7ded38b478cd h1:LtWC4o github.com/cloudquery/arrow/go/v14 v14.0.0-20231002001222-7ded38b478cd/go.mod h1:/SqmdO2dsWqFHqQQeupnsr0ollL8C91n3x0I72rArY8= github.com/cloudquery/cloudquery-api-go v1.2.4 h1:IFu1PajNJEVj1IRN6bYa4fW10gDVfuqvr4HbQACpI7Q= github.com/cloudquery/cloudquery-api-go v1.2.4/go.mod h1:oyNUZZ6CKjPapxMbmE/qR2vdo9/G+tuCdF7uXT1LYuM= -github.com/cloudquery/codegen v0.3.4 h1:i3iz/JoZonnxpTJpUW6W11yFft+9L74cXC/Y/juvtCI= -github.com/cloudquery/codegen v0.3.4/go.mod h1:yBakGbEKNMMPBmLK2IS4r59rTOmVQqB46ueRvTxW1jc= +github.com/cloudquery/codegen v0.3.5 h1:PcWEOM5H8Ck8U3QHjpiACd+Nf7bipbaa/sC1SxZBgFk= +github.com/cloudquery/codegen v0.3.5/go.mod h1:cw8AcB227i6drp3miw3srEuVW49xlm5k/EUd6bbdkVc= +github.com/cloudquery/jsonschema v0.0.0-20231006094144-b42fc04b92bb h1:W3y1G8XTh5NrFloojJqv8oFfAPbzdWlt5i0ik3rU2Pk= +github.com/cloudquery/jsonschema v0.0.0-20231006094144-b42fc04b92bb/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/cloudquery/plugin-pb-go v1.12.1 h1:lxCe/ovcbmY4i0N5ko1fH5PIllosIwbXTvrRqZ/UBeg= github.com/cloudquery/plugin-pb-go v1.12.1/go.mod h1:+20GGdx/k9ApU56ix5QrPIGfzn0WA7VJlExgaHnNGzc= -github.com/cloudquery/plugin-sdk v1.45.0 h1:5vrfQZtaO1dp6ebKt8ouXDmPC7eeLuOB3JMd+FTRSYk= -github.com/cloudquery/plugin-sdk v1.45.0/go.mod h1:9KGuuTGjTCKgh9amKwS+7Zrrqq7/M6lormteOyqoKwg= github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U= github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug= github.com/cloudquery/plugin-sdk/v4 v4.12.3 h1:Gp7FtVhdh+5Ypv81r+j7Ph/2WyNje/4cv3+KraY0SQI= @@ -498,8 +498,6 @@ github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nI github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= @@ -669,8 +667,6 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.11.0 h1:tdAVvos5ttrsYLyEuVymkVVK31EFpwnTu5hWiyYLGWA= -github.com/invopop/jsonschema v0.11.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= @@ -894,8 +890,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= diff --git a/plugins/source/aws/resources/plugin/client.go b/plugins/source/aws/resources/plugin/client.go index 46e8f7380ade18..9ea5bdebbe3bbf 100644 --- a/plugins/source/aws/resources/plugin/client.go +++ b/plugins/source/aws/resources/plugin/client.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/cloudquery/cloudquery/plugins/source/aws/client" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/cloudquery/plugin-sdk/v4/message" "github.com/cloudquery/plugin-sdk/v4/plugin" "github.com/cloudquery/plugin-sdk/v4/scheduler" @@ -28,7 +29,7 @@ type Client struct { } func New(ctx context.Context, logger zerolog.Logger, specBytes []byte, options plugin.NewClientOptions) (plugin.Client, error) { - var spec client.Spec + var s spec.Spec c := &Client{ options: options, logger: logger, @@ -38,22 +39,22 @@ func New(ctx context.Context, logger zerolog.Logger, specBytes []byte, options p return c, nil } var err error - if err := json.Unmarshal(specBytes, &spec); err != nil { + if err := json.Unmarshal(specBytes, &s); err != nil { return nil, err } - spec.SetDefaults() - if err := spec.Validate(); err != nil { + s.SetDefaults() + if err := s.Validate(); err != nil { return nil, err } - c.client, err = client.Configure(ctx, logger, spec) + c.client, err = client.Configure(ctx, logger, s) if err != nil { return nil, err } c.scheduler = scheduler.NewScheduler( - scheduler.WithConcurrency(spec.Concurrency), + scheduler.WithConcurrency(s.Concurrency), scheduler.WithLogger(logger), - scheduler.WithStrategy(spec.Scheduler), + scheduler.WithStrategy(s.Scheduler), ) return c, nil } diff --git a/plugins/source/aws/resources/plugin/plugin.go b/plugins/source/aws/resources/plugin/plugin.go index 389044d13c33fe..a4899a6054ab02 100644 --- a/plugins/source/aws/resources/plugin/plugin.go +++ b/plugins/source/aws/resources/plugin/plugin.go @@ -3,8 +3,9 @@ package plugin import ( "strings" - "github.com/cloudquery/plugin-sdk/plugins/source" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec" "github.com/cloudquery/plugin-sdk/v4/caser" + "github.com/cloudquery/plugin-sdk/v4/docs" "github.com/cloudquery/plugin-sdk/v4/plugin" "github.com/cloudquery/plugin-sdk/v4/schema" ) @@ -93,7 +94,7 @@ var awsExceptions = map[string]string{ func titleTransformer(table *schema.Table) { if table.Title == "" { exceptions := make(map[string]string) - for k, v := range source.DefaultTitleExceptions { + for k, v := range docs.DefaultTitleExceptions { exceptions[k] = v } for k, v := range awsExceptions { @@ -113,5 +114,6 @@ func AWS() *plugin.Plugin { "aws", Version, New, + plugin.WithJSONSchema(spec.JSONSchema), ) } diff --git a/plugins/source/aws/resources/services/accessanalyzer/analyzer_findings.go b/plugins/source/aws/resources/services/accessanalyzer/analyzer_findings.go index 91a806eb60a070..07caaba69d8ba9 100644 --- a/plugins/source/aws/resources/services/accessanalyzer/analyzer_findings.go +++ b/plugins/source/aws/resources/services/accessanalyzer/analyzer_findings.go @@ -9,7 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" ) diff --git a/plugins/source/aws/resources/services/cloudtrail/events.go b/plugins/source/aws/resources/services/cloudtrail/events.go index 5739dc134a775b..0fccb7714762b1 100644 --- a/plugins/source/aws/resources/services/cloudtrail/events.go +++ b/plugins/source/aws/resources/services/cloudtrail/events.go @@ -6,15 +6,14 @@ import ( "time" "github.com/apache/arrow/go/v14/arrow" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" - "github.com/mitchellh/hashstructure/v2" - "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/aws/aws-sdk-go-v2/service/cloudtrail/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" + sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" + "github.com/mitchellh/hashstructure/v2" ) const tableName = "aws_cloudtrail_events" diff --git a/plugins/source/aws/resources/services/cloudwatch/metric_statistics.go b/plugins/source/aws/resources/services/cloudwatch/metric_statistics.go index 5a590131a59e89..6dd816f91b744b 100644 --- a/plugins/source/aws/resources/services/cloudwatch/metric_statistics.go +++ b/plugins/source/aws/resources/services/cloudwatch/metric_statistics.go @@ -9,7 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" cqtypes "github.com/cloudquery/plugin-sdk/v4/types" diff --git a/plugins/source/aws/resources/services/cloudwatch/metrics.go b/plugins/source/aws/resources/services/cloudwatch/metrics.go index 513fb7d4dbc9b6..56ca7c6666e1d4 100644 --- a/plugins/source/aws/resources/services/cloudwatch/metrics.go +++ b/plugins/source/aws/resources/services/cloudwatch/metrics.go @@ -7,7 +7,7 @@ import ( "github.com/apache/arrow/go/v14/arrow" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" cqtypes "github.com/cloudquery/plugin-sdk/v4/types" "github.com/mitchellh/hashstructure/v2" diff --git a/plugins/source/aws/resources/services/cloudwatch/metrics_mock_test.go b/plugins/source/aws/resources/services/cloudwatch/metrics_mock_test.go index ba85cfc9ed5ba6..b2fdf2f9dd9dec 100644 --- a/plugins/source/aws/resources/services/cloudwatch/metrics_mock_test.go +++ b/plugins/source/aws/resources/services/cloudwatch/metrics_mock_test.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" diff --git a/plugins/source/aws/resources/services/ecs/cluster_tasks.go b/plugins/source/aws/resources/services/ecs/cluster_tasks.go index 4fab6c7e784212..6b98109189b02c 100644 --- a/plugins/source/aws/resources/services/ecs/cluster_tasks.go +++ b/plugins/source/aws/resources/services/ecs/cluster_tasks.go @@ -3,16 +3,15 @@ package ecs import ( "context" - "github.com/cloudquery/plugin-sdk/v4/transformers" - sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" - "github.com/apache/arrow/go/v14/arrow" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ecs" "github.com/aws/aws-sdk-go-v2/service/ecs/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/cloudquery/plugin-sdk/v4/transformers" + sdkTypes "github.com/cloudquery/plugin-sdk/v4/types" ) func clusterTasks() *schema.Table { diff --git a/plugins/source/aws/resources/services/inspector2/findings.go b/plugins/source/aws/resources/services/inspector2/findings.go index e1f1bedd5f157e..fc69e569fffbe3 100644 --- a/plugins/source/aws/resources/services/inspector2/findings.go +++ b/plugins/source/aws/resources/services/inspector2/findings.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/inspector2" "github.com/aws/aws-sdk-go-v2/service/inspector2/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" ) diff --git a/plugins/source/aws/resources/services/route53recoveryreadiness/cells_mock_test.go b/plugins/source/aws/resources/services/route53recoveryreadiness/cells_mock_test.go index eff61be8962482..c0cf7d70d2ad2c 100644 --- a/plugins/source/aws/resources/services/route53recoveryreadiness/cells_mock_test.go +++ b/plugins/source/aws/resources/services/route53recoveryreadiness/cells_mock_test.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks" - "github.com/cloudquery/plugin-sdk/faker" + "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) diff --git a/plugins/source/aws/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go b/plugins/source/aws/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go index 4f486f4b399e8c..33d254477aa055 100644 --- a/plugins/source/aws/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go +++ b/plugins/source/aws/resources/services/route53recoveryreadiness/readiness_checks_mock_test.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks" - "github.com/cloudquery/plugin-sdk/faker" + "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) diff --git a/plugins/source/aws/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go b/plugins/source/aws/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go index 0d1e17eb19d3c9..2d4d0c25153ed6 100644 --- a/plugins/source/aws/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go +++ b/plugins/source/aws/resources/services/route53recoveryreadiness/recovery_groups_mock_test.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks" - "github.com/cloudquery/plugin-sdk/faker" + "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) diff --git a/plugins/source/aws/resources/services/route53recoveryreadiness/resource_sets_mock_test.go b/plugins/source/aws/resources/services/route53recoveryreadiness/resource_sets_mock_test.go index 2639842a01769d..d17230fd95e9a8 100644 --- a/plugins/source/aws/resources/services/route53recoveryreadiness/resource_sets_mock_test.go +++ b/plugins/source/aws/resources/services/route53recoveryreadiness/resource_sets_mock_test.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53recoveryreadiness/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" "github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks" - "github.com/cloudquery/plugin-sdk/faker" + "github.com/cloudquery/plugin-sdk/v4/faker" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" ) diff --git a/plugins/source/aws/resources/services/securityhub/findings.go b/plugins/source/aws/resources/services/securityhub/findings.go index 4c26827a7434c3..928b29434a7aef 100644 --- a/plugins/source/aws/resources/services/securityhub/findings.go +++ b/plugins/source/aws/resources/services/securityhub/findings.go @@ -7,7 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/securityhub" "github.com/aws/aws-sdk-go-v2/service/securityhub/types" "github.com/cloudquery/cloudquery/plugins/source/aws/client" - "github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions" + "github.com/cloudquery/cloudquery/plugins/source/aws/client/spec/tableoptions" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/cloudquery/plugin-sdk/v4/transformers" )