From 618d2f776719210d42b6d42fc835e39b0f8ab654 Mon Sep 17 00:00:00 2001 From: Jason Quesenberry Date: Thu, 19 Sep 2024 16:22:12 -0700 Subject: [PATCH] Includes a lot of fixes for comments to the PR. --- .doc_gen/metadata/redshift_metadata.yaml | 38 ++---- gov2/redshift/README.md | 15 +-- gov2/redshift/actions/redshift_actions.go | 54 ++++++-- gov2/redshift/actions/redshiftdata_actions.go | 73 ++++++++--- gov2/redshift/cmd/main.go | 4 +- gov2/redshift/go.mod | 3 +- gov2/redshift/go.sum | 2 + gov2/redshift/hello/hello.go | 14 +- gov2/redshift/scenarios/redshift_basics.go | 124 ++++++++++-------- .../redshift_basics_integration_test.go | 2 +- .../scenarios/redshift_basics_test.go | 32 +++-- gov2/redshift/stubs/redshift_data_stubs.go | 2 +- gov2/redshift/stubs/redshift_stubs.go | 21 +-- gov2/redshift/stubs/secrets_manager_stubs.go | 24 ++++ javav2/example_code/redshift/README.md | 4 +- python/example_code/redshift/README.md | 4 +- 16 files changed, 257 insertions(+), 159 deletions(-) create mode 100644 gov2/redshift/stubs/secrets_manager_stubs.go diff --git a/.doc_gen/metadata/redshift_metadata.yaml b/.doc_gen/metadata/redshift_metadata.yaml index 9cfe4658cc2..2ab4d973613 100644 --- a/.doc_gen/metadata/redshift_metadata.yaml +++ b/.doc_gen/metadata/redshift_metadata.yaml @@ -31,7 +31,7 @@ redshift_Hello: snippet_tags: - python.example_code.redshift.Hello services: - redshift: {describeClusters} + redshift: {DescribeClusters} redshift_ListDatabases: languages: Java: @@ -54,6 +54,8 @@ redshift_CreateCluster: excerpts: - description: snippet_tags: + - gov2.redshift.Imports + - gov2.redshift.ActionsStruct - gov2.redshift.CreateCluster Kotlin: versions: @@ -109,6 +111,8 @@ redshift_DeleteCluster: excerpts: - description: snippet_tags: + - gov2.redshift.Imports + - gov2.redshift.ActionsStruct - gov2.redshift.DeleteCluster Kotlin: versions: @@ -164,6 +168,8 @@ redshift_DescribeClusters: excerpts: - description: snippet_tags: + - gov2.redshift.Imports + - gov2.redshift.ActionsStruct - gov2.redshift.DescribeClusters Kotlin: versions: @@ -219,6 +225,8 @@ redshift_ModifyCluster: excerpts: - description: snippet_tags: + - gov2.redshift.Imports + - gov2.redshift.ActionsStruct - gov2.redshift.ModifyCluster Kotlin: versions: @@ -267,14 +275,6 @@ redshift_ModifyCluster: redshift: {ModifyCluster} redshift_DescribeStatement: languages: - Go: - versions: - - sdk_version: 2 - github: gov2/redshift - excerpts: - - description: - snippet_tags: - - gov2.redshift.DescribeStatement Java: versions: - sdk_version: 2 @@ -301,14 +301,6 @@ redshift_DescribeStatement: redshift: {DescribeStatement} redshift_GetStatementResult: languages: - Go: - versions: - - sdk_version: 2 - github: gov2/redshift - excerpts: - - description: - snippet_tags: - - gov2.redshift.GetStatementResult Java: versions: - sdk_version: 2 @@ -335,14 +327,6 @@ redshift_GetStatementResult: redshift: {GetStatementResult} redshift_ExecuteStatement: languages: - Go: - versions: - - sdk_version: 2 - github: gov2/redshift - excerpts: - - description: - snippet_tags: - - gov2.redshift.CreateTable Java: versions: - sdk_version: 2 @@ -361,7 +345,7 @@ redshift_ExecuteStatement: services: redshift: {ExecuteStatement} redshift_Scenario: - synopsis: learn core operations for &RS; using an &AWS;. + synopsis: learn core operations for &RS; using an &AWS; SDK. category: Basics languages: Go: @@ -407,4 +391,4 @@ redshift_Scenario: - python.example_code.redshift_data.ListDatabases - python.example_code.redshift.DeleteCluster services: - redshift: {createCluster, describeClusters, executeStatement, describeStatement, modifyCluster, getStatementResult, listDatabasesPaginator} + redshift: {CreateCluster, DescribeClusters, ExecuteStatement, DescribeStatement, ModifyCluster, GetStatementResult, ListDatabasesPaginator} diff --git a/gov2/redshift/README.md b/gov2/redshift/README.md index 6f024d860d8..554c5a724fc 100644 --- a/gov2/redshift/README.md +++ b/gov2/redshift/README.md @@ -31,7 +31,7 @@ For prerequisites, see the [README](../README.md#Prerequisites) in the `gov2` fo ### Get started -- [Hello Amazon Redshift](hello/hello.go#L4) (`describeClusters`) +- [Hello Amazon Redshift](hello/hello.go#L4) (`DescribeClusters`) ### Basics @@ -45,13 +45,10 @@ Code examples that show you how to perform the essential operations within a ser Code excerpts that show you how to call individual service functions. -- [CreateCluster](actions/redshift_actions.go#L20) -- [CreateTable](actions/redshiftdata_actions.go#L75) -- [DeleteCluster](actions/redshift_actions.go#L72) -- [DescribeClusters](scenarios/redshift_basics.go#L242) -- [DescribeStatement](actions/redshiftdata_actions.go#L173) -- [GetStatementResult](actions/redshiftdata_actions.go#L199) -- [ModifyCluster](actions/redshift_actions.go#L50) +- [CreateCluster](actions/redshift_actions.go#L29) +- [DeleteCluster](actions/redshift_actions.go#L84) +- [DescribeClusters](actions/redshift_actions.go#L108) +- [ModifyCluster](actions/redshift_actions.go#L58) @@ -83,7 +80,7 @@ go run ./cmd -h ``` #### Learn the basics -This example shows you how to work with Amazon Redshift tables, items, and queries. +This example shows you how to learn core operations for Amazon Redshift using an AWS SDK. diff --git a/gov2/redshift/actions/redshift_actions.go b/gov2/redshift/actions/redshift_actions.go index a7e82a20304..cdc0239caa5 100644 --- a/gov2/redshift/actions/redshift_actions.go +++ b/gov2/redshift/actions/redshift_actions.go @@ -3,6 +3,8 @@ package actions +// snippet-start:[gov2.redshift.Imports] + import ( "context" "errors" @@ -10,13 +12,20 @@ import ( "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" "log" + "time" ) +// snippet-end:[gov2.redshift.Imports] + +// snippet-start:[gov2.redshift.ActionsStruct] + // RedshiftActions wraps Redshift service actions. type RedshiftActions struct { RedshiftClient *redshift.Client } +// snippet-end:[gov2.redshift.ActionsStruct] + // snippet-start:[gov2.redshift.CreateCluster] // CreateCluster sends a request to create a cluster with the given clusterId using the provided credentials. @@ -30,7 +39,6 @@ func (actor RedshiftActions) CreateCluster(ctx context.Context, clusterId string ClusterType: aws.String(clusterType), PubliclyAccessible: aws.Bool(publiclyAccessible), } - var opErr *types.ClusterAlreadyExistsFault output, err := actor.RedshiftClient.CreateCluster(ctx, input) if err != nil && errors.As(err, &opErr) { @@ -57,10 +65,14 @@ func (actor RedshiftActions) ModifyCluster(ctx context.Context, clusterId string PreferredMaintenanceWindow: aws.String(maintenanceWindow), } + var opErr *types.InvalidClusterStateFault output, err := actor.RedshiftClient.ModifyCluster(ctx, input) - if err != nil { + if err != nil && errors.As(err, &opErr) { + log.Println("Cluster is in an invalid state.") + panic(err) + } else if err != nil { log.Printf("Failed to modify Redshift cluster: %v\n", err) - return nil + panic(err) } log.Printf("The cluster was successfully modified and now has %s as the maintenance window\n", *output.Cluster.PreferredMaintenanceWindow) @@ -74,12 +86,16 @@ func (actor RedshiftActions) ModifyCluster(ctx context.Context, clusterId string // DeleteCluster deletes the given cluster. func (actor RedshiftActions) DeleteCluster(ctx context.Context, clusterId string) (bool, error) { // Delete the specified Redshift cluster - input := &redshift.DeleteClusterInput{ - ClusterIdentifier: aws.String(clusterId), - SkipFinalClusterSnapshot: aws.Bool(true), - } - _, err := actor.RedshiftClient.DeleteCluster(ctx, input) - if err != nil { + + waiter := redshift.NewClusterDeletedWaiter(actor.RedshiftClient) + err := waiter.Wait(ctx, &redshift.DescribeClustersInput{ + ClusterIdentifier: aws.String(clusterId), + }, 5*time.Minute) + var opErr *types.ClusterNotFoundFault + if err != nil && errors.As(err, &opErr) { + log.Println("Cluster was not found. Where could it be?") + return false, err + } else if err != nil { log.Printf("Failed to delete Redshift cluster: %v\n", err) return false, err } @@ -88,3 +104,23 @@ func (actor RedshiftActions) DeleteCluster(ctx context.Context, clusterId string } // snippet-end:[gov2.redshift.DeleteCluster] + +// snippet-start:[gov2.redshift.DescribeClusters] + +// DescribeClusters returns information about the given cluster. +func (actor RedshiftActions) DescribeClusters(ctx context.Context, clusterId string) (*redshift.DescribeClustersOutput, error) { + input, err := actor.RedshiftClient.DescribeClusters(ctx, &redshift.DescribeClustersInput{ + ClusterIdentifier: aws.String(clusterId), + }) + var opErr *types.AccessToClusterDeniedFault + if errors.As(err, &opErr) { + println("Access to cluster denied.") + panic(err) + } else if err != nil { + println("Failed to describe Redshift clusters.") + return nil, err + } + return input, nil +} + +// snippet-end:[gov2.redshift.DescribeClusters] diff --git a/gov2/redshift/actions/redshiftdata_actions.go b/gov2/redshift/actions/redshiftdata_actions.go index ae2f3f4dab8..80a58145a0f 100644 --- a/gov2/redshift/actions/redshiftdata_actions.go +++ b/gov2/redshift/actions/redshiftdata_actions.go @@ -8,15 +8,20 @@ import ( "errors" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshiftdata" + "github.com/aws/aws-sdk-go-v2/service/redshiftdata/types" "github.com/awsdocs/aws-doc-sdk-examples/gov2/demotools" "log" ) +// snippet-start:[gov2.redshift.DataActionsStruct] + // RedshiftDataActions wraps RedshiftData actions. type RedshiftDataActions struct { RedshiftDataClient *redshiftdata.Client } +// snippet-end:[gov2.redshift.DataActionsStruct] + // snippet-start:[gov2.redshift.RedshiftQuery.struct] // RedshiftQuery makes it easier to deal with RedshiftQuery objects. @@ -32,9 +37,7 @@ type RedshiftQuery struct { // ExecuteStatement calls the ExecuteStatement operation from the RedshiftDataClient func (actor RedshiftDataActions) ExecuteStatement(ctx context.Context, input redshiftdata.ExecuteStatementInput) (*redshiftdata.ExecuteStatementOutput, error) { - return actor.RedshiftDataClient.ExecuteStatement(ctx, &input) - } // snippet-end:[gov2.redshift.ExecuteStatement @@ -52,19 +55,28 @@ func (actor RedshiftDataActions) ExecuteBatchStatement(ctx context.Context, inpu // ListDatabases lists all databases in the given cluster. func (actor RedshiftDataActions) ListDatabases(ctx context.Context, clusterId string, databaseName string, userName string) error { - input := redshiftdata.ListDatabasesInput{ - ClusterIdentifier: aws.String(clusterId), + + var opErr *types.DatabaseConnectionException + var databaseNames []string + paginator := redshiftdata.NewListDatabasesPaginator(actor.RedshiftDataClient, &redshiftdata.ListDatabasesInput{ Database: aws.String(databaseName), + ClusterIdentifier: aws.String(clusterId), DbUser: aws.String(userName), + }) + for paginator.HasMorePages() { + output, err := paginator.NextPage(ctx) + if err != nil && errors.As(err, &opErr) { + log.Printf("Could not connect to the database.") + panic(err) + } else if err != nil { + log.Printf("Couldn't finish listing the tables. Here's why: %v\n", err) + return err + } else { + databaseNames = append(databaseNames, output.Databases...) + } } - output, err := actor.RedshiftDataClient.ListDatabases(ctx, &input) - if err != nil { - log.Printf("Failed to list databases: %v\n", err) - return err - } - - for _, database := range output.Databases { + for _, database := range databaseNames { log.Printf("The database name is : %s\n", database) } return nil @@ -90,8 +102,11 @@ func (actor RedshiftDataActions) CreateTable(ctx context.Context, clusterId stri Sql: aws.String(sql), } + var opErr *types.DatabaseConnectionException output, err := actor.RedshiftDataClient.ExecuteStatement(ctx, createTableInput) - if err != nil { + if err != nil && errors.As(err, &opErr) { + log.Printf("Could not connect to the database.") + } else if err != nil { log.Printf("Failed to create table: %v\n", err) return err } @@ -114,8 +129,11 @@ func (actor RedshiftDataActions) DeleteTable(ctx context.Context, clusterId stri Sql: aws.String(sql), } + var opErr *types.DatabaseConnectionException output, err := actor.RedshiftDataClient.ExecuteStatement(ctx, deleteTableInput) - if err != nil { + if err != nil && errors.As(err, &opErr) { + log.Printf("Could not connect to the database.") + } else if err != nil { log.Printf("Failed to delete table "+tableName+" from "+databaseName+" database: %v\n", err) return false, err } @@ -162,6 +180,24 @@ func (actor RedshiftDataActions) DeleteDataRows(ctx context.Context, clusterId s // snippet-end:[gov2.redshift.DeleteRows] +// snippet-start:[gov2.redshift.DescribeStatement] + +// DescribeStatement gets information about the given statement. +func (actor RedshiftDataActions) DescribeStatement(query RedshiftQuery) (*redshiftdata.DescribeStatementOutput, error) { + describeOutput, err := actor.RedshiftDataClient.DescribeStatement(query.Context, &query.Input) + var opErr *types.QueryTimeoutException + if errors.As(err, &opErr) { + println("The connection to the redshift data request timed out.") + panic(err) + } else if err != nil { + println("Failed to execute describe statement") + return nil, err + } + return describeOutput, nil +} + +// snippet-end:[gov2.redshift.DescribeStatement] + // snippet-start:[gov2.redshift.WaitForQueryStatus] // WaitForQueryStatus waits until the given RedshiftQuery object has succeeded or failed. @@ -170,8 +206,7 @@ func (actor RedshiftDataActions) WaitForQueryStatus(query RedshiftQuery, pauser attempts := 0 maxWaitCycles := 30 for done == false { - // snippet-start:[gov2.redshift.DescribeStatement] - describeOutput, err := actor.RedshiftDataClient.DescribeStatement(query.Context, &query.Input) + describeOutput, err := actor.DescribeStatement(query) if err != nil { return err } @@ -187,7 +222,6 @@ func (actor RedshiftDataActions) WaitForQueryStatus(query RedshiftQuery, pauser if describeOutput.Status == "FINISHED" { done = true } - // snippet-end:[gov2.redshift.DescribeStatement] attempts++ pauser.Pause(attempts) } @@ -200,10 +234,15 @@ func (actor RedshiftDataActions) WaitForQueryStatus(query RedshiftQuery, pauser // GetStatementResult returns the result of the statement with the given id. func (actor RedshiftDataActions) GetStatementResult(ctx context.Context, statementId string) (*redshiftdata.GetStatementResultOutput, error) { + + var opErr *types.QueryTimeoutException getStatementResultOutput, err := actor.RedshiftDataClient.GetStatementResult(ctx, &redshiftdata.GetStatementResultInput{ Id: aws.String(statementId), }) - if err != nil { + if err != nil && errors.As(err, &opErr) { + log.Printf("Query timed out: %v\n", err) + return nil, err + } else if err != nil { return nil, err } return getStatementResultOutput, nil diff --git a/gov2/redshift/cmd/main.go b/gov2/redshift/cmd/main.go index e0d01ef011d..bdf6faedf02 100644 --- a/gov2/redshift/cmd/main.go +++ b/gov2/redshift/cmd/main.go @@ -22,9 +22,7 @@ import ( // // `-scenario` can be one of the following: // -// - `getstarted` - Runs the interactive get started scenario that shows you how to use -// Amazon Simple Storage Service (Amazon S3) actions to work with -// S3 buckets and objects. +// - `basics` - Runs the interactive Basics scenario to show core Redshift actions. func main() { scenarioMap := map[string]func(sdkConfig aws.Config, helper scenarios.IScenarioHelper){ "basics": runRedshiftBasicsScenario, diff --git a/gov2/redshift/go.mod b/gov2/redshift/go.mod index a3f60979136..ab9296b0fb3 100644 --- a/gov2/redshift/go.mod +++ b/gov2/redshift/go.mod @@ -7,6 +7,8 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.33 github.com/aws/aws-sdk-go-v2/service/redshift v1.46.8 github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.28.2 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.9 + github.com/aws/smithy-go v1.20.4 github.com/awsdocs/aws-doc-sdk-examples/gov2/demotools v0.0.0-20240911175713-48a391575470 github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20240911175713-48a391575470 ) @@ -22,7 +24,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect - github.com/aws/smithy-go v1.20.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect diff --git a/gov2/redshift/go.sum b/gov2/redshift/go.sum index 3536ac43c8e..b97be7dbff3 100644 --- a/gov2/redshift/go.sum +++ b/gov2/redshift/go.sum @@ -20,6 +20,8 @@ github.com/aws/aws-sdk-go-v2/service/redshift v1.46.8 h1:UBqd0JhsXpCDUf/7ulfzYTx github.com/aws/aws-sdk-go-v2/service/redshift v1.46.8/go.mod h1:UdcfC9kA4bn3cdUdFYVCeXZcoPka6WNzbYyRAX/Vpy0= github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.28.2 h1:oNarSIarQfMAZHeUhD2JOkdEpPfUFfoPKmb1GBK17Kc= github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.28.2/go.mod h1:9HLbgBikxAqW0V3Q8eQMQvoW1XRq0J7TjqYe8Lpiwx4= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.9 h1:croIrE67fpV6wff+0M8jbrJZpKSlrqVGrCnqNU5rtoI= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.9/go.mod h1:BYr9P/rrcLNJ8A36nT15p8tpoVDZ5lroHuMn/njecBw= github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4= diff --git a/gov2/redshift/hello/hello.go b/gov2/redshift/hello/hello.go index 25480a9be4b..137954eed03 100644 --- a/gov2/redshift/hello/hello.go +++ b/gov2/redshift/hello/hello.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/redshift" ) @@ -26,20 +27,17 @@ func main() { redshiftClient := redshift.NewFromConfig(sdkConfig) count := 10 fmt.Printf("Let's list up to %v clusters for your account.\n", count) - result, err := redshiftClient.DescribeClusters(context.TODO(), &redshift.DescribeClustersInput{}) + result, err := redshiftClient.DescribeClusters(context.TODO(), &redshift.DescribeClustersInput{MaxRecords: aws.Int32(10)}) if err != nil { fmt.Printf("Couldn't list clusters for your account. Here's why: %v\n", err) return } if len(result.Clusters) == 0 { fmt.Println("You don't have any clusters!") - } else { - if count > len(result.Clusters) { - count = len(result.Clusters) - } - for _, cluster := range result.Clusters[:count] { - fmt.Printf("\t%v : %v\n", *cluster.ClusterIdentifier, *cluster.ClusterStatus) - } + return + } + for _, cluster := range result.Clusters[:count] { + fmt.Printf("\t%v : %v\n", *cluster.ClusterIdentifier, *cluster.ClusterStatus) } } diff --git a/gov2/redshift/scenarios/redshift_basics.go b/gov2/redshift/scenarios/redshift_basics.go index 5fe5810c2fa..e9252d7e51a 100644 --- a/gov2/redshift/scenarios/redshift_basics.go +++ b/gov2/redshift/scenarios/redshift_basics.go @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +// snippet-start:[gov2.redshift.BasicsScenario] + package scenarios import ( @@ -11,6 +13,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" redshift_types "github.com/aws/aws-sdk-go-v2/service/redshift/types" redshiftdata_types "github.com/aws/aws-sdk-go-v2/service/redshiftdata/types" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" "github.com/awsdocs/aws-doc-sdk-examples/gov2/demotools" "github.com/awsdocs/aws-doc-sdk-examples/gov2/redshift/actions" "log" @@ -49,6 +52,12 @@ type RedshiftBasicsScenario struct { filesystem demotools.IFileSystem redshiftActor *actions.RedshiftActions redshiftDataActor *actions.RedshiftDataActions + secretsmanager *SecretsManager +} + +// SecretsManager is used to retrieve username and password information from a secure service. +type SecretsManager struct { + SecretsManagerClient *secretsmanager.Client } // RedshiftBasics constructs a new Redshift Basics runner. @@ -59,6 +68,7 @@ func RedshiftBasics(sdkConfig aws.Config, questioner demotools.IQuestioner, paus questioner: questioner, pauser: pauser, filesystem: filesystem, + secretsmanager: &SecretsManager{SecretsManagerClient: secretsmanager.NewFromConfig(sdkConfig)}, redshiftActor: &actions.RedshiftActions{RedshiftClient: redshift.NewFromConfig(sdkConfig)}, redshiftDataActor: &actions.RedshiftDataActions{RedshiftDataClient: redshiftdata.NewFromConfig(sdkConfig)}, } @@ -76,11 +86,16 @@ type Movie struct { // snippet-end:[gov2.redshift.Movie.struct] -// snippet-start:[gov2.redshift.BasicsScenario] +// User makes it easier to get the User data back from SecretsManager and use it later. +type User struct { + Username string `json:"userName"` + Password string `json:"userPassword"` +} // Run runs the RedshiftBasics interactive example that shows you how to use Amazon // Redshift and how to interact with its common endpoints. // +// 0. Retrieve username and password information to access Redshift. // 1. Create a cluster. // 2. Wait for the cluster to become available. // 3. List the available databases in the region. @@ -97,13 +112,15 @@ type Movie struct { // This package can be found in the ..\..\demotools folder of this repo. func (runner *RedshiftBasicsScenario) Run() { - // Prompt the user for input - userName, userPassword := "awsuser", "AwsUser1000" - clusterId := "test-cluster-1" + user := User{} + secretId := "s3express/basics/secrets" + clusterId := "demo-cluster-1" + maintenanceWindow := "wed:07:30-wed:08:00" databaseName := "dev" tableName := "Movies" fileName := "Movies.json" - runner.helper.GetName() + nodeType := "ra3.xlplus" + clusterType := "single-node" ctx := context.TODO() defer func() { @@ -113,14 +130,28 @@ func (runner *RedshiftBasicsScenario) Run() { if isMock || runner.questioner.AskBool("Do you want to see the full error message (y/n)?", "y") { log.Println(r) } - runner.cleanUpResources(ctx, clusterId, databaseName, tableName, userName, runner.questioner) + runner.cleanUpResources(ctx, clusterId, databaseName, tableName, user.Username, runner.questioner) } }() + // Retrieve the userName and userPassword from SecretsManager + output, err := runner.secretsmanager.SecretsManagerClient.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretId), + }) + if err != nil { + log.Printf("There was a problem getting the secret value: %s", err) + log.Printf("Please make sure to create a secret named 's3express/basics/secrets' with keys of 'userName' and 'userPassword'.") + panic(err) + } + + err = json.Unmarshal([]byte(*output.SecretString), &user) + if err != nil { + log.Printf("There was a problem parsing the secret value from JSON: %s", err) + panic(err) + } + // Create the Redshift cluster - nodeType := "ra3.xlplus" - clusterType := "single-node" - _, err := runner.redshiftActor.CreateCluster(ctx, clusterId, userName, userPassword, nodeType, clusterType, true) + _, err = runner.redshiftActor.CreateCluster(ctx, clusterId, user.Username, user.Password, nodeType, clusterType, true) if err != nil { var clusterAlreadyExistsFault *redshift_types.ClusterAlreadyExistsFault if errors.As(err, &clusterAlreadyExistsFault) { @@ -132,16 +163,29 @@ func (runner *RedshiftBasicsScenario) Run() { } // Wait for the cluster to become available - err = runner.WaitForClusterAvailable(runner.redshiftActor.RedshiftClient, clusterId, runner.questioner, runner.pauser) + waiter := redshift.NewClusterAvailableWaiter(runner.redshiftActor.RedshiftClient) + err = waiter.Wait(ctx, &redshift.DescribeClustersInput{ + ClusterIdentifier: aws.String(clusterId), + }, 5*time.Minute) if err != nil { log.Println("An error occurred waiting for the cluster.") panic(err) } + // Get some info about the cluster + describeOutput, err := runner.redshiftActor.DescribeClusters(ctx, clusterId) + if err != nil { + log.Println("Something went wrong trying to get information about the cluster.") + panic(err) + } + log.Println("Here's some information about the cluster.") + log.Printf("The cluster's status is %s", *describeOutput.Clusters[0].ClusterStatus) + log.Printf("The cluster was created at %s", *describeOutput.Clusters[0].ClusterCreateTime) + // List databases log.Println("List databases in", clusterId) runner.questioner.Ask("Press Enter to continue...") - err = runner.redshiftDataActor.ListDatabases(ctx, clusterId, databaseName, userName) + err = runner.redshiftDataActor.ListDatabases(ctx, clusterId, databaseName, user.Username) if err != nil { log.Printf("Failed to list databases: %v\n", err) panic(err) @@ -150,28 +194,27 @@ func (runner *RedshiftBasicsScenario) Run() { // Create the "Movies" table log.Println("Now you will create a table named " + tableName + ".") runner.questioner.Ask("Press Enter to continue...") - err = runner.redshiftDataActor.CreateTable(ctx, clusterId, databaseName, tableName, userName, []string{"title VARCHAR(256)", "year INT"}) + err = runner.redshiftDataActor.CreateTable(ctx, clusterId, databaseName, tableName, user.Username, []string{"title VARCHAR(256)", "year INT"}) if err != nil { log.Printf("Failed to create table: %v\n", err) panic(err) } // Populate the "Movies" table - runner.PopulateMoviesTable(ctx, clusterId, databaseName, tableName, userName, fileName) + runner.PopulateMoviesTable(ctx, clusterId, databaseName, tableName, user.Username, fileName) // Query the "Movies" table by year log.Println("Query the Movies table by year.") year := runner.questioner.AskInt( fmt.Sprintf("Enter a value between %v and %v:", 2012, 2014), demotools.InIntRange{Lower: 2012, Upper: 2014}) - runner.QueryMoviesByYear(ctx, clusterId, databaseName, tableName, userName, year) + runner.QueryMoviesByYear(ctx, clusterId, databaseName, tableName, user.Username, year) // Modify the cluster's maintenance window - //maintenanceWindow := aws.String("wed:07:30-wed:08:00") - runner.redshiftActor.ModifyCluster(ctx, clusterId, "wed:07:30-wed:08:00") + runner.redshiftActor.ModifyCluster(ctx, clusterId, maintenanceWindow) // Delete the Redshift cluster if confirmed - runner.cleanUpResources(ctx, clusterId, databaseName, tableName, userName, runner.questioner) + runner.cleanUpResources(ctx, clusterId, databaseName, tableName, user.Username, runner.questioner) log.Println("Thanks for watching!") } @@ -204,8 +247,6 @@ func (runner *RedshiftBasicsScenario) cleanUpResources(ctx context.Context, clus } } -// snippet-end:[gov2.redshift.BasicsScenario] - // snippet-start:[gov2.redshift.loadMoviesFromJSON] // loadMoviesFromJSON takes the file and populates a slice of Movie objects. @@ -227,39 +268,6 @@ func (runner *RedshiftBasicsScenario) loadMoviesFromJSON(fileName string, filesy // snippet-end:[gov2.redshift.loadMoviesFromJSON] -// snippet-start:[gov2.redshift.WaitForClusterAvailable] - -// WaitForClusterAvailable loops until a success or failure state is returned about the given cluster. -func (runner *RedshiftBasicsScenario) WaitForClusterAvailable(client *redshift.Client, clusterId string, questioner demotools.IQuestioner, pauser demotools.IPausable) error { - questioner.Ask("Now we will wait until the cluster is available. Press Enter to continue...") - - log.Println("Waiting for cluster to become available. This may take a few minutes.") - describeClusterInput := &redshift.DescribeClustersInput{ - ClusterIdentifier: &clusterId, - } - - for { - // snippet-start:[gov2.redshift.DescribeClusters] - output, err := client.DescribeClusters(context.TODO(), describeClusterInput) - if err != nil { - return err - } - - if output.Clusters[0].ClusterStatus != nil && *output.Clusters[0].ClusterStatus == "available" { - log.Println("Cluster is available! Total Elapsed Time:", time.Since(time.Now().Add(-1*time.Second*time.Duration(output.Clusters[0].ClusterCreateTime.Second())))) - return nil - } - - log.Print("Elapsed Time: ") - log.Println(time.Since(time.Now().Add(-1 * time.Second * time.Duration(output.Clusters[0].ClusterCreateTime.Second())))) - // snippet-end:[gov2.redshift.DescribeClusters] - - pauser.Pause(5) - } -} - -// snippet-end:[gov2.redshift.WaitForClusterAvailable] - // snippet-start:[gov2.redshift.PopulateMoviesTable] // PopulateMoviesTable reads data from the file and inserts records into the "Movies" table. @@ -272,7 +280,7 @@ func (runner *RedshiftBasicsScenario) PopulateMoviesTable(ctx context.Context, c movies, err := runner.loadMoviesFromJSON(fileName, runner.filesystem) if err != nil { log.Printf("Failed to load movies from JSON: %v\n", err) - return + panic(err) } var sqlStatements []string @@ -296,7 +304,7 @@ func (runner *RedshiftBasicsScenario) PopulateMoviesTable(ctx context.Context, c result, err := runner.redshiftDataActor.ExecuteBatchStatement(ctx, *input) if err != nil { log.Printf("Failed to execute batch statement: %v\n", err) - return + panic(err) } describeInput := redshiftdata.DescribeStatementInput{ @@ -337,7 +345,7 @@ func (runner *RedshiftBasicsScenario) QueryMoviesByYear(ctx context.Context, clu result, err := runner.redshiftDataActor.ExecuteStatement(ctx, *input) if err != nil { log.Printf("Failed to query movies: %v\n", err) - return + panic(err) } log.Println("The identifier of the statement is ", *result.Id) @@ -354,14 +362,14 @@ func (runner *RedshiftBasicsScenario) QueryMoviesByYear(ctx context.Context, clu err = runner.redshiftDataActor.WaitForQueryStatus(query, runner.pauser, true) if err != nil { log.Printf("Failed to execute query: %v\n", err) - return + panic(err) } log.Printf("Successfully executed query\n") getResultOutput, err := runner.redshiftDataActor.GetStatementResult(ctx, *result.Id) if err != nil { log.Printf("Failed to query movies: %v\n", err) - return + panic(err) } for _, row := range getResultOutput.Records { for _, col := range row { @@ -376,3 +384,5 @@ func (runner *RedshiftBasicsScenario) QueryMoviesByYear(ctx context.Context, clu } // snippet-end:[gov2.redshift.QueryMoviesByYear] + +// snippet-end:[gov2.redshift.BasicsScenario] diff --git a/gov2/redshift/scenarios/redshift_basics_integration_test.go b/gov2/redshift/scenarios/redshift_basics_integration_test.go index 0b7fa674ac6..e8debb238b0 100644 --- a/gov2/redshift/scenarios/redshift_basics_integration_test.go +++ b/gov2/redshift/scenarios/redshift_basics_integration_test.go @@ -31,7 +31,7 @@ func TestBasicsScenario_Integration(t *testing.T) { outFile := "integ-test.out" mockQuestioner := &demotools.MockQuestioner{ Answers: []string{ - "", "", "", "10", "2013", "n", "y", + "enter", "enter", "10", "2013", "n", "y", }, } diff --git a/gov2/redshift/scenarios/redshift_basics_test.go b/gov2/redshift/scenarios/redshift_basics_test.go index 22682280117..032b7d84518 100644 --- a/gov2/redshift/scenarios/redshift_basics_test.go +++ b/gov2/redshift/scenarios/redshift_basics_test.go @@ -11,7 +11,6 @@ import ( "github.com/awsdocs/aws-doc-sdk-examples/gov2/redshift/stubs" "io" "math/rand" - "os" "testing" "time" @@ -37,7 +36,8 @@ func TestRunBasicsScenario(t *testing.T) { } // httpErr is used to mock an HTTP error. This is required by the download manager, -// which calls GetObject until it receives a 415 status code. +// which calls GetObject until it receives a 415 status code. IDEs may indicate it's +// unused, but it is required for side effects. type httpErr struct { statusCode int } @@ -76,9 +76,13 @@ type BasicsScenarioTest struct { func (scenarioTest *BasicsScenarioTest) SetupDataAndStubs() []testtools.Stub { // set up variables clusterId := "test-cluster-1" - userName, userPassword := "awsuser", "AwsUser1000" + secretId := "s3express/basics/secrets" + user := User{ + Username: "testUser", + Password: "testPassword", + } databaseName := "dev" - nodeType := "test.node" + nodeType := "ra3.xlplus" clusterType := "single-node" publiclyAccessible := true testId := "test-result-id" @@ -87,22 +91,24 @@ func (scenarioTest *BasicsScenarioTest) SetupDataAndStubs() []testtools.Stub { scenarioTest.OutFilename = "test.out" scenarioTest.Answers = []string{ - "enter", "enter", "enter", "2013", "enter", "y", "y", //"3", "4", "5", "6", "7", "8", "9", "10", + "enter", "enter", "10", "2013", "y", "y", "y", } // set up stubs var stubList []testtools.Stub - stubList = append(stubList, stubs.StubCreateCluster(clusterId, userPassword, userName, nodeType, clusterType, publiclyAccessible, nil)) + stubList = append(stubList, stubs.StubGetSecretValue(secretId, user.Username, user.Password, nil)) + stubList = append(stubList, stubs.StubCreateCluster(clusterId, user.Username, user.Password, nodeType, clusterType, publiclyAccessible, nil)) + stubList = append(stubList, stubs.StubDescribeClusters(clusterId, nil)) stubList = append(stubList, stubs.StubDescribeClusters(clusterId, nil)) - stubList = append(stubList, stubs.StubListDatabases(clusterId, databaseName, userName, nil)) - stubList = append(stubList, stubs.StubExecuteStatement(clusterId, databaseName, userName, sql, testId, nil)) - stubList = append(stubList, stubs.StubBatchExecuteStatement(clusterId, databaseName, userName, sqls, testId, nil)) - stubList = append(stubList, stubs.StubDescribeStatement(testId, nil)) // This is where Execute is getting called instead of Describe. Comment this line out to see the second Describe work. - stubList = append(stubList, stubs.StubExecuteStatement(clusterId, databaseName, userName, sql, testId, nil)) + stubList = append(stubList, stubs.StubListDatabases(clusterId, databaseName, user.Username, nil)) + stubList = append(stubList, stubs.StubExecuteStatement(clusterId, databaseName, user.Username, sql, testId, nil)) + stubList = append(stubList, stubs.StubBatchExecuteStatement(clusterId, databaseName, user.Username, sqls, testId, nil)) + stubList = append(stubList, stubs.StubDescribeStatement(testId, nil)) + stubList = append(stubList, stubs.StubExecuteStatement(clusterId, databaseName, user.Username, sql, testId, nil)) stubList = append(stubList, stubs.StubDescribeStatement(testId, nil)) stubList = append(stubList, stubs.StubGetStatementResult(nil)) stubList = append(stubList, stubs.StubModifyCluster(nil)) - stubList = append(stubList, stubs.StubDeleteCluster(clusterId, nil)) + stubList = append(stubList, stubs.StubDeleteCluster(clusterId)) return stubList } @@ -116,7 +122,5 @@ func (scenarioTest *BasicsScenarioTest) RunSubTest(stubber *testtools.AwsmStubbe scenario.Run() } -// Cleanup deletes the output file created by the download test. func (scenarioTest *BasicsScenarioTest) Cleanup() { - _ = os.Remove(scenarioTest.OutFilename) } diff --git a/gov2/redshift/stubs/redshift_data_stubs.go b/gov2/redshift/stubs/redshift_data_stubs.go index 8eed5f0bd4b..574c14d1a15 100644 --- a/gov2/redshift/stubs/redshift_data_stubs.go +++ b/gov2/redshift/stubs/redshift_data_stubs.go @@ -24,7 +24,7 @@ func StubListDatabases(clusterId string, databaseName string, userName string, r } } -func StubExecuteStatement(clusterId string, databaseName string, userName string, sql string, resultId string, raiseErr *testtools.StubError) testtools.Stub { +func StubExecuteStatement(clusterId string, databaseName string, userName, sql string, resultId string, raiseErr *testtools.StubError) testtools.Stub { return testtools.Stub{ OperationName: "ExecuteStatement", Input: &redshiftdata.ExecuteStatementInput{ diff --git a/gov2/redshift/stubs/redshift_stubs.go b/gov2/redshift/stubs/redshift_stubs.go index d1fd99d5593..d4aae8cb128 100644 --- a/gov2/redshift/stubs/redshift_stubs.go +++ b/gov2/redshift/stubs/redshift_stubs.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/redshift" "github.com/aws/aws-sdk-go-v2/service/redshift/types" + "github.com/aws/smithy-go" "github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools" "time" ) @@ -32,7 +33,7 @@ func StubDescribeClusters(clusterId string, raiseErr *testtools.StubError) testt } } -func StubCreateCluster(clusterId string, userPassword string, userName string, nodeType string, clusterType string, publiclyAccessible bool, raiseErr *testtools.StubError) testtools.Stub { +func StubCreateCluster(clusterId string, userName string, userPassword string, nodeType string, clusterType string, publiclyAccessible bool, raiseErr *testtools.StubError) testtools.Stub { input := &redshift.CreateClusterInput{ ClusterIdentifier: aws.String(clusterId), MasterUserPassword: aws.String(userPassword), @@ -70,14 +71,18 @@ func StubModifyCluster(raiseErr *testtools.StubError) testtools.Stub { } } -func StubDeleteCluster(clusterId string, raiseErr *testtools.StubError) testtools.Stub { +func StubDeleteCluster(clusterId string) testtools.Stub { return testtools.Stub{ - OperationName: "DeleteCluster", - Input: &redshift.DeleteClusterInput{ - ClusterIdentifier: aws.String(clusterId), - SkipFinalClusterSnapshot: aws.Bool(true), + OperationName: "DescribeClusters", // Because a waiter is used, this is the actual called mocked. + Input: &redshift.DescribeClustersInput{ + ClusterIdentifier: &clusterId, + }, + SkipErrorTest: true, + Error: &testtools.StubError{ + Err: &smithy.GenericAPIError{ + Code: "ClusterNotFound", + Message: "ClusterNotFound", + }, }, - Output: &redshift.DeleteClusterOutput{}, - Error: raiseErr, } } diff --git a/gov2/redshift/stubs/secrets_manager_stubs.go b/gov2/redshift/stubs/secrets_manager_stubs.go new file mode 100644 index 00000000000..5fcff2fb6a9 --- /dev/null +++ b/gov2/redshift/stubs/secrets_manager_stubs.go @@ -0,0 +1,24 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package stubs + +import ( + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools" +) + +func StubGetSecretValue(secretId string, userName string, userPassword string, raiseErr *testtools.StubError) testtools.Stub { + return testtools.Stub{ + OperationName: "GetSecretValue", + Input: &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretId), + }, + Output: &secretsmanager.GetSecretValueOutput{ + SecretString: aws.String(fmt.Sprintf(`{"userName": "%s", "userPassword": "%s"}`, userName, userPassword)), + }, + Error: raiseErr, + } +} diff --git a/javav2/example_code/redshift/README.md b/javav2/example_code/redshift/README.md index 260d314c44f..74823ff8740 100644 --- a/javav2/example_code/redshift/README.md +++ b/javav2/example_code/redshift/README.md @@ -31,7 +31,7 @@ For prerequisites, see the [README](../../README.md#Prerequisites) in the `javav ### Get started -- [Hello Amazon Redshift](src/main/java/com/example/redshift/HelloRedshift.java#L6) (`describeClusters`) +- [Hello Amazon Redshift](src/main/java/com/example/redshift/HelloRedshift.java#L6) (`DescribeClusters`) ### Basics @@ -73,7 +73,7 @@ This example shows you how to get started using Amazon Redshift. #### Learn the basics -This example shows you how to learn core operations for Amazon Redshift using an AWS. +This example shows you how to learn core operations for Amazon Redshift using an AWS SDK. diff --git a/python/example_code/redshift/README.md b/python/example_code/redshift/README.md index 58107d2d5d2..865bbedb419 100644 --- a/python/example_code/redshift/README.md +++ b/python/example_code/redshift/README.md @@ -36,7 +36,7 @@ python -m pip install -r requirements.txt ### Get started -- [Hello Amazon Redshift](hello.py#L4) (`describeClusters`) +- [Hello Amazon Redshift](hello.py#L4) (`DescribeClusters`) ### Basics @@ -79,7 +79,7 @@ python hello.py #### Learn the basics -This example shows you how to learn core operations for Amazon Redshift using an AWS. +This example shows you how to learn core operations for Amazon Redshift using an AWS SDK.