diff --git a/infra/blueprint-test/pkg/bq/bq.go b/infra/blueprint-test/pkg/bq/bq.go new file mode 100644 index 00000000000..4612f016aec --- /dev/null +++ b/infra/blueprint-test/pkg/bq/bq.go @@ -0,0 +1,165 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package bq provides a set of helpers to interact with bq tool (part of CloudSDK) +package bq + +import ( + "fmt" + "strings" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/shell" + "github.com/mitchellh/go-testing-interface" + "github.com/tidwall/gjson" +) + +type CmdCfg struct { + bqBinary string // path to bq binary + commonArgs []string // common arguments to pass to bq calls + logger *logger.Logger // custom logger +} + +type cmdOption func(*CmdCfg) + +func WithBinary(bqBinary string) cmdOption { + return func(f *CmdCfg) { + f.bqBinary = bqBinary + } +} + +func WithCommonArgs(commonArgs []string) cmdOption { + return func(f *CmdCfg) { + f.commonArgs = commonArgs + } +} + +func WithLogger(logger *logger.Logger) cmdOption { + return func(f *CmdCfg) { + f.logger = logger + } +} + +// newCmdConfig sets defaults and validates values for bq Options. +func newCmdConfig(opts ...cmdOption) (*CmdCfg, error) { + gOpts := &CmdCfg{} + // apply options + for _, opt := range opts { + opt(gOpts) + } + if gOpts.bqBinary == "" { + err := utils.BinaryInPath("bq") + if err != nil { + return nil, err + } + gOpts.bqBinary = "bq" + } + if gOpts.commonArgs == nil { + gOpts.commonArgs = []string{"--format", "json"} + } + if gOpts.logger == nil { + gOpts.logger = utils.GetLoggerFromT() + } + return gOpts, nil +} + +// RunCmd executes a bq command and fails test if there are any errors. +func RunCmd(t testing.TB, cmd string, opts ...cmdOption) string { + op, err := RunCmdE(t, cmd, opts...) + if err != nil { + t.Fatal(err) + } + return op +} + +// RunCmdE executes a bq command and return output. +func RunCmdE(t testing.TB, cmd string, opts ...cmdOption) (string, error) { + gOpts, err := newCmdConfig(opts...) + if err != nil { + t.Fatal(err) + } + // split command into args + args := strings.Fields(cmd) + bqCmd := shell.Command{ + Command: "bq", + Args: append(args, gOpts.commonArgs...), + Logger: gOpts.logger, + } + return shell.RunCommandAndGetStdOutE(t, bqCmd) +} + +// Run executes a bq command and returns value as gjson.Result. +// It fails the test if there are any errors executing the bq command or parsing the output value. +func Run(t testing.TB, cmd string, opts ...cmdOption) gjson.Result { + op := RunCmd(t, cmd, opts...) + if !gjson.Valid(op) { + t.Fatalf("Error parsing output, invalid json: %s", op) + } + return gjson.Parse(op) +} + +// RunWithCmdOptsf executes a bq command and returns value as gjson.Result. +// +// RunWithCmdOptsf(t, ops.., "ls --datasets --project_id=%s", "projectId") +// +// It fails the test if there are any errors executing the bq command or parsing the output value. +func RunWithCmdOptsf(t testing.TB, opts []cmdOption, cmd string, args ...interface{}) gjson.Result { + return Run(t, stringFromTextAndArgs(append([]interface{}{cmd}, args...)...), opts...) +} + +// Runf executes a bq command and returns value as gjson.Result. +// +// Runf(t, "ls --datasets --project_id=%s", "projectId") +// +// It fails the test if there are any errors executing the bq command or parsing the output value. +func Runf(t testing.TB, cmd string, args ...interface{}) gjson.Result { + return Run(t, stringFromTextAndArgs(append([]interface{}{cmd}, args...)...)) +} + +// stringFromTextAndArgs convert msg and args to formatted text +func stringFromTextAndArgs(msgAndArgs ...interface{}) string { + if len(msgAndArgs) == 0 || msgAndArgs == nil { + return "" + } + if len(msgAndArgs) == 1 { + msg := msgAndArgs[0] + if msgAsStr, ok := msg.(string); ok { + return msgAsStr + } + return fmt.Sprintf("%+v", msg) + } + if len(msgAndArgs) > 1 { + return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + } + return "" +} + +// ActivateCredsAndEnvVars activates credentials and exports auth related envvars. +func ActivateCredsAndEnvVars(t testing.TB, creds string) { + credsPath, err := utils.WriteTmpFile(creds) + if err != nil { + t.Fatal(err) + } + RunCmd(t, "auth activate-service-account", WithCommonArgs([]string{"--key-file", credsPath})) + // set auth related env vars + // TF provider auth + utils.SetEnv(t, "GOOGLE_CREDENTIALS", creds) + // bq SDK override + utils.SetEnv(t, "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE", credsPath) + // ADC + utils.SetEnv(t, "GOOGLE_APPLICATION_CREDENTIALS", credsPath) +} \ No newline at end of file diff --git a/infra/blueprint-test/pkg/bq/bq_test.go b/infra/blueprint-test/pkg/bq/bq_test.go new file mode 100644 index 00000000000..7e81d368e43 --- /dev/null +++ b/infra/blueprint-test/pkg/bq/bq_test.go @@ -0,0 +1,83 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package bq provides a set of helpers to interact with bq tool (part of CloudSDK) +package bq + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestActivateCredsAndEnvVars(t *testing.T) { + tests := []struct { + name string + keyEnvVar string + user string + }{ + { + name: "with sa key", + keyEnvVar: "TEST_KEY", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + creds, present := os.LookupEnv(tt.keyEnvVar) + if !present { + t.Logf("Skipping test, %s envvar not set", tt.keyEnvVar) + t.Skip() + } + ActivateCredsAndEnvVars(t, creds) + assert := assert.New(t) + assert.Equal(os.Getenv("GOOGLE_CREDENTIALS"), creds) + pathEnvVars := []string{"CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE", "GOOGLE_APPLICATION_CREDENTIALS"} + for _, v := range pathEnvVars { + c, err := os.ReadFile(os.Getenv(v)) + assert.NoError(err) + assert.Equal(string(c), creds) + } + + }) + } +} + +func TestRunf(t *testing.T) { + tests := []struct { + name string + cmd string + projectIdEnvVar string + }{ + { + name: "Runf", + cmd: "ls --datasets --project_id=%s", + projectIdEnvVar: "bigquery-public-data", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if projectName, present := os.LookupEnv(tt.projectIdEnvVar); present { + op := Runf(t, tt.cmd, projectName) + assert := assert.New(t) + assert.Equal("bigquery#dataset", op.Array()[0].Get("kind").String()) + } else { + t.Logf("Skipping test, %s envvar not set", tt.projectIdEnvVar) + t.Skip() + } + }) + } +}