adding support for bq

infra/blueprint-test/pkg/bq/bq.go

@@ -0,0 +1,165 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package bq provides a set of helpers to interact with bq tool (part of CloudSDK)
+package bq
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/gruntwork-io/terratest/modules/logger"
+	"github.com/gruntwork-io/terratest/modules/shell"
+	"github.com/mitchellh/go-testing-interface"
+	"github.com/tidwall/gjson"
+)
+
+type CmdCfg struct {
+	bqBinary string             // path to bq binary
+	commonArgs   []string       // common arguments to pass to bq calls
+	logger       *logger.Logger // custom logger
+}
+
+type cmdOption func(*CmdCfg)
+
+func WithBinary(bqBinary string) cmdOption {
+	return func(f *CmdCfg) {
+		f.bqBinary = bqBinary
+	}
+}
+
+func WithCommonArgs(commonArgs []string) cmdOption {
+	return func(f *CmdCfg) {
+		f.commonArgs = commonArgs
+	}
+}
+
+func WithLogger(logger *logger.Logger) cmdOption {
+	return func(f *CmdCfg) {
+		f.logger = logger
+	}
+}
+
+// newCmdConfig sets defaults and validates values for bq Options.
+func newCmdConfig(opts ...cmdOption) (*CmdCfg, error) {
+	gOpts := &CmdCfg{}
+	// apply options
+	for _, opt := range opts {
+		opt(gOpts)
+	}
+	if gOpts.bqBinary == "" {
+		err := utils.BinaryInPath("bq")
+		if err != nil {
+			return nil, err
+		}
+		gOpts.bqBinary = "bq"
+	}
+	if gOpts.commonArgs == nil {
+		gOpts.commonArgs = []string{"--format", "json"}
+	}
+	if gOpts.logger == nil {
+		gOpts.logger = utils.GetLoggerFromT()
+	}
+	return gOpts, nil
+}
+
+// RunCmd executes a bq command and fails test if there are any errors.
+func RunCmd(t testing.TB, cmd string, opts ...cmdOption) string {
+	op, err := RunCmdE(t, cmd, opts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return op
+}
+
+// RunCmdE executes a bq command and return output.
+func RunCmdE(t testing.TB, cmd string, opts ...cmdOption) (string, error) {
+	gOpts, err := newCmdConfig(opts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// split command into args
+	args := strings.Fields(cmd)
+	bqCmd := shell.Command{
+		Command: "bq",
+		Args:    append(args, gOpts.commonArgs...),
+		Logger:  gOpts.logger,
+	}
+	return shell.RunCommandAndGetStdOutE(t, bqCmd)
+}
+
+// Run executes a bq command and returns value as gjson.Result.
+// It fails the test if there are any errors executing the bq command or parsing the output value.
+func Run(t testing.TB, cmd string, opts ...cmdOption) gjson.Result {
+	op := RunCmd(t, cmd, opts...)
+	if !gjson.Valid(op) {
+		t.Fatalf("Error parsing output, invalid json: %s", op)
+	}
+	return gjson.Parse(op)
+}
+
+// RunWithCmdOptsf executes a bq command and returns value as gjson.Result.
+//
+// RunWithCmdOptsf(t, ops.., "ls --datasets --project_id=%s", "projectId")
+//
+// It fails the test if there are any errors executing the bq command or parsing the output value.
+func RunWithCmdOptsf(t testing.TB, opts []cmdOption, cmd string, args ...interface{}) gjson.Result {
+	return Run(t, stringFromTextAndArgs(append([]interface{}{cmd}, args...)...), opts...)
+}
+
+// Runf executes a bq command and returns value as gjson.Result.
+//
+// Runf(t, "ls --datasets --project_id=%s", "projectId")
+//
+// It fails the test if there are any errors executing the bq command or parsing the output value.
+func Runf(t testing.TB, cmd string, args ...interface{}) gjson.Result {
+	return Run(t, stringFromTextAndArgs(append([]interface{}{cmd}, args...)...))
+}
+
+// stringFromTextAndArgs convert msg and args to formatted text
+func stringFromTextAndArgs(msgAndArgs ...interface{}) string {
+	if len(msgAndArgs) == 0 || msgAndArgs == nil {
+		return ""
+	}
+	if len(msgAndArgs) == 1 {
+		msg := msgAndArgs[0]
+		if msgAsStr, ok := msg.(string); ok {
+			return msgAsStr
+		}
+		return fmt.Sprintf("%+v", msg)
+	}
+	if len(msgAndArgs) > 1 {
+		return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...)
+	}
+	return ""
+}
+
+// ActivateCredsAndEnvVars activates credentials and exports auth related envvars.
+func ActivateCredsAndEnvVars(t testing.TB, creds string) {
+	credsPath, err := utils.WriteTmpFile(creds)
+	if err != nil {
+		t.Fatal(err)
+	}
+	RunCmd(t, "auth activate-service-account", WithCommonArgs([]string{"--key-file", credsPath}))
+	// set auth related env vars
+	// TF provider auth
+	utils.SetEnv(t, "GOOGLE_CREDENTIALS", creds)
+	// bq SDK override
+	utils.SetEnv(t, "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE", credsPath)
+	// ADC
+	utils.SetEnv(t, "GOOGLE_APPLICATION_CREDENTIALS", credsPath)
+}

infra/blueprint-test/pkg/bq/bq_test.go

@@ -0,0 +1,83 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package bq provides a set of helpers to interact with bq tool (part of CloudSDK)
+package bq
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestActivateCredsAndEnvVars(t *testing.T) {
+	tests := []struct {
+		name      string
+		keyEnvVar string
+		user      string
+	}{
+		{
+			name:      "with sa key",
+			keyEnvVar: "TEST_KEY",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			creds, present := os.LookupEnv(tt.keyEnvVar)
+			if !present {
+				t.Logf("Skipping test, %s envvar not set", tt.keyEnvVar)
+				t.Skip()
+			}
+			ActivateCredsAndEnvVars(t, creds)
+			assert := assert.New(t)
+			assert.Equal(os.Getenv("GOOGLE_CREDENTIALS"), creds)
+			pathEnvVars := []string{"CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE", "GOOGLE_APPLICATION_CREDENTIALS"}
+			for _, v := range pathEnvVars {
+				c, err := os.ReadFile(os.Getenv(v))
+				assert.NoError(err)
+				assert.Equal(string(c), creds)
+			}
+
+		})
+	}
+}
+
+func TestRunf(t *testing.T) {
+	tests := []struct {
+		name            string
+		cmd             string
+		projectIdEnvVar string
+	}{
+		{
+			name:            "Runf",
+			cmd:             "ls --datasets --project_id=%s",
+			projectIdEnvVar: "bigquery-public-data",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if projectName, present := os.LookupEnv(tt.projectIdEnvVar); present {
+				op := Runf(t, tt.cmd, projectName)
+				assert := assert.New(t)
+				assert.Equal("bigquery#dataset", op.Array()[0].Get("kind").String())
+			} else {
+				t.Logf("Skipping test, %s envvar not set", tt.projectIdEnvVar)
+				t.Skip()
+			}
+		})
+	}
+}