Add connection support

.github/workflows/01-powerpipe-pre-release.yaml

@@ -36,6 +36,7 @@ jobs:
         with:
           repository: turbot/pipe-fittings
           path: pipe-fittings
+          ref: v1.6.x
 
        # this is required, check golangci-lint-action docs
       - uses: actions/setup-go@v5
@@ -108,6 +109,7 @@ jobs:
           - "params_and_args"
           - "snapshot"
           - "dashboard_parsing_validation"
+          - "database_precedence"
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout
@@ -555,6 +557,7 @@ jobs:
         with:
           repository: turbot/pipe-fittings
           path: pipe-fittings
+          ref: v1.6.x
 
       - name: Tag Release
         run: |

.github/workflows/02-powerpipe-release.yaml

@@ -128,6 +128,7 @@ jobs:
           - "params_and_args"
           - "snapshot"
           - "dashboard_parsing_validation"
+          - "database_precedence"
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout

.github/workflows/10-test-lint.yaml

@@ -35,6 +35,6 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.54.1
+          version: v1.61.0
           args: --timeout=10m
           working-directory: powerpipe

.github/workflows/11-test-acceptance.yaml

@@ -85,6 +85,7 @@ jobs:
           - "params_and_args"
           - "snapshot"
           - "dashboard_parsing_validation"
+          - "database_precedence"
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout

go.mod

@@ -21,13 +21,13 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.19.0
-	github.com/stevenle/topsort v0.2.0
+	github.com/stevenle/topsort v0.2.0 // indirect
 	github.com/turbot/go-kit v0.10.0-rc.0
 	github.com/turbot/pipe-fittings v1.5.4
 	github.com/turbot/steampipe-plugin-sdk/v5 v5.10.4
 	github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
-	github.com/zclconf/go-cty v1.14.4
+	github.com/zclconf/go-cty v1.14.4 // indirect
 	github.com/zclconf/go-cty-yaml v1.0.3 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	sigs.k8s.io/yaml v1.4.0 // indirect
@@ -38,6 +38,7 @@ require github.com/sethvargo/go-retry v0.3.0 // indirect
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/didip/tollbooth/v7 v7.0.2
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/gin-contrib/gzip v1.0.1
 	github.com/gin-contrib/size v1.0.1
 	github.com/go-sql-driver/mysql v1.8.1
@@ -108,7 +109,6 @@ require (
 	github.com/eko/gocache/store/ristretto/v4 v4.2.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gertd/go-pluralize v0.2.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect

internal/cmd/check.go

@@ -12,7 +12,6 @@ import (
 	"github.com/spf13/viper"
 	"github.com/thediveo/enumflag/v2"
 	"github.com/turbot/go-kit/helpers"
-	"github.com/turbot/pipe-fittings/app_specific"
 	"github.com/turbot/pipe-fittings/cmdconfig"
 	"github.com/turbot/pipe-fittings/constants"
 	"github.com/turbot/pipe-fittings/contexthelpers"
@@ -58,7 +57,7 @@ func checkCmd[T controlinit.CheckTarget]() *cobra.Command {
 	builder.
 		AddCloudFlags().
 		AddModLocationFlag().
-		AddStringFlag(constants.ArgDatabase, app_specific.DefaultDatabase, "Turbot Pipes workspace database").
+		AddStringFlag(constants.ArgDatabase, "", "Turbot Pipes workspace database", cmdconfig.FlagOptions.Deprecated("use a variable or param")).
 		AddBoolFlag(constants.ArgHeader, true, "Include column headers for csv and table output").
 		AddBoolFlag(constants.ArgHelp, false, "Help for run command", cmdconfig.FlagOptions.WithShortHand("h")).
 		AddBoolFlag(constants.ArgInput, true, "Enable interactive prompts").
@@ -365,7 +364,7 @@ func validateCheckArgs(ctx context.Context) error {
 		return fmt.Errorf("only 1 of '--%s' and '--%s' may be set", constants.ArgWhere, constants.ArgTag)
 	}
 
-	return nil
+	return localcmdconfig.ValidateDatabaseArg()
 }
 
 func shouldPrintCheckTiming() bool {

internal/cmd/dashboard.go

@@ -12,7 +12,6 @@ import (
 	"github.com/spf13/viper"
 	"github.com/thediveo/enumflag/v2"
 	"github.com/turbot/go-kit/helpers"
-	"github.com/turbot/pipe-fittings/app_specific"
 	"github.com/turbot/pipe-fittings/cloud"
 	"github.com/turbot/pipe-fittings/cmdconfig"
 	"github.com/turbot/pipe-fittings/constants"
@@ -55,7 +54,7 @@ The current mod is the working directory, or the directory specified by the --mo
 		AddModLocationFlag().
 		AddStringArrayFlag(constants.ArgArg, nil, "Specify the value of a dashboard argument").
 		AddStringSliceFlag(constants.ArgExport, nil, "Export output to file, supported format: pps (snapshot)").
-		AddStringFlag(constants.ArgDatabase, app_specific.DefaultDatabase, "Turbot Pipes workspace database").
+		AddStringFlag(constants.ArgDatabase, "", "Turbot Pipes workspace database", cmdconfig.FlagOptions.Deprecated("use a variable or param")).
 		AddIntFlag(constants.ArgDatabaseQueryTimeout, localconstants.DatabaseDefaultQueryTimeout, "The query timeout").
 		AddBoolFlag(constants.ArgHelp, false, "Help for dashboard", cmdconfig.FlagOptions.WithShortHand("h")).
 		AddBoolFlag(constants.ArgInput, true, "Enable interactive prompts").
@@ -185,7 +184,7 @@ func validateDashboardArgs(ctx context.Context) error {
 		return fmt.Errorf("only one of --share or --snapshot may be set")
 	}
 
-	return nil
+	return localcmdconfig.ValidateDatabaseArg()
 }
 
 func displaySnapshot(snapshot *steampipeconfig.SteampipeSnapshot) {

internal/cmd/mod.go

@@ -20,6 +20,7 @@ import (
 	"github.com/turbot/pipe-fittings/parse"
 	"github.com/turbot/pipe-fittings/plugin"
 	"github.com/turbot/pipe-fittings/utils"
+	localcmdconfig "github.com/turbot/powerpipe/internal/cmdconfig"
 	localconstants "github.com/turbot/powerpipe/internal/constants"
 	"github.com/turbot/powerpipe/internal/db_client"
 	"github.com/turbot/powerpipe/internal/display"
@@ -101,7 +102,7 @@ Examples:
 
 	cmdconfig.OnCmd(cmd).
 		AddBoolFlag(constants.ArgDryRun, false, "Show which mods would be installed/updated/uninstalled without modifying them").
-		AddStringFlag(constants.ArgDatabase, app_specific.DefaultDatabase, "Turbot Pipes workspace database").
+		AddStringFlag(constants.ArgDatabase, "", "Turbot Pipes workspace database", cmdconfig.FlagOptions.Deprecated("use a variable or param")).
 		AddBoolFlag(constants.ArgForce, false, "Install mods even if plugin/cli version requirements are not met (cannot be used with --dry-run)").
 		AddBoolFlag(constants.ArgHelp, false, "Help for install", cmdconfig.FlagOptions.WithShortHand("h")).
 		AddBoolFlag(constants.ArgPrune, true, "Remove unused dependencies after installation is complete").
@@ -124,6 +125,8 @@ func runModInstallCmd(cmd *cobra.Command, args []string) {
 		}
 	}()
 
+	error_helpers.FailOnError(validateModArgs())
+
 	// try to load the workspace mod definition
 	// - if it does not exist, this will return a nil mod and a nil error
 	workspacePath := viper.GetString(constants.ArgModLocation)
@@ -155,8 +158,18 @@ func runModInstallCmd(cmd *cobra.Command, args []string) {
 	fmt.Println(summary) //nolint:forbidigo // intended output
 }
 
+func validateModArgs() error {
+	return localcmdconfig.ValidateDatabaseArg()
+}
+
 func getPluginVersions(ctx context.Context) *plugin.PluginVersionMap {
-	defaultDatabase, _ := db_client.GetDefaultDatabaseConfig()
+	defaultDatabase, _, err := db_client.GetDefaultDatabaseConfig()
+	if err != nil {
+		if !viper.GetBool(constants.ArgForce) {
+			error_helpers.ShowWarning("Could not connect to database - plugin validation will not be performed")
+		}
+		return nil
+	}
 
 	client, err := db_client.NewDbClient(ctx, defaultDatabase)
 	if err != nil {

internal/cmd/query.go

@@ -13,7 +13,6 @@ import (
 	"github.com/spf13/viper"
 	"github.com/thediveo/enumflag/v2"
 	"github.com/turbot/go-kit/helpers"
-	"github.com/turbot/pipe-fittings/app_specific"
 	"github.com/turbot/pipe-fittings/cmdconfig"
 	"github.com/turbot/pipe-fittings/constants"
 	"github.com/turbot/pipe-fittings/error_helpers"
@@ -52,7 +51,7 @@ The current mod is the working directory, or the directory specified by the --mo
 		// NOTE: use StringArrayFlag for ArgQueryInput, not StringSliceFlag
 		// Cobra will interpret values passed to a StringSliceFlag as CSV, where args passed to StringArrayFlag are not parsed and used raw
 		AddStringArrayFlag(constants.ArgArg, nil, "Specify the value of a query argument").
-		AddStringFlag(constants.ArgDatabase, app_specific.DefaultDatabase, "Turbot Pipes workspace database").
+		AddStringFlag(constants.ArgDatabase, "", "Turbot Pipes workspace database", cmdconfig.FlagOptions.Deprecated("use a variable or param")).
 		AddIntFlag(constants.ArgDatabaseQueryTimeout, localconstants.DatabaseDefaultQueryTimeout, "The query timeout").
 		AddStringSliceFlag(constants.ArgExport, nil, "Export output to file, supported formats: csv, html, json, md, nunit3, pps (snapshot), asff").
 		AddBoolFlag(constants.ArgHeader, true, "Include column headers for csv and table output").
@@ -197,7 +196,7 @@ func validateQueryArgs(ctx context.Context) error {
 		return fmt.Errorf("only one of --share or --snapshot may be set")
 	}
 
-	return nil
+	return localcmdconfig.ValidateDatabaseArg()
 }
 
 func queryExporters() []export.Exporter {

internal/cmd/server.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/turbot/pipe-fittings/app_specific"
 	"github.com/turbot/pipe-fittings/cmdconfig"
 	"github.com/turbot/pipe-fittings/constants"
 	"github.com/turbot/pipe-fittings/error_helpers"
@@ -42,9 +41,9 @@ Powerpipe server runs in the foreground; Press Ctrl-C to exit.`,
 		AddIntFlag(constants.ArgPort, dashboardserver.DashboardServerDefaultPort, "Web server port").
 		AddBoolFlag(constants.ArgWatch, true, "Watch mod files for changes when running powerpipe server").
 		AddStringFlag(constants.ArgListen, string(dashboardserver.ListenTypeLocal), "Accept connections from local (localhost only) or network (all interfaces / IP addresses)").
-		AddStringSliceFlag(constants.ArgVariable, []string{}, "Specify the value of a variable. Multiple --var arguments may be passed.").
+		AddStringArrayFlag(constants.ArgVariable, []string{}, "Specify the value of a variable. Multiple --var arguments may be passed.").
 		AddStringFlag(constants.ArgVarFile, "", "Specify a .ppvar file containing variable values.").
-		AddStringFlag(constants.ArgDatabase, app_specific.DefaultDatabase, "Turbot Pipes workspace database").
+		AddStringFlag(constants.ArgDatabase, "", "Turbot Pipes workspace database", cmdconfig.FlagOptions.Deprecated("use a variable or param")).
 		AddIntFlag(constants.ArgDashboardTimeout, 0, "Set a the dashboard execution timeout")
 
 	return cmd
@@ -61,6 +60,7 @@ func runServerCmd(cmd *cobra.Command, _ []string) {
 		return
 	}
 
+	error_helpers.FailOnError(validateServerArgs())
 	// retrieve server params
 	serverPort := dashboardserver.ListenPort(viper.GetInt(constants.ArgPort))
 	error_helpers.FailOnError(serverPort.IsValid())
@@ -107,3 +107,7 @@ func runServerCmd(cmd *cobra.Command, _ []string) {
 
 	<-ctx.Done()
 }
+
+func validateServerArgs() error {
+	return localcmdconfig.ValidateDatabaseArg()
+}

internal/cmdconfig/app_specific.go

@@ -6,12 +6,17 @@ import (
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
+	"github.com/hashicorp/hcl/v2"
 	"github.com/spf13/viper"
 	"github.com/turbot/go-kit/files"
 	"github.com/turbot/pipe-fittings/app_specific"
+	"github.com/turbot/pipe-fittings/app_specific_connection"
 	"github.com/turbot/pipe-fittings/cmdconfig"
+	"github.com/turbot/pipe-fittings/connection"
+	"github.com/turbot/pipe-fittings/constants"
 	"github.com/turbot/pipe-fittings/error_helpers"
 	"github.com/turbot/pipe-fittings/filepaths"
+	"github.com/turbot/pipe-fittings/utils"
 )
 
 // SetAppSpecificConstants sets app specific constants defined in pipe-fittings
@@ -46,8 +51,6 @@ func SetAppSpecificConstants() {
 
 	app_specific.DefaultVarsFileName = "powerpipe.ppvars"
 	app_specific.LegacyDefaultVarsFileName = "steampipe.spvars"
-	// default to local steampipe service
-	app_specific.DefaultDatabase = "postgres://[email protected]:9193/steampipe"
 
 	// extensions
 
@@ -70,4 +73,20 @@ func SetAppSpecificConstants() {
 	app_specific.VersionCheckHost = "hub.powerpipe.io"
 	app_specific.VersionCheckPath = "api/cli/version/latest"
 	app_specific.EnvProfile = "POWERPIPE_PROFILE"
+
+	// register supported connection types
+	registerConnections()
+}
+
+func registerConnections() {
+	app_specific_connection.RegisterConnections(
+		connection.NewSteampipePgConnection,
+		connection.NewPostgresConnection,
+		connection.NewSqliteConnection,
+		connection.NewDuckDbConnection,
+	)
+	// set default connections
+	var defaultSteampipeConnection = connection.NewSteampipePgConnection("default", hcl.Range{})
+	defaultSteampipeConnection.(*connection.SteampipePgConnection).ConnectionString = utils.ToStringPointer(constants.DefaultSteampipeConnectionString)
+	app_specific_connection.DefaultConnections["steampipe"] = defaultSteampipeConnection
 }

internal/cmdconfig/cmd_hooks.go

@@ -24,6 +24,7 @@ import (
 	"github.com/turbot/pipe-fittings/utils"
 	"github.com/turbot/pipe-fittings/workspace_profile"
 	"github.com/turbot/powerpipe/internal/logger"
+	"github.com/turbot/powerpipe/internal/powerpipeconfig"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
 	"github.com/turbot/steampipe-plugin-sdk/v5/sperr"
 )
@@ -122,6 +123,13 @@ func initGlobalConfig() error_helpers.ErrorAndWarnings {
 
 	var cmd = viper.Get(constants.ConfigKeyActiveCommand).(*cobra.Command)
 
+	var config, ew = powerpipeconfig.LoadPowerpipeConfig(filepaths.EnsureConfigDir())
+	if ew.GetError() != nil {
+		return ew
+	}
+
+	powerpipeconfig.GlobalConfig = config
+
 	// set-up viper with defaults from the env and default workspace profile
 
 	cmdconfig.BootstrapViper(loader, cmd,
@@ -142,8 +150,9 @@ func initGlobalConfig() error_helpers.ErrorAndWarnings {
 	// if an explicit workspace profile was set, add to viper as highest precedence default
 	// NOTE: if install_dir/mod_location are set these will already have been passed to viper by BootstrapViper
 	// since the "ConfiguredProfile" is passed in through a cmdline flag, it will always take precedence
-	if loader.ConfiguredProfile != nil {
-		cmdconfig.SetDefaultsFromConfig(loader.ConfiguredProfile.ConfigMap(cmd))
+	wp := loader.ConfiguredProfile
+	if wp != nil {
+		cmdconfig.SetDefaultsFromConfig(wp.ConfigMap(cmd))
 	}
 
 	// now env vars have been processed, set filepaths.PipesInstallDir
@@ -157,6 +166,15 @@ func initGlobalConfig() error_helpers.ErrorAndWarnings {
 		return error_helpers.NewErrorsAndWarning(err)
 	}
 
+	// if the configured workspace is a cloud workspace, create cloud metadata and set the default connection
+	if wp != nil && wp.IsCloudWorkspace() {
+		defaultConnection, ew := wp.GetCloudMetadata()
+		if ew.GetError() != nil {
+			return ew
+		}
+		config.DefaultConnection = defaultConnection
+	}
+
 	// now validate all config values have appropriate values
 	return validateConfig()
 }

internal/cmdconfig/validate.go

@@ -3,6 +3,7 @@ package cmdconfig
 import (
 	"context"
 	"fmt"
+	"github.com/turbot/pipe-fittings/connection"
 	"strings"
 
 	"github.com/spf13/viper"
@@ -11,8 +12,37 @@ import (
 	"github.com/turbot/pipe-fittings/constants"
 	"github.com/turbot/pipe-fittings/error_helpers"
 	"github.com/turbot/pipe-fittings/steampipeconfig"
+	"github.com/turbot/powerpipe/internal/powerpipeconfig"
 )
 
+// ValidateDatabaseArg validates the connection and database arg
+// if both or neither are set, returns an error
+// if connection is set, verify it is in the config and update the database arg with the connection string
+func ValidateDatabaseArg() error {
+	databaseArg := viper.GetString(constants.ArgDatabase)
+	if databaseArg == "" {
+		return nil
+	}
+	if strings.HasPrefix(databaseArg, "connection.") {
+		conn, ok := powerpipeconfig.GlobalConfig.PipelingConnections[strings.TrimPrefix(databaseArg, "connection.")]
+		if !ok {
+			return fmt.Errorf("connection '%s' not found", databaseArg)
+		}
+
+		csp, ok := conn.(connection.ConnectionStringProvider)
+		if !ok {
+			// unexpected - all registered connections should implement this interface
+			return fmt.Errorf("connection '%s' does not implement connection.ConnectionStringProvider", databaseArg)
+		}
+		connectionString := csp.GetConnectionString()
+		powerpipeconfig.GlobalConfig.DefaultConnection = csp
+		// update viper Database arg with the connection string
+		viper.Set(constants.ArgDatabase, connectionString)
+	}
+
+	return nil
+}
+
 func ValidateSnapshotArgs(ctx context.Context) error {
 	// only 1 of 'share' and 'snapshot' may be set
 	share := viper.GetBool(constants.ArgShare)

internal/constants/db.go

@@ -2,4 +2,5 @@ package constants
 
 const (
 	DatabaseDefaultQueryTimeout = 300
+	DefaultConnection           = "connection.steampipe.default"
 )

internal/controlexecute/result_group.go

@@ -259,7 +259,7 @@ func (r *ResultGroup) addDimensionKeys(keys ...string) {
 // onChildDone is a callback that gets called from the children of this result group when they are done
 func (r *ResultGroup) onChildDone() {
 	newCount := atomic.AddUint32(&r.childrenComplete, 1)
-	totalCount := uint32(len(r.ControlRuns) + len(r.Groups))
+	totalCount := uint32(len(r.ControlRuns) + len(r.Groups)) //nolint:gosec // will not overflow
 	if newCount < totalCount {
 		// all children haven't finished execution yet
 		return