Merge pull request #6733 from dolthub/zachmu/timestamp-commit

Moved all environment variables relevant to a customer into a common file so we can collect and document them more easily

.github/workflows/ci-go-tests.yaml

@@ -79,7 +79,6 @@ jobs:
     - name: Test All
       working-directory: ./go
       run: |
-        go test -vet=off -timeout 30m ./libraries/doltcore/sqle/altertests
         go test -vet=off -timeout 30m ./libraries/doltcore/sqle/integration_test
       env:
         MATRIX_OS: ${{ matrix.os }}

go/cmd/dolt/cli/credentials.go

@@ -20,6 +20,7 @@ import (
 
 	"golang.org/x/crypto/ssh/terminal"
 
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 )
 
@@ -29,17 +30,13 @@ type UserPassword struct {
 	Specified bool // If true, the user and password were provided by the user.
 }
 
-const DOLT_ENV_PWD = "DOLT_CLI_PASSWORD"
-const DOLT_ENV_USER = "DOLT_CLI_USER"
-const DOLT_SILENCE_USER_REQ_FOR_TESTING = "DOLT_SILENCE_USER_REQ_FOR_TESTING"
-
 // BuildUserPasswordPrompt builds a UserPassword struct from the parsed args. The user is prompted for a password if one
 // is not provided. If a username is not provided, the default is "root" (which will not be allowed is a password is
 // provided). A new instances of ArgParseResults is returned which does not contain the user or password flags.
 func BuildUserPasswordPrompt(parsedArgs *argparser.ArgParseResults) (newParsedArgs *argparser.ArgParseResults, credentials *UserPassword, err error) {
 	userId, hasUserId := parsedArgs.GetValue(UserFlag)
 	if !hasUserId {
-		envUser, hasEnvUser := os.LookupEnv(DOLT_ENV_USER)
+		envUser, hasEnvUser := os.LookupEnv(dconfig.EnvUser)
 		if hasEnvUser {
 			userId = envUser
 			hasUserId = true
@@ -48,7 +45,7 @@ func BuildUserPasswordPrompt(parsedArgs *argparser.ArgParseResults) (newParsedAr
 
 	password, hasPassword := parsedArgs.GetValue(PasswordFlag)
 	if !hasPassword {
-		envPassword, hasEnvPassword := os.LookupEnv(DOLT_ENV_PWD)
+		envPassword, hasEnvPassword := os.LookupEnv(dconfig.EnvPassword)
 		if hasEnvPassword {
 			password = envPassword
 			hasPassword = true
@@ -69,7 +66,7 @@ func BuildUserPasswordPrompt(parsedArgs *argparser.ArgParseResults) (newParsedAr
 
 	if hasUserId && !hasPassword {
 		password = ""
-		val, hasVal := os.LookupEnv(DOLT_ENV_PWD)
+		val, hasVal := os.LookupEnv(dconfig.EnvPassword)
 		if hasVal {
 			password = val
 		} else {
@@ -83,7 +80,7 @@ func BuildUserPasswordPrompt(parsedArgs *argparser.ArgParseResults) (newParsedAr
 		return newParsedArgs, &UserPassword{Username: userId, Password: password, Specified: true}, nil
 	}
 
-	testOverride, hasTestOverride := os.LookupEnv(DOLT_SILENCE_USER_REQ_FOR_TESTING)
+	testOverride, hasTestOverride := os.LookupEnv(dconfig.EnvSilenceUserReqForTesting)
 	if hasTestOverride && testOverride == "Y" {
 		// Used for BATS testing only. Typical usage will not hit this path, but we have many legacy tests which
 		// do not provide a user, and the DOLT_ENV_PWD is set to avoid the prompt.

go/cmd/dolt/commands/assist.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/cmd/dolt/commands/engine"
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 )
@@ -69,7 +70,7 @@ func (a Assist) Hidden() bool {
 func (a *Assist) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) int {
 	a.messages = make([]string, 0)
 
-	apiKey, ok := os.LookupEnv("OPENAI_API_KEY")
+	apiKey, ok := os.LookupEnv(dconfig.EnvOpenAiKey)
 	if !ok {
 		cli.PrintErrln("Could not find OpenAI API key. Please set the OPENAI_API_KEY environment variable.")
 		return 1
@@ -145,7 +146,7 @@ func (a *Assist) Exec(ctx context.Context, commandStr string, args []string, dEn
 }
 
 func agreeToTerms(scanner *bufio.Scanner) bool {
-	_, ok := os.LookupEnv("DOLT_ASSIST_AGREE")
+	_, ok := os.LookupEnv(dconfig.EnvDoltAssistAgree)
 	if ok {
 		return true
 	}

go/cmd/dolt/commands/clone.go

@@ -20,6 +20,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/store/types"
 
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
@@ -240,7 +241,7 @@ func getRemoteUserAndPassConfig(apr *argparser.ArgParseResults) (*creds.DoltCred
 	if !apr.Contains(cli.UserFlag) {
 		return nil, nil
 	}
-	pass, found := os.LookupEnv("DOLT_REMOTE_PASSWORD")
+	pass, found := os.LookupEnv(dconfig.EnvDoltRemotePassword)
 	if !found {
 		return nil, errhand.BuildDError("error: must set DOLT_REMOTE_PASSWORD environment variable to use --user param").Build()
 	}

go/cmd/dolt/commands/commit.go

@@ -31,6 +31,7 @@ import (
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/cmd/dolt/errhand"
 	"github.com/dolthub/dolt/go/libraries/doltcore/branch_control"
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/doltcore/diff"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
@@ -350,7 +351,7 @@ func getCommitMessageFromEditor(sqlCtx *sql.Context, queryist cli.Queryist, sugg
 
 	backupEd := "vim"
 	// try getting default editor on the user system
-	if ed, edSet := os.LookupEnv("EDITOR"); edSet {
+	if ed, edSet := os.LookupEnv(dconfig.EnvEditor); edSet {
 		backupEd = ed
 	}
 	// try getting Dolt config core.editor
@@ -374,7 +375,7 @@ func getCommitMessageFromEditor(sqlCtx *sql.Context, queryist cli.Queryist, sugg
 func checkIsTerminal() bool {
 	isTerminal := false
 	cli.ExecuteWithStdioRestored(func() {
-		if goisatty.IsTerminal(os.Stdout.Fd()) || os.Getenv("DOLT_TEST_FORCE_OPEN_EDITOR") == "1" {
+		if goisatty.IsTerminal(os.Stdout.Fd()) || os.Getenv(dconfig.EnvTestForceOpenEditor) == "1" {
 			isTerminal = true
 		}
 	})

go/cmd/dolt/commands/engine/sqlengine.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/libraries/doltcore/branch_control"
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	dblr "github.com/dolthub/dolt/go/libraries/doltcore/sqle/binlogreplication"
@@ -184,9 +185,9 @@ func NewSqlEngine(
 		return nil, err
 	}
 
-	if dbg, ok := os.LookupEnv("DOLT_SQL_DEBUG_LOG"); ok && strings.ToLower(dbg) == "true" {
+	if dbg, ok := os.LookupEnv(dconfig.EnvSqlDebugLog); ok && strings.ToLower(dbg) == "true" {
 		engine.Analyzer.Debug = true
-		if verbose, ok := os.LookupEnv("DOLT_SQL_DEBUG_LOG_VERBOSE"); ok && strings.ToLower(verbose) == "true" {
+		if verbose, ok := os.LookupEnv(dconfig.EnvSqlDebugLogVerbose); ok && strings.ToLower(verbose) == "true" {
 			engine.Analyzer.Verbose = true
 		}
 	}

go/cmd/dolt/dolt.go

@@ -50,6 +50,7 @@ import (
 	"github.com/dolthub/dolt/go/cmd/dolt/commands/stashcmds"
 	"github.com/dolthub/dolt/go/cmd/dolt/commands/tblcmds"
 	"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions"
@@ -237,7 +238,7 @@ func runMain() int {
 		return 1
 	}
 
-	if os.Getenv("DOLT_VERBOSE_ASSERT_TABLE_FILES_CLOSED") == "" {
+	if os.Getenv(dconfig.EnvVerboseAssertTableFilesClosed) == "" {
 		nbs.TableIndexGCFinalizerWithStackTrace = false
 	}
 

go/libraries/doltcore/dbfactory/file.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"sync"
 
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 	"github.com/dolthub/dolt/go/store/datas"
 	"github.com/dolthub/dolt/go/store/nbs"
@@ -32,7 +33,7 @@ import (
 
 func init() {
 	// default to chunk journal unless feature flag is set
-	if os.Getenv("DOLT_DISABLE_CHUNK_JOURNAL") != "" {
+	if os.Getenv(dconfig.EnvDisableChunkJournal) != "" {
 		chunkJournalFeatureFlag = false
 	}
 }

go/libraries/doltcore/dbfactory/oss.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/store/blobstore"
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/datas"
@@ -34,9 +35,6 @@ import (
 )
 
 const (
-	ossEndpointEnvKey        = "OSS_ENDPOINT"
-	ossAccessKeyIDEnvKey     = "OSS_ACCESS_KEY_ID"
-	ossAccessKeySecretEnvKey = "OSS_ACCESS_KEY_SECRET"
 
 	// OSSCredsFileParam is a creation parameter that can be used to specify a credential file to use.
 	OSSCredsFileParam = "oss-creds-file"
@@ -149,30 +147,30 @@ func (opt ossCredential) getEndPoint() (string, error) {
 	if opt.Endpoint != "" {
 		return opt.Endpoint, nil
 	}
-	if v := os.Getenv(ossEndpointEnvKey); v != "" {
+	if v := os.Getenv(dconfig.EnvOssEndpoint); v != "" {
 		return v, nil
 	}
-	return "", fmt.Errorf("failed to find endpoint from cred file or env %s", ossEndpointEnvKey)
+	return "", fmt.Errorf("failed to find endpoint from cred file or env %s", dconfig.EnvOssEndpoint)
 }
 
 func (opt ossCredential) getAccessKeyID() (string, error) {
 	if opt.AccessKeyID != "" {
 		return opt.AccessKeyID, nil
 	}
-	if v := os.Getenv(ossAccessKeyIDEnvKey); v != "" {
+	if v := os.Getenv(dconfig.EnvOssAccessKeyID); v != "" {
 		return v, nil
 	}
-	return "", fmt.Errorf("failed to find accessKeyID from cred file or env %s", ossAccessKeyIDEnvKey)
+	return "", fmt.Errorf("failed to find accessKeyID from cred file or env %s", dconfig.EnvOssAccessKeyID)
 }
 
 func (opt ossCredential) getAccessKeySecret() (string, error) {
 	if opt.AccessKeySecret != "" {
 		return opt.AccessKeySecret, nil
 	}
-	if v := os.Getenv(ossAccessKeySecretEnvKey); v != "" {
+	if v := os.Getenv(dconfig.EnvOssAccessKeySecret); v != "" {
 		return v, nil
 	}
-	return "", fmt.Errorf("failed to find accessKeySecret from cred file or env %s", ossAccessKeySecretEnvKey)
+	return "", fmt.Errorf("failed to find accessKeySecret from cred file or env %s", dconfig.EnvOssAccessKeySecret)
 }
 
 func readOSSCredentialsFromFile(credFile string) (ossCredentials, error) {

go/libraries/doltcore/dbfactory/oss_test.go

@@ -20,6 +20,8 @@ import (
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 )
 
 func Test_readOssCredentialsFromFile(t *testing.T) {
@@ -154,10 +156,10 @@ func Test_getOSSClient(t *testing.T) {
 		{
 			name: "get valid oss client from env",
 			before: func() {
-				os.Setenv(ossEndpointEnvKey, "testendpoint")
+				os.Setenv(dconfig.EnvOssEndpoint, "testendpoint")
 			},
 			after: func() {
-				os.Unsetenv(ossEndpointEnvKey)
+				os.Unsetenv(dconfig.EnvOssEndpoint)
 			},
 			args: args{
 				opts: ossCredential{

go/libraries/doltcore/dconfig/envvars.go

@@ -0,0 +1,41 @@
+// Copyright 2023 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dconfig
+
+const (
+	EnvPassword                      = "DOLT_CLI_PASSWORD"
+	EnvUser                          = "DOLT_CLI_USER"
+	EnvSilenceUserReqForTesting      = "DOLT_SILENCE_USER_REQ_FOR_TESTING"
+	EnvOpenAiKey                     = "OPENAI_API_KEY"
+	EnvDoltRemotePassword            = "DOLT_REMOTE_PASSWORD"
+	EnvEditor                        = "EDITOR"
+	EnvSqlDebugLogVerbose            = "DOLT_SQL_DEBUG_LOG_VERBOSE"
+	EnvSqlDebugLog                   = "DOLT_SQL_DEBUG_LOG"
+	EnvHome                          = "HOME"
+	EnvDoltRootPath                  = "DOLT_ROOT_PATH"
+	EnvRemoteVersionDownloadStats    = "DOLT_REMOTE_VERBOSE_DOWNLOAD_STATS"
+	EnvPushLog                       = "PUSH_LOG"
+	EnvDefaultBinFormat              = "DOLT_DEFAULT_BIN_FORMAT"
+	EnvTestForceOpenEditor           = "DOLT_TEST_FORCE_OPEN_EDITOR"
+	EnvDisableChunkJournal           = "DOLT_DISABLE_CHUNK_JOURNAL"
+	EnvOssEndpoint                   = "OSS_ENDPOINT"
+	EnvOssAccessKeyID                = "OSS_ACCESS_KEY_ID"
+	EnvOssAccessKeySecret            = "OSS_ACCESS_KEY_SECRET"
+	EnvVerboseAssertTableFilesClosed = "DOLT_VERBOSE_ASSERT_TABLE_FILES_CLOSED"
+	EnvDisableGcProcedure            = "DOLT_DISABLE_GC_PROCEDURE"
+	EnvEditTableBufferRows           = "DOLT_EDIT_TABLE_BUFFER_ROWS"
+	EnvDisableFixedAccess            = "DOLT_DISABLE_FIXED_ACCESS"
+	EnvDoltAssistAgree               = "DOLT_ASSIST_AGREE"
+)

go/libraries/doltcore/doltdb/envvars.go

@@ -0,0 +1,15 @@
+// Copyright 2023 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package doltdb

go/libraries/doltcore/dtestutils/sql_server_driver/cmd.go

@@ -40,8 +40,10 @@ const TestEmailAddress = "[email protected]"
 const ConnectAttempts = 50
 const RetrySleepDuration = 50 * time.Millisecond
 
+const EnvDoltBinPath = "DOLT_BIN_PATH"
+
 func init() {
-	path := os.Getenv("DOLT_BIN_PATH")
+	path := os.Getenv(EnvDoltBinPath)
 	if path == "" {
 		path = "dolt"
 	}

go/libraries/doltcore/env/paths.go

@@ -20,13 +20,12 @@ import (
 	"path/filepath"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 )
 
 const (
-	homeEnvVar         = "HOME"
-	doltRootPathEnvVar = "DOLT_ROOT_PATH"
-	credsDir           = "creds"
+	credsDir = "creds"
 
 	configFile       = "config.json"
 	GlobalConfigFile = "config_global.json"
@@ -42,12 +41,12 @@ type HomeDirProvider func() (string, error)
 // state will be stored inside of the .dolt directory.  The environment variable DOLT_ROOT_PATH can be used to
 // provide a different directory where the root .dolt directory should be located and global state will be stored there.
 func GetCurrentUserHomeDir() (string, error) {
-	if doltRootPath, ok := os.LookupEnv(doltRootPathEnvVar); ok && doltRootPath != "" {
+	if doltRootPath, ok := os.LookupEnv(dconfig.EnvDoltRootPath); ok && doltRootPath != "" {
 		return filesys.LocalFS.Abs(doltRootPath)
 	}
 
 	var home string
-	if homeEnvPath, ok := os.LookupEnv(homeEnvVar); ok && homeEnvPath != "" {
+	if homeEnvPath, ok := os.LookupEnv(dconfig.EnvHome); ok && homeEnvPath != "" {
 		home = homeEnvPath
 	} else if usr, err := user.Current(); err != nil {
 		return "", err

go/libraries/doltcore/remotestorage/stats.go

@@ -23,6 +23,8 @@ import (
 
 	"github.com/HdrHistogram/hdrhistogram-go"
 	"github.com/fatih/color"
+
+	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
 )
 
 var StatsFactory func() StatsRecorder = NullStatsRecorderFactory
@@ -41,7 +43,7 @@ func StatsFlusherToColorError(r StatsRecorder) {
 }
 
 func init() {
-	if _, ok := os.LookupEnv("DOLT_REMOTE_VERBOSE_DOWNLOAD_STATS"); ok {
+	if _, ok := os.LookupEnv(dconfig.EnvRemoteVersionDownloadStats); ok {
 		StatsFactory = HistogramStatsRecorderFactory
 		StatsFlusher = StatsFlusherToColorError
 	}