Merge pull request #7259 from dolthub/db/sql-bom

support importing files with UTF-8, UTF-16LE, UTF16-BE BOM headers on sql path

go/cmd/dolt/commands/sql.go

@@ -32,6 +32,8 @@ import (
 	"github.com/dolthub/vitess/go/vt/vterrors"
 	"github.com/fatih/color"
 	"github.com/flynn-archive/go-shlex"
+	textunicode "golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	"gopkg.in/src-d/go-errors.v1"
 
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
@@ -238,7 +240,7 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
 
 		_, continueOnError := apr.GetValue(continueFlag)
 
-		input := os.Stdin
+		var input io.Reader = os.Stdin
 		if fileInput, ok := apr.GetValue(fileInputFlag); ok {
 			isTty = false
 			input, err = os.OpenFile(fileInput, os.O_RDONLY, os.ModePerm)
@@ -250,6 +252,8 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
 				return sqlHandleVErrAndExitCode(queryist, errhand.BuildDError("couldn't get file size %s", fileInput).Build(), usage)
 			}
 
+			input = transform.NewReader(input, textunicode.BOMOverride(transform.Nop))
+
 			// initialize fileReadProg global variable if there is a file to process queries from
 			fileReadProg = &fileReadProgress{bytesRead: 0, totalBytes: info.Size(), printed: 0, displayStrLen: 0}
 			defer fileReadProg.close()
@@ -261,6 +265,7 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
 				return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage)
 			}
 		} else {
+			input = transform.NewReader(input, textunicode.BOMOverride(transform.Nop))
 			err := execBatchMode(sqlCtx, queryist, input, continueOnError, format)
 			if err != nil {
 				return sqlHandleVErrAndExitCode(queryist, errhand.VerboseErrorFromError(err), usage)

integration-tests/bats/helper/with_utf8_bom.sql

@@ -0,0 +1 @@
+create table t1 (pk int primary key);

integration-tests/bats/sql.bats

@@ -2899,3 +2899,12 @@ SQL
     mkdir .dolt
     dolt sql -q "select 1"
 }
+
+@test "sql: handle importing files with bom headers" {
+    dolt sql < $BATS_TEST_DIRNAME/helper/with_utf8_bom.sql
+    dolt table rm t1
+    dolt sql < $BATS_TEST_DIRNAME/helper/with_utf16le_bom.sql
+    dolt table rm t1
+    dolt sql < $BATS_TEST_DIRNAME/helper/with_utf16be_bom.sql
+    dolt table rm t1
+}