Merge pull request #6974 from dolthub/zachmu/virtual-merge

Support merging schemas with virtual / generated columns

go/cmd/dolt/commands/indexcmds/rebuild.go

@@ -17,6 +17,8 @@ package indexcmds
 import (
 	"context"
 
+	"github.com/dolthub/go-mysql-server/sql"
+
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/cmd/dolt/commands"
 	"github.com/dolthub/dolt/go/cmd/dolt/errhand"
@@ -102,7 +104,7 @@ func (cmd RebuildCmd) Exec(ctx context.Context, commandStr string, args []string
 	if idxSch == nil {
 		return HandleErr(errhand.BuildDError("the index `%s` does not exist on table `%s`", indexName, tableName).Build(), nil)
 	}
-	indexRowData, err := creation.BuildSecondaryIndex(ctx, table, idxSch, opts)
+	indexRowData, err := creation.BuildSecondaryIndex(sql.NewContext(ctx), table, idxSch, tableName, opts)
 	if err != nil {
 		return HandleErr(errhand.BuildDError("Unable to rebuild index `%s` on table `%s`.", indexName, tableName).AddCause(err).Build(), nil)
 	}

go/go.mod

@@ -59,7 +59,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.17.1-0.20231109211027-734826ff8972
+	github.com/dolthub/go-mysql-server v0.17.1-0.20231110001639-33b593341822
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

go/go.sum

@@ -181,8 +181,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.17.1-0.20231109211027-734826ff8972 h1:+cJcknWqzo1nZ9ZqkKxubiKIK/D+7c6ztOz0RC6dQ2I=
-github.com/dolthub/go-mysql-server v0.17.1-0.20231109211027-734826ff8972/go.mod h1:Z3EbOzC1yoK9MoYBxl6LDksV8GRRyjjHDZTu2lWpT/E=
+github.com/dolthub/go-mysql-server v0.17.1-0.20231110001639-33b593341822 h1:5A5opfcrehlHmaPPBAm6drw4rP3niBoVOci+7YjEg3s=
+github.com/dolthub/go-mysql-server v0.17.1-0.20231110001639-33b593341822/go.mod h1:Z3EbOzC1yoK9MoYBxl6LDksV8GRRyjjHDZTu2lWpT/E=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
 github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ=

go/libraries/doltcore/diff/table_deltas.go

@@ -612,70 +612,6 @@ func GetDataDiffStatement(tableName string, sch schema.Schema, row sql.Row, rowD
 	}
 }
 
-// GenerateCreateTableStatement returns CREATE TABLE statement for given table. This function was made to share the same
-// 'create table' statement logic as GMS. We initially were running `SHOW CREATE TABLE` query to get the statement;
-// however, it cannot be done for cases that need this statement in sql shell mode. Dolt uses its own Schema and
-// Column and other object types which are not directly compatible with GMS, so we try to use as much shared logic
-// as possible with GMS to get 'create table' statement in Dolt.
-func GenerateCreateTableStatement(tblName string, sch schema.Schema, pkSchema sql.PrimaryKeySchema, fks []doltdb.ForeignKey, fksParentSch map[string]schema.Schema) (string, error) {
-	sqlSch := pkSchema.Schema
-	colStmts := make([]string, len(sqlSch))
-
-	// Statement creation parts for each column
-	for i, col := range sch.GetAllCols().GetColumns() {
-		colStmts[i] = sqlfmt.GenerateCreateTableIndentedColumnDefinition(col, sql.CollationID(sch.GetCollation()))
-	}
-
-	primaryKeyCols := sch.GetPKCols().GetColumnNames()
-	if len(primaryKeyCols) > 0 {
-		primaryKey := sql.GenerateCreateTablePrimaryKeyDefinition(primaryKeyCols)
-		colStmts = append(colStmts, primaryKey)
-	}
-
-	indexes := sch.Indexes().AllIndexes()
-	for _, index := range indexes {
-		// The primary key may or may not be declared as an index by the table. Don't print it twice if it's here.
-		if isPrimaryKeyIndex(index, sch) {
-			continue
-		}
-		colStmts = append(colStmts, sqlfmt.GenerateCreateTableIndexDefinition(index))
-	}
-
-	for _, fk := range fks {
-		colStmts = append(colStmts, sqlfmt.GenerateCreateTableForeignKeyDefinition(fk, sch, fksParentSch[fk.ReferencedTableName]))
-	}
-
-	for _, check := range sch.Checks().AllChecks() {
-		colStmts = append(colStmts, sqlfmt.GenerateCreateTableCheckConstraintClause(check))
-	}
-
-	coll := sql.CollationID(sch.GetCollation())
-	createTableStmt := sql.GenerateCreateTableStatement(tblName, colStmts, coll.CharacterSet().Name(), coll.Name())
-	return fmt.Sprintf("%s;", createTableStmt), nil
-}
-
-// isPrimaryKeyIndex returns whether the index given matches the table's primary key columns. Order is not considered.
-func isPrimaryKeyIndex(index schema.Index, sch schema.Schema) bool {
-	var pks = sch.GetPKCols().GetColumns()
-	var pkMap = make(map[string]struct{})
-	for _, c := range pks {
-		pkMap[c.Name] = struct{}{}
-	}
-
-	indexCols := index.ColumnNames()
-	if len(indexCols) != len(pks) {
-		return false
-	}
-
-	for _, c := range index.ColumnNames() {
-		if _, ok := pkMap[c]; !ok {
-			return false
-		}
-	}
-
-	return true
-}
-
 // WorkingSetContainsOnlyIgnoredTables returns true if all changes in working set are ignored tables.
 // Otherwise, if there are any non-ignored changes, returns false.
 // Note that only unstaged tables are subject to dolt_ignore (this is consistent with what git does.)

go/libraries/doltcore/merge/fulltext_rebuild.go

@@ -224,5 +224,5 @@ func createRowIterForTable(ctx *sql.Context, t *doltdb.Table, sch schema.Schema)
 		return nil, err
 	}
 
-	return index.NewProllyRowIter(sch, rows, iter, nil)
+	return index.NewProllyRowIterForMap(sch, rows, iter, nil), nil
 }

go/libraries/doltcore/merge/merge_prolly_indexes.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/dolthub/go-mysql-server/sql"
+
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
@@ -35,7 +37,7 @@ import (
 // changes from the other side of the merge to have been merged in before this
 // function was called. This is safer, but less efficient.
 func mergeProllySecondaryIndexes(
-	ctx context.Context,
+	ctx *sql.Context,
 	tm *TableMerger,
 	leftSet, rightSet durable.IndexSet,
 	finalSch schema.Schema,
@@ -103,7 +105,17 @@ func mergeProllySecondaryIndexes(
 	return mergedIndexSet, nil
 }
 
-func buildIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, postMergeSchema schema.Schema, index schema.Index, m prolly.Map, artEditor *prolly.ArtifactsEditor, theirRootIsh doltdb.Rootish, tblName string) (durable.Index, error) {
+func buildIndex(
+	ctx *sql.Context,
+	vrw types.ValueReadWriter,
+	ns tree.NodeStore,
+	postMergeSchema schema.Schema,
+	index schema.Index,
+	m prolly.Map,
+	artEditor *prolly.ArtifactsEditor,
+	theirRootIsh doltdb.Rootish,
+	tblName string,
+) (durable.Index, error) {
 	if index.IsUnique() {
 		meta, err := makeUniqViolMeta(postMergeSchema, index)
 		if err != nil {
@@ -119,33 +131,26 @@ func buildIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStor
 
 		pkMapping := ordinalMappingFromIndex(index)
 
-		mergedMap, err := creation.BuildUniqueProllyIndex(
-			ctx,
-			vrw,
-			ns,
-			postMergeSchema,
-			index,
-			m,
-			func(ctx context.Context, existingKey, newKey val.Tuple) (err error) {
-				eK := getPKFromSecondaryKey(kb, p, pkMapping, existingKey)
-				nK := getPKFromSecondaryKey(kb, p, pkMapping, newKey)
-				err = replaceUniqueKeyViolation(ctx, artEditor, m, eK, kd, theirRootIsh, vInfo, tblName)
-				if err != nil {
-					return err
-				}
-				err = replaceUniqueKeyViolation(ctx, artEditor, m, nK, kd, theirRootIsh, vInfo, tblName)
-				if err != nil {
-					return err
-				}
-				return nil
-			})
+		mergedMap, err := creation.BuildUniqueProllyIndex(ctx, vrw, ns, postMergeSchema, tblName, index, m, func(ctx context.Context, existingKey, newKey val.Tuple) (err error) {
+			eK := getPKFromSecondaryKey(kb, p, pkMapping, existingKey)
+			nK := getPKFromSecondaryKey(kb, p, pkMapping, newKey)
+			err = replaceUniqueKeyViolation(ctx, artEditor, m, eK, kd, theirRootIsh, vInfo, tblName)
+			if err != nil {
+				return err
+			}
+			err = replaceUniqueKeyViolation(ctx, artEditor, m, nK, kd, theirRootIsh, vInfo, tblName)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
 		if err != nil {
 			return nil, err
 		}
 		return mergedMap, nil
 	}
 
-	mergedIndex, err := creation.BuildSecondaryProllyIndex(ctx, vrw, ns, postMergeSchema, index, m)
+	mergedIndex, err := creation.BuildSecondaryProllyIndex(ctx, vrw, ns, postMergeSchema, tblName, index, m)
 	if err != nil {
 		return nil, err
 	}