Merge pull request #6919 from dolthub/zachmu/virtual

Partial support for virtual columns

go/cmd/dolt/commands/diff.go

@@ -864,6 +864,8 @@ func getTableSchemaAtRef(queryist cli.Queryist, sqlCtx *sql.Context, tableName s
 	return sch, createStmt, nil
 }
 
+// schemaFromCreateTableStmt returns a schema for the CREATE TABLE statement given
+// TODO: this is substantially incorrect, doesn't handle primary key ordering, probably other things too
 func schemaFromCreateTableStmt(createTableStmt string) (schema.Schema, error) {
 	parsed, err := ast.Parse(createTableStmt)
 	if err != nil {
@@ -900,15 +902,17 @@ func schemaFromCreateTableStmt(createTableStmt string) (schema.Schema, error) {
 		if col.Type.Comment != nil {
 			comment = col.Type.Comment.String()
 		}
-		sCol, err := schema.NewColumnWithTypeInfo(
-			col.Name.String(),
-			0,
-			typeInfo,
-			primaryCols[col.Name.Lowered()],
-			defBuf.String(),
-			col.Type.Autoincrement == true,
-			comment,
-		)
+		sCol := schema.Column{
+			Name:          col.Name.String(),
+			Kind:          typeInfo.NomsKind(),
+			IsPartOfPK:    primaryCols[col.Name.Lowered()],
+			TypeInfo:      typeInfo,
+			Default:       defBuf.String(),
+			Generated:     "",    // TODO
+			Virtual:       false, // TODO
+			AutoIncrement: col.Type.Autoincrement == true,
+			Comment:       comment,
+		}
 		cols = append(cols, sCol)
 	}
 

go/libraries/doltcore/doltdb/durable/artifact_index.go

@@ -50,7 +50,7 @@ func RefFromArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, idx Ar
 		return refFromNomsValue(ctx, vrw, b)
 
 	default:
-		return types.Ref{}, errNbfUnkown
+		return types.Ref{}, errNbfUnknown
 	}
 }
 
@@ -69,7 +69,7 @@ func NewEmptyArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return ArtifactIndexFromProllyMap(m), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 
@@ -107,7 +107,7 @@ func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return ArtifactIndexFromProllyMap(m), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 

go/libraries/doltcore/doltdb/durable/conflict_index.go

@@ -40,7 +40,7 @@ func RefFromConflictIndex(ctx context.Context, vrw types.ValueReadWriter, idx Co
 		return types.Ref{}, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
 
 	default:
-		return types.Ref{}, errNbfUnkown
+		return types.Ref{}, errNbfUnknown
 	}
 }
 
@@ -58,7 +58,7 @@ func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return nil, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 
@@ -91,7 +91,7 @@ func conflictIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return nil, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 

go/libraries/doltcore/doltdb/durable/index.go

@@ -88,7 +88,7 @@ func RefFromIndex(ctx context.Context, vrw types.ValueReadWriter, idx Index) (ty
 		return refFromNomsValue(ctx, vrw, b)
 
 	default:
-		return types.Ref{}, errNbfUnkown
+		return types.Ref{}, errNbfUnknown
 	}
 }
 
@@ -115,7 +115,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 		return IndexFromProllyMap(pm), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 
@@ -138,7 +138,7 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 		return IndexFromProllyMap(m), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 

go/libraries/doltcore/doltdb/durable/table.go

@@ -52,7 +52,7 @@ var (
 )
 
 var (
-	errNbfUnkown      = fmt.Errorf("unknown NomsBinFormat")
+	errNbfUnknown     = fmt.Errorf("unknown NomsBinFormat")
 	errNbfUnsupported = fmt.Errorf("operation unsupported for NomsBinFormat")
 )
 

go/libraries/doltcore/doltdb/root_val.go

@@ -50,7 +50,7 @@ type FeatureVersion int64
 
 // DoltFeatureVersion is described in feature_version.md.
 // only variable for testing.
-var DoltFeatureVersion FeatureVersion = 4 // last bumped when adding sql_mode column to dolt_schemas
+var DoltFeatureVersion FeatureVersion = 5 // last bumped when adding virtual columns to schema storage
 
 // RootValue is the value of the Database and is the committed value in every Dolt commit.
 type RootValue struct {

go/libraries/doltcore/migrate/transform.go

@@ -484,15 +484,11 @@ func migrateSchema(ctx context.Context, tableName string, existing schema.Schema
 			case query.Type_TEXT:
 				patched = true
 				info := typeinfo.StringDefaultType
-				cols[i], err = schema.NewColumnWithTypeInfo(
-					c.Name, c.Tag, info, c.IsPartOfPK, c.Default,
-					c.AutoIncrement, c.Comment, c.Constraints...)
+				cols[i], err = schema.NewColumnWithTypeInfo(c.Name, c.Tag, info, c.IsPartOfPK, c.Default, c.AutoIncrement, c.Comment, c.Constraints...)
 			case query.Type_BLOB:
 				patched = true
 				info := typeinfo.VarbinaryDefaultType
-				cols[i], err = schema.NewColumnWithTypeInfo(
-					c.Name, c.Tag, info, c.IsPartOfPK, c.Default,
-					c.AutoIncrement, c.Comment, c.Constraints...)
+				cols[i], err = schema.NewColumnWithTypeInfo(c.Name, c.Tag, info, c.IsPartOfPK, c.Default, c.AutoIncrement, c.Comment, c.Constraints...)
 			}
 			if err != nil {
 				return nil, err
@@ -525,9 +521,7 @@ func migrateSchema(ctx context.Context, tableName string, existing schema.Schema
 			return nil, err
 		}
 
-		cols[i], err = schema.NewColumnWithTypeInfo(
-			c.Name, c.Tag, info, c.IsPartOfPK, c.Default,
-			c.AutoIncrement, c.Comment, c.Constraints...)
+		cols[i], err = schema.NewColumnWithTypeInfo(c.Name, c.Tag, info, c.IsPartOfPK, c.Default, c.AutoIncrement, c.Comment, c.Constraints...)
 		if err != nil {
 			return nil, err
 		}

go/libraries/doltcore/schema/col_coll.go

@@ -37,20 +37,24 @@ var ErrNoPrimaryKeyColumns = errors.New("no primary key columns")
 var ErrNonAutoIncType = errors.New("column type cannot be auto incremented")
 
 var EmptyColColl = &ColCollection{
-	[]Column{},
-	[]uint64{},
-	[]uint64{},
-	map[uint64]Column{},
-	map[string]Column{},
-	map[string]Column{},
-	map[uint64]int{},
+	cols:           []Column{},
+	Tags:           []uint64{},
+	SortedTags:     []uint64{},
+	TagToCol:       map[uint64]Column{},
+	NameToCol:      map[string]Column{},
+	LowerNameToCol: map[string]Column{},
+	TagToIdx:       map[uint64]int{},
 }
 
 // ColCollection is a collection of columns. As a stand-alone collection, all columns in the collection must have unique
 // tags. To be instantiated as a schema for writing to the database, names must also be unique.
 // See schema.ValidateForInsert for details.
 type ColCollection struct {
 	cols []Column
+	// virtualColumns stores the indexes of any virtual columns in the collection
+	virtualColumns []int
+	// storedIndexes stores the indexes of the stored columns in the collection
+	storedIndexes []int
 	// Tags is a list of all the tags in the ColCollection in their original order.
 	Tags []uint64
 	// SortedTags is a list of all the tags in the ColCollection in sorted order.
@@ -63,6 +67,8 @@ type ColCollection struct {
 	LowerNameToCol map[string]Column
 	// TagToIdx is a map from a tag to the column index
 	TagToIdx map[uint64]int
+	// tagToStorageIndex is a map from a tag to the physical storage column index
+	tagToStorageIndex map[uint64]int
 }
 
 // NewColCollection creates a new collection from a list of columns. If any columns have the same tag, by-tag lookups in
@@ -78,8 +84,12 @@ func NewColCollection(cols ...Column) *ColCollection {
 	nameToCol := make(map[string]Column, len(cols))
 	lowerNameToCol := make(map[string]Column, len(cols))
 	tagToIdx := make(map[uint64]int, len(cols))
+	tagToStorageIndex := make(map[uint64]int, len(cols))
+	var virtualColumns []int
 
 	var columns []Column
+	var storedIndexes []int
+	storageIdx := 0
 	for i, col := range cols {
 		// If multiple columns have the same tag, the last one is used for tag lookups.
 		// Columns must have unique tags to pass schema.ValidateForInsert.
@@ -96,18 +106,29 @@ func NewColCollection(cols ...Column) *ColCollection {
 		if _, ok := lowerNameToCol[lowerCaseName]; !ok {
 			lowerNameToCol[lowerCaseName] = cols[i]
 		}
+
+		if col.Virtual {
+			virtualColumns = append(virtualColumns, i)
+		} else {
+			storedIndexes = append(storedIndexes, i)
+			tagToStorageIndex[col.Tag] = storageIdx
+			storageIdx++
+		}
 	}
 
 	sort.Slice(sortedTags, func(i, j int) bool { return sortedTags[i] < sortedTags[j] })
 
 	return &ColCollection{
-		cols:           columns,
-		Tags:           tags,
-		SortedTags:     sortedTags,
-		TagToCol:       tagToCol,
-		NameToCol:      nameToCol,
-		LowerNameToCol: lowerNameToCol,
-		TagToIdx:       tagToIdx,
+		cols:              columns,
+		virtualColumns:    virtualColumns,
+		storedIndexes:     storedIndexes,
+		tagToStorageIndex: tagToStorageIndex,
+		Tags:              tags,
+		SortedTags:        sortedTags,
+		TagToCol:          tagToCol,
+		NameToCol:         nameToCol,
+		LowerNameToCol:    lowerNameToCol,
+		TagToIdx:          tagToIdx,
 	}
 }
 
@@ -220,16 +241,32 @@ func (cc *ColCollection) GetByTag(tag uint64) (Column, bool) {
 	return InvalidCol, false
 }
 
-// GetByIndex returns a column with a given index
+// GetByIndex returns the Nth column in the collection
 func (cc *ColCollection) GetByIndex(idx int) Column {
 	return cc.cols[idx]
 }
 
+// GetByStoredIndex returns the Nth stored column (omitting virtual columns from index calculation)
+func (cc *ColCollection) GetByStoredIndex(idx int) Column {
+	return cc.cols[cc.storedIndexes[idx]]
+}
+
+// StoredIndexByTag returns the storage index of the column with the given tag, ignoring virtual columns
+func (cc *ColCollection) StoredIndexByTag(tag uint64) (int, bool) {
+	idx, ok := cc.tagToStorageIndex[tag]
+	return idx, ok
+}
+
 // Size returns the number of columns in the collection.
 func (cc *ColCollection) Size() int {
 	return len(cc.cols)
 }
 
+// StoredSize returns the number of non-virtual columns in the collection
+func (cc *ColCollection) StoredSize() int {
+	return len(cc.storedIndexes)
+}
+
 // Contains returns whether this column collection contains a column with the name given, case insensitive
 func (cc *ColCollection) Contains(name string) bool {
 	_, ok := cc.GetByNameCaseInsensitive(name)

go/libraries/doltcore/schema/col_coll_test.go

@@ -21,14 +21,14 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
+	typeinfo "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
 	"github.com/dolthub/dolt/go/store/types"
 )
 
-var firstNameCol = Column{"first", 0, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil}
-var lastNameCol = Column{"last", 1, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil}
-var firstNameCapsCol = Column{"FiRsT", 2, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil}
-var lastNameCapsCol = Column{"LAST", 3, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil}
+var firstNameCol = Column{Name: "first", Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType}
+var lastNameCol = Column{Name: "last", Tag: 1, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType}
+var firstNameCapsCol = Column{Name: "FiRsT", Tag: 2, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType}
+var lastNameCapsCol = Column{Name: "LAST", Tag: 3, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType}
 
 func TestGetByNameAndTag(t *testing.T) {
 	cols := []Column{firstNameCol, lastNameCol, firstNameCapsCol, lastNameCapsCol}
@@ -102,18 +102,18 @@ func TestGetByNameCaseInsensitive(t *testing.T) {
 
 func TestAppendAndItrInSortOrder(t *testing.T) {
 	cols := []Column{
-		{"0", 0, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"2", 2, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"4", 4, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"3", 3, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"1", 1, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
+		{Name: "0", Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "2", Tag: 2, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "4", Tag: 4, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "3", Tag: 3, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "1", Tag: 1, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
 	}
 	cols2 := []Column{
-		{"7", 7, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"9", 9, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"5", 5, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"8", 8, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
-		{"6", 6, types.StringKind, false, typeinfo.StringDefaultType, "", false, "", nil},
+		{Name: "7", Tag: 7, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "9", Tag: 9, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "5", Tag: 5, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "8", Tag: 8, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
+		{Name: "6", Tag: 6, Kind: types.StringKind, TypeInfo: typeinfo.StringDefaultType},
 	}
 
 	colColl := NewColCollection(cols...)