From 3a8bd18b6da91db39e10f30672159e96e36b1c1e Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 6 Aug 2024 10:15:59 -0700 Subject: [PATCH 01/19] Simplify the diffPartitionRowIter which never uses diffPartitions --- go/libraries/doltcore/sqle/dtables/diff_iter.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/diff_iter.go b/go/libraries/doltcore/sqle/dtables/diff_iter.go index 5e284a8a30..2fb483231e 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_iter.go +++ b/go/libraries/doltcore/sqle/dtables/diff_iter.go @@ -500,7 +500,6 @@ func maybeTime(t *time.Time) interface{} { var _ sql.RowIter = (*diffPartitionRowIter)(nil) type diffPartitionRowIter struct { - diffPartitions *DiffPartitions ddb *doltdb.DoltDB joiner *rowconv.Joiner currentPartition *sql.Partition @@ -518,13 +517,8 @@ func NewDiffPartitionRowIter(partition sql.Partition, ddb *doltdb.DoltDB, joiner func (itr *diffPartitionRowIter) Next(ctx *sql.Context) (sql.Row, error) { for { if itr.currentPartition == nil { - nextPartition, err := itr.diffPartitions.Next(ctx) - if err != nil { - return nil, err - } - itr.currentPartition = &nextPartition + return nil, io.EOF } - if itr.currentRowIter == nil { dp := (*itr.currentPartition).(DiffPartition) rowIter, err := dp.GetRowIter(ctx, itr.ddb, itr.joiner, sql.IndexLookup{}) @@ -538,12 +532,7 @@ func (itr *diffPartitionRowIter) Next(ctx *sql.Context) (sql.Row, error) { if err == io.EOF { itr.currentPartition = nil itr.currentRowIter = nil - - if itr.diffPartitions == nil { - return nil, err - } - - continue + return nil, err } else if err != nil { return nil, err } else { From 4418333927298663fe30ee4972671a0e075412ce Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 6 Aug 2024 10:21:53 -0700 Subject: [PATCH 02/19] More simplification of DiffPartitioning --- go/libraries/doltcore/sqle/dolt_diff_table_function.go | 2 +- go/libraries/doltcore/sqle/dolt_patch_table_function.go | 2 +- go/libraries/doltcore/sqle/dtables/column_diff_table.go | 2 +- go/libraries/doltcore/sqle/dtables/diff_iter.go | 9 ++++----- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/go/libraries/doltcore/sqle/dolt_diff_table_function.go b/go/libraries/doltcore/sqle/dolt_diff_table_function.go index 6d0b01c861..8200aa99e8 100644 --- a/go/libraries/doltcore/sqle/dolt_diff_table_function.go +++ b/go/libraries/doltcore/sqle/dolt_diff_table_function.go @@ -186,7 +186,7 @@ func (dtf *DiffTableFunction) RowIter(ctx *sql.Context, _ sql.Row) (sql.RowIter, ddb := sqledb.DbData().Ddb dp := dtables.NewDiffPartition(dtf.tableDelta.ToTable, dtf.tableDelta.FromTable, toCommitStr, fromCommitStr, dtf.toDate, dtf.fromDate, dtf.tableDelta.ToSch, dtf.tableDelta.FromSch) - return dtables.NewDiffPartitionRowIter(*dp, ddb, dtf.joiner), nil + return dtables.NewDiffPartitionRowIter(dp, ddb, dtf.joiner), nil } // findMatchingDelta returns the best matching table delta for the table name diff --git a/go/libraries/doltcore/sqle/dolt_patch_table_function.go b/go/libraries/doltcore/sqle/dolt_patch_table_function.go index 6c5bba841b..1a6732e60d 100644 --- a/go/libraries/doltcore/sqle/dolt_patch_table_function.go +++ b/go/libraries/doltcore/sqle/dolt_patch_table_function.go @@ -632,7 +632,7 @@ func getDiffQuery(ctx *sql.Context, dbData env.DbData, td diff.TableDelta, fromR diffQuerySqlSch, projections := getDiffQuerySqlSchemaAndProjections(diffPKSch.Schema, columnsWithDiff) dp := dtables.NewDiffPartition(td.ToTable, td.FromTable, toRefDetails.hashStr, fromRefDetails.hashStr, toRefDetails.commitTime, fromRefDetails.commitTime, td.ToSch, td.FromSch) - ri := dtables.NewDiffPartitionRowIter(*dp, dbData.Ddb, j) + ri := dtables.NewDiffPartitionRowIter(dp, dbData.Ddb, j) return diffQuerySqlSch, projections, ri, nil } diff --git a/go/libraries/doltcore/sqle/dtables/column_diff_table.go b/go/libraries/doltcore/sqle/dtables/column_diff_table.go index 3e18c307c5..27b4417cdf 100644 --- a/go/libraries/doltcore/sqle/dtables/column_diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/column_diff_table.go @@ -527,7 +527,7 @@ func calculateColDelta(ctx *sql.Context, ddb *doltdb.DoltDB, delta *diff.TableDe now := time.Now() // accurate commit time returned elsewhere // TODO: schema name? dp := NewDiffPartition(delta.ToTable, delta.FromTable, delta.ToName.Name, delta.FromName.Name, (*dtypes.Timestamp)(&now), (*dtypes.Timestamp)(&now), delta.ToSch, delta.FromSch) - ri := NewDiffPartitionRowIter(*dp, ddb, j) + ri := NewDiffPartitionRowIter(dp, ddb, j) var resultColNames []string var resultDiffTypes []string diff --git a/go/libraries/doltcore/sqle/dtables/diff_iter.go b/go/libraries/doltcore/sqle/dtables/diff_iter.go index 2fb483231e..c83ee0ee6d 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_iter.go +++ b/go/libraries/doltcore/sqle/dtables/diff_iter.go @@ -502,13 +502,13 @@ var _ sql.RowIter = (*diffPartitionRowIter)(nil) type diffPartitionRowIter struct { ddb *doltdb.DoltDB joiner *rowconv.Joiner - currentPartition *sql.Partition + currentPartition *DiffPartition currentRowIter *sql.RowIter } -func NewDiffPartitionRowIter(partition sql.Partition, ddb *doltdb.DoltDB, joiner *rowconv.Joiner) *diffPartitionRowIter { +func NewDiffPartitionRowIter(partition *DiffPartition, ddb *doltdb.DoltDB, joiner *rowconv.Joiner) *diffPartitionRowIter { return &diffPartitionRowIter{ - currentPartition: &partition, + currentPartition: partition, ddb: ddb, joiner: joiner, } @@ -520,8 +520,7 @@ func (itr *diffPartitionRowIter) Next(ctx *sql.Context) (sql.Row, error) { return nil, io.EOF } if itr.currentRowIter == nil { - dp := (*itr.currentPartition).(DiffPartition) - rowIter, err := dp.GetRowIter(ctx, itr.ddb, itr.joiner, sql.IndexLookup{}) + rowIter, err := itr.currentPartition.GetRowIter(ctx, itr.ddb, itr.joiner, sql.IndexLookup{}) if err != nil { return nil, err } From 7db92a7df3ed518fccd6e387659177326b472225 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 6 Aug 2024 14:46:10 -0700 Subject: [PATCH 03/19] More dirr_iter clean up --- .../doltcore/sqle/dtables/diff_iter.go | 19 ++++++++----------- .../doltcore/sqle/dtables/diff_table.go | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/diff_iter.go b/go/libraries/doltcore/sqle/dtables/diff_iter.go index c83ee0ee6d..af39e8119a 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_iter.go +++ b/go/libraries/doltcore/sqle/dtables/diff_iter.go @@ -34,7 +34,9 @@ import ( "github.com/dolthub/dolt/go/store/val" ) -type diffRowItr struct { +// ldDiffRowItr is a sql.RowIter implementation which iterates over an LD formated DB in order to generate the +// dolt_diff_{table} results. This is legacy code at this point, as the DOLT format is what we'll support going forward. +type ldDiffRowItr struct { ad diff.RowDiffer diffSrc *diff.RowDiffSource joiner *rowconv.Joiner @@ -43,7 +45,7 @@ type diffRowItr struct { toCommitInfo commitInfo } -var _ sql.RowIter = &diffRowItr{} +var _ sql.RowIter = &ldDiffRowItr{} type commitInfo struct { name types.String @@ -52,7 +54,7 @@ type commitInfo struct { dateTag uint64 } -func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner, dp DiffPartition, lookup sql.IndexLookup) (*diffRowItr, error) { +func newLdDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner, dp DiffPartition, lookup sql.IndexLookup) (*ldDiffRowItr, error) { fromData, fromSch, err := tableData(ctx, dp.from, ddb) if err != nil { @@ -110,7 +112,7 @@ func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joine src := diff.NewRowDiffSource(rd, joiner, ctx.Warn) src.AddInputRowConversion(fromConv, toConv) - return &diffRowItr{ + return &ldDiffRowItr{ ad: rd, diffSrc: src, joiner: joiner, @@ -121,7 +123,7 @@ func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joine } // Next returns the next row -func (itr *diffRowItr) Next(ctx *sql.Context) (sql.Row, error) { +func (itr *ldDiffRowItr) Next(ctx *sql.Context) (sql.Row, error) { r, err := itr.diffSrc.NextDiff() if err != nil { @@ -180,7 +182,7 @@ func (itr *diffRowItr) Next(ctx *sql.Context) (sql.Row, error) { } // Close closes the iterator -func (itr *diffRowItr) Close(*sql.Context) (err error) { +func (itr *ldDiffRowItr) Close(*sql.Context) (err error) { defer itr.ad.Close() defer func() { closeErr := itr.diffSrc.Close() @@ -203,7 +205,6 @@ type prollyDiffIter struct { fromSch, toSch schema.Schema targetFromSch, targetToSch schema.Schema fromConverter, toConverter ProllyRowConverter - fromVD, toVD val.TupleDesc keyless bool fromCm commitInfo2 @@ -285,8 +286,6 @@ func newProllyDiffIter(ctx *sql.Context, dp DiffPartition, targetFromSchema, tar return prollyDiffIter{}, err } - fromVD := fsch.GetValueDescriptor() - toVD := tsch.GetValueDescriptor() keyless := schema.IsKeyless(targetFromSchema) && schema.IsKeyless(targetToSchema) child, cancel := context.WithCancel(ctx) iter := prollyDiffIter{ @@ -298,8 +297,6 @@ func newProllyDiffIter(ctx *sql.Context, dp DiffPartition, targetFromSchema, tar targetToSch: targetToSchema, fromConverter: fromConverter, toConverter: toConverter, - fromVD: fromVD, - toVD: toVD, keyless: keyless, fromCm: fromCm, toCm: toCm, diff --git a/go/libraries/doltcore/sqle/dtables/diff_table.go b/go/libraries/doltcore/sqle/dtables/diff_table.go index 1651c4c56b..f20e95f71a 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/diff_table.go @@ -674,7 +674,7 @@ func (dp DiffPartition) GetRowIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner if types.IsFormat_DOLT(ddb.Format()) { return newProllyDiffIter(ctx, dp, dp.fromSch, dp.toSch) } else { - return newNomsDiffIter(ctx, ddb, joiner, dp, lookup) + return newLdDiffIter(ctx, ddb, joiner, dp, lookup) } } From 41be15bdaaca365925bc6e619fa5a2ff5bd6a214 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 5 Aug 2024 08:41:19 -0700 Subject: [PATCH 04/19] Remove unused package local variable --- go/libraries/doltcore/doltdb/system_table.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go/libraries/doltcore/doltdb/system_table.go b/go/libraries/doltcore/doltdb/system_table.go index 87d59e57b2..13521c345e 100644 --- a/go/libraries/doltcore/doltdb/system_table.go +++ b/go/libraries/doltcore/doltdb/system_table.go @@ -185,10 +185,6 @@ var generatedSystemTables = []string{ RemotesTableName, } -var generatedSystemViewPrefixes = []string{ - DoltBlameViewPrefix, -} - var generatedSystemTablePrefixes = []string{ DoltDiffTablePrefix, DoltCommitDiffTablePrefix, From 9f681f5eab1643db15ba80dbd517b7d1f78be212 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Wed, 7 Aug 2024 08:18:38 -0700 Subject: [PATCH 05/19] rename function --- go/libraries/doltcore/sqle/dtables/diff_iter.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/diff_iter.go b/go/libraries/doltcore/sqle/dtables/diff_iter.go index af39e8119a..464c781375 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_iter.go +++ b/go/libraries/doltcore/sqle/dtables/diff_iter.go @@ -369,7 +369,7 @@ func (itr prollyDiffIter) queueRows(ctx context.Context) { // todo(andy): copy string fields func (itr prollyDiffIter) makeDiffRowItr(ctx context.Context, d tree.Diff) (*repeatingRowIter, error) { if !itr.keyless { - r, err := itr.getDiffRow(ctx, d) + r, err := itr.getDiffTableRow(ctx, d) if err != nil { return nil, err } @@ -401,7 +401,7 @@ func (itr prollyDiffIter) getDiffRowAndCardinality(ctx context.Context, d tree.D } } - r, err = itr.getDiffRow(ctx, d) + r, err = itr.getDiffTableRow(ctx, d) if err != nil { return nil, 0, err } @@ -409,7 +409,9 @@ func (itr prollyDiffIter) getDiffRowAndCardinality(ctx context.Context, d tree.D return r, n, nil } -func (itr prollyDiffIter) getDiffRow(ctx context.Context, dif tree.Diff) (row sql.Row, err error) { +// getDiffTableRow returns a row for the diff table given the diff type and the row from the source and target tables. The +// output schema is intended for dolt_diff_* tables and dolt_diff function. +func (itr prollyDiffIter) getDiffTableRow(ctx context.Context, dif tree.Diff) (row sql.Row, err error) { tLen := schemaSize(itr.targetToSch) fLen := schemaSize(itr.targetFromSch) From d413b043c56a5baed9b9f279ba81792ea706dd13 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Wed, 7 Aug 2024 08:28:10 -0700 Subject: [PATCH 06/19] First pass on dolt_workspace_ read operations implemented and tested --- go/libraries/doltcore/doltdb/system_table.go | 3 + go/libraries/doltcore/sqle/database.go | 32 + .../doltcore/sqle/dtables/workspace.go | 594 ++++++++++++++++++ .../sqle/enginetest/dolt_engine_test.go | 5 + .../sqle/enginetest/dolt_engine_tests.go | 10 + .../sqle/enginetest/dolt_queries_workspace.go | 181 ++++++ 6 files changed, 825 insertions(+) create mode 100644 go/libraries/doltcore/sqle/dtables/workspace.go create mode 100644 go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go diff --git a/go/libraries/doltcore/doltdb/system_table.go b/go/libraries/doltcore/doltdb/system_table.go index 13521c345e..909a95d773 100644 --- a/go/libraries/doltcore/doltdb/system_table.go +++ b/go/libraries/doltcore/doltdb/system_table.go @@ -191,6 +191,7 @@ var generatedSystemTablePrefixes = []string{ DoltHistoryTablePrefix, DoltConfTablePrefix, DoltConstViolTablePrefix, + DoltWorkspaceTablePrefix, } const ( @@ -260,6 +261,8 @@ const ( DoltConfTablePrefix = "dolt_conflicts_" // DoltConstViolTablePrefix is the prefix assigned to all the generated constraint violation tables DoltConstViolTablePrefix = "dolt_constraint_violations_" + // DoltWorkspaceTablePrefix is the prefix assigned to all the generated workspace tables + DoltWorkspaceTablePrefix = "dolt_workspace_" ) const ( diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 1151f6f2c5..06fd764327 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -397,6 +397,38 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds return nil, false, err } return dt, true, nil + case strings.HasPrefix(lwrName, doltdb.DoltWorkspaceTablePrefix): + sess := dsess.DSessFromSess(ctx.Session) + /* + adapter := dsess.NewSessionStateAdapter( + sess, db.RevisionQualifiedName(), + concurrentmap.New[string, env.Remote](), + concurrentmap.New[string, env.BranchConfig](), + concurrentmap.New[string, env.Remote](), + ) + */ + if head == nil { + var err error + head, err = ds.GetHeadCommit(ctx, db.RevisionQualifiedName()) + + if err != nil { + return nil, false, err + } + } + baseRoot, err := head.GetRootValue(ctx) + + ws, err := sess.WorkingSet(ctx, db.RevisionQualifiedName()) + if err != nil { + return nil, false, err + } + + suffix := tblName[len(doltdb.DoltWorkspaceTablePrefix):] + + dt, err := dtables.NewWorkspaceTable(ctx, suffix, baseRoot, ws) + if err != nil { + return nil, false, err + } + return dt, true, nil } var dt sql.Table diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go new file mode 100644 index 0000000000..979f8d5080 --- /dev/null +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -0,0 +1,594 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dtables + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/dolthub/dolt/go/libraries/doltcore/diff" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" + "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/dolt/go/store/val" + "github.com/dolthub/go-mysql-server/sql" + sqltypes "github.com/dolthub/go-mysql-server/sql/types" +) + +type WorkspaceTable struct { + base doltdb.RootValue + ws *doltdb.WorkingSet + tableName string + nomsSchema schema.Schema + sqlSchema sql.Schema + stagedDeltas *diff.TableDelta + workingDeltas *diff.TableDelta + + headSchema schema.Schema + + ddb *doltdb.DoltDB +} + +var _ sql.Table = (*WorkspaceTable)(nil) + +// NM4 - drop ctx? error +func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, ws *doltdb.WorkingSet) (sql.Table, error) { + stageDlt, err := diff.GetTableDeltas(ctx, root, ws.StagedRoot()) + if err != nil { + return nil, err + } + var stgDel *diff.TableDelta + for _, delta := range stageDlt { + if delta.ToName.Name == tblName { + stgDel = &delta + break + } + } + + workingDlt, err := diff.GetTableDeltas(ctx, root, ws.WorkingRoot()) + if err != nil { + return nil, err + } + + var wkDel *diff.TableDelta + for _, delta := range workingDlt { + if delta.ToName.Name == tblName { + wkDel = &delta + break + } + } + + if wkDel == nil && stgDel == nil { + emptyTable := emptyWorkspaceTable{tableName: tblName} + return &emptyTable, nil + } + + var toSch, fromSch schema.Schema + if stgDel == nil { + toSch, err = wkDel.ToTable.GetSchema(ctx) + if err != nil { + return nil, err + } + fromSch, err = wkDel.FromTable.GetSchema(ctx) + if err != nil { + return nil, err + } + } else { + toSch, err = stgDel.ToTable.GetSchema(ctx) + if err != nil { + return nil, err + } + fromSch, err = stgDel.FromTable.GetSchema(ctx) + if err != nil { + return nil, err + } + } + + totalSch, err := workspaceSchema(fromSch, toSch) + if err != nil { + return nil, err + } + finalSch, err := sqlutil.FromDoltSchema("", "", totalSch) + if err != nil { + return nil, err + } + + return &WorkspaceTable{ + base: root, + ws: ws, + tableName: tblName, + nomsSchema: totalSch, + sqlSchema: finalSch.Schema, + stagedDeltas: stgDel, + workingDeltas: wkDel, + headSchema: fromSch, // NM4 - convince myself this is correct. + ddb: nil, // NM4 - not sure what this is for. Drop? I think we'll need it eventually.... + }, nil +} + +func (wt *WorkspaceTable) Name() string { + return doltdb.DoltWorkspaceTablePrefix + wt.tableName +} + +func (wt *WorkspaceTable) String() string { + return wt.Name() +} + +func (wt *WorkspaceTable) Schema() sql.Schema { + return wt.sqlSchema +} + +// CalculateDiffSchema returns the schema for the dolt_diff table based on the schemas from the from and to tables. +// Either may be nil, in which case the nil argument will use the schema of the non-nil argument +func workspaceSchema(fromSch, toSch schema.Schema) (schema.Schema, error) { + if fromSch == nil && toSch == nil { + return nil, errors.New("Runtime error:non-nil argument required to CalculateDiffSchema") + } else if fromSch == nil { + fromSch = toSch + } else if toSch == nil { + toSch = fromSch + } + + cols := make([]schema.Column, 0, 2+toSch.GetAllCols().Size()+fromSch.GetAllCols().Size()) + + cols = append(cols, + schema.NewColumn("id", 0, types.UintKind, true), + schema.NewColumn("staged", 1, types.BoolKind, false), + schema.NewColumn("diff_type", 2, types.StringKind, false), + ) + + transformer := func(sch schema.Schema, namer func(string) string) error { + return sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { + c, err := schema.NewColumnWithTypeInfo( + namer(col.Name), + uint64(len(cols)), + col.TypeInfo, + false, + col.Default, + false, + col.Comment) + if err != nil { + return true, err + } + cols = append(cols, c) + return false, nil + }) + } + + err := transformer(toSch, diff.ToColNamer) + if err != nil { + return nil, err + } + err = transformer(fromSch, diff.FromColNamer) + if err != nil { + return nil, err + } + + return schema.UnkeyedSchemaFromCols(schema.NewColCollection(cols...)), nil +} + +func (wt *WorkspaceTable) Collation() sql.CollationID { return sql.Collation_Default } + +type WorkspacePartitionItr struct { + partition *WorkspacePartition +} + +func (w *WorkspacePartitionItr) Close(_ *sql.Context) error { + return nil +} + +func (w *WorkspacePartitionItr) Next(_ *sql.Context) (sql.Partition, error) { + if w.partition == nil { + return nil, io.EOF + } + ans := w.partition + w.partition = nil + return ans, nil +} + +type WorkspacePartition struct { + base *doltdb.Table + baseSch schema.Schema + working *doltdb.Table + workingSch schema.Schema + staging *doltdb.Table + stagingSch schema.Schema +} + +var _ sql.Partition = (*WorkspacePartition)(nil) + +func (w *WorkspacePartition) Key() []byte { + // NM4 - is there ever used? We return the table names in the DiffPartition. What is the purpose of this? + return []byte("hello world") +} + +func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { + _, baseTable, baseTableExists, err := resolve.Table(ctx, wt.base, wt.tableName) + if err != nil { + return nil, err + } + if !baseTableExists { + return nil, sql.ErrTableNotFound.New(wt.tableName) // NM4 - not an error. Just no changes. test/fix. + } + var baseSchema schema.Schema = schema.EmptySchema + if baseSchema, err = baseTable.GetSchema(ctx); err != nil { + return nil, err + } + + _, stagingTable, tableExists, err := resolve.Table(ctx, wt.ws.StagedRoot(), wt.tableName) + if err != nil { + return nil, err + } + if !tableExists { + return nil, sql.ErrTableNotFound.New(tableName) // NM4 - not an error. Just no changes. test/fix. + } + var stagingSchema schema.Schema = schema.EmptySchema + if stagingSchema, err = stagingTable.GetSchema(ctx); err != nil { + return nil, err + } + + _, workingTable, tableExists, err := resolve.Table(ctx, wt.ws.WorkingRoot(), wt.tableName) + if err != nil { + return nil, err + } + if !tableExists { + return nil, sql.ErrTableNotFound.New(tableName) // NM4 - not an error. Just no changes. test/fix. + } + var workingSchema schema.Schema = schema.EmptySchema + if workingSchema, err = workingTable.GetSchema(ctx); err != nil { + return nil, err + } + + part := WorkspacePartition{ + base: baseTable, + baseSch: baseSchema, + staging: stagingTable, + stagingSch: stagingSchema, + working: workingTable, + workingSch: workingSchema, + } + + return &WorkspacePartitionItr{&part}, nil +} + +func (wt *WorkspaceTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) { + wp, ok := part.(*WorkspacePartition) + if !ok { + return nil, fmt.Errorf("Runtime Exception: expected a WorkspacePartition, got %T", part) + } + + return newWorkspaceDiffIter(ctx, *wp) // NM4 - base schema. should we use base and staged? +} + +type workspaceDiffIter struct { + base prolly.Map + working prolly.Map + staging prolly.Map + + baseConverter ProllyRowConverter + workingConverter ProllyRowConverter + stagingConverter ProllyRowConverter + + tgtBaseSch schema.Schema + tgtWorkingSch schema.Schema + tgtStagingSch schema.Schema + + keyless bool + + rows chan sql.Row + errChan chan error + cancel func() +} + +func (itr workspaceDiffIter) Next(ctx *sql.Context) (sql.Row, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case err := <-itr.errChan: + return nil, err + case row, ok := <-itr.rows: + if !ok { + return nil, io.EOF + } + return row, nil + } +} + +func (itr workspaceDiffIter) Close(c *sql.Context) error { + itr.cancel() + return nil +} + +func getWorkspaceRowAndCardinality( + ctx context.Context, + idx int, + staging bool, + toSch schema.Schema, + fromSch schema.Schema, + toConverter ProllyRowConverter, + fromConverter ProllyRowConverter, + d tree.Diff, +) (r sql.Row, n uint64, err error) { + switch d.Type { + case tree.AddedDiff: + n = val.ReadKeylessCardinality(val.Tuple(d.To)) + case tree.RemovedDiff: + n = val.ReadKeylessCardinality(val.Tuple(d.From)) + case tree.ModifiedDiff: + fN := val.ReadKeylessCardinality(val.Tuple(d.From)) + tN := val.ReadKeylessCardinality(val.Tuple(d.To)) + if fN < tN { + n = tN - fN + d.Type = tree.AddedDiff + } else { + n = fN - tN + d.Type = tree.RemovedDiff + } + } + + r, err = getWorkspaceTableRow(ctx, idx, staging, toSch, fromSch, toConverter, fromConverter, d) + if err != nil { + return nil, 0, err + } + + return r, n, nil +} + +// getWorkspaceTableRow returns a row for the diff table given the diff type and the row from the source and target tables. The +// output schema is intended for dolt_workspace_* tables. +func getWorkspaceTableRow( + ctx context.Context, + rowId int, + staged bool, + toSch schema.Schema, + fromSch schema.Schema, + toConverter ProllyRowConverter, + fromConverter ProllyRowConverter, + dif tree.Diff, +) (row sql.Row, err error) { + tLen := schemaSize(toSch) + fLen := schemaSize(fromSch) + + if fLen == 0 && dif.Type == tree.AddedDiff { + fLen = tLen + } else if tLen == 0 && dif.Type == tree.RemovedDiff { + tLen = fLen + } + + row = make(sql.Row, 3+tLen+fLen) + + row[0] = rowId + row[1] = staged + row[2] = diffTypeString(dif) + + idx := 3 + + if dif.Type != tree.RemovedDiff { + err = toConverter.PutConverted(ctx, val.Tuple(dif.Key), val.Tuple(dif.To), row[idx:idx+tLen]) + if err != nil { + return nil, err + } + } + idx += tLen + + if dif.Type != tree.AddedDiff { + err = fromConverter.PutConverted(ctx, val.Tuple(dif.Key), val.Tuple(dif.From), row[idx:idx+fLen]) + if err != nil { + return nil, err + } + } + + return row, nil +} + +// NM4 - Virtually identical to prollyDiffIter.queueRows, but for workspaces. Dedupe this. +func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { + + idx := 0 + + err := prolly.DiffMaps(ctx, itr.base, itr.staging, false, func(ctx context.Context, d tree.Diff) error { + dItr, err := itr.makeWorkspaceRowItr(ctx, idx, true, itr.tgtStagingSch, itr.tgtBaseSch, itr.stagingConverter, itr.baseConverter, d) + if err != nil { + return err + } + for { + r, err := dItr.Next(ctx) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case itr.rows <- r: + idx++ + continue + } + } + }) + + if err != nil && err != io.EOF { + select { + case <-ctx.Done(): + case itr.errChan <- err: + } + return + } + + err = prolly.DiffMaps(ctx, itr.staging, itr.working, false, func(ctx context.Context, d tree.Diff) error { + dItr, err := itr.makeWorkspaceRowItr(ctx, idx, false, itr.tgtWorkingSch, itr.tgtStagingSch, itr.workingConverter, itr.stagingConverter, d) + if err != nil { + return err + } + for { + r, err := dItr.Next(ctx) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case itr.rows <- r: + idx++ + continue + } + } + }) + + // we need to drain itr.rows before returning io.EOF + close(itr.rows) +} + +func (itr *workspaceDiffIter) makeWorkspaceRowItr( + ctx context.Context, + idx int, + staging bool, + toSch schema.Schema, + fromSch schema.Schema, + toConverter ProllyRowConverter, + fromConverter ProllyRowConverter, + d tree.Diff, +) (*repeatingRowIter, error) { + if !itr.keyless { + r, err := getWorkspaceTableRow(ctx, idx, staging, toSch, fromSch, toConverter, fromConverter, d) + if err != nil { + return nil, err + } + return &repeatingRowIter{row: r, n: 1}, nil + } + + r, n, err := getWorkspaceRowAndCardinality(ctx, idx, staging, toSch, fromSch, toConverter, fromConverter, d) + if err != nil { + return nil, err + } + return &repeatingRowIter{row: r, n: n}, nil +} + +// NM4 - virtually identical to newProllyDiffIter, but for workspaces. Dedupe this. +func newWorkspaceDiffIter(ctx *sql.Context, wp WorkspacePartition) (workspaceDiffIter, error) { + var base, working, staging prolly.Map + + if wp.base != nil { + idx, err := wp.base.GetRowData(ctx) + if err != nil { + return workspaceDiffIter{}, err + } + base = durable.ProllyMapFromIndex(idx) + } + + if wp.staging != nil { + idx, err := wp.staging.GetRowData(ctx) + if err != nil { + return workspaceDiffIter{}, err + } + staging = durable.ProllyMapFromIndex(idx) + } + + if wp.working != nil { + idx, err := wp.working.GetRowData(ctx) + if err != nil { + return workspaceDiffIter{}, err + } + working = durable.ProllyMapFromIndex(idx) + } + + nodeStore := wp.base.NodeStore() + + baseConverter, err := NewProllyRowConverter(wp.baseSch, wp.baseSch, ctx.Warn, nodeStore) + if err != nil { + return workspaceDiffIter{}, err + } + + stagingConverter, err := NewProllyRowConverter(wp.stagingSch, wp.stagingSch, ctx.Warn, nodeStore) + if err != nil { + return workspaceDiffIter{}, err + } + + workingConverter, err := NewProllyRowConverter(wp.workingSch, wp.workingSch, ctx.Warn, nodeStore) + if err != nil { + return workspaceDiffIter{}, err + } + + keyless := schema.IsKeyless(wp.baseSch) && schema.IsKeyless(wp.stagingSch) && schema.IsKeyless(wp.workingSch) + child, cancel := context.WithCancel(ctx) + iter := workspaceDiffIter{ + base: base, + working: working, + staging: staging, + + tgtBaseSch: wp.baseSch, + tgtWorkingSch: wp.workingSch, + tgtStagingSch: wp.stagingSch, + + baseConverter: baseConverter, + workingConverter: workingConverter, + stagingConverter: stagingConverter, + + keyless: keyless, + rows: make(chan sql.Row, 64), + errChan: make(chan error), + cancel: cancel, + } + + go func() { + iter.queueWorkspaceRows(child) + }() + + return iter, nil +} + +type emptyWorkspaceTable struct { + tableName string +} + +var _ sql.Table = (*emptyWorkspaceTable)(nil) + +func (e emptyWorkspaceTable) Name() string { + return "dolt_workspace_" + e.tableName +} + +func (e emptyWorkspaceTable) String() string { + return e.Name() +} + +func (e emptyWorkspaceTable) Schema() sql.Schema { + return []*sql.Column{ + {Name: "id", Type: sqltypes.Int32, Nullable: false}, + {Name: "staged", Type: sqltypes.Boolean, Nullable: false}, + } +} + +func (e emptyWorkspaceTable) Collation() sql.CollationID { return sql.Collation_Default } + +func (e emptyWorkspaceTable) Partitions(c *sql.Context) (sql.PartitionIter, error) { + return index.SinglePartitionIterFromNomsMap(nil), nil +} + +func (e emptyWorkspaceTable) PartitionRows(c *sql.Context, partition sql.Partition) (sql.RowIter, error) { + return sql.RowsToRowIter(), nil +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 64ef85798a..4ee3f350c3 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -2052,3 +2052,8 @@ func TestStatsAutoRefreshConcurrency(t *testing.T) { wg.Wait() } } + +func TestDoltWorkspace(t *testing.T) { + harness := newDoltEnginetestHarness(t) + RunDoltWorkspaceTests(t, harness) +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go index c84dc35b33..56cc97d6ce 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go @@ -1960,3 +1960,13 @@ func RunDoltReflogTestsPrepared(t *testing.T, h DoltEnginetestHarness) { }() } } + +func RunDoltWorkspaceTests(t *testing.T, h DoltEnginetestHarness) { + for _, script := range DoltWorkspaceScriptTests { + func() { + h = h.NewHarness(t) + defer h.Close() + enginetest.TestScript(t, h, script) + }() + } +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go new file mode 100644 index 0000000000..2d173b96a7 --- /dev/null +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -0,0 +1,181 @@ +// Copyright 2024 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enginetest + +import ( + "github.com/dolthub/go-mysql-server/enginetest/queries" + "github.com/dolthub/go-mysql-server/sql" +) + +var DoltWorkspaceScriptTests = []queries.ScriptTest{ + { + Name: "dolt_workspace_* multiple edits of a single row", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + + "update tbl set val=51 where pk=42;", + "call dolt_add('tbl');", + }, + + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "modified", 42, 51, 42, 42}, + }, + }, + { + Query: "update tbl set val= 108 where pk = 42;", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "modified", 42, 51, 42, 42}, + {1, false, "modified", 42, 108, 42, 51}, + }, + }, + { + Query: "call dolt_add('tbl');", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "modified", 42, 108, 42, 42}, + }, + }, + }, + }, + { + Name: "dolt_workspace_* single unstaged row", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + + "update tbl set val=51 where pk=42;", + }, + + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "modified", 42, 51, 42, 42}, + }, + }, + }, + }, + { + Name: "dolt_workspace_* inserted row", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "insert into tbl values (44,44);", + }, + + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "added", 44, 44, nil, nil}, + }, + }, + { + Query: "call dolt_add('tbl');", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "added", 44, 44, nil, nil}, + }, + }, + { + Query: "update tbl set val = 108 where pk = 44;", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "added", 44, 44, nil, nil}, + {1, false, "modified", 44, 108, 44, 44}, + }, + }, + }, + }, + + { + Name: "dolt_workspace_* deleted row", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "delete from tbl where pk = 42;", + }, + + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "removed", nil, nil, 42, 42}, + }, + }, + { + Query: "call dolt_add('tbl');", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "removed", nil, nil, 42, 42}, + }, + }, + }, + }, + + { + Name: "dolt_workspace_* clean workspace", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + }, + + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{}, + }, + { + Query: "select * from dolt_workspace_unknowntable", + Expected: []sql.Row{}, + }, + }, + }, +} From 8fa02e0b3aff56b155b9be4388221cd34321c2a0 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Fri, 9 Aug 2024 09:52:02 -0700 Subject: [PATCH 07/19] Create/Drop Table shows modifications in dolt_workspace_* --- .../doltcore/sqle/dtables/workspace.go | 92 ++++++++++++------- .../sqle/enginetest/dolt_queries_workspace.go | 66 +++++++++++-- 2 files changed, 116 insertions(+), 42 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go index 979f8d5080..edba243f70 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace.go +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -59,7 +59,7 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, } var stgDel *diff.TableDelta for _, delta := range stageDlt { - if delta.ToName.Name == tblName { + if delta.FromName.Name == tblName || delta.ToName.Name == tblName { stgDel = &delta break } @@ -72,7 +72,7 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, var wkDel *diff.TableDelta for _, delta := range workingDlt { - if delta.ToName.Name == tblName { + if delta.FromName.Name == tblName || delta.ToName.Name == tblName { wkDel = &delta break } @@ -85,22 +85,38 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, var toSch, fromSch schema.Schema if stgDel == nil { - toSch, err = wkDel.ToTable.GetSchema(ctx) - if err != nil { - return nil, err + if wkDel.FromTable != nil { + fromSch, err = wkDel.FromTable.GetSchema(ctx) + if err != nil { + return nil, err + } } - fromSch, err = wkDel.FromTable.GetSchema(ctx) - if err != nil { - return nil, err + toSch = fromSch + if wkDel.ToTable != nil { + toSch, err = wkDel.ToTable.GetSchema(ctx) + if err != nil { + return nil, err + } + } + if fromSch == nil { + fromSch = toSch } } else { - toSch, err = stgDel.ToTable.GetSchema(ctx) - if err != nil { - return nil, err + if stgDel.FromTable != nil { + fromSch, err = stgDel.FromTable.GetSchema(ctx) + if err != nil { + return nil, err + } } - fromSch, err = stgDel.FromTable.GetSchema(ctx) - if err != nil { - return nil, err + toSch = fromSch + if stgDel.ToTable != nil { + toSch, err = stgDel.ToTable.GetSchema(ctx) + if err != nil { + return nil, err + } + } + if fromSch == nil { + fromSch = toSch } } @@ -149,12 +165,12 @@ func workspaceSchema(fromSch, toSch schema.Schema) (schema.Schema, error) { toSch = fromSch } - cols := make([]schema.Column, 0, 2+toSch.GetAllCols().Size()+fromSch.GetAllCols().Size()) + cols := make([]schema.Column, 0, 3+toSch.GetAllCols().Size()+fromSch.GetAllCols().Size()) cols = append(cols, schema.NewColumn("id", 0, types.UintKind, true), - schema.NewColumn("staged", 1, types.BoolKind, false), - schema.NewColumn("diff_type", 2, types.StringKind, false), + schema.NewColumn("staged", 0, types.BoolKind, false), + schema.NewColumn("diff_type", 0, types.StringKind, false), ) transformer := func(sch schema.Schema, namer func(string) string) error { @@ -227,36 +243,33 @@ func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error if err != nil { return nil, err } - if !baseTableExists { - return nil, sql.ErrTableNotFound.New(wt.tableName) // NM4 - not an error. Just no changes. test/fix. - } var baseSchema schema.Schema = schema.EmptySchema - if baseSchema, err = baseTable.GetSchema(ctx); err != nil { - return nil, err + if baseTableExists { + if baseSchema, err = baseTable.GetSchema(ctx); err != nil { + return nil, err + } } - _, stagingTable, tableExists, err := resolve.Table(ctx, wt.ws.StagedRoot(), wt.tableName) + _, stagingTable, stagingTableExists, err := resolve.Table(ctx, wt.ws.StagedRoot(), wt.tableName) if err != nil { return nil, err } - if !tableExists { - return nil, sql.ErrTableNotFound.New(tableName) // NM4 - not an error. Just no changes. test/fix. - } var stagingSchema schema.Schema = schema.EmptySchema - if stagingSchema, err = stagingTable.GetSchema(ctx); err != nil { - return nil, err + if stagingTableExists { + if stagingSchema, err = stagingTable.GetSchema(ctx); err != nil { + return nil, err + } } - _, workingTable, tableExists, err := resolve.Table(ctx, wt.ws.WorkingRoot(), wt.tableName) + _, workingTable, workingTableExists, err := resolve.Table(ctx, wt.ws.WorkingRoot(), wt.tableName) if err != nil { return nil, err } - if !tableExists { - return nil, sql.ErrTableNotFound.New(tableName) // NM4 - not an error. Just no changes. test/fix. - } var workingSchema schema.Schema = schema.EmptySchema - if workingSchema, err = workingTable.GetSchema(ctx); err != nil { - return nil, err + if workingTableExists { + if workingSchema, err = workingTable.GetSchema(ctx); err != nil { + return nil, err + } } part := WorkspacePartition{ @@ -517,7 +530,16 @@ func newWorkspaceDiffIter(ctx *sql.Context, wp WorkspacePartition) (workspaceDif working = durable.ProllyMapFromIndex(idx) } - nodeStore := wp.base.NodeStore() + var nodeStore tree.NodeStore + if wp.base != nil { + nodeStore = wp.base.NodeStore() + } else if wp.staging != nil { + nodeStore = wp.staging.NodeStore() + } else if wp.working != nil { + nodeStore = wp.working.NodeStore() + } else { + return workspaceDiffIter{}, errors.New("no base, staging, or working table") + } baseConverter, err := NewProllyRowConverter(wp.baseSch, wp.baseSch, ctx.Warn, nodeStore) if err != nil { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index 2d173b96a7..4792b7e8a3 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -33,7 +33,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "update tbl set val=51 where pk=42;", "call dolt_add('tbl');", }, - Assertions: []queries.ScriptTestAssertion{ { Query: "select * from dolt_workspace_tbl", @@ -74,7 +73,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "update tbl set val=51 where pk=42;", }, - Assertions: []queries.ScriptTestAssertion{ { Query: "select * from dolt_workspace_tbl", @@ -95,7 +93,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", "insert into tbl values (44,44);", }, - Assertions: []queries.ScriptTestAssertion{ { Query: "select * from dolt_workspace_tbl", @@ -124,7 +121,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ }, }, }, - { Name: "dolt_workspace_* deleted row", SetUpScript: []string{ @@ -136,7 +132,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", "delete from tbl where pk = 42;", }, - Assertions: []queries.ScriptTestAssertion{ { Query: "select * from dolt_workspace_tbl", @@ -155,7 +150,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ }, }, }, - { Name: "dolt_workspace_* clean workspace", SetUpScript: []string{ @@ -166,7 +160,6 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "insert into tbl values (43,43);", "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", }, - Assertions: []queries.ScriptTestAssertion{ { Query: "select * from dolt_workspace_tbl", @@ -178,4 +171,63 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ }, }, }, + + { + Name: "dolt_workspace_* created table", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "added", 42, 42, nil, nil}, + {1, false, "added", 43, 43, nil, nil}, + }, + }, + { + Query: "call dolt_add('tbl');", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "added", 42, 42, nil, nil}, + {1, true, "added", 43, 43, nil, nil}, + }, + }, + }, + }, + { + Name: "dolt_workspace_* dropped table", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "drop table tbl", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "removed", nil, nil, 42, 42}, + {1, false, "removed", nil, nil, 43, 43}, + }, + }, + { + Query: "call dolt_add('tbl');", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "removed", nil, nil, 42, 42}, + {1, true, "removed", nil, nil, 43, 43}, + }, + }, + }, + }, } From 33eab1af881613e459067ffa97e6f5682c296800 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Fri, 9 Aug 2024 12:42:40 -0700 Subject: [PATCH 08/19] Keyless tables populate dolt_workspace_ tables correctly now --- .../doltcore/sqle/dtables/workspace.go | 108 +++++++----------- .../sqle/enginetest/dolt_queries_workspace.go | 41 +++++++ 2 files changed, 81 insertions(+), 68 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go index edba243f70..75e258136b 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace.go +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -306,8 +306,6 @@ type workspaceDiffIter struct { tgtWorkingSch schema.Schema tgtStagingSch schema.Schema - keyless bool - rows chan sql.Row errChan chan error cancel func() @@ -332,41 +330,6 @@ func (itr workspaceDiffIter) Close(c *sql.Context) error { return nil } -func getWorkspaceRowAndCardinality( - ctx context.Context, - idx int, - staging bool, - toSch schema.Schema, - fromSch schema.Schema, - toConverter ProllyRowConverter, - fromConverter ProllyRowConverter, - d tree.Diff, -) (r sql.Row, n uint64, err error) { - switch d.Type { - case tree.AddedDiff: - n = val.ReadKeylessCardinality(val.Tuple(d.To)) - case tree.RemovedDiff: - n = val.ReadKeylessCardinality(val.Tuple(d.From)) - case tree.ModifiedDiff: - fN := val.ReadKeylessCardinality(val.Tuple(d.From)) - tN := val.ReadKeylessCardinality(val.Tuple(d.To)) - if fN < tN { - n = tN - fN - d.Type = tree.AddedDiff - } else { - n = fN - tN - d.Type = tree.RemovedDiff - } - } - - r, err = getWorkspaceTableRow(ctx, idx, staging, toSch, fromSch, toConverter, fromConverter, d) - if err != nil { - return nil, 0, err - } - - return r, n, nil -} - // getWorkspaceTableRow returns a row for the diff table given the diff type and the row from the source and target tables. The // output schema is intended for dolt_workspace_* tables. func getWorkspaceTableRow( @@ -416,22 +379,20 @@ func getWorkspaceTableRow( // NM4 - Virtually identical to prollyDiffIter.queueRows, but for workspaces. Dedupe this. func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { + k1 := schema.EmptySchema == itr.tgtStagingSch || schema.IsKeyless(itr.tgtStagingSch) + k2 := schema.EmptySchema == itr.tgtBaseSch || schema.IsKeyless(itr.tgtBaseSch) + k3 := schema.EmptySchema == itr.tgtWorkingSch || schema.IsKeyless(itr.tgtWorkingSch) + + keyless := k1 && k2 && k3 idx := 0 err := prolly.DiffMaps(ctx, itr.base, itr.staging, false, func(ctx context.Context, d tree.Diff) error { - dItr, err := itr.makeWorkspaceRowItr(ctx, idx, true, itr.tgtStagingSch, itr.tgtBaseSch, itr.stagingConverter, itr.baseConverter, d) + rows, err := itr.makeWorkspaceRows(ctx, idx, true, itr.tgtStagingSch, itr.tgtBaseSch, keyless, itr.stagingConverter, itr.baseConverter, d) if err != nil { return err } - for { - r, err := dItr.Next(ctx) - if err == io.EOF { - return nil - } - if err != nil { - return err - } + for _, r := range rows { select { case <-ctx.Done(): return ctx.Err() @@ -440,6 +401,7 @@ func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { continue } } + return nil }) if err != nil && err != io.EOF { @@ -451,18 +413,11 @@ func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { } err = prolly.DiffMaps(ctx, itr.staging, itr.working, false, func(ctx context.Context, d tree.Diff) error { - dItr, err := itr.makeWorkspaceRowItr(ctx, idx, false, itr.tgtWorkingSch, itr.tgtStagingSch, itr.workingConverter, itr.stagingConverter, d) + rows, err := itr.makeWorkspaceRows(ctx, idx, false, itr.tgtWorkingSch, itr.tgtStagingSch, keyless, itr.workingConverter, itr.stagingConverter, d) if err != nil { return err } - for { - r, err := dItr.Next(ctx) - if err == io.EOF { - return nil - } - if err != nil { - return err - } + for _, r := range rows { select { case <-ctx.Done(): return ctx.Err() @@ -471,35 +426,54 @@ func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { continue } } + return nil }) // we need to drain itr.rows before returning io.EOF close(itr.rows) } -func (itr *workspaceDiffIter) makeWorkspaceRowItr( +func (itr *workspaceDiffIter) makeWorkspaceRows( ctx context.Context, idx int, staging bool, toSch schema.Schema, fromSch schema.Schema, + keyless bool, toConverter ProllyRowConverter, fromConverter ProllyRowConverter, d tree.Diff, -) (*repeatingRowIter, error) { - if !itr.keyless { +) ([]sql.Row, error) { + n := uint64(1) + if keyless { + switch d.Type { + case tree.AddedDiff: + n = val.ReadKeylessCardinality(val.Tuple(d.To)) + case tree.RemovedDiff: + n = val.ReadKeylessCardinality(val.Tuple(d.From)) + case tree.ModifiedDiff: + fN := val.ReadKeylessCardinality(val.Tuple(d.From)) + tN := val.ReadKeylessCardinality(val.Tuple(d.To)) + if fN < tN { + n = tN - fN + d.Type = tree.AddedDiff + } else { + n = fN - tN + d.Type = tree.RemovedDiff + } + } + } + + ans := make([]sql.Row, n) + for i := uint64(0); i < n; i++ { r, err := getWorkspaceTableRow(ctx, idx, staging, toSch, fromSch, toConverter, fromConverter, d) if err != nil { return nil, err } - return &repeatingRowIter{row: r, n: 1}, nil - } - - r, n, err := getWorkspaceRowAndCardinality(ctx, idx, staging, toSch, fromSch, toConverter, fromConverter, d) - if err != nil { - return nil, err + ans[i] = r + idx++ } - return &repeatingRowIter{row: r, n: n}, nil + return ans, nil } // NM4 - virtually identical to newProllyDiffIter, but for workspaces. Dedupe this. @@ -556,7 +530,6 @@ func newWorkspaceDiffIter(ctx *sql.Context, wp WorkspacePartition) (workspaceDif return workspaceDiffIter{}, err } - keyless := schema.IsKeyless(wp.baseSch) && schema.IsKeyless(wp.stagingSch) && schema.IsKeyless(wp.workingSch) child, cancel := context.WithCancel(ctx) iter := workspaceDiffIter{ base: base, @@ -571,7 +544,6 @@ func newWorkspaceDiffIter(ctx *sql.Context, wp WorkspacePartition) (workspaceDif workingConverter: workingConverter, stagingConverter: stagingConverter, - keyless: keyless, rows: make(chan sql.Row, 64), errChan: make(chan error), cancel: cancel, @@ -591,7 +563,7 @@ type emptyWorkspaceTable struct { var _ sql.Table = (*emptyWorkspaceTable)(nil) func (e emptyWorkspaceTable) Name() string { - return "dolt_workspace_" + e.tableName + return doltdb.DoltWorkspaceTablePrefix + e.tableName } func (e emptyWorkspaceTable) String() string { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index 4792b7e8a3..c6fbb752be 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -20,6 +20,7 @@ import ( ) var DoltWorkspaceScriptTests = []queries.ScriptTest{ + { Name: "dolt_workspace_* multiple edits of a single row", SetUpScript: []string{ @@ -230,4 +231,44 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ }, }, }, + + { + Name: "dolt_workspace_* keyless table", + SetUpScript: []string{ + "create table tbl (x int, y int);", + "insert into tbl values (42,42);", + "insert into tbl values (42,42);", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "added", 42, 42, nil, nil}, + {1, false, "added", 42, 42, nil, nil}, + }, + }, + + { + Query: "call dolt_add('tbl');", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "added", 42, 42, nil, nil}, + {1, true, "added", 42, 42, nil, nil}, + }, + }, + { + Query: "insert into tbl values (42,42);", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "added", 42, 42, nil, nil}, + {1, true, "added", 42, 42, nil, nil}, + {2, false, "added", 42, 42, nil, nil}, + }, + }, + }, + }, } From 4e366e62144cc3f28d13a35f3cf1e1d391b50afc Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 12 Aug 2024 11:54:44 -0700 Subject: [PATCH 09/19] use a doltdb.Roots object rather than a workspace. --- go/libraries/doltcore/sqle/database.go | 27 +++---------------- .../doltcore/sqle/dtables/workspace.go | 18 ++++++------- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 06fd764327..25af5c0ad0 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -399,32 +399,11 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds return dt, true, nil case strings.HasPrefix(lwrName, doltdb.DoltWorkspaceTablePrefix): sess := dsess.DSessFromSess(ctx.Session) - /* - adapter := dsess.NewSessionStateAdapter( - sess, db.RevisionQualifiedName(), - concurrentmap.New[string, env.Remote](), - concurrentmap.New[string, env.BranchConfig](), - concurrentmap.New[string, env.Remote](), - ) - */ - if head == nil { - var err error - head, err = ds.GetHeadCommit(ctx, db.RevisionQualifiedName()) - - if err != nil { - return nil, false, err - } - } - baseRoot, err := head.GetRootValue(ctx) - - ws, err := sess.WorkingSet(ctx, db.RevisionQualifiedName()) - if err != nil { - return nil, false, err - } + roots, _ := sess.GetRoots(ctx, db.RevisionQualifiedName()) - suffix := tblName[len(doltdb.DoltWorkspaceTablePrefix):] + userTable := tblName[len(doltdb.DoltWorkspaceTablePrefix):] - dt, err := dtables.NewWorkspaceTable(ctx, suffix, baseRoot, ws) + dt, err := dtables.NewWorkspaceTable(ctx, userTable, roots) if err != nil { return nil, false, err } diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go index 75e258136b..99720fb68a 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace.go +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -36,8 +36,7 @@ import ( ) type WorkspaceTable struct { - base doltdb.RootValue - ws *doltdb.WorkingSet + roots doltdb.Roots tableName string nomsSchema schema.Schema sqlSchema sql.Schema @@ -52,8 +51,8 @@ type WorkspaceTable struct { var _ sql.Table = (*WorkspaceTable)(nil) // NM4 - drop ctx? error -func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, ws *doltdb.WorkingSet) (sql.Table, error) { - stageDlt, err := diff.GetTableDeltas(ctx, root, ws.StagedRoot()) +func NewWorkspaceTable(ctx *sql.Context, tblName string, roots doltdb.Roots) (sql.Table, error) { + stageDlt, err := diff.GetTableDeltas(ctx, roots.Head, roots.Staged) if err != nil { return nil, err } @@ -65,7 +64,7 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, } } - workingDlt, err := diff.GetTableDeltas(ctx, root, ws.WorkingRoot()) + workingDlt, err := diff.GetTableDeltas(ctx, roots.Head, roots.Working) if err != nil { return nil, err } @@ -130,8 +129,7 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, root doltdb.RootValue, } return &WorkspaceTable{ - base: root, - ws: ws, + roots: roots, tableName: tblName, nomsSchema: totalSch, sqlSchema: finalSch.Schema, @@ -239,7 +237,7 @@ func (w *WorkspacePartition) Key() []byte { } func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { - _, baseTable, baseTableExists, err := resolve.Table(ctx, wt.base, wt.tableName) + _, baseTable, baseTableExists, err := resolve.Table(ctx, wt.roots.Head, wt.tableName) if err != nil { return nil, err } @@ -250,7 +248,7 @@ func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error } } - _, stagingTable, stagingTableExists, err := resolve.Table(ctx, wt.ws.StagedRoot(), wt.tableName) + _, stagingTable, stagingTableExists, err := resolve.Table(ctx, wt.roots.Staged, wt.tableName) if err != nil { return nil, err } @@ -261,7 +259,7 @@ func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error } } - _, workingTable, workingTableExists, err := resolve.Table(ctx, wt.ws.WorkingRoot(), wt.tableName) + _, workingTable, workingTableExists, err := resolve.Table(ctx, wt.roots.Working, wt.tableName) if err != nil { return nil, err } From 734ab11f35a22657c94aba5194d11b6dc5d02586 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 13 Aug 2024 10:11:23 -0700 Subject: [PATCH 10/19] cleanup workspace.go --- .../doltcore/sqle/dtables/workspace.go | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go index 99720fb68a..5352e955b2 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace.go +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -44,13 +44,10 @@ type WorkspaceTable struct { workingDeltas *diff.TableDelta headSchema schema.Schema - - ddb *doltdb.DoltDB } var _ sql.Table = (*WorkspaceTable)(nil) -// NM4 - drop ctx? error func NewWorkspaceTable(ctx *sql.Context, tblName string, roots doltdb.Roots) (sql.Table, error) { stageDlt, err := diff.GetTableDeltas(ctx, roots.Head, roots.Staged) if err != nil { @@ -135,8 +132,7 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, roots doltdb.Roots) (sq sqlSchema: finalSch.Schema, stagedDeltas: stgDel, workingDeltas: wkDel, - headSchema: fromSch, // NM4 - convince myself this is correct. - ddb: nil, // NM4 - not sure what this is for. Drop? I think we'll need it eventually.... + headSchema: fromSch, }, nil } @@ -221,6 +217,7 @@ func (w *WorkspacePartitionItr) Next(_ *sql.Context) (sql.Partition, error) { } type WorkspacePartition struct { + name string base *doltdb.Table baseSch schema.Schema working *doltdb.Table @@ -232,8 +229,7 @@ type WorkspacePartition struct { var _ sql.Partition = (*WorkspacePartition)(nil) func (w *WorkspacePartition) Key() []byte { - // NM4 - is there ever used? We return the table names in the DiffPartition. What is the purpose of this? - return []byte("hello world") + return []byte(w.name) } func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { @@ -271,6 +267,7 @@ func (wt *WorkspaceTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error } part := WorkspacePartition{ + name: wt.Name(), base: baseTable, baseSch: baseSchema, staging: stagingTable, @@ -288,9 +285,10 @@ func (wt *WorkspaceTable) PartitionRows(ctx *sql.Context, part sql.Partition) (s return nil, fmt.Errorf("Runtime Exception: expected a WorkspacePartition, got %T", part) } - return newWorkspaceDiffIter(ctx, *wp) // NM4 - base schema. should we use base and staged? + return newWorkspaceDiffIter(ctx, *wp) } +// workspaceDiffIter enables the iteration over the diff information between the HEAD, STAGING, and WORKING roots. type workspaceDiffIter struct { base prolly.Map working prolly.Map @@ -375,7 +373,9 @@ func getWorkspaceTableRow( return row, nil } -// NM4 - Virtually identical to prollyDiffIter.queueRows, but for workspaces. Dedupe this. +// queueWorkspaceRows is similar to prollyDiffIter.queueRows, but for workspaces. It performs two seperate calls +// to prolly.DiffMaps, one for staging and one for working. The end result is queueing the rows from both maps +// into the "rows" channel of the workspaceDiffIter. func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { k1 := schema.EmptySchema == itr.tgtStagingSch || schema.IsKeyless(itr.tgtStagingSch) k2 := schema.EmptySchema == itr.tgtBaseSch || schema.IsKeyless(itr.tgtBaseSch) @@ -431,6 +431,10 @@ func (itr *workspaceDiffIter) queueWorkspaceRows(ctx context.Context) { close(itr.rows) } +// makeWorkspaceRows takes the diff information from the prolly.DiffMaps and converts it into a slice of rows. In the case +// of tables with a primary key, this method will return a single row. For tables without a primary key, it will return +// 1 or more rows. The rows returned are in the full schema that the workspace table returns, so the workspace table columns +// (id, staged, diff_type) are included in the returned rows with the populated values. func (itr *workspaceDiffIter) makeWorkspaceRows( ctx context.Context, idx int, @@ -474,7 +478,8 @@ func (itr *workspaceDiffIter) makeWorkspaceRows( return ans, nil } -// NM4 - virtually identical to newProllyDiffIter, but for workspaces. Dedupe this. +// newWorkspaceDiffIter takes a WorkspacePartition and returns a workspaceDiffIter. The workspaceDiffIter is used to iterate +// over the diff information from the prolly.DiffMaps. func newWorkspaceDiffIter(ctx *sql.Context, wp WorkspacePartition) (workspaceDiffIter, error) { var base, working, staging prolly.Map From 5022a137f0e2a6bbd9304551d6eab32ff77ae880 Mon Sep 17 00:00:00 2001 From: macneale4 Date: Mon, 12 Aug 2024 19:09:58 +0000 Subject: [PATCH 11/19] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/dtables/workspace.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go index 5352e955b2..2630efbf4d 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace.go +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -20,6 +20,9 @@ import ( "fmt" "io" + "github.com/dolthub/go-mysql-server/sql" + sqltypes "github.com/dolthub/go-mysql-server/sql/types" + "github.com/dolthub/dolt/go/libraries/doltcore/diff" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" @@ -31,8 +34,6 @@ import ( "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/go-mysql-server/sql" - sqltypes "github.com/dolthub/go-mysql-server/sql/types" ) type WorkspaceTable struct { From a598725b7dd66b71ec6dc199bc8f2b26e2b80a91 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 13 Aug 2024 09:06:24 -0700 Subject: [PATCH 12/19] Fix ls --system test to inclued dolt_workspace_ --- integration-tests/bats/ls.bats | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-tests/bats/ls.bats b/integration-tests/bats/ls.bats index 498bff7cc6..6841f4d374 100755 --- a/integration-tests/bats/ls.bats +++ b/integration-tests/bats/ls.bats @@ -60,7 +60,7 @@ teardown() { @test "ls: --system shows system tables" { run dolt ls --system [ "$status" -eq 0 ] - [ "${#lines[@]}" -eq 20 ] + [ "${#lines[@]}" -eq 22 ] [[ "$output" =~ "System tables:" ]] || false [[ "$output" =~ "dolt_status" ]] || false [[ "$output" =~ "dolt_commits" ]] || false @@ -81,6 +81,8 @@ teardown() { [[ "$output" =~ "dolt_conflicts_table_two" ]] || false [[ "$output" =~ "dolt_diff_table_two" ]] || false [[ "$output" =~ "dolt_commit_diff_table_two" ]] || false + [[ "$output" =~ "dolt_workspace_table_one" ]] || false + [[ "$output" =~ "dolt_workspace_table_two" ]] || false } @test "ls: --all shows tables in working set and system tables" { From 49b994cb2e500042fe0eeb5c5cd6e59a7145b266 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 13 Aug 2024 11:32:41 -0700 Subject: [PATCH 13/19] Golang test for listing all system tables fixed --- go/libraries/doltcore/sqle/enginetest/dolt_queries.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index 5d77899d32..b5407407a7 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -7176,6 +7176,7 @@ var DoltSystemVariables = []queries.ScriptTest{ {"dolt_remote_branches"}, {"dolt_remotes"}, {"dolt_status"}, + {"dolt_workspace_test"}, {"test"}, }, }, From 0a95504cdb114b91e650fd1e23b209a2b7977091 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV <46170177+macneale4@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:03:36 -0700 Subject: [PATCH 14/19] Update go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go Co-authored-by: Jason Fulghum --- go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index c6fbb752be..e5f65a20b5 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -29,7 +29,7 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "insert into tbl values (42,42);", "insert into tbl values (43,43);", - "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "call dolt_commit('-am', 'inserting 2 rows at HEAD');", "update tbl set val=51 where pk=42;", "call dolt_add('tbl');", From 810a80d0717c4604049844af45030b9861103910 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV <46170177+macneale4@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:03:47 -0700 Subject: [PATCH 15/19] Update go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go Co-authored-by: Jason Fulghum --- go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index e5f65a20b5..2d7dea77be 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -70,7 +70,7 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "insert into tbl values (42,42);", "insert into tbl values (43,43);", - "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "call dolt_commit('-am', 'inserting 2 rows at HEAD');", "update tbl set val=51 where pk=42;", }, From ea1c0db5fc1f9c372683fff7a220ec9cc5edeb84 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV <46170177+macneale4@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:03:57 -0700 Subject: [PATCH 16/19] Update go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go Co-authored-by: Jason Fulghum --- go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index 2d7dea77be..654e5218de 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -91,7 +91,7 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "insert into tbl values (42,42);", "insert into tbl values (43,43);", - "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "call dolt_commit('-am', 'inserting 2 rows at HEAD');", "insert into tbl values (44,44);", }, Assertions: []queries.ScriptTestAssertion{ From 456a744584a1c9c85e547837486e3761bbc41f82 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV <46170177+macneale4@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:04:09 -0700 Subject: [PATCH 17/19] Update go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go Co-authored-by: Jason Fulghum --- go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index 654e5218de..657d79a413 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -159,7 +159,7 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "insert into tbl values (42,42);", "insert into tbl values (43,43);", - "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "call dolt_commit('-am', 'inserting 2 rows at HEAD');", }, Assertions: []queries.ScriptTestAssertion{ { From b053e4def5de46f59a14516fda4aed51415b8a3f Mon Sep 17 00:00:00 2001 From: Neil Macneale IV <46170177+macneale4@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:04:22 -0700 Subject: [PATCH 18/19] Update go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go Co-authored-by: Jason Fulghum --- go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index 657d79a413..62c3895d65 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -130,7 +130,7 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ "insert into tbl values (42,42);", "insert into tbl values (43,43);", - "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + "call dolt_commit('-am', 'inserting 2 rows at HEAD');", "delete from tbl where pk = 42;", }, Assertions: []queries.ScriptTestAssertion{ From 48f6bc2cdef87896c6366b963b112accc0668e45 Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Tue, 13 Aug 2024 16:21:00 -0700 Subject: [PATCH 19/19] Test adding a column to working and staging kinda works --- .../doltcore/sqle/dtables/workspace.go | 57 ++++++++--------- .../sqle/enginetest/dolt_queries_workspace.go | 62 +++++++++++++++++++ 2 files changed, 88 insertions(+), 31 deletions(-) diff --git a/go/libraries/doltcore/sqle/dtables/workspace.go b/go/libraries/doltcore/sqle/dtables/workspace.go index 2630efbf4d..30481c71be 100644 --- a/go/libraries/doltcore/sqle/dtables/workspace.go +++ b/go/libraries/doltcore/sqle/dtables/workspace.go @@ -80,42 +80,37 @@ func NewWorkspaceTable(ctx *sql.Context, tblName string, roots doltdb.Roots) (sq return &emptyTable, nil } - var toSch, fromSch schema.Schema - if stgDel == nil { - if wkDel.FromTable != nil { - fromSch, err = wkDel.FromTable.GetSchema(ctx) - if err != nil { - return nil, err - } - } - toSch = fromSch - if wkDel.ToTable != nil { - toSch, err = wkDel.ToTable.GetSchema(ctx) - if err != nil { - return nil, err - } - } - if fromSch == nil { - fromSch = toSch + var fromSch schema.Schema + if stgDel != nil && stgDel.FromTable != nil { + fromSch, err = stgDel.FromTable.GetSchema(ctx) + if err != nil { + return nil, err } - } else { - if stgDel.FromTable != nil { - fromSch, err = stgDel.FromTable.GetSchema(ctx) - if err != nil { - return nil, err - } + } else if wkDel != nil && wkDel.FromTable != nil { + fromSch, err = wkDel.FromTable.GetSchema(ctx) + if err != nil { + return nil, err } - toSch = fromSch - if stgDel.ToTable != nil { - toSch, err = stgDel.ToTable.GetSchema(ctx) - if err != nil { - return nil, err - } + } + + toSch := fromSch + if wkDel != nil && wkDel.ToTable != nil { + toSch, err = wkDel.ToTable.GetSchema(ctx) + if err != nil { + return nil, err } - if fromSch == nil { - fromSch = toSch + } else if stgDel != nil && stgDel.ToTable != nil { + toSch, err = stgDel.ToTable.GetSchema(ctx) + if err != nil { + return nil, err } } + if fromSch == nil && toSch == nil { + return nil, errors.New("Runtime error: from and to schemas are both nil") + } + if fromSch == nil { + fromSch = toSch + } totalSch, err := workspaceSchema(fromSch, toSch) if err != nil { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go index 62c3895d65..4b80059204 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_workspace.go @@ -271,4 +271,66 @@ var DoltWorkspaceScriptTests = []queries.ScriptTest{ }, }, }, + { + Name: "dolt_workspace_* schema change", + SetUpScript: []string{ + "create table tbl (pk int primary key, val int);", + "call dolt_commit('-Am', 'creating table t');", + + "insert into tbl values (42,42);", + "insert into tbl values (43,43);", + "call dolt_commit('-am', 'inserting rows 3 rows at HEAD');", + + "update tbl set val=51 where pk=42;", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "modified", 42, 51, 42, 42}, + }, + }, + + { + Query: "ALTER TABLE tbl ADD COLUMN newcol CHAR(36)", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, false, "modified", 42, 51, nil, 42, 42}, + }, + }, + { + Query: "call dolt_add('tbl')", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "modified", 42, 51, nil, 42, 42}, + }, + }, + /* Three schemas are possible by having a schema change staged then altering the schema again. + Currently, it's unclear if/how dolt_workspace_* can/should present this since it's all about data changes, not schema changes. + { + Query: "ALTER TABLE tbl ADD COLUMN newcol2 float", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "modified", 42, 51, nil, 42, 42}, + }, + }, + { + Query: "update tbl set val=59 where pk=42", + }, + { + Query: "select * from dolt_workspace_tbl", + Expected: []sql.Row{ + {0, true, "modified", 42, 51, nil, 42, 42}, + {1, false, "modified", 42, 59, nil, nil, 42, 42}, // + }, + }, + */ + }, + }, }