merge main

README.md

@@ -36,7 +36,7 @@ to see what we're building next.
 
 Lots of things! Dolt is a generally useful tool with countless
 applications. But if you want some ideas, [here's how people are using
-it so far](https://www.dolthub.com/blog/2022-07-11-dolt-case-studies/).
+it so far](https://dolthub.com/blog/2024-10-15-dolt-use-cases/).
 
 Dolt can be [set up as a replica of your existing MySQL or MariaDB](https://www.dolthub.com/blog/2023-02-17-binlog-replication-preview/)
 database using standard MySQL binlog replication. Every write becomes

go/cmd/dolt/cli/arg_parser_helpers.go

@@ -294,6 +294,7 @@ func CreateLogArgParser(isTableFunction bool) *argparser.ArgParser {
 func CreateGCArgParser() *argparser.ArgParser {
 	ap := argparser.NewArgParserWithMaxArgs("gc", 0)
 	ap.SupportsFlag(ShallowFlag, "s", "perform a fast, but incomplete garbage collection pass")
+	ap.SupportsFlag(FullFlag, "f", "perform a full garbage collection, including the old generation")
 	return ap
 }
 

go/cmd/dolt/cli/flags.go

@@ -37,6 +37,7 @@ const (
 	DryRunFlag           = "dry-run"
 	EmptyParam           = "empty"
 	ForceFlag            = "force"
+	FullFlag             = "full"
 	GraphFlag            = "graph"
 	HardResetParam       = "hard"
 	HostFlag             = "host"

go/cmd/dolt/commands/cvcmds/verify_constraints.go

@@ -129,16 +129,16 @@ func (cmd VerifyConstraintsCmd) Exec(ctx context.Context, commandStr string, arg
 		}
 
 		for _, tableName := range tablesWithViolations.AsSortedSlice() {
-			tbl, ok, err := endRoot.GetTable(ctx, doltdb.TableName{Name: tableName})
+			tbl, ok, err := endRoot.GetTable(ctx, tableName)
 			if err != nil {
 				return commands.HandleVErrAndExitCode(errhand.BuildDError("Error loading table.").AddCause(err).Build(), nil)
 			}
 			if !ok {
 				return commands.HandleVErrAndExitCode(errhand.BuildDError("Unable to load table '%s'.", tableName).Build(), nil)
 			}
 			cli.Println("")
-			cli.Println(doltdb.DoltConstViolTablePrefix + tableName)
-			dErr := printViolationsForTable(ctx, dbName, tableName, tbl, eng)
+			cli.Println(doltdb.DoltConstViolTablePrefix + tableName.Name)
+			dErr := printViolationsForTable(ctx, dbName, tableName.Name, tbl, eng)
 			if dErr != nil {
 				return commands.HandleVErrAndExitCode(dErr, nil)
 			}

go/cmd/dolt/commands/gc.go

@@ -33,9 +33,17 @@ var gcDocs = cli.CommandDocumentationContent{
 	ShortDesc: "Cleans up unreferenced data from the repository.",
 	LongDesc: `Searches the repository for data that is no longer referenced and no longer needed.
 
-If the {{.EmphasisLeft}}--shallow{{.EmphasisRight}} flag is supplied, a faster but less thorough garbage collection will be performed.`,
+Dolt GC is generational. When a GC is run, everything reachable from any commit on any branch
+is put into the old generation. Data which is only reachable from uncommited branch HEADs is kept in
+the new generation. By default, Dolt GC will only visit data in the new generation, and so will never
+collect data from deleted branches which has previously made its way to the old generation from being
+copied during a prior garbage collection.
+
+If the {{.EmphasisLeft}}--shallow{{.EmphasisRight}} flag is supplied, a faster but less thorough garbage collection will be performed.
+
+If the {{.EmphasisLeft}}--full{{.EmphasisRight}} flag is supplied, a more thorough garbage collection, fully collecting the old gen and new gen, will be performed.`,
 	Synopsis: []string{
-		"[--shallow]",
+		"[--shallow|--full]",
 	},
 }
 
@@ -83,6 +91,10 @@ func (cmd GarbageCollectionCmd) Exec(ctx context.Context, commandStr string, arg
 	help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, gcDocs, ap))
 	apr := cli.ParseArgsOrDie(ap, args, help)
 
+	if apr.Contains(cli.ShallowFlag) && apr.Contains(cli.FullFlag) {
+		return HandleVErrAndExitCode(errhand.BuildDError("Invalid Argument: --shallow is not compatible with --full").SetPrintUsage().Build(), usage)
+	}
+
 	queryist, sqlCtx, closeFunc, err := cliCtx.QueryEngine(ctx)
 	if err != nil {
 		return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
@@ -110,6 +122,9 @@ func constructDoltGCQuery(apr *argparser.ArgParseResults) (string, error) {
 	if apr.Contains(cli.ShallowFlag) {
 		query += "'--shallow'"
 	}
+	if apr.Contains(cli.FullFlag) {
+		query += "'--full'"
+	}
 	query += ")"
 	return query, nil
 }

go/cmd/dolt/commands/read_tables.go

@@ -152,7 +152,7 @@ func (cmd ReadTablesCmd) Exec(ctx context.Context, commandStr string, args []str
 	}
 
 	for _, tblName := range tblNames {
-		destRoot, verr = pullTableValue(ctx, dEnv, srcDB, srcRoot, destRoot, downloadLanguage, tblName, commitStr)
+		destRoot, verr = pullTableValue(ctx, dEnv, srcDB, srcRoot, destRoot, downloadLanguage, doltdb.TableName{Name: tblName}, commitStr)
 
 		if verr != nil {
 			return HandleVErrAndExitCode(verr, usage)
@@ -168,8 +168,8 @@ func (cmd ReadTablesCmd) Exec(ctx context.Context, commandStr string, args []str
 	return 0
 }
 
-func pullTableValue(ctx context.Context, dEnv *env.DoltEnv, srcDB *doltdb.DoltDB, srcRoot, destRoot doltdb.RootValue, language progLanguage, tblName, commitStr string) (doltdb.RootValue, errhand.VerboseError) {
-	tbl, ok, err := srcRoot.GetTable(ctx, doltdb.TableName{Name: tblName})
+func pullTableValue(ctx context.Context, dEnv *env.DoltEnv, srcDB *doltdb.DoltDB, srcRoot, destRoot doltdb.RootValue, language progLanguage, tblName doltdb.TableName, commitStr string) (doltdb.RootValue, errhand.VerboseError) {
+	tbl, ok, err := srcRoot.GetTable(ctx, tblName)
 	if !ok {
 		return nil, errhand.BuildDError("No table named '%s' at '%s'", tblName, commitStr).Build()
 	} else if err != nil {

go/cmd/dolt/commands/schcmds/import.go

@@ -330,7 +330,7 @@ func putEmptyTableWithSchema(ctx context.Context, tblName string, root doltdb.Ro
 		return nil, errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
 	}
 
-	empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch)
+	empty, err := durable.NewEmptyIndex(ctx, root.VRW(), root.NodeStore(), sch, false)
 	if err != nil {
 		return nil, errhand.BuildDError("error: failed to get table.").AddCause(err).Build()
 	}

go/cmd/dolt/doltversion/version.go

@@ -16,5 +16,5 @@
 package doltversion
 
 const (
-	Version = "1.43.5"
+	Version = "1.43.7"
 )

go/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dolthub/fslock v0.0.3
 	github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
-	github.com/dolthub/vitess v0.0.0-20241010201417-9d4f54b29ccc
+	github.com/dolthub/vitess v0.0.0-20241016191424-d14e107a654e
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fatih/color v1.13.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -57,7 +57,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.18.2-0.20241014204513-d97de8112db7
+	github.com/dolthub/go-mysql-server v0.18.2-0.20241022002146-c5725b1bf340
 	github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2

go/go.sum

@@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662 h1:aC17hZD6iwzBwwfO5M+3oBT5E5gGRiQPdn+vzpDXqIA=
 github.com/dolthub/go-icu-regex v0.0.0-20240916130659-0118adc6b662/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.18.2-0.20241014204513-d97de8112db7 h1:si3h6qLG+4nv2ucNcFYarwAY2pd6lDOgAZPa8fgosOE=
-github.com/dolthub/go-mysql-server v0.18.2-0.20241014204513-d97de8112db7/go.mod h1:Z8tket+3sYcU3d4yW90Ggld2d+C2DUgnpB8cBP0+GvI=
+github.com/dolthub/go-mysql-server v0.18.2-0.20241022002146-c5725b1bf340 h1:OJkZTo67Ikm6Tz6ml4Mn9qbC6he49X9dZfYqIwaZrNM=
+github.com/dolthub/go-mysql-server v0.18.2-0.20241022002146-c5725b1bf340/go.mod h1:z/GGuH2asedC+lkJA4sx+C3oyRH1HRx8ET6N9AGBVms=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
@@ -197,8 +197,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
 github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
 github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
-github.com/dolthub/vitess v0.0.0-20241010201417-9d4f54b29ccc h1:ZZgTRuxEwd3X67njtK30buHeZScLAd4W0rbRV8CORhE=
-github.com/dolthub/vitess v0.0.0-20241010201417-9d4f54b29ccc/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
+github.com/dolthub/vitess v0.0.0-20241016191424-d14e107a654e h1:Ssd/iV0hAOShAgr0c4pJQNgh2E4my2XHblFIIam0D+4=
+github.com/dolthub/vitess v0.0.0-20241016191424-d14e107a654e/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=

go/libraries/doltcore/diff/table_deltas.go

@@ -216,8 +216,7 @@ func GetTableDeltas(ctx context.Context, fromRoot, toRoot doltdb.RootValue) (del
 func getFkParentSchs(ctx context.Context, root doltdb.RootValue, fks ...doltdb.ForeignKey) (map[doltdb.TableName]schema.Schema, error) {
 	schs := make(map[doltdb.TableName]schema.Schema)
 	for _, toFk := range fks {
-		// TODO: schema
-		toRefTable, _, ok, err := doltdb.GetTableInsensitive(ctx, root, doltdb.TableName{Name: toFk.ReferencedTableName})
+		toRefTable, _, ok, err := doltdb.GetTableInsensitive(ctx, root, toFk.ReferencedTableName)
 		if err != nil {
 			return nil, err
 		}
@@ -228,8 +227,7 @@ func getFkParentSchs(ctx context.Context, root doltdb.RootValue, fks ...doltdb.F
 		if err != nil {
 			return nil, err
 		}
-		// TODO: schema name
-		schs[doltdb.TableName{Name: toFk.ReferencedTableName}] = toRefSch
+		schs[toFk.ReferencedTableName] = toRefSch
 	}
 	return schs, nil
 }

go/libraries/doltcore/doltdb/commit_hooks_test.go

@@ -251,7 +251,7 @@ func TestAsyncPushOnWrite(t *testing.T) {
 			assert.NoError(t, err)
 
 			tSchema := createTestSchema(t)
-			rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema)
+			rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema, false)
 			require.NoError(t, err)
 			tbl, err := CreateTestTable(ddb.vrw, ddb.ns, tSchema, rowData)
 			require.NoError(t, err)

go/libraries/doltcore/doltdb/doltdb.go

@@ -1633,7 +1633,7 @@ func (ddb *DoltDB) Rebase(ctx context.Context) error {
 // until no possibly-stale ChunkStore state is retained in memory, or failing
 // certain in-progress operations which cannot be finalized in a timely manner,
 // etc.
-func (ddb *DoltDB) GC(ctx context.Context, safepointF func() error) error {
+func (ddb *DoltDB) GC(ctx context.Context, mode types.GCMode, safepointF func() error) error {
 	collector, ok := ddb.db.Database.(datas.GarbageCollector)
 	if !ok {
 		return fmt.Errorf("this database does not support garbage collection")
@@ -1677,7 +1677,7 @@ func (ddb *DoltDB) GC(ctx context.Context, safepointF func() error) error {
 		return err
 	}
 
-	return collector.GC(ctx, oldGen, newGen, safepointF)
+	return collector.GC(ctx, mode, oldGen, newGen, safepointF)
 }
 
 func (ddb *DoltDB) ShallowGC(ctx context.Context) error {

go/libraries/doltcore/doltdb/doltdb_test.go

@@ -83,7 +83,7 @@ func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schem
 
 func createTestRowData(t *testing.T, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) durable.Index {
 	if types.Format_Default == types.Format_DOLT {
-		idx, err := durable.NewEmptyIndex(context.Background(), vrw, ns, sch)
+		idx, err := durable.NewEmptyIndex(context.Background(), vrw, ns, sch, false)
 		require.NoError(t, err)
 		return idx
 	}
@@ -303,7 +303,7 @@ func TestLDNoms(t *testing.T) {
 
 		ctx := context.Background()
 		tSchema := createTestSchema(t)
-		rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema)
+		rowData, err := durable.NewEmptyIndex(ctx, ddb.vrw, ddb.ns, tSchema, false)
 		if err != nil {
 			t.Fatal("Failed to create new empty index")
 		}

go/libraries/doltcore/doltdb/durable/index.go

@@ -96,11 +96,12 @@ func RefFromIndex(ctx context.Context, vrw types.ValueReadWriter, idx Index) (ty
 }
 
 // indexFromRef reads the types.Ref from storage and returns the Index it points to.
+// This is only used by noms format and can be removed.
 func indexFromRef(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, r types.Ref) (Index, error) {
-	return indexFromAddr(ctx, vrw, ns, sch, r.TargetHash())
+	return indexFromAddr(ctx, vrw, ns, sch, r.TargetHash(), false)
 }
 
-func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, addr hash.Hash) (Index, error) {
+func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, addr hash.Hash, isKeylessTable bool) (Index, error) {
 	v, err := vrw.ReadValue(ctx, addr)
 	if err != nil {
 		return nil, err
@@ -111,7 +112,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 		return IndexFromNomsMap(v.(types.Map), vrw, ns), nil
 
 	case types.Format_DOLT:
-		pm, err := shim.MapFromValue(v, sch, ns)
+		pm, err := shim.MapFromValue(v, sch, ns, isKeylessTable)
 		if err != nil {
 			return nil, err
 		}
@@ -123,7 +124,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 }
 
 // NewEmptyIndex returns an index with no rows.
-func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (Index, error) {
+func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, isKeylessSecondary bool) (Index, error) {
 	switch vrw.Format() {
 	case types.Format_LD_1:
 		m, err := types.NewMap(ctx, vrw)
@@ -134,6 +135,9 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 
 	case types.Format_DOLT:
 		kd, vd := sch.GetMapDescriptors()
+		if isKeylessSecondary {
+			kd = prolly.AddHashToSchema(kd)
+		}
 		m, err := prolly.NewMapFromTuples(ctx, ns, kd, vd)
 		if err != nil {
 			return nil, err
@@ -370,7 +374,7 @@ func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter,
 		return nil, err
 	}
 	for _, index := range sch.Indexes().AllIndexes() {
-		empty, err := NewEmptyIndex(ctx, vrw, ns, index.Schema())
+		empty, err := NewEmptyIndex(ctx, vrw, ns, index.Schema(), false)
 		if err != nil {
 			return nil, err
 		}
@@ -507,7 +511,7 @@ func (is doltDevIndexSet) GetIndex(ctx context.Context, tableSch schema.Schema,
 	if idxSch == nil {
 		idxSch = idx.Schema()
 	}
-	return indexFromAddr(ctx, is.vrw, is.ns, idxSch, foundAddr)
+	return indexFromAddr(ctx, is.vrw, is.ns, idxSch, foundAddr, schema.IsKeyless(tableSch))
 }
 
 func (is doltDevIndexSet) PutIndex(ctx context.Context, name string, idx Index) (IndexSet, error) {

go/libraries/doltcore/doltdb/durable/table.go

@@ -850,7 +850,7 @@ func (t doltDevTable) GetTableRows(ctx context.Context) (Index, error) {
 	if err != nil {
 		return nil, err
 	}
-	m, err := shim.MapFromValue(types.SerialMessage(rowbytes), sch, t.ns)
+	m, err := shim.MapFromValue(types.SerialMessage(rowbytes), sch, t.ns, false)
 	if err != nil {
 		return nil, err
 	}