Skip to content

Commit

Permalink
Merge pull request #8186 from dolthub/daylon/doltgres-indexes
Browse files Browse the repository at this point in the history 
Add support for Doltgres indexes
  • Loading branch information
@Hydrocharged
Hydrocharged authored Aug 27, 2024
2 parents 3259561 + 24b49d0 commit e4bb9ca
Show file tree
Hide file tree
Showing 11 changed files with 394 additions and 59 deletions.
2 changes: 1 addition & 1 deletion go/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0
github.com/creasty/defaults v1.6.0
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
github.com/dolthub/go-mysql-server v0.18.2-0.20240826213655-024a764d305f
github.com/dolthub/go-mysql-server v0.18.2-0.20240827100900-3bf086dd5c18
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
github.com/dolthub/swiss v0.1.0
github.com/goccy/go-json v0.10.2
Expand Down
4 changes: 2 additions & 2 deletions go/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
github.com/dolthub/go-mysql-server v0.18.2-0.20240826213655-024a764d305f h1:veiMzylffumQ1XX4vYuyk/iXV06o7CgDTY+YrQxtfNY=
github.com/dolthub/go-mysql-server v0.18.2-0.20240826213655-024a764d305f/go.mod h1:nbdOzd0ceWONE80vbfwoRBjut7z3CIj69ZgDF/cKuaA=
github.com/dolthub/go-mysql-server v0.18.2-0.20240827100900-3bf086dd5c18 h1:1lgwZvnecrjoc9v0iqxjdKBvaasAPiQzty40uTKOHsE=
github.com/dolthub/go-mysql-server v0.18.2-0.20240827100900-3bf086dd5c18/go.mod h1:nbdOzd0ceWONE80vbfwoRBjut7z3CIj69ZgDF/cKuaA=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
Expand Down
4 changes: 2 additions & 2 deletions go/go.work.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,8 +320,8 @@ github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dolthub/go-mysql-server v0.18.2-0.20240812011431-f3892cc42bbf h1:F4OT8cjaQzGlLne9vp7/q0i5QFsQE2OUWIaL5thO5qA=
github.com/dolthub/go-mysql-server v0.18.2-0.20240812011431-f3892cc42bbf/go.mod h1:PwuemL+YK+YiWcUFhknixeqNLjJNfCx7KDsHNajx9fM=
github.com/dolthub/vitess v0.0.0-20240807181005-71d735078e24 h1:/zCd98CLZURqK85jQ+qRmEMx/dpXz85F1/Et7gqMGkk=
github.com/dolthub/vitess v0.0.0-20240807181005-71d735078e24/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
github.com/dolthub/go-mysql-server v0.18.2-0.20240827100900-3bf086dd5c18 h1:1lgwZvnecrjoc9v0iqxjdKBvaasAPiQzty40uTKOHsE=
github.com/dolthub/go-mysql-server v0.18.2-0.20240827100900-3bf086dd5c18/go.mod h1:nbdOzd0ceWONE80vbfwoRBjut7z3CIj69ZgDF/cKuaA=
github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
Expand Down
15 changes: 9 additions & 6 deletions go/libraries/doltcore/sqle/dtables/commit_diff_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,11 +151,15 @@ func (dt *CommitDiffTable) Partitions(ctx *sql.Context) (sql.PartitionIter, erro
}

func (dt *CommitDiffTable) LookupPartitions(ctx *sql.Context, i sql.IndexLookup) (sql.PartitionIter, error) {
if len(i.Ranges) != 1 || len(i.Ranges[0]) != 2 {
ranges, ok := i.Ranges.(sql.MySQLRangeCollection)
if !ok {
return nil, fmt.Errorf("commit diff table requires MySQL ranges")
}
if len(ranges) != 1 || len(ranges[0]) != 2 {
return nil, ErrInvalidCommitDiffTableArgs
}
to := i.Ranges[0][0]
from := i.Ranges[0][1]
to := ranges[0][0]
from := ranges[0][1]
switch to.UpperBound.(type) {
case sql.Above, sql.Below:
default:
Expand All @@ -166,16 +170,15 @@ func (dt *CommitDiffTable) LookupPartitions(ctx *sql.Context, i sql.IndexLookup)
default:
return nil, ErrInvalidCommitDiffTableArgs
}
toCommit, _, err := to.Typ.Convert(sql.GetRangeCutKey(to.UpperBound))
toCommit, _, err := to.Typ.Convert(sql.GetMySQLRangeCutKey(to.UpperBound))
if err != nil {
return nil, err
}
var ok bool
dt.toCommit, ok = toCommit.(string)
if !ok {
return nil, fmt.Errorf("to_commit must be string, found %T", toCommit)
}
fromCommit, _, err := from.Typ.Convert(sql.GetRangeCutKey(from.UpperBound))
fromCommit, _, err := from.Typ.Convert(sql.GetMySQLRangeCutKey(from.UpperBound))
if err != nil {
return nil, err
}
Expand Down
54 changes: 31 additions & 23 deletions go/libraries/doltcore/sqle/index/dolt_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,22 @@ type CommitIndex struct {
func (p *CommitIndex) CanSupport(ranges ...sql.Range) bool {
var selects []string
for _, r := range ranges {
if len(r) != 1 {
mysqlRange, ok := r.(sql.MySQLRange)
if !ok {
return false
}
lb, ok := r[0].LowerBound.(sql.Below)
if len(mysqlRange) != 1 {
return false
}
lb, ok := mysqlRange[0].LowerBound.(sql.Below)
if !ok {
return false
}
lk, ok := lb.Key.(string)
if !ok {
return false
}
ub, ok := r[0].UpperBound.(sql.Above)
ub, ok := mysqlRange[0].UpperBound.(sql.Above)
if !ok {
return false
}
Expand Down Expand Up @@ -685,7 +689,7 @@ func (di *doltIndex) getDurableState(ctx *sql.Context, ti DoltTableable) (*durab
return ret, nil
}

func (di *doltIndex) prollyRanges(ctx *sql.Context, ns tree.NodeStore, ranges ...sql.Range) ([]prolly.Range, error) {
func (di *doltIndex) prollyRanges(ctx *sql.Context, ns tree.NodeStore, ranges ...sql.MySQLRange) ([]prolly.Range, error) {
//todo(max): it is important that *doltIndexLookup maintains a reference
// to empty sqlRanges, otherwise the analyzer will dismiss the index and
// chose a less optimal lookup index. This is a GMS concern, so GMS should
Expand All @@ -704,12 +708,12 @@ func (di *doltIndex) prollyRanges(ctx *sql.Context, ns tree.NodeStore, ranges ..
return pranges, nil
}

func (di *doltIndex) nomsRanges(ctx *sql.Context, iranges ...sql.Range) ([]*noms.ReadRange, error) {
func (di *doltIndex) nomsRanges(ctx *sql.Context, iranges ...sql.MySQLRange) ([]*noms.ReadRange, error) {
// This might remain nil if the given nomsRanges each contain an EmptyRange for one of the columns. This will just
// cause the lookup to return no rows, which is the desired behavior.
var readRanges []*noms.ReadRange

ranges := make([]sql.Range, len(iranges))
ranges := make([]sql.MySQLRange, len(iranges))

for i := range iranges {
ranges[i] = DropTrailingAllColumnExprs(iranges[i])
Expand All @@ -729,7 +733,7 @@ RangeLoop:
var lowerKeys []interface{}
for _, rangeColumnExpr := range rang {
if rangeColumnExpr.HasLowerBound() {
lowerKeys = append(lowerKeys, sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
lowerKeys = append(lowerKeys, sql.GetMySQLRangeCutKey(rangeColumnExpr.LowerBound))
} else {
break
}
Expand All @@ -753,7 +757,7 @@ RangeLoop:
// We promote each type as the value has already been validated against the type
promotedType := di.columns[i].TypeInfo.Promote()
if rangeColumnExpr.HasLowerBound() {
key := sql.GetRangeCutKey(rangeColumnExpr.LowerBound)
key := sql.GetMySQLRangeCutKey(rangeColumnExpr.LowerBound)
val, err := promotedType.ConvertValueToNomsValue(ctx, di.vrw, key)
if err != nil {
return nil, err
Expand All @@ -770,7 +774,7 @@ RangeLoop:
cb.boundsCase = boundsCase_infinity_infinity
}
if rangeColumnExpr.HasUpperBound() {
key := sql.GetRangeCutKey(rangeColumnExpr.UpperBound)
key := sql.GetMySQLRangeCutKey(rangeColumnExpr.UpperBound)
val, err := promotedType.ConvertValueToNomsValue(ctx, di.vrw, key)
if err != nil {
return nil, err
Expand Down Expand Up @@ -1082,8 +1086,8 @@ func maybeGetKeyBuilder(idx durable.Index) *val.TupleBuilder {
return nil
}

func pruneEmptyRanges(sqlRanges []sql.Range) (pruned []sql.Range, err error) {
pruned = make([]sql.Range, 0, len(sqlRanges))
func pruneEmptyRanges(sqlRanges []sql.MySQLRange) (pruned []sql.MySQLRange, err error) {
pruned = make([]sql.MySQLRange, 0, len(sqlRanges))
for _, sr := range sqlRanges {
empty := false
for _, colExpr := range sr {
Expand Down Expand Up @@ -1137,10 +1141,10 @@ func (di *doltIndex) valueReadWriter() types.ValueReadWriter {
return di.vrw
}

func (di *doltIndex) prollySpatialRanges(ranges []sql.Range) ([]prolly.Range, error) {
func (di *doltIndex) prollySpatialRanges(ranges []sql.MySQLRange) ([]prolly.Range, error) {
// should be exactly one range
rng := ranges[0][0]
lower, upper := sql.GetRangeCutKey(rng.LowerBound), sql.GetRangeCutKey(rng.UpperBound)
lower, upper := sql.GetMySQLRangeCutKey(rng.LowerBound), sql.GetMySQLRangeCutKey(rng.UpperBound)

minPoint, ok := lower.(sqltypes.Point)
if !ok {
Expand Down Expand Up @@ -1190,7 +1194,7 @@ func (di *doltIndex) prollySpatialRanges(ranges []sql.Range) ([]prolly.Range, er
return pRanges, nil
}

func (di *doltIndex) prollyRangesFromSqlRanges(ctx context.Context, ns tree.NodeStore, ranges []sql.Range, tb *val.TupleBuilder) ([]prolly.Range, error) {
func (di *doltIndex) prollyRangesFromSqlRanges(ctx context.Context, ns tree.NodeStore, ranges []sql.MySQLRange, tb *val.TupleBuilder) ([]prolly.Range, error) {
var err error
if !di.spatial {
ranges, err = pruneEmptyRanges(ranges)
Expand Down Expand Up @@ -1309,7 +1313,7 @@ func (di *doltIndex) prollyRangesFromSqlRanges(ctx context.Context, ns tree.Node
return pranges, nil
}

func rangeCutIsBinding(c sql.RangeCut) bool {
func rangeCutIsBinding(c sql.MySQLRangeCut) bool {
switch c.(type) {
case sql.Below, sql.Above, sql.AboveNull:
return true
Expand All @@ -1320,11 +1324,11 @@ func rangeCutIsBinding(c sql.RangeCut) bool {
}
}

func getRangeCutValue(cut sql.RangeCut, typ sql.Type) (interface{}, error) {
func getRangeCutValue(cut sql.MySQLRangeCut, typ sql.Type) (interface{}, error) {
if _, ok := cut.(sql.AboveNull); ok {
return nil, nil
}
ret, oob, err := typ.Convert(sql.GetRangeCutKey(cut))
ret, oob, err := typ.Convert(sql.GetMySQLRangeCutKey(cut))
if oob == sql.OutOfRange {
return ret, nil
}
Expand All @@ -1335,7 +1339,7 @@ func getRangeCutValue(cut sql.RangeCut, typ sql.Type) (interface{}, error) {
//
// Sometimes when we construct read ranges against laid out index structures,
// we want to ignore these trailing clauses.
func DropTrailingAllColumnExprs(r sql.Range) sql.Range {
func DropTrailingAllColumnExprs(r sql.MySQLRange) sql.MySQLRange {
i := len(r)
for i > 0 {
if r[i-1].Type() != sql.RangeType_All {
Expand All @@ -1352,8 +1356,8 @@ func DropTrailingAllColumnExprs(r sql.Range) sql.Range {
//
// This is for building physical scans against storage which does not store
// NULL contiguous and ordered < non-NULL values.
func SplitNullsFromRange(r sql.Range) ([]sql.Range, error) {
res := []sql.Range{{}}
func SplitNullsFromRange(r sql.MySQLRange) ([]sql.MySQLRange, error) {
res := []sql.MySQLRange{{}}

for _, rce := range r {
if _, ok := rce.LowerBound.(sql.BelowNull); ok {
Expand Down Expand Up @@ -1395,8 +1399,8 @@ func SplitNullsFromRange(r sql.Range) ([]sql.Range, error) {
}

// SplitNullsFromRanges splits nulls from ranges.
func SplitNullsFromRanges(rs []sql.Range) ([]sql.Range, error) {
var ret []sql.Range
func SplitNullsFromRanges(rs []sql.MySQLRange) ([]sql.MySQLRange, error) {
var ret []sql.MySQLRange
for _, r := range rs {
nr, err := SplitNullsFromRange(r)
if err != nil {
Expand All @@ -1412,7 +1416,11 @@ func SplitNullsFromRanges(rs []sql.Range) ([]sql.Range, error) {
// to convert.
func LookupToPointSelectStr(lookup sql.IndexLookup) ([]string, bool) {
var selects []string
for _, r := range lookup.Ranges {
mysqlRanges, ok := lookup.Ranges.(sql.MySQLRangeCollection)
if !ok {
return nil, false
}
for _, r := range mysqlRanges {
if len(r) != 1 {
return nil, false
}
Expand Down
16 changes: 8 additions & 8 deletions go/libraries/doltcore/sqle/index/dolt_index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1058,7 +1058,7 @@ func TestDoltIndexBetween(t *testing.T) {
expectedRows := convertSqlRowToInt64(test.expectedRows)

exprs := idx.Expressions()
sqlIndex := sql.NewIndexBuilder(idx)
sqlIndex := sql.NewMySQLIndexBuilder(idx)
for i := range test.greaterThanOrEqual {
sqlIndex = sqlIndex.GreaterOrEqual(ctx, exprs[i], test.greaterThanOrEqual[i]).LessOrEqual(ctx, exprs[i], test.lessThanOrEqual[i])
}
Expand Down Expand Up @@ -1294,7 +1294,7 @@ func requireUnorderedRowsEqual(t *testing.T, s sql.Schema, rows1, rows2 []sql.Ro
func testDoltIndex(t *testing.T, ctx *sql.Context, root doltdb.RootValue, keys []interface{}, expectedRows []sql.Row, idx index.DoltIndex, cmp indexComp) {
ctx = sql.NewEmptyContext()
exprs := idx.Expressions()
builder := sql.NewIndexBuilder(idx)
builder := sql.NewMySQLIndexBuilder(idx)
for i, key := range keys {
switch cmp {
case indexComp_Eq:
Expand Down Expand Up @@ -1460,15 +1460,15 @@ func convertSqlRowToInt64(sqlRows []sql.Row) []sql.Row {

func TestSplitNullsFromRange(t *testing.T) {
t.Run("EmptyRange", func(t *testing.T) {
r, err := index.SplitNullsFromRange(sql.Range{})
r, err := index.SplitNullsFromRange(sql.MySQLRange{})
assert.NoError(t, err)
assert.NotNil(t, r)
assert.Len(t, r, 1)
assert.Len(t, r[0], 0)
})

t.Run("ThreeColumnNoNullsRange", func(t *testing.T) {
r := sql.Range{sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8), sql.NotNullRangeColumnExpr(types.Int8)}
r := sql.MySQLRange{sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8), sql.NotNullRangeColumnExpr(types.Int8)}
rs, err := index.SplitNullsFromRange(r)
assert.NoError(t, err)
assert.NotNil(t, rs)
Expand All @@ -1478,7 +1478,7 @@ func TestSplitNullsFromRange(t *testing.T) {
})

t.Run("LastColumnOnlyNull", func(t *testing.T) {
r := sql.Range{sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8), sql.NullRangeColumnExpr(types.Int8)}
r := sql.MySQLRange{sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8), sql.NullRangeColumnExpr(types.Int8)}
rs, err := index.SplitNullsFromRange(r)
assert.NoError(t, err)
assert.NotNil(t, rs)
Expand All @@ -1488,7 +1488,7 @@ func TestSplitNullsFromRange(t *testing.T) {
})

t.Run("LastColumnAll", func(t *testing.T) {
r := sql.Range{sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8), sql.AllRangeColumnExpr(types.Int8)}
r := sql.MySQLRange{sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8), sql.AllRangeColumnExpr(types.Int8)}
rs, err := index.SplitNullsFromRange(r)
assert.NoError(t, err)
assert.NotNil(t, rs)
Expand All @@ -1502,7 +1502,7 @@ func TestSplitNullsFromRange(t *testing.T) {
})

t.Run("FirstColumnAll", func(t *testing.T) {
r := sql.Range{sql.AllRangeColumnExpr(types.Int8), sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8)}
r := sql.MySQLRange{sql.AllRangeColumnExpr(types.Int8), sql.LessThanRangeColumnExpr(10, types.Int8), sql.GreaterThanRangeColumnExpr(16, types.Int8)}
rs, err := index.SplitNullsFromRange(r)
assert.NoError(t, err)
assert.NotNil(t, rs)
Expand All @@ -1516,7 +1516,7 @@ func TestSplitNullsFromRange(t *testing.T) {
})

t.Run("AllColumnAll", func(t *testing.T) {
r := sql.Range{sql.AllRangeColumnExpr(types.Int8), sql.AllRangeColumnExpr(types.Int8), sql.AllRangeColumnExpr(types.Int8)}
r := sql.MySQLRange{sql.AllRangeColumnExpr(types.Int8), sql.AllRangeColumnExpr(types.Int8), sql.AllRangeColumnExpr(types.Int8)}
rs, err := index.SplitNullsFromRange(r)
assert.NoError(t, err)
assert.NotNil(t, rs)
Expand Down
Loading

0 comments on commit e4bb9ca

Please sign in to comment.