Added storage capabilities for GMS Extended Types

go/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dolthub/fslock v0.0.3
 	github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
-	github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c
+	github.com/dolthub/vitess v0.0.0-20240207121055-c057d2347007
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fatih/color v1.13.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -57,7 +57,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.17.1-0.20240207001922-0aaa10ab846e
+	github.com/dolthub/go-mysql-server v0.17.1-0.20240207124505-c0f397a6aaca
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2
 	github.com/google/go-github/v57 v57.0.0

go/go.sum

@@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.17.1-0.20240207001922-0aaa10ab846e h1:adkEQion8xhowSQSk9NenvrpGt0U6sG6Yr1OaOA7Xjs=
-github.com/dolthub/go-mysql-server v0.17.1-0.20240207001922-0aaa10ab846e/go.mod h1:Kjfapc/1feJ3/WgTgiY0LoR5x/3ax5PJ26brPnvYUO8=
+github.com/dolthub/go-mysql-server v0.17.1-0.20240207124505-c0f397a6aaca h1:tI3X4fIUTOT0N8n+GYkPNa384WlJoOBcztK5c5mBzjU=
+github.com/dolthub/go-mysql-server v0.17.1-0.20240207124505-c0f397a6aaca/go.mod h1:ANK0a6tyjrZ2cOzDJT3nFsDp80xksI4UfeijFlvnjwE=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
 github.com/dolthub/jsonpath v0.0.2-0.20240201003050-392940944c15 h1:sfTETOpsrNJPDn2KydiCtDgVu6Xopq8k3JP8PjFT22s=
@@ -195,10 +195,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
 github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
 github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
-github.com/dolthub/vitess v0.0.0-20240205203605-9e6c6d650813 h1:tGwsoLAMFQ+7FDEyIWOIJ1Vc/nptbFi0Fh7SQahB8ro=
-github.com/dolthub/vitess v0.0.0-20240205203605-9e6c6d650813/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw=
-github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c h1:Zt23BHsxvPHGfpHV9k/FcsHqWZjfybyQQux2OLpRni8=
-github.com/dolthub/vitess v0.0.0-20240206204925-6acf16fa777c/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw=
+github.com/dolthub/vitess v0.0.0-20240207121055-c057d2347007 h1:MvFoe0FnHhxQLyp4Ldw0HRj1yu83YErbtbr7XxhaIFk=
+github.com/dolthub/vitess v0.0.0-20240207121055-c057d2347007/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=

go/libraries/doltcore/migrate/tuples.go

@@ -175,6 +175,9 @@ func translateNomsField(ctx context.Context, ns tree.NodeStore, value types.Valu
 	case types.BlobKind:
 		return translateBlobField(ctx, ns, value.(types.Blob), idx, b)
 
+	case types.ExtendedKind:
+		return fmt.Errorf("extended types are invalid during migration")
+
 	default:
 		return fmt.Errorf("encountered unexpected NomsKind %s",
 			types.KindToString[nk])

go/libraries/doltcore/schema/encoding/serialization.go

@@ -638,6 +638,15 @@ func sqlTypeString(t typeinfo.TypeInfo) string {
 		return typ.String()
 	}
 
+	// Extended types are string serializable, so we'll just prepend a tag
+	if extendedType, ok := typ.(sqltypes.ExtendedType); ok {
+		serializedType, err := sqltypes.SerializeTypeToString(extendedType)
+		if err != nil {
+			panic(err)
+		}
+		return planbuilder.ExtendedTypeTag + serializedType
+	}
+
 	return typ.String()
 }
 
@@ -650,7 +659,7 @@ func typeinfoFromSqlType(s string) (typeinfo.TypeInfo, error) {
 }
 
 func encodingFromTypeinfo(t typeinfo.TypeInfo) serial.Encoding {
-	return schema.EncodingFromSqlType(t.ToSqlType().Type())
+	return schema.EncodingFromSqlType(t.ToSqlType())
 }
 
 func constraintsFromSerialColumn(col *serial.Column) (cc []schema.ColConstraint) {

go/libraries/doltcore/schema/schema_impl.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/dolthub/go-mysql-server/sql"
+	gmstypes "github.com/dolthub/go-mysql-server/sql/types"
 	"github.com/dolthub/vitess/go/vt/proto/query"
 
 	"github.com/dolthub/dolt/go/gen/fb/serial"
@@ -438,6 +439,7 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
 	}
 
 	var tt []val.Type
+	var handlers []val.TupleTypeHandler
 	useCollations := false // We only use collations if a string exists
 	var collations []sql.CollationID
 	_ = si.GetPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) {
@@ -452,12 +454,12 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
 
 		if convertAddressColumns && !contentHashedField && queryType == query.Type_BLOB {
 			t = val.Type{
-				Enc:      val.Encoding(EncodingFromSqlType(query.Type_VARBINARY)),
+				Enc:      val.Encoding(EncodingFromQueryType(query.Type_VARBINARY)),
 				Nullable: columnMissingNotNullConstraint(col),
 			}
 		} else if convertAddressColumns && !contentHashedField && queryType == query.Type_TEXT {
 			t = val.Type{
-				Enc:      val.Encoding(EncodingFromSqlType(query.Type_VARCHAR)),
+				Enc:      val.Encoding(EncodingFromQueryType(query.Type_VARCHAR)),
 				Nullable: columnMissingNotNullConstraint(col),
 			}
 		} else if convertAddressColumns && !contentHashedField && queryType == query.Type_GEOMETRY {
@@ -467,7 +469,7 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
 			}
 		} else {
 			t = val.Type{
-				Enc:      val.Encoding(EncodingFromSqlType(queryType)),
+				Enc:      val.Encoding(EncodingFromSqlType(sqlType)),
 				Nullable: columnMissingNotNullConstraint(col),
 			}
 		}
@@ -478,6 +480,12 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
 		} else {
 			collations = append(collations, sql.Collation_Unspecified)
 		}
+
+		if extendedType, ok := sqlType.(gmstypes.ExtendedType); ok {
+			handlers = append(handlers, extendedType)
+		} else {
+			handlers = append(handlers, nil)
+		}
 		return
 	})
 
@@ -486,18 +494,20 @@ func (si *schemaImpl) getKeyColumnsDescriptor(convertAddressColumns bool) val.Tu
 			panic(fmt.Errorf("cannot create tuple descriptor from %d collations and %d types", len(collations), len(tt)))
 		}
 		cmp := CollationTupleComparator{Collations: collations}
-		return val.NewTupleDescriptorWithComparator(cmp, tt...)
+		return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Comparator: cmp, Handlers: handlers}, tt...)
 	} else {
-		return val.NewTupleDescriptor(tt...)
+		return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Handlers: handlers}, tt...)
 	}
 }
 
 // GetValueDescriptor implements the Schema interface.
 func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
 	var tt []val.Type
+	var handlers []val.TupleTypeHandler
 	var collations []sql.CollationID
 	if IsKeyless(si) {
 		tt = []val.Type{val.KeylessCardType}
+		handlers = []val.TupleTypeHandler{nil}
 		collations = []sql.CollationID{sql.Collation_Unspecified}
 	}
 
@@ -510,7 +520,7 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
 		sqlType := col.TypeInfo.ToSqlType()
 		queryType := sqlType.Type()
 		tt = append(tt, val.Type{
-			Enc:      val.Encoding(EncodingFromSqlType(queryType)),
+			Enc:      val.Encoding(EncodingFromSqlType(sqlType)),
 			Nullable: col.IsNullable(),
 		})
 		if queryType == query.Type_CHAR || queryType == query.Type_VARCHAR {
@@ -519,6 +529,12 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
 		} else {
 			collations = append(collations, sql.Collation_Unspecified)
 		}
+
+		if extendedType, ok := sqlType.(gmstypes.ExtendedType); ok {
+			handlers = append(handlers, extendedType)
+		} else {
+			handlers = append(handlers, nil)
+		}
 		return
 	})
 
@@ -527,9 +543,9 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
 			panic(fmt.Errorf("cannot create tuple descriptor from %d collations and %d types", len(collations), len(tt)))
 		}
 		cmp := CollationTupleComparator{Collations: collations}
-		return val.NewTupleDescriptorWithComparator(cmp, tt...)
+		return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Comparator: cmp, Handlers: handlers}, tt...)
 	} else {
-		return val.NewTupleDescriptor(tt...)
+		return val.NewTupleDescriptorWithArgs(val.TupleDescriptorArgs{Handlers: handlers}, tt...)
 	}
 }