[PostGIS] Supporting Geography (#92)

artie-labs · Feb 12, 2024 · 0da275b · 0da275b
1 parent 57ee7d7
commit 0da275b
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 7 deletions.
diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/DataDog/datadog-go v4.8.3+incompatible
-	github.com/artie-labs/transfer v0.0.0-20240208000914-6c8f7e418eda
+	github.com/artie-labs/transfer v1.22.7
 	github.com/aws/aws-sdk-go v1.44.327
 	github.com/aws/aws-sdk-go-v2/config v1.18.19
 	github.com/getsentry/sentry-go v0.26.0
@@ -38,6 +38,7 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/kr/pretty v0.3.1 // indirect

diff --git a/go.sum b/go.sum
@@ -10,8 +10,8 @@ github.com/alecthomas/assert/v2 v2.4.0 h1:/ZiZ0NnriAWPYYO+4eOjgzNELrFQLaHNr92mHS
 github.com/alecthomas/assert/v2 v2.4.0/go.mod h1:fw5suVxB+wfYJ3291t0hRTqtGzFYdSwstnRQdaQx2DM=
 github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8=
 github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/artie-labs/transfer v0.0.0-20240208000914-6c8f7e418eda h1:msNLtQ3rvrcRAUeUABX8a4RY2l+wowd0mPuXEhbEKcs=
-github.com/artie-labs/transfer v0.0.0-20240208000914-6c8f7e418eda/go.mod h1:QCJNat9BLIK/jm+o/oC7Cwu+WKrEWILDEATyO492RsM=
+github.com/artie-labs/transfer v1.22.7 h1:80bDJlDFPNf7B+N9CU/RDk+zpq9dwFg343E0sdzG2iE=
+github.com/artie-labs/transfer v1.22.7/go.mod h1:QCJNat9BLIK/jm+o/oC7Cwu+WKrEWILDEATyO492RsM=
 github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY=
 github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts=
@@ -127,6 +127,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
 github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -275,6 +277,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

diff --git a/lib/postgres/config.go b/lib/postgres/config.go
@@ -53,6 +53,8 @@ func (c *Config) UpdateCols(colName, colKind string, precision, scale *string, u
 			c.Fields.AddField(colName, debezium.HStore, nil)
 		} else if udtName != nil && *udtName == "geometry" {
 			c.Fields.AddField(colName, debezium.Geometry, nil)
+		} else if udtName != nil && *udtName == "geography" {
+			c.Fields.AddField(colName, debezium.Geography, nil)
 		} else {
 			c.Fields.AddField(colName, debezium.UserDefinedText, nil)
 		}

diff --git a/lib/postgres/debezium/data_type.go b/lib/postgres/debezium/data_type.go
@@ -47,6 +47,7 @@ const (
 	// PostGIS
 	Point
 	Geometry
+	Geography
 )
 
 type Result struct {
@@ -56,6 +57,11 @@ type Result struct {
 
 func (d DataType) ToDebeziumType() Result {
 	switch d {
+	case Geography:
+		return Result{
+			DebeziumType: string(debezium.GeographyType),
+			Type:         "struct",
+		}
 	case Geometry:
 		return Result{
 			DebeziumType: string(debezium.GeometryType),

diff --git a/lib/postgres/parse.go b/lib/postgres/parse.go
@@ -47,15 +47,15 @@ func (c *Config) ParseValue(args ParseValueArgs) (ValueWrapper, error) {
 
 	colKind := c.Fields.GetDataType(args.ColName)
 	switch colKind {
-	case debezium.Geometry:
+	case debezium.Geometry, debezium.Geography:
 		valString, isOk := args.Value().(string)
 		if !isOk {
-			return NewValueWrapper(nil), fmt.Errorf("value: %v not of string type for geometry", args.Value())
+			return NewValueWrapper(nil), fmt.Errorf("value: %v not of string type for geometry / geography", args.Value())
 		}
 
 		geometry, err := parse.ToGeography([]byte(valString))
 		if err != nil {
-			return NewValueWrapper(nil), fmt.Errorf("failed to parse geometry: %w", err)
+			return NewValueWrapper(nil), fmt.Errorf("failed to parse geometry / geography: %w", err)
 		}
 
 		return NewValueWrapper(geometry), nil

diff --git a/lib/postgres/parse_test.go b/lib/postgres/parse_test.go
@@ -1,6 +1,7 @@
 package postgres
 
 import (
+	"github.com/artie-labs/transfer/lib/ptr"
 	"testing"
 	"time"
 
@@ -11,6 +12,7 @@ func TestParse(t *testing.T) {
 	type _testCase struct {
 		colName       string
 		colKind       string
+		udtName       *string
 		parseTime     bool
 		value         ValueWrapper
 		expectErr     bool
@@ -76,11 +78,23 @@ func TestParse(t *testing.T) {
 			},
 			expectedValue: `{"foo":"bar"}`,
 		},
+		{
+			colName: "geography",
+			colKind: "user-defined",
+			udtName: ptr.ToString("geography"),
+			value: ValueWrapper{
+				Value: "0101000020E61000000000000000804B4000000000008040C0",
+			},
+			expectedValue: map[string]interface{}{
+				"srid": nil,
+				"wkb":  "AQEAACDmEAAAAAAAAACAS0AAAAAAAIBAwA==",
+			},
+		},
 	}
 
 	for _, tc := range tcs {
 		cfg := NewPostgresConfig()
-		cfg.UpdateCols(tc.colName, tc.colKind, nil, nil, nil)
+		cfg.UpdateCols(tc.colName, tc.colKind, nil, nil, tc.udtName)
 
 		value, err := cfg.ParseValue(ParseValueArgs{
 			ColName:      tc.colName,