From 0da275bcc5a53702dab6232faa1c94926d4974fc Mon Sep 17 00:00:00 2001 From: Robin Tang Date: Mon, 12 Feb 2024 15:49:12 -0700 Subject: [PATCH] [PostGIS] Supporting Geography (#92) --- go.mod | 3 ++- go.sum | 7 +++++-- lib/postgres/config.go | 2 ++ lib/postgres/debezium/data_type.go | 6 ++++++ lib/postgres/parse.go | 6 +++--- lib/postgres/parse_test.go | 16 +++++++++++++++- 6 files changed, 33 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index db9fc51b..d6946fbb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/DataDog/datadog-go v4.8.3+incompatible - github.com/artie-labs/transfer v0.0.0-20240208000914-6c8f7e418eda + github.com/artie-labs/transfer v1.22.7 github.com/aws/aws-sdk-go v1.44.327 github.com/aws/aws-sdk-go-v2/config v1.18.19 github.com/getsentry/sentry-go v0.26.0 @@ -38,6 +38,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jessevdk/go-flags v1.5.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/go.sum b/go.sum index 2080ce25..d41b6871 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,8 @@ github.com/alecthomas/assert/v2 v2.4.0 h1:/ZiZ0NnriAWPYYO+4eOjgzNELrFQLaHNr92mHS github.com/alecthomas/assert/v2 v2.4.0/go.mod h1:fw5suVxB+wfYJ3291t0hRTqtGzFYdSwstnRQdaQx2DM= github.com/alecthomas/repr v0.3.0 h1:NeYzUPfjjlqHY4KtzgKJiWd6sVq2eNUPTi34PiFGjY8= github.com/alecthomas/repr v0.3.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/artie-labs/transfer v0.0.0-20240208000914-6c8f7e418eda h1:msNLtQ3rvrcRAUeUABX8a4RY2l+wowd0mPuXEhbEKcs= -github.com/artie-labs/transfer v0.0.0-20240208000914-6c8f7e418eda/go.mod h1:QCJNat9BLIK/jm+o/oC7Cwu+WKrEWILDEATyO492RsM= +github.com/artie-labs/transfer v1.22.7 h1:80bDJlDFPNf7B+N9CU/RDk+zpq9dwFg343E0sdzG2iE= +github.com/artie-labs/transfer v1.22.7/go.mod h1:QCJNat9BLIK/jm+o/oC7Cwu+WKrEWILDEATyO492RsM= github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= @@ -127,6 +127,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -275,6 +277,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/lib/postgres/config.go b/lib/postgres/config.go index 4fbb6fc5..69c3eeb7 100644 --- a/lib/postgres/config.go +++ b/lib/postgres/config.go @@ -53,6 +53,8 @@ func (c *Config) UpdateCols(colName, colKind string, precision, scale *string, u c.Fields.AddField(colName, debezium.HStore, nil) } else if udtName != nil && *udtName == "geometry" { c.Fields.AddField(colName, debezium.Geometry, nil) + } else if udtName != nil && *udtName == "geography" { + c.Fields.AddField(colName, debezium.Geography, nil) } else { c.Fields.AddField(colName, debezium.UserDefinedText, nil) } diff --git a/lib/postgres/debezium/data_type.go b/lib/postgres/debezium/data_type.go index ec0641bb..5bc75a75 100644 --- a/lib/postgres/debezium/data_type.go +++ b/lib/postgres/debezium/data_type.go @@ -47,6 +47,7 @@ const ( // PostGIS Point Geometry + Geography ) type Result struct { @@ -56,6 +57,11 @@ type Result struct { func (d DataType) ToDebeziumType() Result { switch d { + case Geography: + return Result{ + DebeziumType: string(debezium.GeographyType), + Type: "struct", + } case Geometry: return Result{ DebeziumType: string(debezium.GeometryType), diff --git a/lib/postgres/parse.go b/lib/postgres/parse.go index 3fed0e25..ef914afd 100644 --- a/lib/postgres/parse.go +++ b/lib/postgres/parse.go @@ -47,15 +47,15 @@ func (c *Config) ParseValue(args ParseValueArgs) (ValueWrapper, error) { colKind := c.Fields.GetDataType(args.ColName) switch colKind { - case debezium.Geometry: + case debezium.Geometry, debezium.Geography: valString, isOk := args.Value().(string) if !isOk { - return NewValueWrapper(nil), fmt.Errorf("value: %v not of string type for geometry", args.Value()) + return NewValueWrapper(nil), fmt.Errorf("value: %v not of string type for geometry / geography", args.Value()) } geometry, err := parse.ToGeography([]byte(valString)) if err != nil { - return NewValueWrapper(nil), fmt.Errorf("failed to parse geometry: %w", err) + return NewValueWrapper(nil), fmt.Errorf("failed to parse geometry / geography: %w", err) } return NewValueWrapper(geometry), nil diff --git a/lib/postgres/parse_test.go b/lib/postgres/parse_test.go index 19adfd14..33b4f568 100644 --- a/lib/postgres/parse_test.go +++ b/lib/postgres/parse_test.go @@ -1,6 +1,7 @@ package postgres import ( + "github.com/artie-labs/transfer/lib/ptr" "testing" "time" @@ -11,6 +12,7 @@ func TestParse(t *testing.T) { type _testCase struct { colName string colKind string + udtName *string parseTime bool value ValueWrapper expectErr bool @@ -76,11 +78,23 @@ func TestParse(t *testing.T) { }, expectedValue: `{"foo":"bar"}`, }, + { + colName: "geography", + colKind: "user-defined", + udtName: ptr.ToString("geography"), + value: ValueWrapper{ + Value: "0101000020E61000000000000000804B4000000000008040C0", + }, + expectedValue: map[string]interface{}{ + "srid": nil, + "wkb": "AQEAACDmEAAAAAAAAACAS0AAAAAAAIBAwA==", + }, + }, } for _, tc := range tcs { cfg := NewPostgresConfig() - cfg.UpdateCols(tc.colName, tc.colKind, nil, nil, nil) + cfg.UpdateCols(tc.colName, tc.colKind, nil, nil, tc.udtName) value, err := cfg.ParseValue(ParseValueArgs{ ColName: tc.colName,