diff --git a/.gitignore b/.gitignore index 2503b135..e479b4ad 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,7 @@ go.work # Application configs .personal + +# GoReleaser +dist/ + diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 00000000..fac80a06 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,73 @@ +project_name: artie-reader + +# .goreleaser.yaml +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy +builds: + - binary: reader + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + +dockers: + - image_templates: + - "artielabs/reader:latest" + - "artielabs/reader:{{ .Tag }}" + # You can have multiple Docker images. + # GOOS of the built binaries/packages that should be used. + # Default: `linux`. + goos: linux + + # GOARCH of the built binaries/packages that should be used. + # Default: `amd64`. + goarch: amd64 + + # Skips the docker push. + skip_push: false + + # Path to the Dockerfile (from the project root). + # Defaults to `Dockerfile`. + dockerfile: goreleaser.dockerfile + + # Set the "backend" for the Docker pipe. + # Valid options are: docker, buildx, podman. + # Defaults to docker. + use: docker + build_flag_templates: + - "--pull" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--platform=linux/amd64" + + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + +# The lines beneath this are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/Makefile b/Makefile index 3c50bebd..b2867f4a 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,11 @@ test: .PHONY: race race: go test -race ./... + +.PHONY: build +build: + goreleaser build --clean + +.PHONY: release +release: + goreleaser release --clean diff --git a/go.mod b/go.mod index 20c2d010..26f5b2e3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/artie-labs/reader go 1.19 require ( + github.com/artie-labs/transfer v0.0.0-20230816211232-759a2f13025c + github.com/aws/aws-sdk-go v1.44.327 github.com/aws/aws-sdk-go-v2/config v1.18.19 github.com/evalphobia/logrus_sentry v0.8.2 github.com/segmentio/kafka-go v0.4.38 @@ -13,8 +15,6 @@ require ( ) require ( - github.com/artie-labs/transfer v0.0.0-20230816211232-759a2f13025c // indirect - github.com/aws/aws-sdk-go v1.44.327 // indirect github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect diff --git a/go.sum b/go.sum index b67033b3..afcb37ae 100644 --- a/go.sum +++ b/go.sum @@ -52,10 +52,12 @@ github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFN github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -82,12 +84,15 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -95,6 +100,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -116,10 +122,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/goreleaser.dockerfile b/goreleaser.dockerfile new file mode 100644 index 00000000..9b06d770 --- /dev/null +++ b/goreleaser.dockerfile @@ -0,0 +1,2 @@ +FROM --platform=linux/amd64 alpine:3.16 +COPY reader /reader diff --git a/lib/dynamo/message.go b/lib/dynamo/message.go index 32d17f2c..1c7fdeda 100644 --- a/lib/dynamo/message.go +++ b/lib/dynamo/message.go @@ -13,8 +13,6 @@ import ( "time" ) -const maxPublishCount = 5 - type Message struct { rowData map[string]interface{} primaryKey map[string]interface{} @@ -27,6 +25,8 @@ func stringToFloat64(s string) (float64, error) { return strconv.ParseFloat(s, 64) } +// transformAttributeValue converts a DynamoDB AttributeValue to a Go type. +// References: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html func transformAttributeValue(attr *dynamodb.AttributeValue) interface{} { switch { case attr.S != nil: diff --git a/sources/dynamodb/dynamodb_suite_test.go b/sources/dynamodb/dynamodb_suite_test.go new file mode 100644 index 00000000..492c826c --- /dev/null +++ b/sources/dynamodb/dynamodb_suite_test.go @@ -0,0 +1,22 @@ +package dynamodb + +import ( + "context" + "github.com/artie-labs/reader/config" + "github.com/stretchr/testify/suite" + "testing" +) + +type DynamoDBTestSuite struct { + suite.Suite + ctx context.Context +} + +func (d *DynamoDBTestSuite) SetupTest() { + d.ctx = context.Background() + d.ctx = config.InjectIntoContext(d.ctx, &config.Settings{}) +} + +func TestDynamoDBTestSuite(t *testing.T) { + suite.Run(t, new(DynamoDBTestSuite)) +} diff --git a/sources/dynamodb/offsets_test.go b/sources/dynamodb/offsets_test.go new file mode 100644 index 00000000..e254e5b4 --- /dev/null +++ b/sources/dynamodb/offsets_test.go @@ -0,0 +1,30 @@ +package dynamodb + +import ( + "github.com/stretchr/testify/assert" + "os" +) + +func (d *DynamoDBTestSuite) TestOffsets_Complete() { + offsetsFilePath := "/tmp/offsets-test" + err := os.RemoveAll(offsetsFilePath) + assert.NoError(d.T(), err) + + s := Store{ + offsetFilePath: "/tmp/offsets-test", + } + + originalLastProcessedSeqNumbers := map[string]string{ + "shard-1": "123", + "shard-2": "456", + "shard-3": "789", + } + + s.lastProcessedSeqNumbers = originalLastProcessedSeqNumbers + s.saveOffsets(d.ctx) + + s.lastProcessedSeqNumbers = map[string]string{} + s.loadOffsets(d.ctx) + + assert.Equal(d.T(), originalLastProcessedSeqNumbers, s.lastProcessedSeqNumbers) +}