From 955c31febf486aecca10b5c8741a882ba5f45e44 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 16 Sep 2024 11:45:20 -0500 Subject: [PATCH] Add basic get/put operations to dynamo service --- aws/dynamo/marshal.go | 63 +++++++++++++++++++++++++++++++++++++ aws/dynamo/service.go | 43 +++++++++++++++++++++++-- aws/dynamo/service_test.go | 64 +++++++++++++++++++++++++++++++++++--- go.mod | 12 ++++--- go.sum | 24 ++++++++------ 5 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 aws/dynamo/marshal.go diff --git a/aws/dynamo/marshal.go b/aws/dynamo/marshal.go new file mode 100644 index 0000000..33b953e --- /dev/null +++ b/aws/dynamo/marshal.go @@ -0,0 +1,63 @@ +package dynamo + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +type Marshaler interface { + MarshalDynamo() (map[string]types.AttributeValue, error) +} + +type Unmarshaler interface { + UnmarshalDynamo(map[string]types.AttributeValue) error +} + +func marshal(v any) (map[string]types.AttributeValue, error) { + marshaler, ok := v.(Marshaler) + if ok { + return marshaler.MarshalDynamo() + } + + return attributevalue.MarshalMap(v) +} + +func unmarshal(m map[string]types.AttributeValue, v any) error { + unmarshaler, ok := v.(Unmarshaler) + if ok { + return unmarshaler.UnmarshalDynamo(m) + } + + return attributevalue.UnmarshalMap(m, v) +} + +func MarshalJSONGZ(v any) ([]byte, error) { + buf := &bytes.Buffer{} + w := gzip.NewWriter(buf) + + if err := json.NewEncoder(w).Encode(v); err != nil { + return nil, fmt.Errorf("error encoding value as json+gzip: %w", err) + } + + w.Close() + + return buf.Bytes(), nil +} + +func UnmarshalJSONGZ(d []byte, v any) error { + r, err := gzip.NewReader(bytes.NewReader(d)) + if err != nil { + return err + } + + if err := json.NewDecoder(r).Decode(v); err != nil { + return fmt.Errorf("error decoding value from json+gzip: %w", err) + } + + return nil +} diff --git a/aws/dynamo/service.go b/aws/dynamo/service.go index d33ea6d..c0e14b3 100644 --- a/aws/dynamo/service.go +++ b/aws/dynamo/service.go @@ -2,11 +2,13 @@ package dynamo import ( "context" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" ) // Service is simple abstraction layer to work with a DynamoDB-compatible database @@ -39,11 +41,48 @@ func NewService(accessKey, secretKey, region, endpoint, tablePrefix string) (*Se return &Service{Client: client, tablePrefix: tablePrefix}, nil } -func (s *Service) Test(ctx context.Context, table string) error { - _, err := s.Client.DescribeTable(ctx, &dynamodb.DescribeTableInput{TableName: aws.String(s.TableName(table))}) +// Test checks if the service is working by trying to list tables +func (s *Service) Test(ctx context.Context) error { + _, err := s.Client.ListTables(ctx, &dynamodb.ListTablesInput{}) return err } +// TableName returns the full table name with the prefix func (s *Service) TableName(base string) string { return s.tablePrefix + base } + +// GetItem retrieves an item from the given table +func (s *Service) GetItem(ctx context.Context, table string, key map[string]types.AttributeValue, dst any) error { + resp, err := s.Client.GetItem(ctx, &dynamodb.GetItemInput{ + TableName: aws.String(s.TableName(table)), + Key: key, + }) + if err != nil { + return fmt.Errorf("error getting item from dynamo: %w", err) + } + + if err := unmarshal(resp.Item, dst); err != nil { + return fmt.Errorf("error unmarshaling dynamo item: %w", err) + } + + return nil +} + +// PutItem puts an item into the given table +func (s *Service) PutItem(ctx context.Context, table string, v any) error { + item, err := marshal(v) + if err != nil { + return fmt.Errorf("error marshaling dynamo item: %w", err) + } + + _, err = s.Client.PutItem(ctx, &dynamodb.PutItemInput{ + TableName: aws.String(s.TableName(table)), + Item: item, + }) + if err != nil { + return fmt.Errorf("error putting item to dynamo: %w", err) + } + + return nil +} diff --git a/aws/dynamo/service_test.go b/aws/dynamo/service_test.go index 11cd960..c28db44 100644 --- a/aws/dynamo/service_test.go +++ b/aws/dynamo/service_test.go @@ -2,23 +2,72 @@ package dynamo_test import ( "context" + "fmt" "testing" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/nyaruka/gocommon/aws/dynamo" + "github.com/nyaruka/gocommon/uuids" "github.com/stretchr/testify/assert" ) +type Thing struct { + uuid uuids.UUID + name string + extra map[string]any +} + +type dyThing struct { + UUID uuids.UUID `dynamodbav:"UUID"` + Name string `dynamodbav:"Name"` + Extra []byte `dynamodbav:"Extra"` +} + +func (t *Thing) MarshalDynamo() (map[string]types.AttributeValue, error) { + e, err := dynamo.MarshalJSONGZ(t.extra) + if err != nil { + return nil, fmt.Errorf("error marshaling extra: %w", err) + } + + d := dyThing{UUID: t.uuid, Name: t.name, Extra: e} + + return attributevalue.MarshalMap(d) +} + +func (t *Thing) UnmarshalDynamo(m map[string]types.AttributeValue) error { + d := &dyThing{} + + if err := attributevalue.UnmarshalMap(m, d); err != nil { + return fmt.Errorf("error unmarshaling thing: %w", err) + } + + t.uuid = d.UUID + t.name = d.Name + + if err := dynamo.UnmarshalJSONGZ(d.Extra, &t.extra); err != nil { + return fmt.Errorf("error unmarshaling extra: %w", err) + } + + return nil +} + func TestService(t *testing.T) { ctx := context.Background() - svc, err := dynamo.NewService("root", "tembatemba", "us-east-1", "http://localhost:6000", "Test") + svc, err := dynamo.NewService("root", "badkey", "us-east-1", "http://localhost:6666", "Test") + assert.NoError(t, err) + + err = svc.Test(ctx) + assert.ErrorContains(t, err, "exceeded maximum number of attempts, 3") + + svc, err = dynamo.NewService("root", "tembatemba", "us-east-1", "http://localhost:6000", "Test") assert.NoError(t, err) - err = svc.Test(ctx, "Things") - assert.ErrorContains(t, err, "ResourceNotFoundException") + err = svc.Test(ctx) + assert.NoError(t, err) _, err = svc.Client.CreateTable(ctx, &dynamodb.CreateTableInput{ TableName: aws.String("TestThings"), @@ -32,8 +81,15 @@ func TestService(t *testing.T) { }) assert.NoError(t, err) - err = svc.Test(ctx, "Things") + thing1 := &Thing{uuid: "9142d9d2-bbc3-4412-b0d5-25c729c4f231", name: "One", extra: map[string]any{"foo": "bar"}} + + err = svc.PutItem(ctx, "Things", thing1) + assert.NoError(t, err) + + thing2 := &Thing{} + err = svc.GetItem(ctx, "Things", map[string]types.AttributeValue{"UUID": &types.AttributeValueMemberS{Value: "9142d9d2-bbc3-4412-b0d5-25c729c4f231"}}, thing2) assert.NoError(t, err) + assert.Equal(t, thing1, thing2) _, err = svc.Client.DeleteTable(ctx, &dynamodb.DeleteTableInput{TableName: aws.String("TestThings")}) assert.NoError(t, err) diff --git a/go.mod b/go.mod index b89daf8..9cf3660 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module github.com/nyaruka/gocommon go 1.22 require ( - github.com/aws/aws-sdk-go-v2 v1.30.4 + github.com/aws/aws-sdk-go-v2 v1.30.5 github.com/aws/aws-sdk-go-v2/config v1.27.28 github.com/aws/aws-sdk-go-v2/credentials v1.17.28 - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.5 + github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.3 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.9 github.com/aws/aws-sdk-go-v2/service/s3 v1.60.0 github.com/gabriel-vasile/mimetype v1.4.5 github.com/go-chi/chi/v5 v5.1.0 @@ -30,13 +31,14 @@ require ( require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.22.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.18 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect diff --git a/go.sum b/go.sum index 4228a5b..294ff5f 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,35 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= -github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= github.com/aws/aws-sdk-go-v2/config v1.27.28 h1:OTxWGW/91C61QlneCtnD62NLb4W616/NM1jA8LhJqbg= github.com/aws/aws-sdk-go-v2/config v1.27.28/go.mod h1:uzVRVtJSU5EFv6Fu82AoVFKozJi2ZCY6WRCXj06rbvs= github.com/aws/aws-sdk-go-v2/credentials v1.17.28 h1:m8+AHY/ND8CMHJnPoH7PJIRakWGa4gbfbxuY9TGTUXM= github.com/aws/aws-sdk-go-v2/credentials v1.17.28/go.mod h1:6TF7dSc78ehD1SL6KpRIPKMA1GyyWflIkjqg+qmf4+c= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.3 h1:/BPXKQ6n1cDWPmc5FWF6fCSaUtK+dWkWd0x9dI4dgaI= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.3/go.mod h1:qabLXChRlJREypX5RN/Z47GU+RaMsjotNCZfZ85oD0M= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.5 h1:Cm77yt+/CV7A6DglkENsWA3H1hq8+4ItJnFKrhxHkvg= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.5/go.mod h1:s2fYaueBuCnwv1XQn6T8TfShxJWusv5tWPMcL+GY6+g= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.9 h1:jbqgtdKfAXebx2/l2UhDEe/jmmCIhaCO3HFK71M7VzM= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.9/go.mod h1:N3YdUYxyxhiuAelUgCpSVBuBI1klobJxZrDtL+olu10= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.22.7 h1:VTBHXWkSeFgT3sfYB4U92qMgzHl0nz9H1tYNHHutLg0= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.22.7/go.mod h1:F/ybU7YfgFcktSp+biKgiHjyscGhlZxOz4QFFQqHXGw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.17 h1:HDJGz1jlV7RokVgTPfx1UHBHANC0N5Uk++xgyYgz5E0= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.17/go.mod h1:5szDu6TWdRDytfDxUQVv2OYfpTQMKApVFyqpm+TcA98= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.18 h1:GACdEPdpBE59I7pbfvu0/Mw1wzstlP3QtPHklUxybFE= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.18/go.mod h1:K+xV06+Wni4TSaOOJ1Y35e5tYOCUBYbebLKmJQQa8yY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I=