Merge pull request #131 from nyaruka/s3x

Add new s3x package to replace storage

.github/workflows/ci.yml

@@ -22,6 +22,14 @@ jobs:
         ports:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      minio:
+        image: bitnami/minio:latest
+        env:
+          MINIO_ROOT_USER: root
+          MINIO_ROOT_PASSWORD: tembatemba
+        ports:
+          - 9000:9000     
+        options: --health-cmd "mc ready local" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - name: Checkout code

s3x/s3.go

@@ -0,0 +1,78 @@
+package s3x
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// Service is simple abstraction layer to work with a S3-compatible storage service
+type Service struct {
+	client *s3.S3
+	urler  ObjectURLer
+}
+
+func NewService(client *s3.S3, urler ObjectURLer) *Service {
+	return &Service{client: client, urler: urler}
+}
+
+func (s *Service) HeadBucket(ctx context.Context, bucket string) error {
+	_, err := s.client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)})
+	if err != nil {
+		return fmt.Errorf("error heading bucket: %w", err)
+	}
+	return nil
+}
+
+func (s *Service) CreateBucket(ctx context.Context, bucket string) error {
+	_, err := s.client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	if err != nil {
+		return fmt.Errorf("error creating bucket: %w", err)
+	}
+	return nil
+}
+
+func (s *Service) DeleteBucket(ctx context.Context, bucket string) error {
+	_, err := s.client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+	if err != nil {
+		return fmt.Errorf("error deleting bucket: %w", err)
+	}
+	return nil
+}
+
+func (s *Service) GetObject(ctx context.Context, bucket, key string) (string, []byte, error) {
+	out, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return "", nil, fmt.Errorf("error getting S3 object: %w", err)
+	}
+
+	body, err := io.ReadAll(out.Body)
+	if err != nil {
+		return "", nil, fmt.Errorf("error reading S3 object: %w", err)
+	}
+
+	return aws.StringValue(out.ContentType), body, nil
+}
+
+// PutObject writes the passed in file to the given bucket with the passed in content type and ACL
+func (s *Service) PutObject(ctx context.Context, bucket, key string, contentType string, body []byte, acl string) (string, error) {
+	_, err := s.client.PutObjectWithContext(ctx, &s3.PutObjectInput{
+		Bucket:      aws.String(bucket),
+		Body:        bytes.NewReader(body),
+		Key:         aws.String(key),
+		ContentType: aws.String(contentType),
+		ACL:         aws.String(acl),
+	})
+	if err != nil {
+		return "", fmt.Errorf("error putting S3 object: %w", err)
+	}
+
+	return s.urler(key), nil
+}

s3x/s3_test.go

@@ -0,0 +1,59 @@
+package s3x_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/nyaruka/gocommon/s3x"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetAndPutObject(t *testing.T) {
+	ctx := context.Background()
+
+	config := &aws.Config{
+		Endpoint:         aws.String("http://localhost:9000"),
+		Region:           aws.String("us-east-1"),
+		Credentials:      credentials.NewStaticCredentials("root", "tembatemba", ""),
+		S3ForcePathStyle: aws.Bool(true),
+	}
+	s, err := session.NewSession(config)
+	require.NoError(t, err)
+
+	client := s3.New(s)
+	require.NotNil(t, client)
+
+	svc := s3x.NewService(client, s3x.MinioURLer("http://localhost:9000", "mybucket"))
+
+	err = svc.HeadBucket(ctx, "gocommon-tests")
+	assert.ErrorContains(t, err, "error heading bucket: NotFound: Not Found\n\tstatus code: 404")
+
+	err = svc.CreateBucket(ctx, "gocommon-tests")
+	assert.NoError(t, err)
+
+	err = svc.HeadBucket(ctx, "gocommon-tests")
+	assert.NoError(t, err)
+
+	url, err := svc.PutObject(ctx, "gocommon-tests", "test.txt", "text/plain", []byte("hello world"), s3.BucketCannedACLPublicRead)
+	assert.NoError(t, err)
+	assert.Equal(t, "http://localhost:9000/mybucket/test.txt", url)
+
+	contentType, body, err := svc.GetObject(ctx, "gocommon-tests", "test.txt")
+	assert.NoError(t, err)
+	assert.Equal(t, "text/plain", contentType)
+	assert.Equal(t, []byte("hello world"), body)
+
+	_, err = client.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String("gocommon-tests"), Key: aws.String("test.txt")})
+	assert.NoError(t, err)
+
+	err = svc.DeleteBucket(ctx, "gocommon-tests")
+	assert.NoError(t, err)
+
+	err = svc.HeadBucket(ctx, "gocommon-tests")
+	assert.Error(t, err)
+}

s3x/urls.go

@@ -0,0 +1,21 @@
+package s3x
+
+import (
+	"fmt"
+	"net/url"
+)
+
+// ObjectURLer is a function that takes a key and returns the publicly accessible URL for that object
+type ObjectURLer func(string) string
+
+func AWSURLer(region, bucket string) ObjectURLer {
+	return func(key string) string {
+		return fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", bucket, region, url.PathEscape(key))
+	}
+}
+
+func MinioURLer(endpoint, bucket string) ObjectURLer {
+	return func(key string) string {
+		return fmt.Sprintf("%s/%s/%s", endpoint, bucket, url.PathEscape(key))
+	}
+}

s3x/urls_test.go

@@ -0,0 +1,16 @@
+package s3x_test
+
+import (
+	"testing"
+
+	"github.com/nyaruka/gocommon/s3x"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestURLers(t *testing.T) {
+	urler := s3x.AWSURLer("us-east-1", "mybucket")
+	assert.Equal(t, "https://mybucket.s3.us-east-1.amazonaws.com/hello%20world.txt", urler("hello world.txt"))
+
+	urler = s3x.MinioURLer("http://localhost:9000", "mybucket")
+	assert.Equal(t, "http://localhost:9000/mybucket/hello%20world.txt", urler("hello world.txt"))
+}