From 273b30df24541b930db2b8746926df9150d863f2 Mon Sep 17 00:00:00 2001 From: Benjamin Texier Date: Wed, 15 Dec 2021 09:53:31 +0100 Subject: [PATCH] feat(storage): add s3 external uri Changelog: Allow to use S3 Exernal URI Ticket: MEN-5280 Signed-off-by: Benjamin Texier --- api/http/api_deployments_test.go | 52 ++++++++++++++-------- app/app.go | 53 ++++++++++++++++++----- app/app_test.go | 13 +++--- app/settings_test.go | 26 ++++++----- config.yaml | 8 +++- config/config.go | 1 + docs/internal_api.yml | 6 ++- model/storage_settings.go | 2 + store/mongo/datastore_mongo.go | 2 + tests/tests/test_settings_api_internal.py | 7 +++ 10 files changed, 120 insertions(+), 50 deletions(-) diff --git a/api/http/api_deployments_test.go b/api/http/api_deployments_test.go index c5eacee24..e92649df0 100644 --- a/api/http/api_deployments_test.go +++ b/api/http/api_deployments_test.go @@ -1515,15 +1515,29 @@ func TestPutTenantStorageSettings(t *testing.T) { }, httpStatus: http.StatusNoContent, }, + "ok external-uri": { + tenantID: "", + settings: &model.StorageSettings{ + Region: "region", + Key: "secretkey", + Secret: "secret", + Bucket: "bucket", + Uri: "https://example.com", + ExternalUri: "https://external.example.com", + Token: "token", + }, + httpStatus: http.StatusNoContent, + }, "ok multi-tenant": { tenantID: "tenant1", settings: &model.StorageSettings{ - Region: "region", - Key: "secretkey", - Secret: "secret", - Bucket: "bucket", - Uri: "https://example.com", - Token: "token", + Region: "region", + Key: "secretkey", + Secret: "secret", + Bucket: "bucket", + Uri: "https://example.com", + ExternalUri: "https://external.example.com", + Token: "token", }, httpStatus: http.StatusNoContent, }, @@ -1558,12 +1572,13 @@ func TestPutTenantStorageSettings(t *testing.T) { "error app err": { tenantID: "", settings: &model.StorageSettings{ - Region: "region", - Key: "secretkey", - Secret: "secret", - Bucket: "bucket", - Uri: "https://example.com", - Token: "token", + Region: "region", + Key: "secretkey", + Secret: "secret", + Bucket: "bucket", + Uri: "https://example.com", + ExternalUri: "https://external.example.com", + Token: "token", }, err: errors.New("generic error"), httpStatus: http.StatusInternalServerError, @@ -1571,12 +1586,13 @@ func TestPutTenantStorageSettings(t *testing.T) { "error app err multi-tenant": { tenantID: "tenant1", settings: &model.StorageSettings{ - Region: "region", - Key: "secretkey", - Secret: "secret", - Bucket: "bucket", - Uri: "https://example.com", - Token: "token", + Region: "region", + Key: "secretkey", + Secret: "secret", + Bucket: "bucket", + Uri: "https://example.com", + ExternalUri: "https://external.example.com", + Token: "token", }, err: errors.New("generic error"), httpStatus: http.StatusInternalServerError, diff --git a/app/app.go b/app/app.go index 10e903215..1410b9ed3 100644 --- a/app/app.go +++ b/app/app.go @@ -201,7 +201,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error { if err != nil { return errors.Wrap(err, "error reaching MongoDB") } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return err } @@ -232,7 +232,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error { return nil } -func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error) { +func (d *Deployments) getFileStorage(ctx context.Context, external bool) (s3.FileStorage, error) { settings, err := d.db.GetStorageSettings(ctx) if err != nil { return nil, err @@ -240,9 +240,11 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error region := config.Config.GetString(dconfig.SettingAwsS3Region) key := config.Config.GetString(dconfig.SettingAwsAuthKeyId) secret := config.Config.GetString(dconfig.SettingAwsAuthSecret) - uri := config.Config.GetString(dconfig.SettingAwsURI) + internalUri := config.Config.GetString(dconfig.SettingAwsURI) + externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI) token := config.Config.GetString(dconfig.SettingAwsAuthToken) tagArtifact := config.Config.GetBool(dconfig.SettingsAwsTagArtifact) + if settings.Region != "" { region = settings.Region } @@ -253,12 +255,17 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error secret = settings.Secret } if settings.Uri != "" { - uri = settings.Uri + internalUri = settings.Uri } if settings.Token != "" { token = settings.Token } + uri := internalUri + if external && externalUri != "" { + uri = externalUri + } + return s3.NewSimpleStorageServiceStatic( settings.Bucket, key, @@ -272,6 +279,21 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error ) } + externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI) + + if external && externalUri != "" { + return s3.NewSimpleStorageServiceStatic( + config.Config.GetString(dconfig.SettingAwsS3Bucket), + config.Config.GetString(dconfig.SettingAwsAuthKeyId), + config.Config.GetString(dconfig.SettingAwsAuthSecret), + config.Config.GetString(dconfig.SettingAwsS3Region), + config.Config.GetString(dconfig.SettingAwsAuthToken), + externalUri, + config.Config.GetBool(dconfig.SettingsAwsTagArtifact), + config.Config.GetBool(dconfig.SettingAwsS3ForcePathStyle), + config.Config.GetBool(dconfig.SettingAwsS3UseAccelerate), + ) + } return d.fileStorage, nil } @@ -309,7 +331,7 @@ func (d *Deployments) CreateImage(ctx context.Context, artifactID, err := d.handleArtifact(ctx, multipartUploadMsg) // try to remove artifact file from file storage on error if err != nil { - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return "", err } @@ -354,7 +376,7 @@ func (d *Deployments) handleArtifact(ctx context.Context, //nolint:errcheck go func() (err error) { defer func() { ch <- err }() - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return err } @@ -443,10 +465,11 @@ func (d *Deployments) GenerateImage(ctx context.Context, multipartGenerateImageMsg.TenantID = id.Tenant } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return "", err } + link, err := fileStorage.GetRequest( ctx, imgID, @@ -551,7 +574,7 @@ func (d *Deployments) handleRawFile(ctx context.Context, LimitError: ErrModelArtifactFileTooLarge, } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return "", err } @@ -611,7 +634,7 @@ func (d *Deployments) DeleteImage(ctx context.Context, imageID string) error { // Delete image file (call to external service) // Noop for not existing file - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return err } @@ -695,7 +718,7 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string, return nil, nil } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return nil, err } @@ -710,7 +733,13 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string, } fileName := image.ArtifactMeta.Name + ".mender" - link, err := fileStorage.GetRequest(ctx, imageID, + + externalFileStorage, err := d.getFileStorage(ctx, true) + if err != nil { + return nil, err + } + + link, err := externalFileStorage.GetRequest(ctx, imageID, expire, ArtifactContentType, fileName) if err != nil { return nil, errors.Wrap(err, "Generating download link") @@ -1370,7 +1399,7 @@ func (d *Deployments) GetDeploymentForDeviceWithCurrent(ctx context.Context, dev return nil, nil } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, true) if err != nil { return nil, err } diff --git a/app/app_test.go b/app/app_test.go index 688172d81..ec0a8e8f2 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -107,12 +107,13 @@ func TestHealthCheck(t *testing.T) { Return(tc.DataStoreError) mDStore.On("GetStorageSettings", ctx). Return(&model.StorageSettings{ - Region: config.Config.GetString(dconfig.SettingAwsS3Region), - Uri: config.Config.GetString(dconfig.SettingAwsURI), - Bucket: config.Config.GetString(dconfig.SettingAwsS3Bucket), - Key: config.Config.GetString(dconfig.SettingAwsAuthKeyId), - Secret: config.Config.GetString(dconfig.SettingAwsAuthSecret), - Token: config.Config.GetString(dconfig.SettingAwsAuthToken)}, nil) + Region: config.Config.GetString(dconfig.SettingAwsS3Region), + ExternalUri: config.Config.GetString(dconfig.SettingAwsExternalURI), + Uri: config.Config.GetString(dconfig.SettingAwsURI), + Bucket: config.Config.GetString(dconfig.SettingAwsS3Bucket), + Key: config.Config.GetString(dconfig.SettingAwsAuthKeyId), + Secret: config.Config.GetString(dconfig.SettingAwsAuthSecret), + Token: config.Config.GetString(dconfig.SettingAwsAuthToken)}, nil) } err := dep.HealthCheck(ctx) switch { diff --git a/app/settings_test.go b/app/settings_test.go index e21e1ccbd..fbe42e3c6 100644 --- a/app/settings_test.go +++ b/app/settings_test.go @@ -33,22 +33,24 @@ func TestGetStorageSettings(t *testing.T) { }{ "ok": { settings: &model.StorageSettings{ - Region: "region", - Key: "secretkey", - Secret: "secret", - Bucket: "bucket", - Uri: "https://example.com", - Token: "token", + Region: "region", + Key: "secretkey", + Secret: "secret", + Bucket: "bucket", + Uri: "https://example.com", + ExternalUri: "https://external.example.com", + Token: "token", }, }, "error": { settings: &model.StorageSettings{ - Region: "region", - Key: "secretkey", - Secret: "secret", - Bucket: "bucket", - Uri: "https://example.com", - Token: "token", + Region: "region", + Key: "secretkey", + Secret: "secret", + Bucket: "bucket", + Uri: "https://example.com", + ExternalUri: "https://external.example.com", + Token: "token", }, err: errors.New("generic error"), }, diff --git a/config.yaml b/config.yaml index fe51f189d..b5f881f30 100644 --- a/config.yaml +++ b/config.yaml @@ -126,12 +126,18 @@ aws: # use_accelerate: false - # S3 URI + # S3 URI (for mender-deployment) # Defaults to: none (s3.amazonaws.com) # Overwrite with environment variable: DEPLOYMENTS_AWS_URI # uri: example.com + # S3 EXTERNAL URI (for devices) + # Defaults to: none (S3 URI) + # Overwrite with environment variable: DEPLOYMENTS_AWS_EXTERNAL_URI + + # uri: external.example.com + # Maximum image size # Defaults to: 10GB # Overwrite with environment variable: DEPLOYMENTS_AWS_MAX_IMAGE_SIZE diff --git a/config/config.go b/config/config.go index f488f873c..cc8cd1548 100644 --- a/config/config.go +++ b/config/config.go @@ -43,6 +43,7 @@ const ( SettingAwsS3MaxImageSize = SettingsAws + ".max_image_size" SettingAwsS3MaxImageSizeDefault = 10737418240 SettingAwsURI = SettingsAws + ".uri" + SettingAwsExternalURI = SettingsAws + ".external_uri" SettingsAwsTagArtifact = SettingsAws + ".tag_artifact" SettingsAwsTagArtifactDefault = false diff --git a/docs/internal_api.yml b/docs/internal_api.yml index a9a5043ef..0394dc06f 100644 --- a/docs/internal_api.yml +++ b/docs/internal_api.yml @@ -441,6 +441,9 @@ definitions: uri: type: string description: Endpoint URI. + external_uri: + type: string + description: External Endpoint URI (if differ from uri). key: type: string description: Access key id (for S3 - AWS_ACCESS_KEY_ID). @@ -464,7 +467,8 @@ definitions: example: region: us-east-1 bucket: mender-artifacts-unique-bucket-name - uri: example.com + uri: example.internal:9000 + external_uri: example.com key: secret: token: diff --git a/model/storage_settings.go b/model/storage_settings.go index 7caaee6a8..83d61308e 100644 --- a/model/storage_settings.go +++ b/model/storage_settings.go @@ -26,6 +26,7 @@ type StorageSettings struct { Region string `json:"region" bson:"region"` Bucket string `json:"bucket" bson:"bucket"` Uri string `json:"uri" bson:"uri"` + ExternalUri string `json:"external_uri" bson:"external_uri"` Key string `json:"key" bson:"key"` Secret string `json:"secret" bson:"secret"` Token string `json:"token" bson:"token"` @@ -60,6 +61,7 @@ func (s StorageSettings) Validate() error { validation.Field(&s.Key, validation.Length(5, 50)), validation.Field(&s.Secret, validation.Length(5, 100)), validation.Field(&s.Uri, validation.Length(3, 2000)), + validation.Field(&s.ExternalUri, validation.Length(3, 2000)), validation.Field(&s.Token, validation.Length(5, 100)), ) } diff --git a/store/mongo/datastore_mongo.go b/store/mongo/datastore_mongo.go index 22dc49888..289f4575e 100644 --- a/store/mongo/datastore_mongo.go +++ b/store/mongo/datastore_mongo.go @@ -390,6 +390,7 @@ const ( StorageKeyStorageSettingsKey = "key" StorageKeyStorageSettingsSecret = "secret" StorageKeyStorageSettingsURI = "uri" + StorageKeyStorageSettingsExternalURI = "external_uri" StorageKeyStorageSettingsToken = "token" StorageKeyStorageSettingsForcePathStyle = "force_path_style" StorageKeyStorageSettingsUseAccelerate = "use_accelerate" @@ -2279,6 +2280,7 @@ func (db *DataStoreMongo) SetStorageSettings( StorageKeyStorageSettingsKey: storageSettings.Key, StorageKeyStorageSettingsSecret: storageSettings.Secret, StorageKeyStorageSettingsURI: storageSettings.Uri, + StorageKeyStorageSettingsExternalURI: storageSettings.ExternalUri, StorageKeyStorageSettingsRegion: storageSettings.Region, StorageKeyStorageSettingsToken: storageSettings.Token, StorageKeyStorageSettingsForcePathStyle: storageSettings.ForcePathStyle, diff --git a/tests/tests/test_settings_api_internal.py b/tests/tests/test_settings_api_internal.py index 84f9a921c..0f1285288 100644 --- a/tests/tests/test_settings_api_internal.py +++ b/tests/tests/test_settings_api_internal.py @@ -25,6 +25,7 @@ def test_ok(self, api_client_int): "region": "region", "bucket": "bucket", "uri": "https://example.com", + "external_uri": "https://external.example.com", "key": "long_key", "secret": "secret", "token": "token", @@ -41,6 +42,7 @@ def test_data_update(self, api_client_int): "region": "region", "bucket": "bucket", "uri": "https://example.com", + "external_uri": "https://external.example.com", "key": "long_key", "secret": "secret", "token": "token", @@ -51,6 +53,7 @@ def test_data_update(self, api_client_int): "region": "region", "bucket": "new_bucket", "uri": "https://example.com", + "external_uri": "https://external.example.com", "key": "long_key", "secret": "secret", "token": "token", @@ -66,6 +69,7 @@ def test_update_to_empty_data_set(self, api_client_int): "region": "region", "bucket": "bucket", "uri": "https://example.com", + "external_uri": "https://external.example.com", "key": "long_key", "secret": "secret", "token": "token", @@ -76,6 +80,7 @@ def test_update_to_empty_data_set(self, api_client_int): "region": "", "bucket": "", "uri": "", + "external_uri": "", "key": "", "secret": "", "token": "", @@ -90,6 +95,7 @@ def test_failed_data_key_length(self, api_client_int): "region": "region", "bucket": "bucket", "uri": "https://example.com", + "external_uri": "https://external.example.com", "key": "key", "secret": "secret", "token": "token", @@ -102,6 +108,7 @@ def test_failed_data_missing_bucket(self, api_client_int): data = { "region": "region", "uri": "https://example.com", + "external_uri": "https://external.example.com", "key": "long_key", "secret": "secret", "token": "token",