From 1b099354b022b6f3539655391c44671e37683730 Mon Sep 17 00:00:00 2001 From: Benjamin Texier Date: Wed, 15 Dec 2021 09:53:31 +0100 Subject: [PATCH] feat(storage): add s3 external uri changelog: Allow to use S3 Exernal URI Signed-off-by: Benjamin Texier --- api/http/api_deployments_test.go | 1 + app/app.go | 57 +++++++++++++++++++++++++------- app/images_test.go | 14 ++++---- app/settings_test.go | 1 + config/config.go | 1 + docs/internal_api.yml | 6 +++- model/storage_settings.go | 2 ++ store/mongo/datastore_mongo.go | 2 ++ 8 files changed, 64 insertions(+), 20 deletions(-) diff --git a/api/http/api_deployments_test.go b/api/http/api_deployments_test.go index 001b6f6c7..0006db3c8 100644 --- a/api/http/api_deployments_test.go +++ b/api/http/api_deployments_test.go @@ -1451,6 +1451,7 @@ func TestGetTenantStorageSettings(t *testing.T) { app := &mapp.App{} app.On("GetStorageSettings", mock.MatchedBy(func(ctx context.Context) bool { return true }), + false, ).Return(tc.settings, tc.err) restView := new(view.RESTView) diff --git a/app/app.go b/app/app.go index 79226e9d9..78399524d 100644 --- a/app/app.go +++ b/app/app.go @@ -198,7 +198,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error { if err != nil { return errors.Wrap(err, "error reaching MongoDB") } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return err } @@ -229,7 +229,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error { return nil } -func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error) { +func (d *Deployments) getFileStorage(ctx context.Context, external bool) (s3.FileStorage, error) { settings, err := d.db.GetStorageSettings(ctx) if err != nil { return nil, err @@ -237,9 +237,11 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error region := config.Config.GetString(dconfig.SettingAwsS3Region) key := config.Config.GetString(dconfig.SettingAwsAuthKeyId) secret := config.Config.GetString(dconfig.SettingAwsAuthSecret) - uri := config.Config.GetString(dconfig.SettingAwsURI) + internalUri := config.Config.GetString(dconfig.SettingAwsURI) + externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI) token := config.Config.GetString(dconfig.SettingAwsAuthToken) tagArtifact := config.Config.GetBool(dconfig.SettingsAwsTagArtifact) + if settings.Region != "" { region = settings.Region } @@ -250,12 +252,17 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error secret = settings.Secret } if settings.Uri != "" { - uri = settings.Uri + internalUri = settings.Uri } if settings.Token != "" { token = settings.Token } + uri := internalUri + if external && externalUri != "" { + uri = externalUri + } + return s3.NewSimpleStorageServiceStatic( settings.Bucket, key, @@ -269,6 +276,21 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error ) } + externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI) + + if external && externalUri != "" { + return s3.NewSimpleStorageServiceStatic( + config.Config.GetString(dconfig.SettingAwsS3Bucket), + config.Config.GetString(dconfig.SettingAwsAuthKeyId), + config.Config.GetString(dconfig.SettingAwsAuthSecret), + config.Config.GetString(dconfig.SettingAwsS3Region), + config.Config.GetString(dconfig.SettingAwsAuthToken), + externalUri, + config.Config.GetBool(dconfig.SettingsAwsTagArtifact), + config.Config.GetBool(dconfig.SettingAwsS3ForcePathStyle), + config.Config.GetBool(dconfig.SettingAwsS3UseAccelerate), + ) + } return d.fileStorage, nil } @@ -307,7 +329,7 @@ func (d *Deployments) CreateImage(ctx context.Context, artifactID, err := d.handleArtifact(ctx, multipartUploadMsg) // try to remove artifact file from file storage on error if err != nil { - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return "", err } @@ -352,7 +374,7 @@ func (d *Deployments) handleArtifact(ctx context.Context, //nolint:errcheck go func() (err error) { defer func() { ch <- err }() - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return err } @@ -441,10 +463,11 @@ func (d *Deployments) GenerateImage(ctx context.Context, multipartGenerateImageMsg.TenantID = id.Tenant } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return "", err } + link, err := fileStorage.GetRequest( ctx, imgID, @@ -549,7 +572,7 @@ func (d *Deployments) handleRawFile(ctx context.Context, LimitError: ErrModelArtifactFileTooLarge, } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return "", err } @@ -609,7 +632,7 @@ func (d *Deployments) DeleteImage(ctx context.Context, imageID string) error { // Delete image file (call to external service) // Noop for not existing file - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return err } @@ -693,7 +716,7 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string, return nil, nil } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, false) if err != nil { return nil, err } @@ -708,8 +731,18 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string, } fileName := image.ArtifactMeta.Name + ".mender" - link, err := fileStorage.GetRequest(ctx, imageID, + + externalFileStorage, err := d.getFileStorage(ctx, true) + if err != nil { + return nil, err + } + + link, err := externalFileStorage.GetRequest(ctx, imageID, expire, ArtifactContentType, fileName) + if err != nil { + return nil, errors.Wrap(err, "Searching for image file") + } + if err != nil { return nil, errors.Wrap(err, "Generating download link") } @@ -1337,7 +1370,7 @@ func (d *Deployments) GetDeploymentForDeviceWithCurrent(ctx context.Context, dev return nil, nil } - fileStorage, err := d.getFileStorage(ctx) + fileStorage, err := d.getFileStorage(ctx, true) if err != nil { return nil, err } diff --git a/app/images_test.go b/app/images_test.go index 77b69507f..3906a4e12 100644 --- a/app/images_test.go +++ b/app/images_test.go @@ -145,7 +145,7 @@ func TestGenerateImageErrorWhileUploading(t *testing.T) { ).Return(errors.New("error while uploading")) db.On("GetStorageSettings", - ctx, + ctx, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), @@ -200,7 +200,7 @@ func TestGenerateImageErrorS3GetRequest(t *testing.T) { ).Return(true, nil) db.On("GetStorageSettings", - ctx, + ctx, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), @@ -257,7 +257,7 @@ func TestGenerateImageErrorS3DeleteRequest(t *testing.T) { ).Return(true, nil) db.On("GetStorageSettings", - ctx, + ctx, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), @@ -354,7 +354,7 @@ func TestGenerateImageErrorWhileStartingWorkflow(t *testing.T) { ).Return(true, nil) db.On("GetStorageSettings", - ctx, + ctx, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), @@ -436,7 +436,7 @@ func TestGenerateImageErrorWhileStartingWorkflowAndFailsWhenCleaningUp(t *testin ).Return(true, nil) db.On("GetStorageSettings", - ctx, + ctx, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), @@ -522,7 +522,7 @@ func TestGenerateImageSuccessful(t *testing.T) { ).Return(true, nil) db.On("GetStorageSettings", - ctx, + ctx, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), @@ -599,7 +599,7 @@ func TestGenerateImageSuccessfulWithTenant(t *testing.T) { ctxWithIdentity := identity.WithContext(ctx, identityObject) db.On("GetStorageSettings", - ctxWithIdentity, + ctxWithIdentity, false, ).Return(&model.StorageSettings{ Region: config.Config.GetString(dconfig.SettingAwsS3Region), Uri: config.Config.GetString(dconfig.SettingAwsURI), diff --git a/app/settings_test.go b/app/settings_test.go index e21e1ccbd..5018b8551 100644 --- a/app/settings_test.go +++ b/app/settings_test.go @@ -59,6 +59,7 @@ func TestGetStorageSettings(t *testing.T) { db := mocks.DataStore{} db.On("GetStorageSettings", mock.MatchedBy(func(ctx context.Context) bool { return true }), + false, ).Return(tc.settings, tc.err) ctx := context.Background() diff --git a/config/config.go b/config/config.go index 0edf49ec2..3cd25dc7b 100644 --- a/config/config.go +++ b/config/config.go @@ -41,6 +41,7 @@ const ( SettingAwsS3UseAccelerate = SettingsAws + ".use_accelerate" SettingAwsS3UseAccelerateDefault = false SettingAwsURI = SettingsAws + ".uri" + SettingAwsExternalURI = SettingsAws + ".external_uri" SettingsAwsTagArtifact = SettingsAws + ".tag_artifact" SettingsAwsTagArtifactDefault = false diff --git a/docs/internal_api.yml b/docs/internal_api.yml index a9a5043ef..52dcf5019 100644 --- a/docs/internal_api.yml +++ b/docs/internal_api.yml @@ -441,6 +441,9 @@ definitions: uri: type: string description: Endpoint URI. + external_uri: + type: string + description: External Endpoint URI (default: uri). key: type: string description: Access key id (for S3 - AWS_ACCESS_KEY_ID). @@ -464,7 +467,8 @@ definitions: example: region: us-east-1 bucket: mender-artifacts-unique-bucket-name - uri: example.com + uri: example.internal:9000 + external_uri: example.com key: secret: token: diff --git a/model/storage_settings.go b/model/storage_settings.go index 7caaee6a8..83d61308e 100644 --- a/model/storage_settings.go +++ b/model/storage_settings.go @@ -26,6 +26,7 @@ type StorageSettings struct { Region string `json:"region" bson:"region"` Bucket string `json:"bucket" bson:"bucket"` Uri string `json:"uri" bson:"uri"` + ExternalUri string `json:"external_uri" bson:"external_uri"` Key string `json:"key" bson:"key"` Secret string `json:"secret" bson:"secret"` Token string `json:"token" bson:"token"` @@ -60,6 +61,7 @@ func (s StorageSettings) Validate() error { validation.Field(&s.Key, validation.Length(5, 50)), validation.Field(&s.Secret, validation.Length(5, 100)), validation.Field(&s.Uri, validation.Length(3, 2000)), + validation.Field(&s.ExternalUri, validation.Length(3, 2000)), validation.Field(&s.Token, validation.Length(5, 100)), ) } diff --git a/store/mongo/datastore_mongo.go b/store/mongo/datastore_mongo.go index 90b071f87..cbb2bbd73 100644 --- a/store/mongo/datastore_mongo.go +++ b/store/mongo/datastore_mongo.go @@ -357,6 +357,7 @@ const ( StorageKeyStorageSettingsKey = "key" StorageKeyStorageSettingsSecret = "secret" StorageKeyStorageSettingsURI = "uri" + StorageKeyStorageSettingsExternalURI = "external_uri" StorageKeyStorageSettingsToken = "token" StorageKeyStorageSettingsForcePathStyle = "force_path_style" StorageKeyStorageSettingsUseAccelerate = "use_accelerate" @@ -2219,6 +2220,7 @@ func (db *DataStoreMongo) SetStorageSettings( StorageKeyStorageSettingsKey: storageSettings.Key, StorageKeyStorageSettingsSecret: storageSettings.Secret, StorageKeyStorageSettingsURI: storageSettings.Uri, + StorageKeyStorageSettingsExternalURI: storageSettings.ExternalUri, StorageKeyStorageSettingsRegion: storageSettings.Region, StorageKeyStorageSettingsToken: storageSettings.Token, StorageKeyStorageSettingsForcePathStyle: storageSettings.ForcePathStyle,