feat(storage): add s3 external uri

api/http/api_deployments_test.go

@@ -1437,6 +1437,7 @@ func TestGetTenantStorageSettings(t *testing.T) {
 			app := &mapp.App{}
 			app.On("GetStorageSettings",
 				mock.MatchedBy(func(ctx context.Context) bool { return true }),
+				false,
 			).Return(tc.settings, tc.err)
 
 			restView := new(view.RESTView)

app/app.go

@@ -197,7 +197,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "error reaching MongoDB")
 	}
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -228,17 +228,19 @@ func (d *Deployments) HealthCheck(ctx context.Context) error {
 	return nil
 }
 
-func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error) {
+func (d *Deployments) getFileStorage(ctx context.Context, external bool) (s3.FileStorage, error) {
 	settings, err := d.db.GetStorageSettings(ctx)
 	if err != nil {
 		return nil, err
 	} else if settings != nil && settings.Bucket != "" {
 		region := config.Config.GetString(dconfig.SettingAwsS3Region)
 		key := config.Config.GetString(dconfig.SettingAwsAuthKeyId)
 		secret := config.Config.GetString(dconfig.SettingAwsAuthSecret)
-		uri := config.Config.GetString(dconfig.SettingAwsURI)
+		internalUri := config.Config.GetString(dconfig.SettingAwsURI)
+		externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI)
 		token := config.Config.GetString(dconfig.SettingAwsAuthToken)
 		tagArtifact := config.Config.GetBool(dconfig.SettingsAwsTagArtifact)
+
 		if settings.Region != "" {
 			region = settings.Region
 		}
@@ -249,12 +251,17 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error
 			secret = settings.Secret
 		}
 		if settings.Uri != "" {
-			uri = settings.Uri
+			internalUri = settings.Uri
 		}
 		if settings.Token != "" {
 			token = settings.Token
 		}
 
+		uri := internalUri
+		if external && externalUri != "" {
+			uri = externalUri
+		}
+
 		return s3.NewSimpleStorageServiceStatic(
 			settings.Bucket,
 			key,
@@ -268,6 +275,21 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error
 		)
 	}
 
+	externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI)
+
+	if external && externalUri != "" {
+		return s3.NewSimpleStorageServiceStatic(
+			config.Config.GetString(dconfig.SettingAwsS3Bucket),
+			config.Config.GetString(dconfig.SettingAwsAuthKeyId),
+			config.Config.GetString(dconfig.SettingAwsAuthSecret),
+			config.Config.GetString(dconfig.SettingAwsS3Region),
+			config.Config.GetString(dconfig.SettingAwsAuthToken),
+			externalUri,
+			config.Config.GetBool(dconfig.SettingsAwsTagArtifact),
+			config.Config.GetBool(dconfig.SettingAwsS3ForcePathStyle),
+			config.Config.GetBool(dconfig.SettingAwsS3UseAccelerate),
+		)
+	}
 	return d.fileStorage, nil
 }
 
@@ -306,7 +328,7 @@ func (d *Deployments) CreateImage(ctx context.Context,
 	artifactID, err := d.handleArtifact(ctx, multipartUploadMsg)
 	// try to remove artifact file from file storage on error
 	if err != nil {
-		fileStorage, err := d.getFileStorage(ctx)
+		fileStorage, err := d.getFileStorage(ctx, false)
 		if err != nil {
 			return "", err
 		}
@@ -351,7 +373,7 @@ func (d *Deployments) handleArtifact(ctx context.Context,
 	//nolint:errcheck
 	go func() (err error) {
 		defer func() { ch <- err }()
-		fileStorage, err := d.getFileStorage(ctx)
+		fileStorage, err := d.getFileStorage(ctx, false)
 		if err != nil {
 			return err
 		}
@@ -440,10 +462,11 @@ func (d *Deployments) GenerateImage(ctx context.Context,
 		multipartGenerateImageMsg.TenantID = id.Tenant
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return "", err
 	}
+
 	link, err := fileStorage.GetRequest(
 		ctx,
 		imgID,
@@ -548,7 +571,7 @@ func (d *Deployments) handleRawFile(ctx context.Context,
 		LimitError: ErrModelArtifactFileTooLarge,
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return "", err
 	}
@@ -608,7 +631,7 @@ func (d *Deployments) DeleteImage(ctx context.Context, imageID string) error {
 
 	// Delete image file (call to external service)
 	// Noop for not existing file
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -692,7 +715,7 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string,
 		return nil, nil
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return nil, err
 	}
@@ -709,6 +732,7 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string,
 	fileName := image.ArtifactMeta.Name + ".mender"
 	link, err := fileStorage.GetRequest(ctx, imageID,
 		expire, ArtifactContentType, fileName)
+
 	if err != nil {
 		return nil, errors.Wrap(err, "Generating download link")
 	}
@@ -1336,7 +1360,7 @@ func (d *Deployments) GetDeploymentForDeviceWithCurrent(ctx context.Context, dev
 		return nil, nil
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, true)
 	if err != nil {
 		return nil, err
 	}

app/images_test.go

@@ -145,7 +145,7 @@ func TestGenerateImageErrorWhileUploading(t *testing.T) {
 	).Return(errors.New("error while uploading"))
 
 	db.On("GetStorageSettings",
-		ctx,
+		ctx, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),
@@ -200,7 +200,7 @@ func TestGenerateImageErrorS3GetRequest(t *testing.T) {
 	).Return(true, nil)
 
 	db.On("GetStorageSettings",
-		ctx,
+		ctx, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),
@@ -257,7 +257,7 @@ func TestGenerateImageErrorS3DeleteRequest(t *testing.T) {
 	).Return(true, nil)
 
 	db.On("GetStorageSettings",
-		ctx,
+		ctx, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),
@@ -354,7 +354,7 @@ func TestGenerateImageErrorWhileStartingWorkflow(t *testing.T) {
 	).Return(true, nil)
 
 	db.On("GetStorageSettings",
-		ctx,
+		ctx, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),
@@ -436,7 +436,7 @@ func TestGenerateImageErrorWhileStartingWorkflowAndFailsWhenCleaningUp(t *testin
 	).Return(true, nil)
 
 	db.On("GetStorageSettings",
-		ctx,
+		ctx, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),
@@ -522,7 +522,7 @@ func TestGenerateImageSuccessful(t *testing.T) {
 	).Return(true, nil)
 
 	db.On("GetStorageSettings",
-		ctx,
+		ctx, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),
@@ -599,7 +599,7 @@ func TestGenerateImageSuccessfulWithTenant(t *testing.T) {
 	ctxWithIdentity := identity.WithContext(ctx, identityObject)
 
 	db.On("GetStorageSettings",
-		ctxWithIdentity,
+		ctxWithIdentity, false,
 	).Return(&model.StorageSettings{
 		Region: config.Config.GetString(dconfig.SettingAwsS3Region),
 		Uri:    config.Config.GetString(dconfig.SettingAwsURI),

app/settings_test.go

@@ -59,6 +59,7 @@ func TestGetStorageSettings(t *testing.T) {
 			db := mocks.DataStore{}
 			db.On("GetStorageSettings",
 				mock.MatchedBy(func(ctx context.Context) bool { return true }),
+				false,
 			).Return(tc.settings, tc.err)
 			ctx := context.Background()
 

config/config.go

@@ -41,6 +41,7 @@ const (
 	SettingAwsS3UseAccelerate         = SettingsAws + ".use_accelerate"
 	SettingAwsS3UseAccelerateDefault  = false
 	SettingAwsURI                     = SettingsAws + ".uri"
+	SettingAwsExternalURI             = SettingsAws + ".external_uri"
 	SettingsAwsTagArtifact            = SettingsAws + ".tag_artifact"
 	SettingsAwsTagArtifactDefault     = false
 

docs/internal_api.yml

@@ -441,6 +441,9 @@ definitions:
       uri:
         type: string
         description: Endpoint URI.
+      external_uri:
+        type: string
+        description: External Endpoint URI (default: uri).
       key:
         type: string
         description: Access key id (for S3 - AWS_ACCESS_KEY_ID).
@@ -464,7 +467,8 @@ definitions:
     example:
       region: us-east-1
       bucket: mender-artifacts-unique-bucket-name
-      uri: example.com
+      uri: example.internal:9000
+      external_uri: example.com
       key: <key>
       secret: <secret>
       token: <token>

model/storage_settings.go

@@ -26,6 +26,7 @@ type StorageSettings struct {
 	Region         string `json:"region" bson:"region"`
 	Bucket         string `json:"bucket" bson:"bucket"`
 	Uri            string `json:"uri" bson:"uri"`
+	ExternalUri    string `json:"external_uri" bson:"external_uri"`
 	Key            string `json:"key" bson:"key"`
 	Secret         string `json:"secret" bson:"secret"`
 	Token          string `json:"token" bson:"token"`
@@ -60,6 +61,7 @@ func (s StorageSettings) Validate() error {
 		validation.Field(&s.Key, validation.Length(5, 50)),
 		validation.Field(&s.Secret, validation.Length(5, 100)),
 		validation.Field(&s.Uri, validation.Length(3, 2000)),
+		validation.Field(&s.ExternalUri, validation.Length(3, 2000)),
 		validation.Field(&s.Token, validation.Length(5, 100)),
 	)
 }

store/mongo/datastore_mongo.go

@@ -357,6 +357,7 @@ const (
 	StorageKeyStorageSettingsKey            = "key"
 	StorageKeyStorageSettingsSecret         = "secret"
 	StorageKeyStorageSettingsURI            = "uri"
+	StorageKeyStorageSettingsExternalURI    = "external_uri"
 	StorageKeyStorageSettingsToken          = "token"
 	StorageKeyStorageSettingsForcePathStyle = "force_path_style"
 	StorageKeyStorageSettingsUseAccelerate  = "use_accelerate"
@@ -2219,6 +2220,7 @@ func (db *DataStoreMongo) SetStorageSettings(
 			StorageKeyStorageSettingsKey:            storageSettings.Key,
 			StorageKeyStorageSettingsSecret:         storageSettings.Secret,
 			StorageKeyStorageSettingsURI:            storageSettings.Uri,
+			StorageKeyStorageSettingsExternalURI:    storageSettings.ExternalUri,
 			StorageKeyStorageSettingsRegion:         storageSettings.Region,
 			StorageKeyStorageSettingsToken:          storageSettings.Token,
 			StorageKeyStorageSettingsForcePathStyle: storageSettings.ForcePathStyle,