feat(storage): add external s3 uri

api/http/api_deployments_test.go

@@ -1515,15 +1515,29 @@ func TestPutTenantStorageSettings(t *testing.T) {
 			},
 			httpStatus: http.StatusNoContent,
 		},
+		"ok external-uri": {
+			tenantID: "",
+			settings: &model.StorageSettings{
+				Region:      "region",
+				Key:         "secretkey",
+				Secret:      "secret",
+				Bucket:      "bucket",
+				Uri:         "https://example.com",
+				ExternalUri: "https://external.example.com",
+				Token:       "token",
+			},
+			httpStatus: http.StatusNoContent,
+		},
 		"ok multi-tenant": {
 			tenantID: "tenant1",
 			settings: &model.StorageSettings{
-				Region: "region",
-				Key:    "secretkey",
-				Secret: "secret",
-				Bucket: "bucket",
-				Uri:    "https://example.com",
-				Token:  "token",
+				Region:      "region",
+				Key:         "secretkey",
+				Secret:      "secret",
+				Bucket:      "bucket",
+				Uri:         "https://example.com",
+				ExternalUri: "https://external.example.com",
+				Token:       "token",
 			},
 			httpStatus: http.StatusNoContent,
 		},
@@ -1558,25 +1572,27 @@ func TestPutTenantStorageSettings(t *testing.T) {
 		"error app err": {
 			tenantID: "",
 			settings: &model.StorageSettings{
-				Region: "region",
-				Key:    "secretkey",
-				Secret: "secret",
-				Bucket: "bucket",
-				Uri:    "https://example.com",
-				Token:  "token",
+				Region:      "region",
+				Key:         "secretkey",
+				Secret:      "secret",
+				Bucket:      "bucket",
+				Uri:         "https://example.com",
+				ExternalUri: "https://external.example.com",
+				Token:       "token",
 			},
 			err:        errors.New("generic error"),
 			httpStatus: http.StatusInternalServerError,
 		},
 		"error app err multi-tenant": {
 			tenantID: "tenant1",
 			settings: &model.StorageSettings{
-				Region: "region",
-				Key:    "secretkey",
-				Secret: "secret",
-				Bucket: "bucket",
-				Uri:    "https://example.com",
-				Token:  "token",
+				Region:      "region",
+				Key:         "secretkey",
+				Secret:      "secret",
+				Bucket:      "bucket",
+				Uri:         "https://example.com",
+				ExternalUri: "https://external.example.com",
+				Token:       "token",
 			},
 			err:        errors.New("generic error"),
 			httpStatus: http.StatusInternalServerError,

app/app.go

@@ -201,7 +201,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "error reaching MongoDB")
 	}
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -232,17 +232,19 @@ func (d *Deployments) HealthCheck(ctx context.Context) error {
 	return nil
 }
 
-func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error) {
+func (d *Deployments) getFileStorage(ctx context.Context, external bool) (s3.FileStorage, error) {
 	settings, err := d.db.GetStorageSettings(ctx)
 	if err != nil {
 		return nil, err
 	} else if settings != nil && settings.Bucket != "" {
 		region := config.Config.GetString(dconfig.SettingAwsS3Region)
 		key := config.Config.GetString(dconfig.SettingAwsAuthKeyId)
 		secret := config.Config.GetString(dconfig.SettingAwsAuthSecret)
-		uri := config.Config.GetString(dconfig.SettingAwsURI)
+		internalUri := config.Config.GetString(dconfig.SettingAwsURI)
+		externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI)
 		token := config.Config.GetString(dconfig.SettingAwsAuthToken)
 		tagArtifact := config.Config.GetBool(dconfig.SettingsAwsTagArtifact)
+
 		if settings.Region != "" {
 			region = settings.Region
 		}
@@ -253,12 +255,17 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error
 			secret = settings.Secret
 		}
 		if settings.Uri != "" {
-			uri = settings.Uri
+			internalUri = settings.Uri
 		}
 		if settings.Token != "" {
 			token = settings.Token
 		}
 
+		uri := internalUri
+		if external && externalUri != "" {
+			uri = externalUri
+		}
+
 		return s3.NewSimpleStorageServiceStatic(
 			settings.Bucket,
 			key,
@@ -272,6 +279,21 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error
 		)
 	}
 
+	externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI)
+
+	if external && externalUri != "" {
+		return s3.NewSimpleStorageServiceStatic(
+			config.Config.GetString(dconfig.SettingAwsS3Bucket),
+			config.Config.GetString(dconfig.SettingAwsAuthKeyId),
+			config.Config.GetString(dconfig.SettingAwsAuthSecret),
+			config.Config.GetString(dconfig.SettingAwsS3Region),
+			config.Config.GetString(dconfig.SettingAwsAuthToken),
+			externalUri,
+			config.Config.GetBool(dconfig.SettingsAwsTagArtifact),
+			config.Config.GetBool(dconfig.SettingAwsS3ForcePathStyle),
+			config.Config.GetBool(dconfig.SettingAwsS3UseAccelerate),
+		)
+	}
 	return d.fileStorage, nil
 }
 
@@ -309,7 +331,7 @@ func (d *Deployments) CreateImage(ctx context.Context,
 	artifactID, err := d.handleArtifact(ctx, multipartUploadMsg)
 	// try to remove artifact file from file storage on error
 	if err != nil {
-		fileStorage, err := d.getFileStorage(ctx)
+		fileStorage, err := d.getFileStorage(ctx, false)
 		if err != nil {
 			return "", err
 		}
@@ -354,7 +376,7 @@ func (d *Deployments) handleArtifact(ctx context.Context,
 	//nolint:errcheck
 	go func() (err error) {
 		defer func() { ch <- err }()
-		fileStorage, err := d.getFileStorage(ctx)
+		fileStorage, err := d.getFileStorage(ctx, false)
 		if err != nil {
 			return err
 		}
@@ -443,10 +465,11 @@ func (d *Deployments) GenerateImage(ctx context.Context,
 		multipartGenerateImageMsg.TenantID = id.Tenant
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return "", err
 	}
+
 	link, err := fileStorage.GetRequest(
 		ctx,
 		imgID,
@@ -551,7 +574,7 @@ func (d *Deployments) handleRawFile(ctx context.Context,
 		LimitError: ErrModelArtifactFileTooLarge,
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return "", err
 	}
@@ -611,7 +634,7 @@ func (d *Deployments) DeleteImage(ctx context.Context, imageID string) error {
 
 	// Delete image file (call to external service)
 	// Noop for not existing file
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -695,7 +718,7 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string,
 		return nil, nil
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, false)
 	if err != nil {
 		return nil, err
 	}
@@ -710,7 +733,13 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string,
 	}
 
 	fileName := image.ArtifactMeta.Name + ".mender"
-	link, err := fileStorage.GetRequest(ctx, imageID,
+
+	externalFileStorage, err := d.getFileStorage(ctx, true)
+	if err != nil {
+		return nil, err
+	}
+
+	link, err := externalFileStorage.GetRequest(ctx, imageID,
 		expire, ArtifactContentType, fileName)
 	if err != nil {
 		return nil, errors.Wrap(err, "Generating download link")
@@ -1370,7 +1399,7 @@ func (d *Deployments) GetDeploymentForDeviceWithCurrent(ctx context.Context, dev
 		return nil, nil
 	}
 
-	fileStorage, err := d.getFileStorage(ctx)
+	fileStorage, err := d.getFileStorage(ctx, true)
 	if err != nil {
 		return nil, err
 	}

app/app_test.go

@@ -107,12 +107,13 @@ func TestHealthCheck(t *testing.T) {
 					Return(tc.DataStoreError)
 				mDStore.On("GetStorageSettings", ctx).
 					Return(&model.StorageSettings{
-						Region: config.Config.GetString(dconfig.SettingAwsS3Region),
-						Uri:    config.Config.GetString(dconfig.SettingAwsURI),
-						Bucket: config.Config.GetString(dconfig.SettingAwsS3Bucket),
-						Key:    config.Config.GetString(dconfig.SettingAwsAuthKeyId),
-						Secret: config.Config.GetString(dconfig.SettingAwsAuthSecret),
-						Token:  config.Config.GetString(dconfig.SettingAwsAuthToken)}, nil)
+						Region:      config.Config.GetString(dconfig.SettingAwsS3Region),
+						ExternalUri: config.Config.GetString(dconfig.SettingAwsExternalURI),
+						Uri:         config.Config.GetString(dconfig.SettingAwsURI),
+						Bucket:      config.Config.GetString(dconfig.SettingAwsS3Bucket),
+						Key:         config.Config.GetString(dconfig.SettingAwsAuthKeyId),
+						Secret:      config.Config.GetString(dconfig.SettingAwsAuthSecret),
+						Token:       config.Config.GetString(dconfig.SettingAwsAuthToken)}, nil)
 			}
 			err := dep.HealthCheck(ctx)
 			switch {

app/settings_test.go

@@ -33,22 +33,24 @@ func TestGetStorageSettings(t *testing.T) {
 	}{
 		"ok": {
 			settings: &model.StorageSettings{
-				Region: "region",
-				Key:    "secretkey",
-				Secret: "secret",
-				Bucket: "bucket",
-				Uri:    "https://example.com",
-				Token:  "token",
+				Region:      "region",
+				Key:         "secretkey",
+				Secret:      "secret",
+				Bucket:      "bucket",
+				Uri:         "https://example.com",
+				ExternalUri: "https://external.example.com",
+				Token:       "token",
 			},
 		},
 		"error": {
 			settings: &model.StorageSettings{
-				Region: "region",
-				Key:    "secretkey",
-				Secret: "secret",
-				Bucket: "bucket",
-				Uri:    "https://example.com",
-				Token:  "token",
+				Region:      "region",
+				Key:         "secretkey",
+				Secret:      "secret",
+				Bucket:      "bucket",
+				Uri:         "https://example.com",
+				ExternalUri: "https://external.example.com",
+				Token:       "token",
 			},
 			err: errors.New("generic error"),
 		},

config.yaml

@@ -126,12 +126,18 @@ aws:
 
     # use_accelerate: false
 
-    # S3 URI
+    # S3 URI (for mender-deployment)
     # Defaults to: none (s3.amazonaws.com)
     # Overwrite with environment variable: DEPLOYMENTS_AWS_URI
 
     # uri: example.com
 
+    # S3 EXTERNAL URI (for devices)
+    # Defaults to: none (S3 URI)
+    # Overwrite with environment variable: DEPLOYMENTS_AWS_EXTERNAL_URI
+
+    # uri: external.example.com
+
     # Maximum image size
     # Defaults to: 10GB
     # Overwrite with environment variable: DEPLOYMENTS_AWS_MAX_IMAGE_SIZE

config/config.go

@@ -43,6 +43,7 @@ const (
 	SettingAwsS3MaxImageSize          = SettingsAws + ".max_image_size"
 	SettingAwsS3MaxImageSizeDefault   = 10737418240
 	SettingAwsURI                     = SettingsAws + ".uri"
+	SettingAwsExternalURI             = SettingsAws + ".external_uri"
 	SettingsAwsTagArtifact            = SettingsAws + ".tag_artifact"
 	SettingsAwsTagArtifactDefault     = false
 

docs/internal_api.yml

@@ -441,6 +441,9 @@ definitions:
       uri:
         type: string
         description: Endpoint URI.
+      external_uri:
+        type: string
+        description: External Endpoint URI (if differ from uri).
       key:
         type: string
         description: Access key id (for S3 - AWS_ACCESS_KEY_ID).
@@ -464,7 +467,8 @@ definitions:
     example:
       region: us-east-1
       bucket: mender-artifacts-unique-bucket-name
-      uri: example.com
+      uri: example.internal:9000
+      external_uri: example.com
       key: <key>
       secret: <secret>
       token: <token>

model/storage_settings.go

@@ -26,6 +26,7 @@ type StorageSettings struct {
 	Region         string `json:"region" bson:"region"`
 	Bucket         string `json:"bucket" bson:"bucket"`
 	Uri            string `json:"uri" bson:"uri"`
+	ExternalUri    string `json:"external_uri" bson:"external_uri"`
 	Key            string `json:"key" bson:"key"`
 	Secret         string `json:"secret" bson:"secret"`
 	Token          string `json:"token" bson:"token"`
@@ -60,6 +61,7 @@ func (s StorageSettings) Validate() error {
 		validation.Field(&s.Key, validation.Length(5, 50)),
 		validation.Field(&s.Secret, validation.Length(5, 100)),
 		validation.Field(&s.Uri, validation.Length(3, 2000)),
+		validation.Field(&s.ExternalUri, validation.Length(3, 2000)),
 		validation.Field(&s.Token, validation.Length(5, 100)),
 	)
 }

store/mongo/datastore_mongo.go

@@ -390,6 +390,7 @@ const (
 	StorageKeyStorageSettingsKey            = "key"
 	StorageKeyStorageSettingsSecret         = "secret"
 	StorageKeyStorageSettingsURI            = "uri"
+	StorageKeyStorageSettingsExternalURI    = "external_uri"
 	StorageKeyStorageSettingsToken          = "token"
 	StorageKeyStorageSettingsForcePathStyle = "force_path_style"
 	StorageKeyStorageSettingsUseAccelerate  = "use_accelerate"
@@ -2279,6 +2280,7 @@ func (db *DataStoreMongo) SetStorageSettings(
 			StorageKeyStorageSettingsKey:            storageSettings.Key,
 			StorageKeyStorageSettingsSecret:         storageSettings.Secret,
 			StorageKeyStorageSettingsURI:            storageSettings.Uri,
+			StorageKeyStorageSettingsExternalURI:    storageSettings.ExternalUri,
 			StorageKeyStorageSettingsRegion:         storageSettings.Region,
 			StorageKeyStorageSettingsToken:          storageSettings.Token,
 			StorageKeyStorageSettingsForcePathStyle: storageSettings.ForcePathStyle,