feat(storage): add s3 external uri

changelog: Allow to use S3 Exernal URI

Signed-off-by: Benjamin Texier <[email protected]>

api/http/api_deployments_test.go

@@ -1468,6 +1468,7 @@ func TestGetTenantStorageSettings(t *testing.T) {
  app := &mapp.App{}
  app.On("GetStorageSettings",
  mock.MatchedBy(func(ctx context.Context) bool { return true }),
+ false,
  ).Return(tc.settings, tc.err)
 
  restView := new(view.RESTView)

app/app.go

@@ -201,7 +201,7 @@ func (d *Deployments) HealthCheck(ctx context.Context) error {
  if err != nil {
  return errors.Wrap(err, "error reaching MongoDB")
  }
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return err
  }
@@ -232,17 +232,19 @@ func (d *Deployments) HealthCheck(ctx context.Context) error {
  return nil
 }
 
-func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error) {
+func (d *Deployments) getFileStorage(ctx context.Context, external bool) (s3.FileStorage, error) {
  settings, err := d.db.GetStorageSettings(ctx)
  if err != nil {
  return nil, err
  } else if settings != nil && settings.Bucket != "" {
  region := config.Config.GetString(dconfig.SettingAwsS3Region)
  key := config.Config.GetString(dconfig.SettingAwsAuthKeyId)
  secret := config.Config.GetString(dconfig.SettingAwsAuthSecret)
- uri := config.Config.GetString(dconfig.SettingAwsURI)
+ internalUri := config.Config.GetString(dconfig.SettingAwsURI)
+ externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI)
  token := config.Config.GetString(dconfig.SettingAwsAuthToken)
  tagArtifact := config.Config.GetBool(dconfig.SettingsAwsTagArtifact)
+
  if settings.Region != "" {
  region = settings.Region
  }
@@ -253,12 +255,17 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error
  secret = settings.Secret
  }
  if settings.Uri != "" {
- uri = settings.Uri
+ internalUri = settings.Uri
  }
  if settings.Token != "" {
  token = settings.Token
  }
 
+ uri := internalUri
+ if external && externalUri != "" {
+ uri = externalUri
+ }
+
  return s3.NewSimpleStorageServiceStatic(
  settings.Bucket,
  key,
@@ -272,6 +279,21 @@ func (d *Deployments) getFileStorage(ctx context.Context) (s3.FileStorage, error
  )
  }
 
+ externalUri := config.Config.GetString(dconfig.SettingAwsExternalURI)
+
+ if external && externalUri != "" {
+ return s3.NewSimpleStorageServiceStatic(
+ config.Config.GetString(dconfig.SettingAwsS3Bucket),
+ config.Config.GetString(dconfig.SettingAwsAuthKeyId),
+ config.Config.GetString(dconfig.SettingAwsAuthSecret),
+ config.Config.GetString(dconfig.SettingAwsS3Region),
+ config.Config.GetString(dconfig.SettingAwsAuthToken),
+ externalUri,
+ config.Config.GetBool(dconfig.SettingsAwsTagArtifact),
+ config.Config.GetBool(dconfig.SettingAwsS3ForcePathStyle),
+ config.Config.GetBool(dconfig.SettingAwsS3UseAccelerate),
+ )
+ }
  return d.fileStorage, nil
 }
 
@@ -309,7 +331,7 @@ func (d *Deployments) CreateImage(ctx context.Context,
  artifactID, err := d.handleArtifact(ctx, multipartUploadMsg)
  // try to remove artifact file from file storage on error
  if err != nil {
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return "", err
  }
@@ -354,7 +376,7 @@ func (d *Deployments) handleArtifact(ctx context.Context,
  //nolint:errcheck
  go func() (err error) {
  defer func() { ch <- err }()
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return err
  }
@@ -443,10 +465,11 @@ func (d *Deployments) GenerateImage(ctx context.Context,
  multipartGenerateImageMsg.TenantID = id.Tenant
  }
 
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return "", err
  }
+
  link, err := fileStorage.GetRequest(
  ctx,
  imgID,
@@ -551,7 +574,7 @@ func (d *Deployments) handleRawFile(ctx context.Context,
  LimitError: ErrModelArtifactFileTooLarge,
  }
 
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return "", err
  }
@@ -611,7 +634,7 @@ func (d *Deployments) DeleteImage(ctx context.Context, imageID string) error {
 
  // Delete image file (call to external service)
  // Noop for not existing file
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return err
  }
@@ -695,7 +718,7 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string,
  return nil, nil
  }
 
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, false)
  if err != nil {
  return nil, err
  }
@@ -710,8 +733,18 @@ func (d *Deployments) DownloadLink(ctx context.Context, imageID string,
  }
 
  fileName := image.ArtifactMeta.Name + ".mender"
- link, err := fileStorage.GetRequest(ctx, imageID,
+
+ externalFileStorage, err := d.getFileStorage(ctx, true)
+ if err != nil {
+ return nil, err
+ }
+
+ link, err := externalFileStorage.GetRequest(ctx, imageID,
  expire, ArtifactContentType, fileName)
+ if err != nil {
+ return nil, errors.Wrap(err, "Searching for image file")
+ }
+
  if err != nil {
  return nil, errors.Wrap(err, "Generating download link")
  }
@@ -1370,7 +1403,7 @@ func (d *Deployments) GetDeploymentForDeviceWithCurrent(ctx context.Context, dev
  return nil, nil
  }
 
- fileStorage, err := d.getFileStorage(ctx)
+ fileStorage, err := d.getFileStorage(ctx, true)
  if err != nil {
  return nil, err
  }

app/images_test.go

@@ -145,7 +145,7 @@ func TestGenerateImageErrorWhileUploading(t *testing.T) {
  ).Return(errors.New("error while uploading"))
 
  db.On("GetStorageSettings",
- ctx,
+ ctx, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),
@@ -200,7 +200,7 @@ func TestGenerateImageErrorS3GetRequest(t *testing.T) {
  ).Return(true, nil)
 
  db.On("GetStorageSettings",
- ctx,
+ ctx, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),
@@ -257,7 +257,7 @@ func TestGenerateImageErrorS3DeleteRequest(t *testing.T) {
  ).Return(true, nil)
 
  db.On("GetStorageSettings",
- ctx,
+ ctx, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),
@@ -354,7 +354,7 @@ func TestGenerateImageErrorWhileStartingWorkflow(t *testing.T) {
  ).Return(true, nil)
 
  db.On("GetStorageSettings",
- ctx,
+ ctx, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),
@@ -436,7 +436,7 @@ func TestGenerateImageErrorWhileStartingWorkflowAndFailsWhenCleaningUp(t *testin
  ).Return(true, nil)
 
  db.On("GetStorageSettings",
- ctx,
+ ctx, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),
@@ -522,7 +522,7 @@ func TestGenerateImageSuccessful(t *testing.T) {
  ).Return(true, nil)
 
  db.On("GetStorageSettings",
- ctx,
+ ctx, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),
@@ -599,7 +599,7 @@ func TestGenerateImageSuccessfulWithTenant(t *testing.T) {
  ctxWithIdentity := identity.WithContext(ctx, identityObject)
 
  db.On("GetStorageSettings",
- ctxWithIdentity,
+ ctxWithIdentity, false,
  ).Return(&model.StorageSettings{
  Region: config.Config.GetString(dconfig.SettingAwsS3Region),
  Uri: config.Config.GetString(dconfig.SettingAwsURI),

app/settings_test.go

@@ -59,6 +59,7 @@ func TestGetStorageSettings(t *testing.T) {
  db := mocks.DataStore{}
  db.On("GetStorageSettings",
  mock.MatchedBy(func(ctx context.Context) bool { return true }),
+ false,
  ).Return(tc.settings, tc.err)
  ctx := context.Background()
 

config/config.go

@@ -43,6 +43,7 @@ const (
  SettingAwsS3MaxImageSize = SettingsAws + ".max_image_size"
  SettingAwsS3MaxImageSizeDefault = 10737418240
  SettingAwsURI = SettingsAws + ".uri"
+ SettingAwsExternalURI = SettingsAws + ".external_uri"
  SettingsAwsTagArtifact = SettingsAws + ".tag_artifact"
  SettingsAwsTagArtifactDefault = false
 

docs/internal_api.yml

@@ -441,6 +441,9 @@ definitions:
  uri:
  type: string
  description: Endpoint URI.
+ external_uri:
+ type: string
+ description: External Endpoint URI (default: uri).
  key:
  type: string
  description: Access key id (for S3 - AWS_ACCESS_KEY_ID).
@@ -464,7 +467,8 @@ definitions:
  example:
  region: us-east-1
  bucket: mender-artifacts-unique-bucket-name
- uri: example.com
+ uri: example.internal:9000
+ external_uri: example.com
  key: <key>
  secret: <secret>
  token: <token>

model/storage_settings.go

@@ -26,6 +26,7 @@ type StorageSettings struct {
  Region string `json:"region" bson:"region"`
  Bucket string `json:"bucket" bson:"bucket"`
  Uri string `json:"uri" bson:"uri"`
+ ExternalUri string `json:"external_uri" bson:"external_uri"`
  Key string `json:"key" bson:"key"`
  Secret string `json:"secret" bson:"secret"`
  Token string `json:"token" bson:"token"`
@@ -60,6 +61,7 @@ func (s StorageSettings) Validate() error {
  validation.Field(&s.Key, validation.Length(5, 50)),
  validation.Field(&s.Secret, validation.Length(5, 100)),
  validation.Field(&s.Uri, validation.Length(3, 2000)),
+ validation.Field(&s.ExternalUri, validation.Length(3, 2000)),
  validation.Field(&s.Token, validation.Length(5, 100)),
  )
 }

store/mongo/datastore_mongo.go

@@ -390,6 +390,7 @@ const (
  StorageKeyStorageSettingsKey = "key"
  StorageKeyStorageSettingsSecret = "secret"
  StorageKeyStorageSettingsURI = "uri"
+ StorageKeyStorageSettingsExternalURI = "external_uri"
  StorageKeyStorageSettingsToken = "token"
  StorageKeyStorageSettingsForcePathStyle = "force_path_style"
  StorageKeyStorageSettingsUseAccelerate = "use_accelerate"
@@ -2279,6 +2280,7 @@ func (db *DataStoreMongo) SetStorageSettings(
  StorageKeyStorageSettingsKey: storageSettings.Key,
  StorageKeyStorageSettingsSecret: storageSettings.Secret,
  StorageKeyStorageSettingsURI: storageSettings.Uri,
+ StorageKeyStorageSettingsExternalURI: storageSettings.ExternalUri,
  StorageKeyStorageSettingsRegion: storageSettings.Region,
  StorageKeyStorageSettingsToken: storageSettings.Token,
  StorageKeyStorageSettingsForcePathStyle: storageSettings.ForcePathStyle,