From 85529d27dd76542235d948ec62c4ab7ff3bc6145 Mon Sep 17 00:00:00 2001 From: Andrei Lukyanchyk <125263040+andrei-lukyanchyk@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:03:31 +0200 Subject: [PATCH] CDI-623: support s3 bucket in origin group (#125) * CDI-623: upd CDNOriginGroup resource to support S3 storages * CDI-623: add examples of S3 CDNOriginGroup * CDI-623: upd gcorelabscdn version * CDI-623: upd Origin Group docs --- docs/resources/cdn_origingroup.md | 43 +++- .../gcore_cdn_origingroup/resource.tf | 22 ++ gcore/resource_gcore_cdn_origin_group.go | 214 ++++++++++++++++-- go.mod | 8 +- go.sum | 2 + 5 files changed, 270 insertions(+), 19 deletions(-) diff --git a/docs/resources/cdn_origingroup.md b/docs/resources/cdn_origingroup.md index a6353d1e..2cf5b824 100644 --- a/docs/resources/cdn_origingroup.md +++ b/docs/resources/cdn_origingroup.md @@ -30,6 +30,28 @@ resource "gcore_cdn_origingroup" "origin_group_1" { backup = true } } + +resource "gcore_cdn_origingroup" "amazon_s3_origin_group" { + name = "amazon_s3_origin_group" + auth { + s3_type = "amazon" + s3_access_key_id = "123*******************" + s3_secret_access_key = "123*******************" + s3_bucket_name = "bucket-name" + s3_region = "eu-south-2" + } +} + +resource "gcore_cdn_origingroup" "other_s3_origin_group" { + name = "other_s3_origin_group" + auth { + s3_type = "other" + s3_storage_hostname = "s3.example.com" + s3_access_key_id = "123*******************" + s3_secret_access_key = "123*******************" + s3_bucket_name = "bucket-name" + } +} ``` @@ -38,17 +60,34 @@ resource "gcore_cdn_origingroup" "origin_group_1" { ### Required - `name` (String) Name of the origin group -- `origin` (Block Set, Min: 1) Contains information about all IP address or Domain names of your origin and the port if custom (see [below for nested schema](#nestedblock--origin)) -- `use_next` (Boolean) This options have two possible values: true — The option is active. In case the origin responds with 4XX or 5XX codes, use the next origin from the list. false — The option is disabled. ### Optional +- `auth` (Block List, Max: 1) Authentication configuration for S3 storage. This field is required unless `origin` is specified. `auth` and `origin` cannot both be specified simultaneously. (see [below for nested schema](#nestedblock--auth)) +- `origin` (Block Set) Contains information about all IP address or Domain names of your origin and the port if custom. This field is required unless `auth` is specified. `origin` and `auth` cannot both be specified simultaneously. (see [below for nested schema](#nestedblock--origin)) - `proxy_next_upstream` (Set of String) Available values: error, timeout, invalid_header, http_403, http_404, http_429, http_500, http_502, http_503, http_504. +- `use_next` (Boolean) This options have two possible values: true — The option is active. In case the origin responds with 4XX or 5XX codes, use the next origin from the list. false — The option is disabled. ### Read-Only - `id` (String) The ID of this resource. + +### Nested Schema for `auth` + +Required: + +- `s3_access_key_id` (String, Sensitive) Access key ID for the S3 storage +- `s3_bucket_name` (String) Bucket name of the S3 storage +- `s3_secret_access_key` (String, Sensitive) Secret access key for the S3 storage +- `s3_type` (String) Type of the S3 storage, accepted values: 'other' or 'amazon' + +Optional: + +- `s3_region` (String) Region of the S3 storage, required if s3_type is 'amazon' +- `s3_storage_hostname` (String) Hostname of the S3 storage, required if s3_type is 'other' + + ### Nested Schema for `origin` diff --git a/examples/resources/gcore_cdn_origingroup/resource.tf b/examples/resources/gcore_cdn_origingroup/resource.tf index 8636eee7..61f5d79c 100644 --- a/examples/resources/gcore_cdn_origingroup/resource.tf +++ b/examples/resources/gcore_cdn_origingroup/resource.tf @@ -15,3 +15,25 @@ resource "gcore_cdn_origingroup" "origin_group_1" { backup = true } } + +resource "gcore_cdn_origingroup" "amazon_s3_origin_group" { + name = "amazon_s3_origin_group" + auth { + s3_type = "amazon" + s3_access_key_id = "123*******************" + s3_secret_access_key = "123*******************" + s3_bucket_name = "bucket-name" + s3_region = "eu-south-2" + } +} + +resource "gcore_cdn_origingroup" "other_s3_origin_group" { + name = "other_s3_origin_group" + auth { + s3_type = "other" + s3_storage_hostname = "s3.example.com" + s3_access_key_id = "123*******************" + s3_secret_access_key = "123*******************" + s3_bucket_name = "bucket-name" + } +} diff --git a/gcore/resource_gcore_cdn_origin_group.go b/gcore/resource_gcore_cdn_origin_group.go index 56f5cce2..ba2f3ba1 100644 --- a/gcore/resource_gcore_cdn_origin_group.go +++ b/gcore/resource_gcore_cdn_origin_group.go @@ -12,6 +12,7 @@ import ( "github.com/G-Core/gcorelabscdn-go/origingroups" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" ) func resourceCDNOriginGroup() *schema.Resource { @@ -27,13 +28,21 @@ func resourceCDNOriginGroup() *schema.Resource { }, "use_next": { Type: schema.TypeBool, - Required: true, + Optional: true, + Default: true, Description: "This options have two possible values: true — The option is active. In case the origin responds with 4XX or 5XX codes, use the next origin from the list. false — The option is disabled.", }, + "proxy_next_upstream": { + Type: schema.TypeSet, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + Computed: true, + Description: "Available values: error, timeout, invalid_header, http_403, http_404, http_429, http_500, http_502, http_503, http_504.", + }, "origin": { Type: schema.TypeSet, - Required: true, - Description: "Contains information about all IP address or Domain names of your origin and the port if custom", + Optional: true, + Description: "Contains information about all IP address or Domain names of your origin and the port if custom. This field is required unless `auth` is specified. `origin` and `auth` cannot both be specified simultaneously.", Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "source": { @@ -56,12 +65,48 @@ func resourceCDNOriginGroup() *schema.Resource { }, }, }, - "proxy_next_upstream": { - Type: schema.TypeSet, - Elem: &schema.Schema{Type: schema.TypeString}, + "auth": { + Type: schema.TypeList, Optional: true, - Computed: true, - Description: "Available values: error, timeout, invalid_header, http_403, http_404, http_429, http_500, http_502, http_503, http_504.", + MaxItems: 1, + Description: "Authentication configuration for S3 storage. This field is required unless `origin` is specified. `auth` and `origin` cannot both be specified simultaneously.", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "s3_type": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"other", "amazon"}, false), + Description: "Type of the S3 storage, accepted values: 'other' or 'amazon'", + }, + "s3_storage_hostname": { + Type: schema.TypeString, + Optional: true, + Description: "Hostname of the S3 storage, required if s3_type is 'other'", + }, + "s3_access_key_id": { + Type: schema.TypeString, + Required: true, + Sensitive: true, + Description: "Access key ID for the S3 storage", + }, + "s3_secret_access_key": { + Type: schema.TypeString, + Required: true, + Sensitive: true, + Description: "Secret access key for the S3 storage", + }, + "s3_region": { + Type: schema.TypeString, + Optional: true, + Description: "Region of the S3 storage, required if s3_type is 'amazon'", + }, + "s3_bucket_name": { + Type: schema.TypeString, + Required: true, + Description: "Bucket name of the S3 storage", + }, + }, + }, }, }, CreateContext: resourceCDNOriginGroupCreate, @@ -69,9 +114,46 @@ func resourceCDNOriginGroup() *schema.Resource { UpdateContext: resourceCDNOriginGroupUpdate, DeleteContext: resourceCDNOriginGroupDelete, Description: "Represent origin group", + CustomizeDiff: validateCDNOriginGroupConfig, } } +func validateCDNOriginGroupConfig(ctx context.Context, diff *schema.ResourceDiff, v interface{}) error { + _, originExists := diff.GetOk("origin") + authRaw, authExists := diff.GetOk("auth") + + if !originExists && !authExists { + return fmt.Errorf("One of `origin` or `auth` must be specified") + } + + if originExists && authExists { + return fmt.Errorf("Both `origin` and `auth` cannot be specified at the same time") + } + + if authExists { + authList := authRaw.([]interface{}) + + if len(authList) > 0 { + auth := authList[0].(map[string]interface{}) + s3Type := auth["s3_type"].(string) + + if s3Type == "other" { + if storageHostname, ok := auth["s3_storage_hostname"].(string); !ok || storageHostname == "" { + return fmt.Errorf("`s3_storage_hostname` is required when `s3_type` is 'other'") + } + } + + if s3Type == "amazon" { + if s3Region, ok := auth["s3_region"].(string); !ok || s3Region == "" { + return fmt.Errorf("`s3_region` is required when `s3_type` is 'amazon'") + } + } + } + } + + return nil +} + func resourceCDNOriginGroupCreate(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { log.Println("[DEBUG] Start CDN OriginGroup creating") config := m.(*Config) @@ -80,7 +162,15 @@ func resourceCDNOriginGroupCreate(ctx context.Context, d *schema.ResourceData, m var req origingroups.GroupRequest req.Name = d.Get("name").(string) req.UseNext = d.Get("use_next").(bool) - req.Sources = setToSourceRequests(d.Get("origin").(*schema.Set)) + + if originSet, ok := d.GetOk("origin"); ok { + req.AuthType = "none" + req.Sources = setToSourceRequests(originSet.(*schema.Set)) + } else { + req.AuthType = "awsSignatureV4" + req.Auth = listToAuthS3(d.Get("auth").([]interface{})) + req.Sources = nil + } proxyNextUpstream, ok := d.Get("proxy_next_upstream").(*schema.Set) if ok && proxyNextUpstream.Len() > 0 { @@ -96,7 +186,16 @@ func resourceCDNOriginGroupCreate(ctx context.Context, d *schema.ResourceData, m } d.SetId(fmt.Sprintf("%d", result.ID)) - resourceCDNOriginGroupRead(ctx, d, m) + diags := resourceCDNOriginGroupRead(ctx, d, m) + + if diags.HasError() { + return diags + } + + if _, ok := d.GetOk("auth"); ok { + d.Set("auth.0.s3_secret_access_key", req.Auth.S3SecretAccessKey) + d.Set("auth.0.s3_access_key_id", req.Auth.S3AccessKeyID) + } log.Printf("[DEBUG] Finish CDN OriginGroup creating (id=%d)\n", result.ID) return nil @@ -125,6 +224,24 @@ func resourceCDNOriginGroupRead(ctx context.Context, d *schema.ResourceData, m i } d.Set("proxy_next_upstream", result.ProxyNextUpstream) + // keep s3_secret_access_key and s3_access_key_id unchanged by API response + currentSecretAccessKey, keyExists := d.GetOk("auth.0.s3_secret_access_key") + currentAccessKeyID, keyIDExists := d.GetOk("auth.0.s3_access_key_id") + if err := d.Set("auth", authToList(result.Auth)); err != nil { + return diag.FromErr(err) + } + if keyExists && keyIDExists { + authList := d.Get("auth").([]interface{}) + if len(authList) > 0 { + authMap := authList[0].(map[string]interface{}) + authMap["s3_secret_access_key"] = currentSecretAccessKey + authMap["s3_access_key_id"] = currentAccessKeyID + if err := d.Set("auth", []interface{}{authMap}); err != nil { + return diag.FromErr(err) + } + } + } + log.Println("[DEBUG] Finish CDN OriginGroup reading") return nil } @@ -143,7 +260,15 @@ func resourceCDNOriginGroupUpdate(ctx context.Context, d *schema.ResourceData, m var req origingroups.GroupRequest req.Name = d.Get("name").(string) req.UseNext = d.Get("use_next").(bool) - req.Sources = setToSourceRequests(d.Get("origin").(*schema.Set)) + + if originSet, ok := d.GetOk("origin"); ok { + req.AuthType = "none" + req.Sources = setToSourceRequests(originSet.(*schema.Set)) + } else { + req.AuthType = "awsSignatureV4" + req.Auth = listToAuthS3(d.Get("auth").([]interface{})) + req.Sources = nil + } if req.UseNext == true { proxyNextUpstream, ok := d.Get("proxy_next_upstream").(*schema.Set) @@ -159,9 +284,26 @@ func resourceCDNOriginGroupUpdate(ctx context.Context, d *schema.ResourceData, m return diag.FromErr(err) } - log.Println("[DEBUG] Finish CDN OriginGroup updating") + diags := resourceCDNOriginGroupRead(ctx, d, m) + + if diags.HasError() { + return diags + } + + if authList, ok := d.GetOk("auth"); ok { + if len(authList.([]interface{})) > 0 { + authConfig := authList.([]interface{})[0].(map[string]interface{}) + if secretAccessKey, ok := authConfig["s3_secret_access_key"].(string); ok { + d.Set("s3_secret_access_key", secretAccessKey) + } + if accessKeyID, ok := authConfig["s3_access_key_id"].(string); ok { + d.Set("s3_access_key_id", accessKeyID) + } + } + } - return resourceCDNOriginGroupRead(ctx, d, m) + log.Println("[DEBUG] Finish CDN OriginGroup updating") + return nil } func resourceCDNOriginGroupDelete(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { @@ -232,3 +374,49 @@ func originSetIDFunc(i interface{}) int { return int(binary.BigEndian.Uint64(h.Sum(nil))) } + +func listToAuthS3(authList []interface{}) *origingroups.AuthS3 { + if len(authList) == 0 { + return nil + } + + authConfig := authList[0].(map[string]interface{}) + + auth := &origingroups.AuthS3{ + S3Type: authConfig["s3_type"].(string), + S3AccessKeyID: authConfig["s3_access_key_id"].(string), + S3SecretAccessKey: authConfig["s3_secret_access_key"].(string), + S3BucketName: authConfig["s3_bucket_name"].(string), + } + + if s3StorageHostname, ok := authConfig["s3_storage_hostname"]; ok { + auth.S3StorageHostname = s3StorageHostname.(string) + } + + if s3Region, ok := authConfig["s3_region"]; ok { + auth.S3Region = s3Region.(string) + } + + return auth +} + +func authToList(auth *origingroups.AuthS3) []interface{} { + if auth == nil { + return nil + } + + authMap := map[string]interface{}{ + "s3_type": auth.S3Type, + "s3_bucket_name": auth.S3BucketName, + } + + if auth.S3StorageHostname != "" { + authMap["s3_storage_hostname"] = auth.S3StorageHostname + } + + if auth.S3Region != "" { + authMap["s3_region"] = auth.S3Region + } + + return []interface{}{authMap} +} diff --git a/go.mod b/go.mod index 5f15099c..b43341ba 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/AlekSi/pointer v1.2.0 github.com/G-Core/gcore-dns-sdk-go v0.2.9 github.com/G-Core/gcore-storage-sdk-go v0.1.34 - github.com/G-Core/gcorelabscdn-go v1.0.15 + github.com/G-Core/gcorelabscdn-go v1.0.16 github.com/G-Core/gcorelabscloud-go v0.8.4 github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 github.com/hashicorp/terraform-plugin-sdk/v2 v2.27.0 @@ -84,15 +84,15 @@ require ( golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect - google.golang.org/grpc v1.56.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/grpc v1.56.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( github.com/go-openapi/runtime v0.26.0 // indirect - github.com/hashicorp/terraform v1.5.7 + github.com/hashicorp/terraform v1.5.2 github.com/hashicorp/terraform-exec v0.21.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect go.mongodb.org/mongo-driver v1.12.0 // indirect diff --git a/go.sum b/go.sum index 0120820c..39c7b33e 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/G-Core/gcorelabscdn-go v1.0.14 h1:s34XWrMeuR/TvmnN0jrb6vsC9IzQFGFhb+q github.com/G-Core/gcorelabscdn-go v1.0.14/go.mod h1:iSGXaTvZBzDHQW+rKFS918BgFVpONcyLEijwh8WsXpE= github.com/G-Core/gcorelabscdn-go v1.0.15 h1:KIsZk2gadIlX3kSQJNRHS+HMtabHsdw0f0ARJSYlcxI= github.com/G-Core/gcorelabscdn-go v1.0.15/go.mod h1:iSGXaTvZBzDHQW+rKFS918BgFVpONcyLEijwh8WsXpE= +github.com/G-Core/gcorelabscdn-go v1.0.16 h1:Sxr/8krN/dMikoFd4lYQjJKGbK9LlsnKnkTpLKmSsCo= +github.com/G-Core/gcorelabscdn-go v1.0.16/go.mod h1:iSGXaTvZBzDHQW+rKFS918BgFVpONcyLEijwh8WsXpE= github.com/G-Core/gcorelabscloud-go v0.8.0 h1:6w+Mikiz+GbHJs1PD+tPD1gIR88Xl3UPkJuvQVuG7bs= github.com/G-Core/gcorelabscloud-go v0.8.0/go.mod h1:13Z1USxlxPbDFuYQyWqfNexlk4kUvOYTXbnvV/Z1lZo= github.com/G-Core/gcorelabscloud-go v0.8.4 h1:Yf0c0ZFOTxu0VjMgMVooHp2k2fYYKLLU5jqFRuEVEcg=