Add disabling TLS Certificate Verification support for Secure S3 Stor…

…age (#1554) Signed-off-by: Anisur Rahman <[email protected]>
stashed · Feb 9, 2024 · 8fa0835 · 8fa0835
1 parent 6d5ad36
commit 8fa0835
Show file tree

Hide file tree

Showing 24 changed files with 196 additions and 62 deletions.
diff --git a/go.mod b/go.mod
@@ -39,13 +39,13 @@ require (
  kmodules.xyz/constants v0.0.0-20230304030334-d2d1f28732a5
  kmodules.xyz/csi-utils v0.25.4
  kmodules.xyz/custom-resources v0.29.0
- kmodules.xyz/objectstore-api v0.29.0
+ kmodules.xyz/objectstore-api v0.29.1
  kmodules.xyz/offshoot-api v0.29.0
  kmodules.xyz/openshift v0.29.0
  kmodules.xyz/prober v0.29.0
  kmodules.xyz/webhook-runtime v0.29.0
  sigs.k8s.io/controller-runtime v0.16.3
- stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2
+ stash.appscode.dev/apimachinery v0.32.1-0.20240206075719-41610d0ce38f
 )
 
 require (

diff --git a/go.sum b/go.sum
@@ -1649,8 +1649,8 @@ kmodules.xyz/custom-resources v0.29.0 h1:RaDM2+wSVXiwIvLqmkTVYpwoH83AC8wruXe2p2r
 kmodules.xyz/custom-resources v0.29.0/go.mod h1:MzZyXtxdg1PDxGk3RTTO1Xv3KiVqZnIonSwmxVbagOY=
 kmodules.xyz/go-containerregistry v0.0.12 h1:Tl32QGmSqRVm9PUEb/f3dgDeu9zW5fVzt3qmAFIE37I=
 kmodules.xyz/go-containerregistry v0.0.12/go.mod h1:KgeNg0hDsgeda+qc0NzWk0iVRdF0+ZIg/oRzGoYh78I=
-kmodules.xyz/objectstore-api v0.29.0 h1:dK53fQXdoboyW/EyBBAMjykT8u7jstKrM1DS4RJvhEU=
-kmodules.xyz/objectstore-api v0.29.0/go.mod h1:Kxmv6F7Kd/7EoKX3X2xIzhHT++zlj2qdXLcp/8avUYI=
+kmodules.xyz/objectstore-api v0.29.1 h1:uUsjf8KU0w4LYowSEOnl0AbHT3hsHIu1wNLHqGe1o6s=
+kmodules.xyz/objectstore-api v0.29.1/go.mod h1:xG+5awH1SXYKxwN/+k1FEQvzixd5tgNqEN/1LEiB2FE=
 kmodules.xyz/offshoot-api v0.29.0 h1:GHLhxxT9jU1N8+FvOCCeJNyU5g0duYS46UGrs6AHNLY=
 kmodules.xyz/offshoot-api v0.29.0/go.mod h1:5NxhBblXoDHWStx9HCDJR2KFTwYjEZ7i1Id3jelIunw=
 kmodules.xyz/openshift v0.29.0 h1:8PXjeQ+usUGkLnYUMSbZrMg+i1OIYBe9UWOeBf2FRzU=
@@ -1684,7 +1684,7 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
-stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2 h1:dePrbjp7o57sKe33K1ppaWQK/Ely4QgxmCcab0sDOpY=
-stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2/go.mod h1:hTslVqyx20fF2i2s/m0rqXD+pZwnI2oG3k5zPzsDnXQ=
+stash.appscode.dev/apimachinery v0.32.1-0.20240206075719-41610d0ce38f h1:0B35Biy3T5cNWlfMKr2LPgDaXv6C7IeDBYHOeqBaLko=
+stash.appscode.dev/apimachinery v0.32.1-0.20240206075719-41610d0ce38f/go.mod h1:5ZunLyvEChKp4LpPJq8mTKQss3bsT93S/Tqu9BRvQTA=
 x-helm.dev/apimachinery v0.0.16 h1:Eb160xcdH9fMVHak5QSWYWxoaReytch+A7kk25QWjx0=
 x-helm.dev/apimachinery v0.0.16/go.mod h1:05brgFw5oWOX7OTXT090SQojqXjbttqWfqoJo+ejBU4=
diff --git a/pkg/cmds/backup_pvc.go b/pkg/cmds/backup_pvc.go
@@ -140,6 +140,7 @@ func NewCmdBackupPVC() *cobra.Command {
  cmd.Flags().StringVar(&opt.setupOpt.Provider, "provider", opt.setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
  cmd.Flags().StringVar(&opt.setupOpt.Bucket, "bucket", opt.setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
  cmd.Flags().StringVar(&opt.setupOpt.Endpoint, "endpoint", opt.setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+ cmd.Flags().BoolVar(&opt.setupOpt.InsecureTLS, "insecure-tls", opt.setupOpt.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
  cmd.Flags().StringVar(&opt.setupOpt.Region, "region", opt.setupOpt.Region, "Region for s3/s3 compatible backend")
  cmd.Flags().StringVar(&opt.setupOpt.Path, "path", opt.setupOpt.Path, "Directory inside the bucket where backed up data will be stored")
  cmd.Flags().StringVar(&opt.setupOpt.ScratchDir, "scratch-dir", opt.setupOpt.ScratchDir, "Temporary directory")

diff --git a/pkg/cmds/restore_pvc.go b/pkg/cmds/restore_pvc.go
@@ -113,6 +113,7 @@ func NewCmdRestorePVC() *cobra.Command {
  cmd.Flags().StringVar(&opt.setupOpt.Provider, "provider", opt.setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
  cmd.Flags().StringVar(&opt.setupOpt.Bucket, "bucket", opt.setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
  cmd.Flags().StringVar(&opt.setupOpt.Endpoint, "endpoint", opt.setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+ cmd.Flags().BoolVar(&opt.setupOpt.InsecureTLS, "insecure-tls", opt.setupOpt.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
  cmd.Flags().StringVar(&opt.setupOpt.Region, "region", opt.setupOpt.Region, "Region for s3/s3 compatible backend")
  cmd.Flags().StringVar(&opt.setupOpt.Path, "path", opt.setupOpt.Path, "Directory inside the bucket where backed up data will be stored")
  cmd.Flags().StringVar(&opt.setupOpt.ScratchDir, "scratch-dir", opt.setupOpt.ScratchDir, "Temporary directory")

diff --git a/pkg/cmds/update_status.go b/pkg/cmds/update_status.go
@@ -84,6 +84,7 @@ func NewCmdUpdateStatus() *cobra.Command {
  cmd.Flags().StringVar(&opt.SetupOpt.Provider, "provider", opt.SetupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
  cmd.Flags().StringVar(&opt.SetupOpt.Bucket, "bucket", opt.SetupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
  cmd.Flags().StringVar(&opt.SetupOpt.Endpoint, "endpoint", opt.SetupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+ cmd.Flags().BoolVar(&opt.SetupOpt.InsecureTLS, "insecure-tls", opt.SetupOpt.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
  cmd.Flags().StringVar(&opt.SetupOpt.Region, "region", opt.SetupOpt.Region, "Region for s3/s3 compatible backend")
  cmd.Flags().StringVar(&opt.SetupOpt.Path, "path", opt.SetupOpt.Path, "Directory inside the bucket where backed up data will be stored")
  cmd.Flags().StringVar(&opt.SetupOpt.ScratchDir, "scratch-dir", opt.SetupOpt.ScratchDir, "Temporary directory")

diff --git a/pkg/resolver/inputs.go b/pkg/resolver/inputs.go
@@ -164,8 +164,11 @@ func (r *TaskOptions) setRepositoryVariables() error {
  vars[apis.RepositorySecretName] = r.Repository.Spec.Backend.StorageSecretName
  vars[apis.RepositorySecretNamespace] = r.Repository.Namespace
  }
- if r.Repository.Spec.Backend.S3 != nil && r.Repository.Spec.Backend.S3.Endpoint != "" {
- vars[apis.RepositoryEndpoint] = r.Repository.Spec.Backend.S3.Endpoint
+ if r.Repository.Spec.Backend.S3 != nil {
+ vars[apis.RepositoryInsecureTLS] = strconv.FormatBool(r.Repository.Spec.Backend.S3.InsecureTLS)
+ if r.Repository.Spec.Backend.S3.Endpoint != "" {
+ vars[apis.RepositoryEndpoint] = r.Repository.Spec.Backend.S3.Endpoint
+ }
  }
  endpoint, found := r.Repository.Spec.Backend.Endpoint()
  if found {

diff --git a/pkg/util/crds.go b/pkg/util/crds.go
@@ -100,6 +100,7 @@ func updateStatusFunction(image docker.Docker) *api_v1beta1.Function {
  "--provider=${REPOSITORY_PROVIDER:=}",
  "--bucket=${REPOSITORY_BUCKET:=}",
  "--endpoint=${REPOSITORY_ENDPOINT:=}",
+ "--insecure-tls=${REPOSITORY_INSECURE_TLS:=}",
  "--region=${REPOSITORY_REGION:=}",
  "--path=${REPOSITORY_PREFIX:=}",
  "--enable-cache=${ENABLE_CACHE:=true}",
@@ -134,6 +135,7 @@ func pvcBackupFunction(image docker.Docker) *api_v1beta1.Function {
  "--provider=${REPOSITORY_PROVIDER:=}",
  "--bucket=${REPOSITORY_BUCKET:=}",
  "--endpoint=${REPOSITORY_ENDPOINT:=}",
+ "--insecure-tls=${REPOSITORY_INSECURE_TLS:=}",
  "--region=${REPOSITORY_REGION:=}",
  "--path=${REPOSITORY_PREFIX:=}",
  "--enable-cache=${ENABLE_CACHE:=true}",
@@ -184,6 +186,7 @@ func pvcRestoreFunction(image docker.Docker) *api_v1beta1.Function {
  "--provider=${REPOSITORY_PROVIDER:=}",
  "--bucket=${REPOSITORY_BUCKET:=}",
  "--endpoint=${REPOSITORY_ENDPOINT:=}",
+ "--insecure-tls=${REPOSITORY_INSECURE_TLS:=}",
  "--region=${REPOSITORY_REGION:=}",
  "--path=${REPOSITORY_PREFIX:=}",
  "--enable-cache=${ENABLE_CACHE:=true}",

diff --git a/pkg/util/options.go b/pkg/util/options.go
@@ -95,13 +95,15 @@ func SetupOptionsForRepository(repository api_v1alpha1.Repository, extraOpt Extr
  }
  endpoint, _ := repository.Spec.Backend.Endpoint()
  region, _ := repository.Spec.Backend.Region()
+ insecureTLS := repository.Spec.Backend.InsecureTLS()
 
  return restic.SetupOptions{
  Provider: provider,
  Bucket: bucket,
  Path: prefix,
  Endpoint: endpoint,
  Region: region,
+ InsecureTLS: insecureTLS,
  CacertFile: extraOpt.CacertFile,
  StorageSecret: extraOpt.StorageSecret,
  ScratchDir: extraOpt.ScratchDir,

diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/generated.pb.go b/vendor/kmodules.xyz/objectstore-api/api/v1/generated.pb.go
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/generated.proto b/vendor/kmodules.xyz/objectstore-api/api/v1/generated.proto
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go b/vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go
@@ -163,3 +163,11 @@ func (backend Backend) Region() (string, bool) {
  }
  return "", false
 }
+
+// InsecureTLS returns insecureTLS of S3/S3 compatible backend
+func (backend Backend) InsecureTLS() bool {
+ if backend.S3 != nil {
+ return backend.S3.InsecureTLS
+ }
+ return false
+}
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/openapi_generated.go b/vendor/kmodules.xyz/objectstore-api/api/v1/openapi_generated.go
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/types.go b/vendor/kmodules.xyz/objectstore-api/api/v1/types.go
@@ -98,10 +98,11 @@ type LocalSpec struct {
 }
 
 type S3Spec struct {
- Endpoint string `json:"endpoint" protobuf:"bytes,1,opt,name=endpoint"`
- Bucket string `json:"bucket" protobuf:"bytes,2,opt,name=bucket"`
- Prefix string `json:"prefix,omitempty" protobuf:"bytes,3,opt,name=prefix"`
- Region string `json:"region,omitempty" protobuf:"bytes,4,opt,name=region"`
+ Endpoint string `json:"endpoint" protobuf:"bytes,1,opt,name=endpoint"`
+ Bucket string `json:"bucket" protobuf:"bytes,2,opt,name=bucket"`
+ Prefix string `json:"prefix,omitempty" protobuf:"bytes,3,opt,name=prefix"`
+ Region string `json:"region,omitempty" protobuf:"bytes,4,opt,name=region"`
+ InsecureTLS bool `json:"insecureTLS,omitempty" protobuf:"varint,5,opt,name=insecureTLS"`
 }
 
 type GCSSpec struct {