From b0fcceb66188b6cfce280dc8084e0fc96a3fc467 Mon Sep 17 00:00:00 2001 From: lgtm <1gtm@users.noreply.github.com> Date: Fri, 9 Feb 2024 01:44:57 -0800 Subject: [PATCH] Add support Disabling TLS Certificate Verification for Secure S3 Storage (#2017) (#2027) Signed-off-by: Anisur Rahman --- go.mod | 2 +- go.sum | 4 +-- pkg/backup.go | 1 + pkg/restore.go | 1 + vendor/modules.txt | 2 +- .../v1alpha1/openapi_generated.go | 6 +++++ .../apis/stash/v1alpha1/openapi_generated.go | 6 +++++ .../apis/stash/v1beta1/openapi_generated.go | 6 +++++ .../apis/ui/v1alpha1/openapi_generated.go | 6 +++++ .../stash.appscode.com_backupblueprints.yaml | 2 ++ .../crds/stash.appscode.com_repositories.yaml | 2 ++ .../apimachinery/pkg/restic/commands.go | 23 +++++++++++++++++ .../apimachinery/pkg/restic/config.go | 1 + .../apimachinery/pkg/util/addon.go | 25 +++++++++++++++++-- 14 files changed, 81 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d17f6c350..6ecd1e81b 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( kmodules.xyz/custom-resources v0.29.0 kmodules.xyz/offshoot-api v0.29.0 kubedb.dev/apimachinery v0.40.1-0.20240101000103-032b27211164 - stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2 + stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8 ) require ( diff --git a/go.sum b/go.sum index c757a93a4..6be9a6ae8 100644 --- a/go.sum +++ b/go.sum @@ -558,5 +558,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= -stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2 h1:dePrbjp7o57sKe33K1ppaWQK/Ely4QgxmCcab0sDOpY= -stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2/go.mod h1:hTslVqyx20fF2i2s/m0rqXD+pZwnI2oG3k5zPzsDnXQ= +stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8 h1:p3kd1saM5Ehz9mkqJhVrcFDhY0Bjv8S0XYuq2dZb3KI= +stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8/go.mod h1:cPdkM9Upe5hxgxrV+WlIQCq7K0Tpx//VUwFs0NKB7ek= diff --git a/pkg/backup.go b/pkg/backup.go index c2df415c7..dcfd1284f 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -173,6 +173,7 @@ func NewCmdBackup() *cobra.Command { cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)") cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)") cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL") + cmd.Flags().BoolVar(&opt.setupOptions.InsecureTLS, "insecure-tls", opt.setupOptions.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend") cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend") cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored") cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory") diff --git a/pkg/restore.go b/pkg/restore.go index 4f9e740bc..85a01c1b6 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -134,6 +134,7 @@ func NewCmdRestore() *cobra.Command { cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)") cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)") cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL") + cmd.Flags().BoolVar(&opt.setupOptions.InsecureTLS, "insecure-tls", opt.setupOptions.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend") cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend") cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored") cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory") diff --git a/vendor/modules.txt b/vendor/modules.txt index b0e210bc9..dd4f60b0b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -705,7 +705,7 @@ sigs.k8s.io/structured-merge-diff/v4/value ## explicit; go 1.12 sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 -# stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2 +# stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8 ## explicit; go 1.21.5 stash.appscode.dev/apimachinery/apis stash.appscode.dev/apimachinery/apis/repositories diff --git a/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go index dc8ba7d2d..f3cc64e39 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/repositories/v1alpha1/openapi_generated.go @@ -19446,6 +19446,12 @@ func schema_kmodulesxyz_objectstore_api_api_v1_S3Spec(ref common.ReferenceCallba Format: "", }, }, + "insecureTLS": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"endpoint", "bucket"}, }, diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go index 6790e1e02..5c383e8c5 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1alpha1/openapi_generated.go @@ -19451,6 +19451,12 @@ func schema_kmodulesxyz_objectstore_api_api_v1_S3Spec(ref common.ReferenceCallba Format: "", }, }, + "insecureTLS": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"endpoint", "bucket"}, }, diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go index 4f81a6742..9ea7459b1 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/openapi_generated.go @@ -19497,6 +19497,12 @@ func schema_kmodulesxyz_objectstore_api_api_v1_S3Spec(ref common.ReferenceCallba Format: "", }, }, + "insecureTLS": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"endpoint", "bucket"}, }, diff --git a/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go b/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go index 057cfd604..95e8ba721 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/ui/v1alpha1/openapi_generated.go @@ -19446,6 +19446,12 @@ func schema_kmodulesxyz_objectstore_api_api_v1_S3Spec(ref common.ReferenceCallba Format: "", }, }, + "insecureTLS": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, }, Required: []string{"endpoint", "bucket"}, }, diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupblueprints.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupblueprints.yaml index b700f6bc2..e35c1d971 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupblueprints.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupblueprints.yaml @@ -1794,6 +1794,8 @@ spec: type: string endpoint: type: string + insecureTLS: + type: boolean prefix: type: string region: diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_repositories.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_repositories.yaml index 879a6435a..0321c1a7a 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_repositories.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_repositories.yaml @@ -1801,6 +1801,8 @@ spec: type: string endpoint: type: string + insecureTLS: + type: boolean prefix: type: string region: diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/restic/commands.go b/vendor/stash.appscode.dev/apimachinery/pkg/restic/commands.go index 2723ec351..fd4a83774 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/restic/commands.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/restic/commands.go @@ -80,6 +80,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error) result := make([]Snapshot, 0) args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--quiet", "--no-lock"}) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) for _, id := range snapshotIDs { args = append(args, id) @@ -95,6 +96,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error) func (w *ResticWrapper) deleteSnapshots(snapshotIDs []string) ([]byte, error) { args := w.appendCacheDirFlag([]interface{}{"forget", "--quiet", "--prune"}) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) for _, id := range snapshotIDs { args = append(args, id) @@ -107,6 +109,7 @@ func (w *ResticWrapper) repositoryExist() bool { klog.Infoln("Checking whether the backend repository exist or not....") args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--no-lock"}) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) if _, err := w.run(Command{Name: ResticCMD, Args: args}); err == nil { return true @@ -122,6 +125,7 @@ func (w *ResticWrapper) initRepository() error { args := w.appendCacheDirFlag([]interface{}{"init"}) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) _, err := w.run(Command{Name: ResticCMD, Args: args}) return err @@ -151,6 +155,7 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendCleanupCacheFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) @@ -174,6 +179,7 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendCleanupCacheFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) commands = append(commands, Command{Name: ResticCMD, Args: args}) @@ -248,6 +254,7 @@ func (w *ResticWrapper) tryCleanup(retentionPolicy v1alpha1.RetentionPolicy, hos if len(args) > 1 { args = w.appendCacheDirFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) @@ -294,6 +301,7 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) { } args = w.appendCacheDirFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) @@ -325,6 +333,7 @@ func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) // first add restic command, then add StdoutPipeCommands @@ -339,6 +348,7 @@ func (w *ResticWrapper) check() ([]byte, error) { klog.Infoln("Checking integrity of repository") args := w.appendCacheDirFlag([]interface{}{"check", "--no-lock"}) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) args = w.appendMaxConnectionsFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) @@ -353,6 +363,7 @@ func (w *ResticWrapper) stats(snapshotID string) ([]byte, error) { args = w.appendMaxConnectionsFlag(args) args = append(args, "--quiet", "--json", "--mode", "raw-data", "--no-lock") args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) } @@ -362,6 +373,7 @@ func (w *ResticWrapper) unlock() ([]byte, error) { args := w.appendCacheDirFlag([]interface{}{"unlock", "--remove-all"}) args = w.appendMaxConnectionsFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) } @@ -515,6 +527,7 @@ func (w *ResticWrapper) addKey(params keyParams) ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendMaxConnectionsFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) } @@ -527,6 +540,7 @@ func (w *ResticWrapper) listKey() ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendMaxConnectionsFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) } @@ -543,6 +557,7 @@ func (w *ResticWrapper) updateKey(params keyParams) ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendMaxConnectionsFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) } @@ -555,6 +570,14 @@ func (w *ResticWrapper) removeKey(params keyParams) ([]byte, error) { args = w.appendCacheDirFlag(args) args = w.appendMaxConnectionsFlag(args) args = w.appendCaCertFlag(args) + args = w.appendInsecureTLSFlag(args) return w.run(Command{Name: ResticCMD, Args: args}) } + +func (w *ResticWrapper) appendInsecureTLSFlag(args []interface{}) []interface{} { + if w.config.InsecureTLS { + return append(args, "--insecure-tls") + } + return args +} diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/restic/config.go b/vendor/stash.appscode.dev/apimachinery/pkg/restic/config.go index a070db441..11b295fab 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/restic/config.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/restic/config.go @@ -87,6 +87,7 @@ type SetupOptions struct { CacertFile string ScratchDir string EnableCache bool + InsecureTLS bool MaxConnections int64 StorageSecret *core.Secret Nice *ofst.NiceSettings diff --git a/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go b/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go index cff35cbf4..bb8d762c4 100644 --- a/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go +++ b/vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go @@ -60,8 +60,8 @@ func ExtractAddonInfo(appClient appcatalog_cs.Interface, task v1beta1.TaskRef, t addon.RestoreTask.Name = task.Name } if len(task.Params) != 0 { - addon.BackupTask.Params = getTaskParams(task) - addon.RestoreTask.Params = getTaskParams(task) + addon.BackupTask.Params = upsertParams(addon.BackupTask.Params, getTaskParams(task)) + addon.RestoreTask.Params = upsertParams(addon.BackupTask.Params, getTaskParams(task)) } return &addon, nil @@ -75,3 +75,24 @@ func getTaskParams(task v1beta1.TaskRef) []appcat.Param { } return params } + +func upsertParams(oldParams, newParams []appcat.Param) []appcat.Param { + paramMap := make(map[string]appcat.Param) + + for _, newParam := range newParams { + paramMap[newParam.Name] = newParam + } + + for _, oldParam := range oldParams { + if _, found := paramMap[oldParam.Name]; !found { + paramMap[oldParam.Name] = oldParam + } + } + + var updatedParams []appcat.Param + for _, param := range paramMap { + updatedParams = append(updatedParams, param) + } + + return updatedParams +}