From 25de160234c8540ae655d3a6019052ebba2e6c12 Mon Sep 17 00:00:00 2001 From: lgtm <1gtm@users.noreply.github.com> Date: Tue, 27 Aug 2024 04:56:02 -0700 Subject: [PATCH] Fix MongoDB url connection str for external databases (#2138) (#2144) Signed-off-by: Anisur Rahman Signed-off-by: sayedppqq Co-authored-by: sayedppqq --- pkg/backup.go | 77 +++++++++++++++++++++++++++++++++++++++----------- pkg/restore.go | 69 +++++++++++++++++++++++++++++++------------- pkg/utils.go | 63 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 35 deletions(-) diff --git a/pkg/backup.go b/pkg/backup.go index dcfd1284f..e5825e64f 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -267,11 +267,24 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic return nil, err } - port, err := appBinding.Port() - if err != nil { - return nil, err + var isSrv bool + port := int32(27017) + if appBinding.Spec.ClientConfig.URL != nil { + isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL) + if err != nil { + return nil, err + } } + // Checked for Altlas and DigitalOcean srv format connection string don't give port. + // mongodump not support both --uri and --port. + + if !isSrv { + port, err = appBinding.Port() + if err != nil { + return nil, err + } + } waitForDBReady(hostname, port, opt.waitTimeout) // unmarshal parameter is the field has value @@ -318,7 +331,12 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic } } + var tlsEnable bool if appBinding.Spec.ClientConfig.CABundle != nil { + tlsEnable = true + } + + if tlsEnable { if tlsSecret == nil { return nil, errors.Wrap(err, "spec.tlsSecret needs to be set in appbinding for TLS secured database.") } @@ -333,8 +351,8 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic } dumpCreds = []interface{}{ "--ssl", - "--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName), - "--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName), + fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)), + fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)), } // get certificate secret to get client certificate @@ -361,9 +379,9 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic return nil, errors.Wrap(err, "unable to get user from ssl.") } userAuth := []interface{}{ - "-u", user, - "--authenticationMechanism", "MONGODB-X509", - "--authenticationDatabase", "$external", + fmt.Sprintf("--username=%s", user), + "--authenticationMechanism=MONGODB-X509", + "--authenticationDatabase=$external", } mongoCreds = append(mongoCreds, userAuth...) dumpCreds = append(dumpCreds, userAuth...) @@ -372,7 +390,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic userAuth := []interface{}{ fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]), fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]), - "--authenticationDatabase", opt.authenticationDatabase, + fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase), } mongoCreds = append(mongoCreds, userAuth...) dumpCreds = append(dumpCreds, userAuth...) @@ -387,19 +405,32 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic BackupPaths: opt.defaultBackupOptions.BackupPaths, } + uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable) + // setup pipe command backupCmd := restic.Command{ Name: MongoDumpCMD, - Args: append([]interface{}{ - "--host", mongoDSN, + Args: []interface{}{ + "--uri", fmt.Sprintf("\"%s\"", uri), "--archive", - }, dumpCreds...), + }, } - userArgs := strings.Fields(opt.mongoArgs) - if isStandalone { - backupCmd.Args = append(backupCmd.Args, fmt.Sprintf("--port=%d", port)) - } else { + if tlsEnable { + backupCmd.Args = append(backupCmd.Args, + fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")), + fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile"))) + } + + var userArgs []string + for _, arg := range strings.Fields(opt.mongoArgs) { + // illegal argument combination: cannot specify --db and --uri + if !strings.Contains(arg, "--db") { + userArgs = append(userArgs, arg) + } + } + + if !isStandalone { // - port is already added in mongoDSN with replicasetName/host:port format. // - oplog is enabled automatically for replicasets. // Don't use --oplog if user specify any of these arguments through opt.mongoArgs @@ -558,6 +589,20 @@ func cleanup() { } } +func getOptionValue(args []interface{}, option string) string { + for _, arg := range args { + strArg, ok := arg.(string) + if !ok { + continue + } + // assuming value has '=' + if strings.HasPrefix(strArg, option+"=") { + return strings.TrimPrefix(strArg, option+"=") + } + } + return "" +} + func (opt *mongoOptions) getHostBackupStats(err error) []api_v1beta1.HostBackupStats { var backupStats []api_v1beta1.HostBackupStats diff --git a/pkg/restore.go b/pkg/restore.go index 85a01c1b6..046a844b9 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -204,9 +204,23 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti return nil, err } - port, err := appBinding.Port() - if err != nil { - return nil, err + var isSrv bool + port := int32(27017) + if appBinding.Spec.ClientConfig.URL != nil { + isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL) + if err != nil { + return nil, err + } + } + + // Checked for Altlas and DigitalOcean srv format connection string don't give port. + // mongodump --uri format not support port. + + if !isSrv { + port, err = appBinding.Port() + if err != nil { + return nil, err + } } // unmarshal parameter is the field has value @@ -249,8 +263,12 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti return nil, err } } - + var tlsEnable bool if appBinding.Spec.ClientConfig.CABundle != nil { + tlsEnable = true + } + + if tlsEnable { if err := os.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil { return nil, errors.Wrap(err, "failed to write key for CA certificate") } @@ -261,8 +279,8 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti } dumpCreds = []interface{}{ "--ssl", - "--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName), - "--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName), + fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)), + fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)), } // get certificate secret to get client certificate @@ -289,9 +307,9 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti return nil, errors.Wrap(err, "unable to get user from ssl.") } userAuth := []interface{}{ - "-u", user, - "--authenticationMechanism", "MONGODB-X509", - "--authenticationDatabase", "$external", + fmt.Sprintf("--username=%s", user), + "--authenticationMechanism=MONGODB-X509", + "--authenticationDatabase=$external", } mongoCreds = append(mongoCreds, userAuth...) dumpCreds = append(dumpCreds, userAuth...) @@ -300,7 +318,7 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti userAuth := []interface{}{ fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]), fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]), - "--authenticationDatabase", opt.authenticationDatabase, + fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase), } mongoCreds = append(mongoCreds, userAuth...) dumpCreds = append(dumpCreds, userAuth...) @@ -314,19 +332,32 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti FileName: opt.defaultDumpOptions.FileName, Snapshot: opt.getSnapshotForHost(hostKey, restoreSession.Spec.Target.Rules), } + + uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable) + // setup pipe command restoreCmd := restic.Command{ Name: MongoRestoreCMD, - Args: append([]interface{}{ - "--host", mongoDSN, + Args: []interface{}{ + "--uri", fmt.Sprintf("\"%s\"", uri), "--archive", - }, dumpCreds...), + }, + } + if tlsEnable { + restoreCmd.Args = append(restoreCmd.Args, + fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")), + fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile"))) + } + + var userArgs []string + for _, arg := range strings.Fields(opt.mongoArgs) { + // illegal argument combination: cannot specify --db and --uri + if !strings.Contains(arg, "--db") { + userArgs = append(userArgs, arg) + } } - userArgs := strings.Fields(opt.mongoArgs) - if isStandalone { - restoreCmd.Args = append(restoreCmd.Args, fmt.Sprintf("--port=%d", port)) - } else { + if !isStandalone { // - port is already added in mongoDSN with replicasetName/host:port format. // - oplog is enabled automatically for replicasets. // Don't use --oplogReplay if user specify any of these arguments through opt.mongoArgs @@ -369,11 +400,11 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti // ref: https://docs.mongodb.com/manual/tutorial/backup-sharded-cluster-with-database-dumps/ if parameters.ConfigServer != "" { - opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(parameters.ConfigServer, MongoConfigSVRHostKey, false)) + opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(parameters.ConfigServer), MongoConfigSVRHostKey, false)) } for key, host := range parameters.ReplicaSets { - opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(host, key, false)) + opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(host), key, false)) } // if parameters.ReplicaSets is nil, then perform normal backup with clientconfig.Service.Name mongo dsn diff --git a/pkg/utils.go b/pkg/utils.go index f13e086fb..0f6874f0a 100644 --- a/pkg/utils.go +++ b/pkg/utils.go @@ -18,6 +18,7 @@ package pkg import ( "fmt" + "net/url" "os/exec" "strings" "time" @@ -114,3 +115,65 @@ func getTime(t string) (time.Time, error) { } return parsedTime, nil } + +func isSrvConnection(connectionString string) (bool, error) { + parsedURL, err := url.Parse(connectionString) + if err != nil { + return false, err + } + + // Check if the scheme is "mongodb+srv" + return parsedURL.Scheme == "mongodb+srv", nil +} + +func (opt *mongoOptions) buildMongoURI(mongoDSN string, port int32, isStandalone, isSrv, tlsEnable bool) string { + prefix, ssl := "mongodb", "" + portStr := fmt.Sprintf(":%d", port) + if isSrv { + prefix += "+srv" + } + if !isStandalone || isSrv { + portStr = "" + } + + backupDb := getBackupDB(opt.mongoArgs) // "" stands for all databases. + authDbName := getOptionValue(dumpCreds, "--authenticationDatabase") + userName := getOptionValue(dumpCreds, "--username") + password := getOptionValue(dumpCreds, "--password") + authMechanism := getOptionValue(dumpCreds, "--authenticationMechanism") + + if password != "" { + password = fmt.Sprintf(":%s", password) + } + if authMechanism == "" { + authMechanism = "SCRAM-SHA-256" + } + if tlsEnable { + ssl = "&ssl=true" + } + + return fmt.Sprintf("%s://%s%s@%s%s/%s?authSource=%s&authMechanism=%s%s", + prefix, userName, password, mongoDSN, portStr, backupDb, authDbName, authMechanism, ssl) +} + +// remove "shard0/" prefix from shard0/simple-shard0-0.simple-shard0-pods.demo.svc:27017,simple-shard0-1.simple-shard0-pods.demo.svc:27017 +func extractHost(host string) string { + index := strings.Index(host, "/") + if index != -1 { + host = host[index+1:] + } + return host +} + +func getBackupDB(mongoArgs string) string { + backupdb := "" // full + if strings.Contains(mongoArgs, "--db") { + args := strings.Fields(mongoArgs) + for _, arg := range args { + if strings.Contains(arg, "--db") { + backupdb = strings.Split(arg, "=")[1] + } + } + } + return backupdb +}