Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[cherry-pick] Fix MongoDB url connection str for external databases (#2138) #2144

Merged
merged 1 commit into from
Aug 27, 2024
Merged

[cherry-pick] Fix MongoDB url connection str for external databases (#2138) #2144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 61 additions & 16 deletions pkg/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,11 +267,24 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
return nil, err
}

port, err := appBinding.Port()
if err != nil {
return nil, err
var isSrv bool
port := int32(27017)
if appBinding.Spec.ClientConfig.URL != nil {
isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL)
if err != nil {
return nil, err
}
}

// Checked for Altlas and DigitalOcean srv format connection string don't give port.
// mongodump not support both --uri and --port.

if !isSrv {
port, err = appBinding.Port()
if err != nil {
return nil, err
}
}
waitForDBReady(hostname, port, opt.waitTimeout)

// unmarshal parameter is the field has value
Expand Down Expand Up @@ -318,7 +331,12 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
}
}

var tlsEnable bool
if appBinding.Spec.ClientConfig.CABundle != nil {
tlsEnable = true
}

if tlsEnable {
if tlsSecret == nil {
return nil, errors.Wrap(err, "spec.tlsSecret needs to be set in appbinding for TLS secured database.")
}
Expand All @@ -333,8 +351,8 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
}
dumpCreds = []interface{}{
"--ssl",
"--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName),
"--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName),
fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)),
fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)),
}

// get certificate secret to get client certificate
Expand All @@ -361,9 +379,9 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
return nil, errors.Wrap(err, "unable to get user from ssl.")
}
userAuth := []interface{}{
"-u", user,
"--authenticationMechanism", "MONGODB-X509",
"--authenticationDatabase", "$external",
fmt.Sprintf("--username=%s", user),
"--authenticationMechanism=MONGODB-X509",
"--authenticationDatabase=$external",
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -372,7 +390,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
userAuth := []interface{}{
fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]),
fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]),
"--authenticationDatabase", opt.authenticationDatabase,
fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase),
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -387,19 +405,32 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
BackupPaths: opt.defaultBackupOptions.BackupPaths,
}

uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable)

// setup pipe command
backupCmd := restic.Command{
Name: MongoDumpCMD,
Args: append([]interface{}{
"--host", mongoDSN,
Args: []interface{}{
"--uri", fmt.Sprintf("\"%s\"", uri),
"--archive",
}, dumpCreds...),
},
}
userArgs := strings.Fields(opt.mongoArgs)

if isStandalone {
backupCmd.Args = append(backupCmd.Args, fmt.Sprintf("--port=%d", port))
} else {
if tlsEnable {
backupCmd.Args = append(backupCmd.Args,
fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")),
fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile")))
}

var userArgs []string
for _, arg := range strings.Fields(opt.mongoArgs) {
// illegal argument combination: cannot specify --db and --uri
if !strings.Contains(arg, "--db") {
userArgs = append(userArgs, arg)
}
}

if !isStandalone {
// - port is already added in mongoDSN with replicasetName/host:port format.
// - oplog is enabled automatically for replicasets.
// Don't use --oplog if user specify any of these arguments through opt.mongoArgs
Expand Down Expand Up @@ -558,6 +589,20 @@ func cleanup() {
}
}

func getOptionValue(args []interface{}, option string) string {
for _, arg := range args {
strArg, ok := arg.(string)
if !ok {
continue
}
// assuming value has '='
if strings.HasPrefix(strArg, option+"=") {
return strings.TrimPrefix(strArg, option+"=")
}
}
return ""
}

func (opt *mongoOptions) getHostBackupStats(err error) []api_v1beta1.HostBackupStats {
var backupStats []api_v1beta1.HostBackupStats

Expand Down
69 changes: 50 additions & 19 deletions pkg/restore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,23 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
return nil, err
}

port, err := appBinding.Port()
if err != nil {
return nil, err
var isSrv bool
port := int32(27017)
if appBinding.Spec.ClientConfig.URL != nil {
isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL)
if err != nil {
return nil, err
}
}

// Checked for Altlas and DigitalOcean srv format connection string don't give port.
// mongodump --uri format not support port.

if !isSrv {
port, err = appBinding.Port()
if err != nil {
return nil, err
}
}

// unmarshal parameter is the field has value
Expand Down Expand Up @@ -249,8 +263,12 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
return nil, err
}
}

var tlsEnable bool
if appBinding.Spec.ClientConfig.CABundle != nil {
tlsEnable = true
}

if tlsEnable {
if err := os.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
return nil, errors.Wrap(err, "failed to write key for CA certificate")
}
Expand All @@ -261,8 +279,8 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
}
dumpCreds = []interface{}{
"--ssl",
"--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName),
"--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName),
fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)),
fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)),
}

// get certificate secret to get client certificate
Expand All @@ -289,9 +307,9 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
return nil, errors.Wrap(err, "unable to get user from ssl.")
}
userAuth := []interface{}{
"-u", user,
"--authenticationMechanism", "MONGODB-X509",
"--authenticationDatabase", "$external",
fmt.Sprintf("--username=%s", user),
"--authenticationMechanism=MONGODB-X509",
"--authenticationDatabase=$external",
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -300,7 +318,7 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
userAuth := []interface{}{
fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]),
fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]),
"--authenticationDatabase", opt.authenticationDatabase,
fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase),
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -314,19 +332,32 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
FileName: opt.defaultDumpOptions.FileName,
Snapshot: opt.getSnapshotForHost(hostKey, restoreSession.Spec.Target.Rules),
}

uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable)

// setup pipe command
restoreCmd := restic.Command{
Name: MongoRestoreCMD,
Args: append([]interface{}{
"--host", mongoDSN,
Args: []interface{}{
"--uri", fmt.Sprintf("\"%s\"", uri),
"--archive",
}, dumpCreds...),
},
}
if tlsEnable {
restoreCmd.Args = append(restoreCmd.Args,
fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")),
fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile")))
}

var userArgs []string
for _, arg := range strings.Fields(opt.mongoArgs) {
// illegal argument combination: cannot specify --db and --uri
if !strings.Contains(arg, "--db") {
userArgs = append(userArgs, arg)
}
}

userArgs := strings.Fields(opt.mongoArgs)
if isStandalone {
restoreCmd.Args = append(restoreCmd.Args, fmt.Sprintf("--port=%d", port))
} else {
if !isStandalone {
// - port is already added in mongoDSN with replicasetName/host:port format.
// - oplog is enabled automatically for replicasets.
// Don't use --oplogReplay if user specify any of these arguments through opt.mongoArgs
Expand Down Expand Up @@ -369,11 +400,11 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
// ref: https://docs.mongodb.com/manual/tutorial/backup-sharded-cluster-with-database-dumps/

if parameters.ConfigServer != "" {
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(parameters.ConfigServer, MongoConfigSVRHostKey, false))
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(parameters.ConfigServer), MongoConfigSVRHostKey, false))
}

for key, host := range parameters.ReplicaSets {
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(host, key, false))
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(host), key, false))
}

// if parameters.ReplicaSets is nil, then perform normal backup with clientconfig.Service.Name mongo dsn
Expand Down
63 changes: 63 additions & 0 deletions pkg/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package pkg

import (
"fmt"
"net/url"
"os/exec"
"strings"
"time"
Expand Down Expand Up @@ -114,3 +115,65 @@ func getTime(t string) (time.Time, error) {
}
return parsedTime, nil
}

func isSrvConnection(connectionString string) (bool, error) {
parsedURL, err := url.Parse(connectionString)
if err != nil {
return false, err
}

// Check if the scheme is "mongodb+srv"
return parsedURL.Scheme == "mongodb+srv", nil
}

func (opt *mongoOptions) buildMongoURI(mongoDSN string, port int32, isStandalone, isSrv, tlsEnable bool) string {
prefix, ssl := "mongodb", ""
portStr := fmt.Sprintf(":%d", port)
if isSrv {
prefix += "+srv"
}
if !isStandalone || isSrv {
portStr = ""
}

backupDb := getBackupDB(opt.mongoArgs) // "" stands for all databases.
authDbName := getOptionValue(dumpCreds, "--authenticationDatabase")
userName := getOptionValue(dumpCreds, "--username")
password := getOptionValue(dumpCreds, "--password")
authMechanism := getOptionValue(dumpCreds, "--authenticationMechanism")

if password != "" {
password = fmt.Sprintf(":%s", password)
}
if authMechanism == "" {
authMechanism = "SCRAM-SHA-256"
}
if tlsEnable {
ssl = "&ssl=true"
}

return fmt.Sprintf("%s://%s%s@%s%s/%s?authSource=%s&authMechanism=%s%s",
prefix, userName, password, mongoDSN, portStr, backupDb, authDbName, authMechanism, ssl)
}

// remove "shard0/" prefix from shard0/simple-shard0-0.simple-shard0-pods.demo.svc:27017,simple-shard0-1.simple-shard0-pods.demo.svc:27017
func extractHost(host string) string {
index := strings.Index(host, "/")
if index != -1 {
host = host[index+1:]
}
return host
}

func getBackupDB(mongoArgs string) string {
backupdb := "" // full
if strings.Contains(mongoArgs, "--db") {
args := strings.Fields(mongoArgs)
for _, arg := range args {
if strings.Contains(arg, "--db") {
backupdb = strings.Split(arg, "=")[1]
}
}
}
return backupdb
}
Loading