diff --git a/go/cmd/vtbackup/cli/vtbackup.go b/go/cmd/vtbackup/cli/vtbackup.go index 4700d93eea1..1f6f62f7ad1 100644 --- a/go/cmd/vtbackup/cli/vtbackup.go +++ b/go/cmd/vtbackup/cli/vtbackup.go @@ -204,7 +204,7 @@ func init() { Main.Flags().StringVar(&initKeyspace, "init_keyspace", initKeyspace, "(init parameter) keyspace to use for this tablet") Main.Flags().StringVar(&initShard, "init_shard", initShard, "(init parameter) shard to use for this tablet") Main.Flags().IntVar(&concurrency, "concurrency", concurrency, "(init restore parameter) how many concurrent files to restore at once") - Main.Flags().StringVar(&incrementalFromPos, "incremental_from_pos", incrementalFromPos, "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position") + Main.Flags().StringVar(&incrementalFromPos, "incremental_from_pos", incrementalFromPos, "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.") // mysqlctld-like flags Main.Flags().IntVar(&mysqlPort, "mysql_port", mysqlPort, "mysql port") diff --git a/go/cmd/vtctldclient/command/backups.go b/go/cmd/vtctldclient/command/backups.go index aa0a2ea41c0..ea439ded70e 100644 --- a/go/cmd/vtctldclient/command/backups.go +++ b/go/cmd/vtctldclient/command/backups.go @@ -35,7 +35,7 @@ import ( var ( // Backup makes a Backup gRPC call to a vtctld. Backup = &cobra.Command{ - Use: "Backup [--concurrency ] [--allow-primary] [--incremental-from-pos=|auto] [--upgrade-safe] ", + Use: "Backup [--concurrency ] [--allow-primary] [--incremental-from-pos=||auto] [--upgrade-safe] ", Short: "Uses the BackupStorage service on the given tablet to create and store a new backup.", DisableFlagsInUseLine: true, Args: cobra.ExactArgs(1), @@ -43,7 +43,7 @@ var ( } // BackupShard makes a BackupShard gRPC call to a vtctld. BackupShard = &cobra.Command{ - Use: "BackupShard [--concurrency ] [--allow-primary] [--incremental-from-pos=|auto] [--upgrade-safe] ", + Use: "BackupShard [--concurrency ] [--allow-primary] [--incremental-from-pos=||auto] [--upgrade-safe] ", Short: "Finds the most up-to-date REPLICA, RDONLY, or SPARE tablet in the given shard and uses the BackupStorage service on that tablet to create and store a new backup.", Long: `Finds the most up-to-date REPLICA, RDONLY, or SPARE tablet in the given shard and uses the BackupStorage service on that tablet to create and store a new backup. @@ -281,14 +281,14 @@ func commandRestoreFromBackup(cmd *cobra.Command, args []string) error { func init() { Backup.Flags().BoolVar(&backupOptions.AllowPrimary, "allow-primary", false, "Allow the primary of a shard to be used for the backup. WARNING: If using the builtin backup engine, this will shutdown mysqld on the primary and stop writes for the duration of the backup.") Backup.Flags().Int32Var(&backupOptions.Concurrency, "concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously.") - Backup.Flags().StringVar(&backupOptions.IncrementalFromPos, "incremental-from-pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position") + Backup.Flags().StringVar(&backupOptions.IncrementalFromPos, "incremental-from-pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.") Backup.Flags().BoolVar(&backupOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.") Root.AddCommand(Backup) BackupShard.Flags().BoolVar(&backupShardOptions.AllowPrimary, "allow-primary", false, "Allow the primary of a shard to be used for the backup. WARNING: If using the builtin backup engine, this will shutdown mysqld on the primary and stop writes for the duration of the backup.") BackupShard.Flags().Int32Var(&backupShardOptions.Concurrency, "concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously.") - BackupShard.Flags().StringVar(&backupShardOptions.IncrementalFromPos, "incremental-from-pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position") + BackupShard.Flags().StringVar(&backupShardOptions.IncrementalFromPos, "incremental-from-pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.") BackupShard.Flags().BoolVar(&backupOptions.UpgradeSafe, "upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.") Root.AddCommand(BackupShard) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 866564c7d60..ea98ad70b88 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -137,7 +137,7 @@ Flags: --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) --grpc_prometheus Enable gRPC monitoring with Prometheus. -h, --help help for vtbackup - --incremental_from_pos string Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position + --incremental_from_pos string Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position. --init_db_name_override string (init parameter) override the name of the db used by vttablet --init_db_sql_file string path to .sql file to run after mysql_install_db --init_keyspace string (init parameter) keyspace to use for this tablet diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index e470652f7ee..442e87adc43 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1255,13 +1255,9 @@ func waitForNumBackups(t *testing.T, expectNumBackups int) []string { } } -func testReplicaIncrementalBackup(t *testing.T, replica *cluster.Vttablet, incrementalFromPos replication.Position, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) { +func testReplicaIncrementalBackup(t *testing.T, replica *cluster.Vttablet, incrementalFromPos string, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) { numBackups := len(waitForNumBackups(t, -1)) - incrementalFromPosArg := "auto" - if !incrementalFromPos.IsZero() { - incrementalFromPosArg = replication.EncodePosition(incrementalFromPos) - } - output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("Backup", "--incremental-from-pos", incrementalFromPosArg, replica.Alias) + output, err := localCluster.VtctldClientProcess.ExecuteCommandWithOutput("Backup", "--incremental-from-pos", incrementalFromPos, replica.Alias) if expectError != "" { require.Errorf(t, err, "expected: %v", expectError) require.Contains(t, output, expectError) @@ -1278,7 +1274,7 @@ func testReplicaIncrementalBackup(t *testing.T, replica *cluster.Vttablet, incre return readManifestFile(t, backupLocation), backupName } -func TestReplicaIncrementalBackup(t *testing.T, replicaIndex int, incrementalFromPos replication.Position, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) { +func TestReplicaIncrementalBackup(t *testing.T, replicaIndex int, incrementalFromPos string, expectError string) (manifest *mysqlctl.BackupManifest, backupName string) { replica := getReplica(t, replicaIndex) return testReplicaIncrementalBackup(t, replica, incrementalFromPos, expectError) } diff --git a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go index 8b9014e7f8c..5effcc339d8 100644 --- a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go +++ b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go @@ -48,6 +48,14 @@ const ( operationFlushAndPurge ) +type incrementalFromPosType int + +const ( + incrementalFromPosPosition incrementalFromPosType = iota + incrementalFromPosAuto + incrementalFromPosBackupName +) + type PITRTestCase struct { Name string SetupType int @@ -106,6 +114,7 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) } var fullBackupPos replication.Position + var lastBackupName string t.Run("full backup", func(t *testing.T) { InsertRowOnPrimary(t, "before-full-backup") waitForReplica(t, 0) @@ -118,6 +127,8 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) pos := replication.EncodePosition(fullBackupPos) backupPositions = append(backupPositions, pos) rowsPerPosition[pos] = len(msgs) + + lastBackupName = manifest.BackupName }) lastBackupPos := fullBackupPos @@ -127,50 +138,57 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) name string writeBeforeBackup bool fromFullPosition bool - autoPosition bool + incrementalFrom incrementalFromPosType expectError string }{ { - name: "first incremental backup", + name: "first incremental backup", + incrementalFrom: incrementalFromPosPosition, }, { - name: "fail1", - expectError: "no binary logs to backup", + name: "fail1", + incrementalFrom: incrementalFromPosPosition, + expectError: "no binary logs to backup", }, { - name: "fail2", - expectError: "no binary logs to backup", + name: "fail2", + incrementalFrom: incrementalFromPosPosition, + expectError: "no binary logs to backup", }, { name: "make writes, succeed", writeBeforeBackup: true, + incrementalFrom: incrementalFromPosPosition, }, { - name: "fail, no binary logs to backup", - expectError: "no binary logs to backup", + name: "fail, no binary logs to backup", + incrementalFrom: incrementalFromPosPosition, + expectError: "no binary logs to backup", }, { name: "make writes again, succeed", writeBeforeBackup: true, + incrementalFrom: incrementalFromPosBackupName, }, { name: "auto position, succeed", writeBeforeBackup: true, - autoPosition: true, + incrementalFrom: incrementalFromPosAuto, }, { - name: "fail auto position, no binary logs to backup", - autoPosition: true, - expectError: "no binary logs to backup", + name: "fail auto position, no binary logs to backup", + incrementalFrom: incrementalFromPosAuto, + expectError: "no binary logs to backup", }, { name: "auto position, make writes again, succeed", writeBeforeBackup: true, - autoPosition: true, + incrementalFrom: incrementalFromPosAuto, }, { name: "from full backup position", fromFullPosition: true, + incrementalFrom: incrementalFromPosPosition, }, } var fromFullPositionBackups []string @@ -192,11 +210,16 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) // - auto // - explicit last backup pos // - back in history to the original full backup - var incrementalFromPos replication.Position - if !tc.autoPosition { - incrementalFromPos = lastBackupPos + var incrementalFromPos string + switch tc.incrementalFrom { + case incrementalFromPosAuto: + incrementalFromPos = mysqlctl.AutoIncrementalFromPos + case incrementalFromPosBackupName: + incrementalFromPos = lastBackupName + case incrementalFromPosPosition: + incrementalFromPos = replication.EncodePosition(lastBackupPos) if tc.fromFullPosition { - incrementalFromPos = fullBackupPos + incrementalFromPos = replication.EncodePosition(fullBackupPos) } } // always use same 1st replica @@ -206,6 +229,7 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) } defer func() { lastBackupPos = manifest.Position + lastBackupName = manifest.BackupName }() if tc.fromFullPosition { fromFullPositionBackups = append(fromFullPositionBackups, backupName) @@ -219,8 +243,10 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) fromPositionIncludingPurged := manifest.FromPosition.GTIDSet.Union(gtidPurgedPos.GTIDSet) expectFromPosition := lastBackupPos.GTIDSet - if !incrementalFromPos.IsZero() { - expectFromPosition = incrementalFromPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) + if tc.incrementalFrom == incrementalFromPosPosition { + pos, err := replication.DecodePosition(incrementalFromPos) + assert.NoError(t, err) + expectFromPosition = pos.GTIDSet.Union(gtidPurgedPos.GTIDSet) } require.Equalf(t, expectFromPosition, fromPositionIncludingPurged, "expected: %v, found: %v, gtid_purged: %v, manifest.Position: %v", expectFromPosition, fromPositionIncludingPurged, gtidPurgedPos, manifest.Position) }) @@ -304,6 +330,7 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes testedBackups := []testedBackupTimestampInfo{} var fullBackupPos replication.Position + var lastBackupName string t.Run("full backup", func(t *testing.T) { insertRowOnPrimary(t, "before-full-backup") waitForReplica(t, 0) @@ -314,6 +341,8 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes // rows := ReadRowsFromReplica(t, 0) testedBackups = append(testedBackups, testedBackupTimestampInfo{len(rows), time.Now()}) + + lastBackupName = manifest.BackupName }) lastBackupPos := fullBackupPos @@ -323,50 +352,57 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes name string writeBeforeBackup bool fromFullPosition bool - autoPosition bool + incrementalFrom incrementalFromPosType expectError string }{ { - name: "first incremental backup", + name: "first incremental backup", + incrementalFrom: incrementalFromPosPosition, }, { - name: "fail1", - expectError: "no binary logs to backup", + name: "fail1", + incrementalFrom: incrementalFromPosPosition, + expectError: "no binary logs to backup", }, { - name: "fail2", - expectError: "no binary logs to backup", + name: "fail2", + incrementalFrom: incrementalFromPosPosition, + expectError: "no binary logs to backup", }, { name: "make writes, succeed", writeBeforeBackup: true, + incrementalFrom: incrementalFromPosPosition, }, { - name: "fail, no binary logs to backup", - expectError: "no binary logs to backup", + name: "fail, no binary logs to backup", + incrementalFrom: incrementalFromPosPosition, + expectError: "no binary logs to backup", }, { name: "make writes again, succeed", writeBeforeBackup: true, + incrementalFrom: incrementalFromPosBackupName, }, { name: "auto position, succeed", writeBeforeBackup: true, - autoPosition: true, + incrementalFrom: incrementalFromPosAuto, }, { - name: "fail auto position, no binary logs to backup", - autoPosition: true, - expectError: "no binary logs to backup", + name: "fail auto position, no binary logs to backup", + incrementalFrom: incrementalFromPosAuto, + expectError: "no binary logs to backup", }, { name: "auto position, make writes again, succeed", writeBeforeBackup: true, - autoPosition: true, + incrementalFrom: incrementalFromPosAuto, }, { name: "from full backup position", fromFullPosition: true, + incrementalFrom: incrementalFromPosPosition, }, } var fromFullPositionBackups []string @@ -386,11 +422,16 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes // - auto // - explicit last backup pos // - back in history to the original full backup - var incrementalFromPos replication.Position - if !tc.autoPosition { - incrementalFromPos = lastBackupPos + var incrementalFromPos string + switch tc.incrementalFrom { + case incrementalFromPosAuto: + incrementalFromPos = mysqlctl.AutoIncrementalFromPos + case incrementalFromPosBackupName: + incrementalFromPos = lastBackupName + case incrementalFromPosPosition: + incrementalFromPos = replication.EncodePosition(lastBackupPos) if tc.fromFullPosition { - incrementalFromPos = fullBackupPos + incrementalFromPos = replication.EncodePosition(fullBackupPos) } } manifest, backupName := TestReplicaIncrementalBackup(t, 0, incrementalFromPos, tc.expectError) @@ -405,6 +446,7 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes testedBackups = append(testedBackups, testedBackupTimestampInfo{len(rowsBeforeBackup), time.Now()}) defer func() { lastBackupPos = manifest.Position + lastBackupName = manifest.BackupName }() if tc.fromFullPosition { fromFullPositionBackups = append(fromFullPositionBackups, backupName) @@ -434,8 +476,10 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes fromPositionIncludingPurged := manifest.FromPosition.GTIDSet.Union(gtidPurgedPos.GTIDSet) expectFromPosition := lastBackupPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) - if !incrementalFromPos.IsZero() { - expectFromPosition = incrementalFromPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) + if tc.incrementalFrom == incrementalFromPosPosition { + pos, err := replication.DecodePosition(incrementalFromPos) + assert.NoError(t, err) + expectFromPosition = pos.GTIDSet.Union(gtidPurgedPos.GTIDSet) } require.Equalf(t, expectFromPosition, fromPositionIncludingPurged, "expected: %v, found: %v, gtid_purged: %v, manifest.Position: %v", expectFromPosition, fromPositionIncludingPurged, gtidPurgedPos, manifest.Position) }) @@ -663,8 +707,7 @@ func ExecTestIncrementalBackupOnTwoTablets(t *testing.T, tcase *PITRTestCase) { lastBackupPos = fullBackupPos case operationIncrementalBackup: - var incrementalFromPos replication.Position // keep zero, we will use "auto" - manifest, _ := TestReplicaIncrementalBackup(t, tc.replicaIndex, incrementalFromPos, tc.expectError) + manifest, _ := TestReplicaIncrementalBackup(t, tc.replicaIndex, "auto", tc.expectError) if tc.expectError != "" { return } diff --git a/go/vt/mysqlctl/backupengine.go b/go/vt/mysqlctl/backupengine.go index cf6065cd733..0fbafb9a4a8 100644 --- a/go/vt/mysqlctl/backupengine.go +++ b/go/vt/mysqlctl/backupengine.go @@ -269,6 +269,9 @@ type IncrementalBackupDetails struct { // their own custom fields by embedding this struct anonymously into their own // custom struct, as long as their custom fields don't have conflicting names. type BackupManifest struct { + // BackupName is the name of the backup, which is also the name of the directory + BackupName string + // BackupMethod is the name of the backup engine that created this backup. // If this is empty, the backup engine is assumed to be "builtin" since that // was the only engine that ever left this field empty. All new backup @@ -408,9 +411,9 @@ func (p *RestorePath) String() string { return sb.String() } -// FindLatestSuccessfulBackup returns the handle and manifest for the last good backup, +// findLatestSuccessfulBackup returns the handle and manifest for the last good backup, // which can be either full or increment -func FindLatestSuccessfulBackup(ctx context.Context, logger logutil.Logger, bhs []backupstorage.BackupHandle, excludeBackupName string) (backupstorage.BackupHandle, *BackupManifest, error) { +func findLatestSuccessfulBackup(ctx context.Context, logger logutil.Logger, bhs []backupstorage.BackupHandle, excludeBackupName string) (backupstorage.BackupHandle, *BackupManifest, error) { for index := len(bhs) - 1; index >= 0; index-- { bh := bhs[index] if bh.Name() == excludeBackupName { @@ -431,8 +434,8 @@ func FindLatestSuccessfulBackup(ctx context.Context, logger logutil.Logger, bhs return nil, nil, ErrNoCompleteBackup } -// FindLatestSuccessfulBackupPosition returns the position of the last known successful backup -func FindLatestSuccessfulBackupPosition(ctx context.Context, params BackupParams, excludeBackupName string) (backupName string, pos replication.Position, err error) { +// findLatestSuccessfulBackupPosition returns the position of the last known successful backup +func findLatestSuccessfulBackupPosition(ctx context.Context, params BackupParams, excludeBackupName string) (backupName string, pos replication.Position, err error) { bs, err := backupstorage.GetBackupStorage() if err != nil { return "", pos, err @@ -446,7 +449,7 @@ func FindLatestSuccessfulBackupPosition(ctx context.Context, params BackupParams if err != nil { return "", pos, vterrors.Wrap(err, "ListBackups failed") } - bh, manifest, err := FindLatestSuccessfulBackup(ctx, params.Logger, bhs, excludeBackupName) + bh, manifest, err := findLatestSuccessfulBackup(ctx, params.Logger, bhs, excludeBackupName) if err != nil { return "", pos, vterrors.Wrap(err, "FindLatestSuccessfulBackup failed") } @@ -454,6 +457,32 @@ func FindLatestSuccessfulBackupPosition(ctx context.Context, params BackupParams return bh.Name(), pos, nil } +// findBackupPosition returns the position of a given backup, assuming the backup exists. +func findBackupPosition(ctx context.Context, params BackupParams, backupName string) (pos replication.Position, err error) { + bs, err := backupstorage.GetBackupStorage() + if err != nil { + return pos, err + } + defer bs.Close() + + backupDir := GetBackupDir(params.Keyspace, params.Shard) + bhs, err := bs.ListBackups(ctx, backupDir) + if err != nil { + return pos, vterrors.Wrap(err, "ListBackups failed") + } + for _, bh := range bhs { + if bh.Name() != backupName { + continue + } + manifest, err := GetBackupManifest(ctx, bh) + if err != nil { + return pos, vterrors.Wrapf(err, "GetBackupManifest failed for backup: %v", backupName) + } + return manifest.Position, nil + } + return pos, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "could not find backup %q for %s/%s", backupName, params.Keyspace, params.Shard) +} + // FindBackupToRestore returns a path, a sequence of backup handles, to be restored. // The returned handles stand for valid backups with complete manifests. func FindBackupToRestore(ctx context.Context, params RestoreParams, bhs []backupstorage.BackupHandle) (restorePath *RestorePath, err error) { diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 57f55cefb2f..be4b33a4e4c 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -58,7 +58,7 @@ import ( const ( builtinBackupEngineName = "builtin" - autoIncrementalFromPos = "auto" + AutoIncrementalFromPos = "auto" dataDictionaryFile = "mysql.ibd" ) @@ -245,10 +245,14 @@ func (be *BuiltinBackupEngine) executeIncrementalBackup(ctx context.Context, par return false, vterrors.Wrap(err, "can't get MySQL version") } + // We now need to figure out the GTIDSet from which we want to take the incremental backup. The user may have + // specified a position, or they may have specified "auto", or they may have specified a backup name, in which + // case we need to find the position of that backup. var fromBackupName string - if params.IncrementalFromPos == autoIncrementalFromPos { + if params.IncrementalFromPos == AutoIncrementalFromPos { + // User has supplied "auto". params.Logger.Infof("auto evaluating incremental_from_pos") - backupName, pos, err := FindLatestSuccessfulBackupPosition(ctx, params, bh.Name()) + backupName, pos, err := findLatestSuccessfulBackupPosition(ctx, params, bh.Name()) if err != nil { return false, err } @@ -257,17 +261,16 @@ func (be *BuiltinBackupEngine) executeIncrementalBackup(ctx context.Context, par params.Logger.Infof("auto evaluated incremental_from_pos: %s", params.IncrementalFromPos) } - // @@gtid_purged - getPurgedGTIDSet := func() (replication.Position, replication.Mysql56GTIDSet, error) { - gtidPurged, err := params.Mysqld.GetGTIDPurged(ctx) + if _, err := replication.DecodePositionDefaultFlavor(params.IncrementalFromPos, replication.Mysql56FlavorID); err != nil { + // This does not seem to be a valid position. Maybe it's a backup name? + backupName := params.IncrementalFromPos + pos, err := findBackupPosition(ctx, params, backupName) if err != nil { - return gtidPurged, nil, vterrors.Wrap(err, "can't get @@gtid_purged") - } - purgedGTIDSet, ok := gtidPurged.GTIDSet.(replication.Mysql56GTIDSet) - if !ok { - return gtidPurged, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "cannot get MySQL GTID purged value: %v", gtidPurged) + return false, err } - return gtidPurged, purgedGTIDSet, nil + fromBackupName = backupName + params.IncrementalFromPos = replication.EncodePosition(pos) + params.Logger.Infof("evaluated incremental_from_pos using backup name %q: %s", backupName, params.IncrementalFromPos) } // params.IncrementalFromPos is a string. We want to turn that into a MySQL GTID @@ -275,7 +278,7 @@ func (be *BuiltinBackupEngine) executeIncrementalBackup(ctx context.Context, par if err != nil { return false, err } - // OK, we now have the formal MySQL GTID from which we want to take the incremental backip. + // OK, we now have the formal MySQL GTID from which we want to take the incremental backup. // binlogs may not contain information about purged GTIDs. e.g. some binlog.000003 may have // previous GTIDs like 00021324-1111-1111-1111-111111111111:30-60, ie 1-29 range is missing. This can happen @@ -291,6 +294,18 @@ func (be *BuiltinBackupEngine) executeIncrementalBackup(ctx context.Context, par if err != nil { return false, vterrors.Wrapf(err, "cannot get binary logs in incremental backup") } + + getPurgedGTIDSet := func() (replication.Position, replication.Mysql56GTIDSet, error) { + gtidPurged, err := params.Mysqld.GetGTIDPurged(ctx) + if err != nil { + return gtidPurged, nil, vterrors.Wrap(err, "can't get @@gtid_purged") + } + purgedGTIDSet, ok := gtidPurged.GTIDSet.(replication.Mysql56GTIDSet) + if !ok { + return gtidPurged, nil, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "failed to parse a valid MySQL GTID set from value: %v", gtidPurged) + } + return gtidPurged, purgedGTIDSet, nil + } // gtid_purged is important information. The restore flow uses this info to to complement binary logs' Previous-GTIDs. // It is important to only get gtid_purged _after_ we've rotated into the new binary log, because the `FLUSH BINARY LOGS` // command may also purge old logs, hence affecting the value of gtid_purged. @@ -647,6 +662,7 @@ func (be *BuiltinBackupEngine) backupFiles( bm := &builtinBackupManifest{ // Common base fields BackupManifest: BackupManifest{ + BackupName: bh.Name(), BackupMethod: builtinBackupEngineName, Position: backupPosition, PurgedPosition: purgedPosition, diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go index a61551c3dcf..4aa2ab96b00 100644 --- a/go/vt/mysqlctl/xtrabackupengine.go +++ b/go/vt/mysqlctl/xtrabackupengine.go @@ -244,6 +244,7 @@ func (be *XtrabackupEngine) executeFullBackup(ctx context.Context, params Backup bm := &xtraBackupManifest{ // Common base fields BackupManifest: BackupManifest{ + BackupName: bh.Name(), BackupMethod: xtrabackupEngineName, Position: replicationPosition, PurgedPosition: replicationPosition, diff --git a/go/vt/vtctl/backup.go b/go/vt/vtctl/backup.go index 991038963c8..e832b1f79d1 100644 --- a/go/vt/vtctl/backup.go +++ b/go/vt/vtctl/backup.go @@ -72,7 +72,7 @@ func init() { func commandBackup(ctx context.Context, wr *wrangler.Wrangler, subFlags *pflag.FlagSet, args []string) error { concurrency := subFlags.Int32("concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously") allowPrimary := subFlags.Bool("allow_primary", false, "Allows backups to be taken on primary. Warning!! If you are using the builtin backup engine, this will shutdown your primary mysql for as long as it takes to create a backup.") - incrementalFromPos := subFlags.String("incremental_from_pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position") + incrementalFromPos := subFlags.String("incremental_from_pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.") upgradeSafe := subFlags.Bool("upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.") if err := subFlags.Parse(args); err != nil { @@ -114,7 +114,7 @@ func (b *backupEventStreamLogger) Send(resp *vtctldatapb.BackupResponse) error { func commandBackupShard(ctx context.Context, wr *wrangler.Wrangler, subFlags *pflag.FlagSet, args []string) error { concurrency := subFlags.Int32("concurrency", 4, "Specifies the number of compression/checksum jobs to run simultaneously") allowPrimary := subFlags.Bool("allow_primary", false, "Whether to use primary tablet for backup. Warning!! If you are using the builtin backup engine, this will shutdown your primary mysql for as long as it takes to create a backup.") - incrementalFromPos := subFlags.String("incremental_from_pos", "", "Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position") + incrementalFromPos := subFlags.String("incremental_from_pos", "", "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.") upgradeSafe := subFlags.Bool("upgrade-safe", false, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.") if err := subFlags.Parse(args); err != nil {