[Release 18.0] Backport of #17174 (#14210)

Signed-off-by: Rohit Nayak <[email protected]>
vitessio · Oct 8, 2023 · 2b28e01 · 2b28e01
1 parent 0f9fd92
commit 2b28e01
Show file tree

Hide file tree

Showing 20 changed files with 13,766 additions and 4,741 deletions.
diff --git a/go/cmd/vtctldclient/command/root.go b/go/cmd/vtctldclient/command/root.go
@@ -26,17 +26,20 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"vitess.io/vitess/go/trace"
+	"vitess.io/vitess/go/vt/logutil"
+	"vitess.io/vitess/go/vt/servenv"
+	"vitess.io/vitess/go/vt/vtctl/vtctldclient"
+
 	// These imports ensure init()s within them get called and they register their commands/subcommands.
 	vreplcommon "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/lookupvindex"
+	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/migrate"
+	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/mount"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/movetables"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/reshard"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/vdiff"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/workflow"
-	"vitess.io/vitess/go/trace"
-	"vitess.io/vitess/go/vt/logutil"
-	"vitess.io/vitess/go/vt/servenv"
-	"vitess.io/vitess/go/vt/vtctl/vtctldclient"
 )
 
 var (

diff --git a/go/cmd/vtctldclient/command/vreplication/migrate/migrate.go b/go/cmd/vtctldclient/command/vreplication/migrate/migrate.go
@@ -0,0 +1,134 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrate
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"vitess.io/vitess/go/cmd/vtctldclient/cli"
+	"vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
+
+	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
+)
+
+var (
+	// migrate is the base command for all actions related to the migrate command.
+	migrate = &cobra.Command{
+		Use:                   "Migrate --workflow <workflow> --target-keyspace <keyspace> [command] [command-flags]",
+		Short:                 "Migrate is used to import data from an external cluster into the current cluster.",
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"migrate"},
+		Args:                  cobra.ExactArgs(1),
+	}
+)
+
+var createOptions = struct {
+	MountName      string
+	SourceKeyspace string
+	AllTables      bool
+	IncludeTables  []string
+	ExcludeTables  []string
+	SourceTimeZone string
+	NoRoutingRules bool
+}{}
+
+var createCommand = &cobra.Command{
+	Use:                   "create",
+	Short:                 "Create and optionally run a Migrate VReplication workflow.",
+	Example:               `vtctldclient --server localhost:15999 migrate --workflow import --target-keyspace customer create --source-keyspace commerce --mount-name ext1 --tablet-types replica`,
+	SilenceUsage:          true,
+	DisableFlagsInUseLine: true,
+	Aliases:               []string{"Create"},
+	Args:                  cobra.NoArgs,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		// Either specific tables or the all tables flags are required.
+		if !cmd.Flags().Lookup("tables").Changed && !cmd.Flags().Lookup("all-tables").Changed {
+			return fmt.Errorf("tables or all-tables are required to specify which tables to move")
+		}
+		if err := common.ParseAndValidateCreateOptions(cmd); err != nil {
+			return err
+		}
+		return nil
+	},
+	RunE: commandCreate,
+}
+
+func commandCreate(cmd *cobra.Command, args []string) error {
+	tsp := common.GetTabletSelectionPreference(cmd)
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.MigrateCreateRequest{
+		Workflow:                  common.BaseOptions.Workflow,
+		TargetKeyspace:            common.BaseOptions.TargetKeyspace,
+		SourceKeyspace:            createOptions.SourceKeyspace,
+		MountName:                 createOptions.MountName,
+		SourceTimeZone:            createOptions.SourceTimeZone,
+		Cells:                     common.CreateOptions.Cells,
+		TabletTypes:               common.CreateOptions.TabletTypes,
+		TabletSelectionPreference: tsp,
+		AllTables:                 createOptions.AllTables,
+		IncludeTables:             createOptions.IncludeTables,
+		ExcludeTables:             createOptions.ExcludeTables,
+		OnDdl:                     common.CreateOptions.OnDDL,
+		DeferSecondaryKeys:        common.CreateOptions.DeferSecondaryKeys,
+		AutoStart:                 common.CreateOptions.AutoStart,
+		StopAfterCopy:             common.CreateOptions.StopAfterCopy,
+		NoRoutingRules:            createOptions.NoRoutingRules,
+	}
+
+	_, err := common.GetClient().MigrateCreate(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func addCreateFlags(cmd *cobra.Command) {
+	common.AddCommonCreateFlags(cmd)
+	cmd.Flags().StringVar(&createOptions.SourceKeyspace, "source-keyspace", "", "Keyspace where the tables are being moved from.")
+	cmd.MarkFlagRequired("source-keyspace")
+	cmd.Flags().StringVar(&createOptions.MountName, "mount-name", "", "Name external cluster is mounted as.")
+	cmd.MarkFlagRequired("mount-name")
+	cmd.Flags().StringVar(&createOptions.SourceTimeZone, "source-time-zone", "", "Specifying this causes any DATETIME fields to be converted from the given time zone into UTC.")
+	cmd.Flags().BoolVar(&createOptions.AllTables, "all-tables", false, "Copy all tables from the source.")
+	cmd.Flags().StringSliceVar(&createOptions.IncludeTables, "tables", nil, "Source tables to copy.")
+	cmd.Flags().StringSliceVar(&createOptions.ExcludeTables, "exclude-tables", nil, "Source tables to exclude from copying.")
+	cmd.Flags().BoolVar(&createOptions.NoRoutingRules, "no-routing-rules", false, "(Advanced) Do not create routing rules while creating the workflow. See the reference documentation for limitations if you use this flag.")
+
+}
+
+func registerCommands(root *cobra.Command) {
+	common.AddCommonFlags(migrate)
+	root.AddCommand(migrate)
+	addCreateFlags(createCommand)
+	migrate.AddCommand(createCommand)
+	opts := &common.SubCommandsOpts{
+		SubCommand: "Migrate",
+		Workflow:   "import",
+	}
+	migrate.AddCommand(common.GetCompleteCommand(opts))
+	migrate.AddCommand(common.GetCancelCommand(opts))
+	migrate.AddCommand(common.GetShowCommand(opts))
+	migrate.AddCommand(common.GetStatusCommand(opts))
+}
+
+func init() {
+	common.RegisterCommandHandler("Migrate", registerCommands)
+}
diff --git a/go/cmd/vtctldclient/command/vreplication/mount/mount.go b/go/cmd/vtctldclient/command/vreplication/mount/mount.go
@@ -0,0 +1,176 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrate
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"vitess.io/vitess/go/cmd/vtctldclient/cli"
+	"vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
+
+	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
+)
+
+var (
+	// mount is the base command for all actions related to the mount action.
+	mount = &cobra.Command{
+		Use:                   "Mount [command] [command-flags]",
+		Short:                 "Mount is used to link an external Vitess cluster in order to migrate data from it.",
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"mount"},
+		Args:                  cobra.ExactArgs(1),
+	}
+)
+
+var mountOptions struct {
+	TopoType   string
+	TopoServer string
+	TopoRoot   string
+}
+
+var register = &cobra.Command{
+	Use:                   "register",
+	Short:                 "Register an external Vitess Cluster.",
+	Example:               `vtctldclient --server localhost:15999 mount register --topo-type etcd2 --topo-server localhost:12379 --topo-root /vitess/global ext1`,
+	DisableFlagsInUseLine: true,
+	Aliases:               []string{"Register"},
+	Args:                  cobra.ExactArgs(1),
+	RunE:                  commandRegister,
+}
+
+func commandRegister(cmd *cobra.Command, args []string) error {
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.MountRegisterRequest{
+		TopoType:   mountOptions.TopoType,
+		TopoServer: mountOptions.TopoServer,
+		TopoRoot:   mountOptions.TopoRoot,
+		Name:       cmd.Flags().Arg(0),
+	}
+	_, err := common.GetClient().MountRegister(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Mount %s registered successfully\n", req.Name)
+	return nil
+}
+
+var unregister = &cobra.Command{
+	Use:                   "unregister",
+	Short:                 "Unregister a previously mounted external Vitess Cluster.",
+	Example:               `vtctldclient --server localhost:15999 mount unregister ext1`,
+	DisableFlagsInUseLine: true,
+	Aliases:               []string{"Unregister"},
+	Args:                  cobra.ExactArgs(1),
+	RunE:                  commandUnregister,
+}
+
+func commandUnregister(cmd *cobra.Command, args []string) error {
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.MountUnregisterRequest{
+		Name: args[0],
+	}
+	_, err := common.GetClient().MountUnregister(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Mount %s unregistered successfully\n", req.Name)
+	return nil
+}
+
+var show = &cobra.Command{
+	Use:                   "show",
+	Short:                 "Show attributes of a previously mounted external Vitess Cluster.",
+	Example:               `vtctldclient --server localhost:15999 Mount Show ext1`,
+	DisableFlagsInUseLine: true,
+	Aliases:               []string{"Show"},
+	Args:                  cobra.ExactArgs(1),
+	RunE:                  commandShow,
+}
+
+func commandShow(cmd *cobra.Command, args []string) error {
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.MountShowRequest{
+		Name: args[0],
+	}
+	resp, err := common.GetClient().MountShow(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+	data, err := json.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%s\n", string(data))
+	return nil
+}
+
+var list = &cobra.Command{
+	Use:                   "list",
+	Short:                 "List all mounted external Vitess Clusters.",
+	Example:               `vtctldclient --server localhost:15999 mount list`,
+	DisableFlagsInUseLine: true,
+	Aliases:               []string{"List"},
+	Args:                  cobra.NoArgs,
+	RunE:                  commandList,
+}
+
+func commandList(cmd *cobra.Command, args []string) error {
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.MountListRequest{}
+	resp, err := common.GetClient().MountList(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+	if err != nil {
+		return err
+	}
+	data, err := json.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%s\n", string(data))
+	return nil
+}
+
+func registerCommands(root *cobra.Command) {
+	root.AddCommand(mount)
+	addRegisterFlags(register)
+	mount.AddCommand(register)
+	mount.AddCommand(unregister)
+	mount.AddCommand(show)
+	mount.AddCommand(list)
+}
+
+func addRegisterFlags(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&mountOptions.TopoType, "topo-type", "", "Topo server implementation to use.")
+	cmd.Flags().StringVar(&mountOptions.TopoServer, "topo-server", "", "Topo server address.")
+	cmd.Flags().StringVar(&mountOptions.TopoRoot, "topo-root", "", "Topo server root path.")
+	cmd.MarkFlagRequired("topo-type")
+	cmd.MarkFlagRequired("topo-server")
+	cmd.MarkFlagRequired("topo-root")
+}
+
+func init() {
+	common.RegisterCommandHandler("Mount", registerCommands)
+}
diff --git a/go/flags/endtoend/vtctldclient.txt b/go/flags/endtoend/vtctldclient.txt
@@ -52,6 +52,8 @@ Available Commands:
   GetWorkflows                Gets all vreplication workflows (Reshard, MoveTables, etc) in the given keyspace.
   LegacyVtctlCommand          Invoke a legacy vtctlclient command. Flag parsing is best effort.
   LookupVindex                Perform commands related to creating, backfilling, and externalizing Lookup Vindexes using VReplication workflows.
+  Migrate                     Migrate is used to import data from an external cluster into the current cluster.
+  Mount                       Mount is used to link an external Vitess cluster in order to migrate data from it.
   MoveTables                  Perform commands related to moving tables from a source keyspace to a target keyspace.
   OnlineDDL                   Operates on online DDL (schema migrations).
   PingTablet                  Checks that the specified tablet is awake and responding to RPCs. This command can be blocked by other in-flight operations.