[release-18.0] Migrate CreateLookupVindex and ExternalizeVindex to vtctldclient (#14086)

go/cmd/vtctldclient/command/root.go

@@ -28,6 +28,7 @@ import (
 
 	// These imports ensure init()s within them get called and they register their commands/subcommands.
 	vreplcommon "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
+	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/lookupvindex"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/movetables"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/reshard"
 	_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/vdiff"

go/cmd/vtctldclient/command/vreplication/common/complete.go

@@ -20,9 +20,10 @@ var CompleteOptions = struct {
 
 func GetCompleteCommand(opts *SubCommandsOpts) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:                   "complete",
-		Short:                 "Complete a MoveTables VReplication workflow.",
-		Example:               `vtctldclient --server localhost:15999 movetables --workflow commerce2customer --target-keyspace customer complete`,
+		Use:   "complete",
+		Short: fmt.Sprintf("Complete a %s VReplication workflow.", opts.SubCommand),
+		Example: fmt.Sprintf(`vtctldclient --server localhost:15999 %s --workflow %s --target-keyspace customer complete`,
+			opts.SubCommand, opts.Workflow),
 		DisableFlagsInUseLine: true,
 		Aliases:               []string{"Complete"},
 		Args:                  cobra.NoArgs,
@@ -68,7 +69,7 @@ func commandComplete(cmd *cobra.Command, args []string) error {
 		}
 		output = tout.Bytes()
 	}
-	fmt.Printf("%s\n", output)
+	fmt.Println(string(output))
 
 	return nil
 }

go/cmd/vtctldclient/command/vreplication/lookupvindex/lookupvindex.go

@@ -0,0 +1,321 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lookupvindex
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"vitess.io/vitess/go/cmd/vtctldclient/cli"
+	"vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
+
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
+	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
+	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
+	topoprotopb "vitess.io/vitess/go/vt/topo/topoproto"
+)
+
+var (
+	tabletTypesDefault = []topodatapb.TabletType{
+		topodatapb.TabletType_REPLICA,
+		topodatapb.TabletType_PRIMARY,
+	}
+
+	baseOptions = struct {
+		// This is where the lookup table and VReplicaiton workflow
+		// will be created.
+		TableKeyspace string
+		// This will be the name of the Lookup Vindex and the name
+		// of the VReplication workflow.
+		Name    string
+		Vschema *vschemapb.Keyspace
+	}{}
+
+	// base is the base command for all actions related to Lookup Vindexes.
+	base = &cobra.Command{
+		Use:                   "LookupVindex --name <name> --table-keyspace <keyspace> [command] [command-flags]",
+		Short:                 "Perform commands related to creating, backfilling, and externalizing Lookup Vindexes using VReplication workflows.",
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"lookupvindex"},
+		Args:                  cobra.NoArgs,
+	}
+
+	createOptions = struct {
+		Keyspace                     string
+		Type                         string
+		TableOwner                   string
+		TableOwnerColumns            []string
+		TableName                    string
+		TableVindexType              string
+		Cells                        []string
+		TabletTypes                  []topodatapb.TabletType
+		TabletTypesInPreferenceOrder bool
+		IgnoreNulls                  bool
+		ContinueAfterCopyWithOwner   bool
+	}{}
+
+	externalizeOptions = struct {
+		Keyspace string
+	}{}
+
+	parseAndValidateCreate = func(cmd *cobra.Command, args []string) error {
+		if createOptions.TableName == "" { // Use vindex name
+			createOptions.TableName = baseOptions.Name
+		}
+		if !strings.Contains(createOptions.Type, "lookup") {
+			return fmt.Errorf("vindex type must be a lookup vindex")
+		}
+		baseOptions.Vschema = &vschemapb.Keyspace{
+			Vindexes: map[string]*vschemapb.Vindex{
+				baseOptions.Name: {
+					Type: createOptions.Type,
+					Params: map[string]string{
+						"table":        baseOptions.TableKeyspace + "." + createOptions.TableName,
+						"from":         strings.Join(createOptions.TableOwnerColumns, ","),
+						"to":           "keyspace_id",
+						"ignore_nulls": fmt.Sprintf("%t", createOptions.IgnoreNulls),
+					},
+					Owner: createOptions.TableOwner,
+				},
+			},
+			Tables: map[string]*vschemapb.Table{
+				createOptions.TableOwner: {
+					ColumnVindexes: []*vschemapb.ColumnVindex{
+						{
+							Name:    baseOptions.Name,
+							Columns: createOptions.TableOwnerColumns,
+						},
+					},
+				},
+				createOptions.TableName: {
+					ColumnVindexes: []*vschemapb.ColumnVindex{
+						{
+							// If the vindex name/type is empty then we'll fill this in
+							// later using the defult for the column types.
+							Name:    createOptions.TableVindexType,
+							Columns: createOptions.TableOwnerColumns,
+						},
+					},
+				},
+			},
+		}
+
+		// VReplication specific flags.
+		ttFlag := cmd.Flags().Lookup("tablet-types")
+		if ttFlag != nil && ttFlag.Changed {
+			createOptions.TabletTypes = tabletTypesDefault
+		}
+		cFlag := cmd.Flags().Lookup("cells")
+		if cFlag != nil && cFlag.Changed {
+			for i, cell := range createOptions.Cells {
+				createOptions.Cells[i] = strings.TrimSpace(cell)
+			}
+		}
+		return nil
+	}
+
+	// cancel makes a WorkflowDelete call to a vtctld.
+	cancel = &cobra.Command{
+		Use:                   "cancel",
+		Short:                 "Cancel the VReplication workflow that backfills the Lookup Vindex.",
+		Example:               `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer cancel`,
+		SilenceUsage:          true,
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"Cancel"},
+		Args:                  cobra.NoArgs,
+		RunE:                  commandCancel,
+	}
+
+	// create makes a LookupVindexCreate call to a vtctld.
+	create = &cobra.Command{
+		Use:                   "create",
+		Short:                 "Create the Lookup Vindex in the specified keyspace and backfill it with a VReplication workflow.",
+		Example:               `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer create --keyspace customer --type consistent_lookup_unique --table-owner corder --table-owner-columns sku --table-name corder_lookup_tbl --table-vindex-type unicode_loose_xxhash`,
+		SilenceUsage:          true,
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"Create"},
+		Args:                  cobra.NoArgs,
+		PreRunE:               parseAndValidateCreate,
+		RunE:                  commandCreate,
+	}
+
+	// externalize makes a LookupVindexExternalize call to a vtctld.
+	externalize = &cobra.Command{
+		Use:                   "externalize",
+		Short:                 "Externalize the Lookup Vindex. If the Vindex has an owner the VReplication workflow will also be deleted.",
+		Example:               `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer externalize`,
+		SilenceUsage:          true,
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"Externalize"},
+		Args:                  cobra.NoArgs,
+		RunE:                  commandExternalize,
+	}
+
+	// show makes a GetWorkflows call to a vtctld.
+	show = &cobra.Command{
+		Use:                   "show",
+		Short:                 "Show the status of the VReplication workflow that backfills the Lookup Vindex.",
+		Example:               `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer show`,
+		SilenceUsage:          true,
+		DisableFlagsInUseLine: true,
+		Aliases:               []string{"Show"},
+		Args:                  cobra.NoArgs,
+		RunE:                  commandShow,
+	}
+)
+
+func commandCancel(cmd *cobra.Command, args []string) error {
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.WorkflowDeleteRequest{
+		Keyspace: baseOptions.TableKeyspace,
+		Workflow: baseOptions.Name,
+	}
+	_, err := common.GetClient().WorkflowDelete(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+
+	output := fmt.Sprintf("LookupVindex %s left in place and the %s VReplication wokflow has been deleted",
+		baseOptions.Name, baseOptions.Name)
+	fmt.Println(output)
+
+	return nil
+}
+
+func commandCreate(cmd *cobra.Command, args []string) error {
+	tsp := common.GetTabletSelectionPreference(cmd)
+	cli.FinishedParsing(cmd)
+
+	_, err := common.GetClient().LookupVindexCreate(common.GetCommandCtx(), &vtctldatapb.LookupVindexCreateRequest{
+		Workflow:                   baseOptions.Name,
+		Keyspace:                   createOptions.Keyspace,
+		Vindex:                     baseOptions.Vschema,
+		ContinueAfterCopyWithOwner: createOptions.ContinueAfterCopyWithOwner,
+		Cells:                      createOptions.Cells,
+		TabletTypes:                createOptions.TabletTypes,
+		TabletSelectionPreference:  tsp,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	output := fmt.Sprintf("LookupVindex %s created in the %s keyspace and the %s VReplication wokflow scheduled on the %s shards, use show to view progress",
+		baseOptions.Name, createOptions.Keyspace, baseOptions.Name, baseOptions.TableKeyspace)
+	fmt.Println(output)
+
+	return nil
+}
+
+func commandExternalize(cmd *cobra.Command, args []string) error {
+	if externalizeOptions.Keyspace == "" {
+		externalizeOptions.Keyspace = baseOptions.TableKeyspace
+	}
+	cli.FinishedParsing(cmd)
+
+	resp, err := common.GetClient().LookupVindexExternalize(common.GetCommandCtx(), &vtctldatapb.LookupVindexExternalizeRequest{
+		Keyspace: externalizeOptions.Keyspace,
+		// The name of the workflow and lookup vindex.
+		Name: baseOptions.Name,
+		// Where the lookup table and VReplication workflow were created.
+		TableKeyspace: baseOptions.TableKeyspace,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	output := fmt.Sprintf("LookupVindex %s has been externalized", baseOptions.Name)
+	if resp.WorkflowDeleted {
+		output = output + fmt.Sprintf(" and the %s VReplication workflow has been deleted", baseOptions.Name)
+	}
+	fmt.Println(output)
+
+	return nil
+}
+
+func commandShow(cmd *cobra.Command, args []string) error {
+	cli.FinishedParsing(cmd)
+
+	req := &vtctldatapb.GetWorkflowsRequest{
+		Keyspace: baseOptions.TableKeyspace,
+		Workflow: baseOptions.Name,
+	}
+	resp, err := common.GetClient().GetWorkflows(common.GetCommandCtx(), req)
+	if err != nil {
+		return err
+	}
+
+	data, err := cli.MarshalJSONPretty(resp)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s\n", data)
+
+	return nil
+}
+
+func registerCommands(root *cobra.Command) {
+	base.PersistentFlags().StringVar(&baseOptions.Name, "name", "", "The name of the Lookup Vindex to create. This will also be the name of the VReplication workflow created to backfill the Lookup Vindex.")
+	base.MarkPersistentFlagRequired("name")
+	base.PersistentFlags().StringVar(&baseOptions.TableKeyspace, "table-keyspace", "", "The keyspace to create the lookup table in. This is also where the VReplication workflow is created to backfill the Lookup Vindex.")
+	base.MarkPersistentFlagRequired("table-keyspace")
+	root.AddCommand(base)
+
+	// This will create the lookup vindex in the specified keyspace
+	// and setup a VReplication workflow to backfill its lookup table.
+	create.Flags().StringVar(&createOptions.Keyspace, "keyspace", "", "The keyspace to create the Lookup Vindex in. This is also where the table-owner must exist.")
+	create.MarkFlagRequired("keyspace")
+	create.Flags().StringVar(&createOptions.Type, "type", "", "The type of Lookup Vindex to create.")
+	create.MarkFlagRequired("type")
+	create.Flags().StringVar(&createOptions.TableOwner, "table-owner", "", "The table holding the data which we should use to backfill the Lookup Vindex. This must exist in the same keyspace as the Lookup Vindex.")
+	create.MarkFlagRequired("table-owner")
+	create.Flags().StringSliceVar(&createOptions.TableOwnerColumns, "table-owner-columns", nil, "The columns to read from the owner table. These will be used to build the hash which gets stored as the keyspace_id value in the lookup table.")
+	create.MarkFlagRequired("table-owner-columns")
+	create.Flags().StringVar(&createOptions.TableName, "table-name", "", "The name of the lookup table. If not specified, then it will be created using the same name as the Lookup Vindex.")
+	create.Flags().StringVar(&createOptions.TableVindexType, "table-vindex-type", "", "The primary vindex name/type to use for the lookup table, if the table-keyspace is sharded. This must match the name of a vindex defined in the table-keyspace. If no value is provided then the default type will be used based on the table-owner-columns types.")
+	create.Flags().BoolVar(&createOptions.IgnoreNulls, "ignore-nulls", false, "Do not add corresponding records in the lookup table if any of the owner table's 'from' fields are NULL.")
+	create.Flags().BoolVar(&createOptions.ContinueAfterCopyWithOwner, "continue-after-copy-with-owner", true, "Vindex will continue materialization after the backfill completes when an owner is provided.")
+	// VReplication specific flags.
+	create.Flags().StringSliceVar(&createOptions.Cells, "cells", nil, "Cells to look in for source tablets to replicate from.")
+	create.Flags().Var((*topoprotopb.TabletTypeListFlag)(&createOptions.TabletTypes), "tablet-types", "Source tablet types to replicate from.")
+	create.Flags().BoolVar(&createOptions.TabletTypesInPreferenceOrder, "tablet-types-in-preference-order", true, "When performing source tablet selection, look for candidates in the type order as they are listed in the tablet-types flag.")
+	base.AddCommand(create)
+
+	// This will show the output of GetWorkflows client call
+	// for the VReplication workflow used.
+	base.AddCommand(show)
+
+	// This will also delete the VReplication workflow if the
+	// vindex has an owner as the lookup vindex will then be
+	// managed by VTGate.
+	externalize.Flags().StringVar(&externalizeOptions.Keyspace, "keyspace", "", "The keyspace containing the Lookup Vindex. If no value is specified then the table-keyspace will be used.")
+	base.AddCommand(externalize)
+
+	// The cancel command deletes the VReplication workflow used
+	// to backfill the lookup vindex. It ends up making a
+	// WorkflowDelete VtctldServer call.
+	base.AddCommand(cancel)
+}
+
+func init() {
+	common.RegisterCommandHandler("LookupVindex", registerCommands)
+}

go/cmd/vtctldclient/command/vreplication/movetables/movetables.go

@@ -25,10 +25,8 @@ import (
 var (
 	// moveTables is the base command for all actions related to moveTables.
 	moveTables = &cobra.Command{
-		Use:   "MoveTables --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
-		Short: "Perform commands related to moving tables from a source keyspace to a target keyspace.",
-		Long: `MoveTables commands: Create, Show, Status, SwitchTraffic, ReverseTraffic, Stop, Start, Cancel, and Delete.
-See the --help output for each command for more details.`,
+		Use:                   "MoveTables --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
+		Short:                 "Perform commands related to moving tables from a source keyspace to a target keyspace.",
 		DisableFlagsInUseLine: true,
 		Aliases:               []string{"movetables"},
 		Args:                  cobra.ExactArgs(1),

go/cmd/vtctldclient/command/vreplication/reshard/create.go

@@ -35,8 +35,8 @@ var (
 	// reshardCreate makes a ReshardCreate gRPC call to a vtctld.
 	reshardCreate = &cobra.Command{
 		Use:                   "create",
-		Short:                 "Create and optionally run a reshard VReplication workflow.",
-		Example:               `vtctldclient --server localhost:15999 reshard --workflow customer2customer --target-keyspace customer create --source_shards="0" --target_shards="-80,80-" --cells zone1 --cells zone2 --tablet-types replica`,
+		Short:                 "Create and optionally run a Reshard VReplication workflow.",
+		Example:               `vtctldclient --server localhost:15999 reshard --workflow customer2customer --target-keyspace customer create --source-shards="0" --target-shards="-80,80-" --cells zone1 --cells zone2 --tablet-types replica`,
 		SilenceUsage:          true,
 		DisableFlagsInUseLine: true,
 		Aliases:               []string{"Create"},

go/cmd/vtctldclient/command/vreplication/reshard/reshard.go

@@ -25,10 +25,8 @@ import (
 var (
 	// reshard is the base command for all actions related to reshard.
 	reshard = &cobra.Command{
-		Use:   "Reshard --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
-		Short: "Perform commands related to resharding a keyspace.",
-		Long: `Reshard commands: Create, Show, Status, SwitchTraffic, ReverseTraffic, Stop, Start, Cancel, and Delete.
-See the --help output for each command for more details.`,
+		Use:                   "Reshard --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
+		Short:                 "Perform commands related to resharding a keyspace.",
 		DisableFlagsInUseLine: true,
 		Aliases:               []string{"reshard"},
 		Args:                  cobra.ExactArgs(1),