Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement --max-report-sample-rows for VDiff #14437

Merged
merged 10 commits into from
Nov 24, 2023
Merged

implement --max-report-sample-rows for VDiff #14437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
b0f8cd2
implement `--max-vdiff-report-rows` for VDiff
mcrauwel Nov 2, 2023
5f2773a
refactor variables to max-report-sample-rows
mcrauwel Nov 2, 2023
ba14b04
refactor counter ints to unsigned int64's
mcrauwel Nov 2, 2023
a6ada3f
implement `--max-report-sample-rows` in vtctldclient
mcrauwel Nov 2, 2023
c5f1375
Duration.Seconds() is an int64 and not a uint64
mcrauwel Nov 2, 2023
c41431e
remove flag in old vtctlclient
mcrauwel Nov 2, 2023
736bafe
revert int64 to uint64 changes to comply with Vitess project policy
mcrauwel Nov 3, 2023
022e47b
rename `MaxReportSampleRows` to `MaxSampleRows`, fix int-validation
mcrauwel Nov 16, 2023
0c0b702
fix broken test
mcrauwel Nov 16, 2023
35d2bfa
Trigger rebuild
rohit-nayak-ps Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions go/cmd/vtctldclient/command/vreplication/vdiff/vdiff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ var (
Limit uint32 // We only accept positive values but pass on an int64
FilteredReplicationWaitTime time.Duration
DebugQuery bool
MaxReportSampleRows uint32 // We only accept positive values but pass on an int64
OnlyPKs bool
UpdateTableStats bool
MaxExtraRowsToCompare uint32 // We only accept positive values but pass on an int64
Expand Down Expand Up @@ -279,6 +280,7 @@ func commandCreate(cmd *cobra.Command, args []string) error {
Wait: createOptions.Wait,
WaitUpdateInterval: protoutil.DurationToProto(createOptions.WaitUpdateInterval),
AutoRetry: createOptions.AutoRetry,
MaxReportSampleRows: int64(createOptions.MaxReportSampleRows),
})

if err != nil {
Expand Down Expand Up @@ -863,6 +865,7 @@ func registerCommands(root *cobra.Command) {
create.Flags().DurationVar(&createOptions.FilteredReplicationWaitTime, "filtered-replication-wait-time", 30*time.Second, "Specifies the maximum time to wait, in seconds, for replication to catch up when syncing tablet streams.")
create.Flags().Uint32Var(&createOptions.Limit, "limit", math.MaxUint32, "Max rows to stop comparing after.")
create.Flags().BoolVar(&createOptions.DebugQuery, "debug-query", false, "Adds a mysql query to the report that can be used for further debugging.")
create.Flags().Uint32Var(&createOptions.MaxReportSampleRows, "max-report-sample-rows", 10, "Maximum number of row differences to report (0 for all differences). NOTE: when increasing this value it is highly recommended to also specify --only-pks")
create.Flags().BoolVar(&createOptions.OnlyPKs, "only-pks", false, "When reporting missing rows, only show primary keys in the report.")
create.Flags().StringSliceVar(&createOptions.Tables, "tables", nil, "Only run vdiff for these tables in the workflow.")
create.Flags().Uint32Var(&createOptions.MaxExtraRowsToCompare, "max-extra-rows-to-compare", 1000, "If there are collation differences between the source and target, you can have rows that are identical but simply returned in a different order from MySQL. We will do a second pass to compare the rows for any actual differences in this case and this flag allows you to control the resources used for this operation.")
Expand Down
271 changes: 271 additions & 0 deletions go/cmd/vtctldclient/command/vreplication/vdiff/vdiff_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,277 @@ func TestVDiffUnsharded(t *testing.T) {
}
}
]`),
}, {
id: "9", // --max-vdiff-report-rows=20 --only-pks
result: sqltypes.MakeTestResult(fields,
"completed||t1|"+UUID+"|completed|30|"+starttime+"|30|"+comptime+"|1|"+
`{"TableName": "t1", "MatchingRows": 10, "ProcessedRows": 30, "MismatchedRows": 20, "ExtraRowsSource": 0, `+
`"ExtraRowsTarget": 0, "MismatchedRowsSample": [`+
`{"Source": {"Row": {"c1": "2"}}, "Target": {"Row": {"c1": "2"}}},`+
`{"Source": {"Row": {"c1": "3"}}, "Target": {"Row": {"c1": "3"}}},`+
`{"Source": {"Row": {"c1": "4"}}, "Target": {"Row": {"c1": "4"}}},`+
`{"Source": {"Row": {"c1": "5"}}, "Target": {"Row": {"c1": "5"}}},`+
`{"Source": {"Row": {"c1": "6"}}, "Target": {"Row": {"c1": "6"}}},`+
`{"Source": {"Row": {"c1": "7"}}, "Target": {"Row": {"c1": "7"}}},`+
`{"Source": {"Row": {"c1": "8"}}, "Target": {"Row": {"c1": "8"}}},`+
`{"Source": {"Row": {"c1": "9"}}, "Target": {"Row": {"c1": "9"}}},`+
`{"Source": {"Row": {"c1": "10"}}, "Target": {"Row": {"c1": "10"}}},`+
`{"Source": {"Row": {"c1": "11"}}, "Target": {"Row": {"c1": "11"}}},`+
`{"Source": {"Row": {"c1": "12"}}, "Target": {"Row": {"c1": "12"}}},`+
`{"Source": {"Row": {"c1": "13"}}, "Target": {"Row": {"c1": "13"}}},`+
`{"Source": {"Row": {"c1": "14"}}, "Target": {"Row": {"c1": "14"}}},`+
`{"Source": {"Row": {"c1": "15"}}, "Target": {"Row": {"c1": "15"}}},`+
`{"Source": {"Row": {"c1": "16"}}, "Target": {"Row": {"c1": "16"}}},`+
`{"Source": {"Row": {"c1": "17"}}, "Target": {"Row": {"c1": "17"}}},`+
`{"Source": {"Row": {"c1": "18"}}, "Target": {"Row": {"c1": "18"}}},`+
`{"Source": {"Row": {"c1": "19"}}, "Target": {"Row": {"c1": "19"}}},`+
`{"Source": {"Row": {"c1": "20"}}, "Target": {"Row": {"c1": "20"}}},`+
`{"Source": {"Row": {"c1": "21"}}, "Target": {"Row": {"c1": "21"}}}`+
`]}`),
report: fmt.Sprintf(badReportfmt,
env.targetKeyspace, UUID, 30, true, starttime, comptime, 30, 10, 20, 0, 0, 30, 10, 20, 0, 0,
`"MismatchedRowsSample": [
{
"Source": {
"Row": {
"c1": "2"
}
},
"Target": {
"Row": {
"c1": "2"
}
}
},
{
"Source": {
"Row": {
"c1": "3"
}
},
"Target": {
"Row": {
"c1": "3"
}
}
},
{
"Source": {
"Row": {
"c1": "4"
}
},
"Target": {
"Row": {
"c1": "4"
}
}
},
{
"Source": {
"Row": {
"c1": "5"
}
},
"Target": {
"Row": {
"c1": "5"
}
}
},
{
"Source": {
"Row": {
"c1": "6"
}
},
"Target": {
"Row": {
"c1": "6"
}
}
},
{
"Source": {
"Row": {
"c1": "7"
}
},
"Target": {
"Row": {
"c1": "7"
}
}
},
{
"Source": {
"Row": {
"c1": "8"
}
},
"Target": {
"Row": {
"c1": "8"
}
}
},
{
"Source": {
"Row": {
"c1": "9"
}
},
"Target": {
"Row": {
"c1": "9"
}
}
},
{
"Source": {
"Row": {
"c1": "10"
}
},
"Target": {
"Row": {
"c1": "10"
}
}
},
{
"Source": {
"Row": {
"c1": "11"
}
},
"Target": {
"Row": {
"c1": "11"
}
}
},
{
"Source": {
"Row": {
"c1": "12"
}
},
"Target": {
"Row": {
"c1": "12"
}
}
},
{
"Source": {
"Row": {
"c1": "13"
}
},
"Target": {
"Row": {
"c1": "13"
}
}
},
{
"Source": {
"Row": {
"c1": "14"
}
},
"Target": {
"Row": {
"c1": "14"
}
}
},
{
"Source": {
"Row": {
"c1": "15"
}
},
"Target": {
"Row": {
"c1": "15"
}
}
},
{
"Source": {
"Row": {
"c1": "16"
}
},
"Target": {
"Row": {
"c1": "16"
}
}
},
{
"Source": {
"Row": {
"c1": "17"
}
},
"Target": {
"Row": {
"c1": "17"
}
}
},
{
"Source": {
"Row": {
"c1": "18"
}
},
"Target": {
"Row": {
"c1": "18"
}
}
},
{
"Source": {
"Row": {
"c1": "19"
}
},
"Target": {
"Row": {
"c1": "19"
}
}
},
{
"Source": {
"Row": {
"c1": "20"
}
},
"Target": {
"Row": {
"c1": "20"
}
}
},
{
"Source": {
"Row": {
"c1": "21"
}
},
"Target": {
"Row": {
"c1": "21"
}
}
}
]`),
},
}

Expand Down
Loading
Loading