From 36a981a81147bd6bd45db42c35307c5c19f30332 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 29 Nov 2023 13:24:17 +0200 Subject: [PATCH 1/2] Materializer: normalize schema via schemadiff on --atomic-copy Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/vreplication/fk_test.go | 2 +- go/vt/vtctl/workflow/materializer.go | 17 +++++++++++++++++ go/vt/wrangler/materializer.go | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vreplication/fk_test.go b/go/test/endtoend/vreplication/fk_test.go index 06e94e0222d..4798edfb975 100644 --- a/go/test/endtoend/vreplication/fk_test.go +++ b/go/test/endtoend/vreplication/fk_test.go @@ -88,7 +88,7 @@ func TestFKWorkflow(t *testing.T) { } targetKeyspace := "fktarget" targetTabletId := 200 - vc.AddKeyspace(t, []*Cell{cell}, targetKeyspace, shardName, initialFKTargetVSchema, initialFKSchema, 0, 0, targetTabletId, sourceKsOpts) + vc.AddKeyspace(t, []*Cell{cell}, targetKeyspace, shardName, initialFKTargetVSchema, "", 0, 0, targetTabletId, sourceKsOpts) vtgate.WaitForStatusOfTabletInShard(fmt.Sprintf("%s.%s.primary", targetKeyspace, shardName), 1, 30*time.Second) workflowName := "fk" diff --git a/go/vt/vtctl/workflow/materializer.go b/go/vt/vtctl/workflow/materializer.go index 6be5ac7f445..a04ab0a95b9 100644 --- a/go/vt/vtctl/workflow/materializer.go +++ b/go/vt/vtctl/workflow/materializer.go @@ -29,6 +29,7 @@ import ( "vitess.io/vitess/go/vt/key" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/tmutils" + "vitess.io/vitess/go/vt/schemadiff" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/vtctl/schematools" @@ -444,7 +445,23 @@ func (mz *materializer) deploySchema() error { } if len(applyDDLs) > 0 { + if mz.ms.AtomicCopy { + // AtomicCopy suggests we may be interested in Foreign Key support. As such, we want to + // normalize the source schema: ensure the order of table definitions is compatible with + // the constraints graph. We want to first create the parents, then the children. + // We use schemadiff to normalize the schema. + // For now, and because this is could have wider implications, we ignore any errors in + // reading the source schema. + schema, err := schemadiff.NewSchemaFromQueries(applyDDLs) + if err != nil { + log.Error(vterrors.Wrapf(err, "AtomicCopy: failed to normalize schema via schemadiff")) + } else { + applyDDLs = schema.ToQueries() + log.Infof("AtomicCopy used, and schema was normalized via schemadiff. New queries: %v", applyDDLs) + } + } sql := strings.Join(applyDDLs, ";\n") + log.Infof("materializer.deploySchema: sql=%v", sql) _, err = mz.tmc.ApplySchema(mz.ctx, targetTablet.Tablet, &tmutils.SchemaChange{ SQL: sql, diff --git a/go/vt/wrangler/materializer.go b/go/vt/wrangler/materializer.go index 990492bd191..901b01697a9 100644 --- a/go/vt/wrangler/materializer.go +++ b/go/vt/wrangler/materializer.go @@ -39,6 +39,7 @@ import ( "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/tmutils" "vitess.io/vitess/go/vt/schema" + "vitess.io/vitess/go/vt/schemadiff" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/topo" "vitess.io/vitess/go/vt/topo/topoproto" @@ -1260,7 +1261,23 @@ func (mz *materializer) deploySchema(ctx context.Context) error { } if len(applyDDLs) > 0 { + if mz.ms.AtomicCopy { + // AtomicCopy suggests we may be interested in Foreign Key support. As such, we want to + // normalize the source schema: ensure the order of table definitions is compatible with + // the constraints graph. We want to first create the parents, then the children. + // We use schemadiff to normalize the schema. + // For now, and because this is could have wider implications, we ignore any errors in + // reading the source schema. + schema, err := schemadiff.NewSchemaFromQueries(applyDDLs) + if err != nil { + log.Error(vterrors.Wrapf(err, "AtomicCopy: failed to normalize schema via schemadiff")) + } else { + applyDDLs = schema.ToQueries() + log.Infof("AtomicCopy used, and schema was normalized via schemadiff. New queries: %v", applyDDLs) + } + } sql := strings.Join(applyDDLs, ";\n") + log.Infof("materializer.deploySchema: sql=%v", sql) _, err = mz.wr.tmc.ApplySchema(ctx, targetTablet.Tablet, &tmutils.SchemaChange{ SQL: sql, From 4a58be6a18314c71b3a3f58fac91fd9ccd5cef88 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 29 Nov 2023 13:36:37 +0200 Subject: [PATCH 2/2] reduce logging Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vtctl/workflow/materializer.go | 3 +-- go/vt/wrangler/materializer.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go/vt/vtctl/workflow/materializer.go b/go/vt/vtctl/workflow/materializer.go index a04ab0a95b9..52196661eb5 100644 --- a/go/vt/vtctl/workflow/materializer.go +++ b/go/vt/vtctl/workflow/materializer.go @@ -457,11 +457,10 @@ func (mz *materializer) deploySchema() error { log.Error(vterrors.Wrapf(err, "AtomicCopy: failed to normalize schema via schemadiff")) } else { applyDDLs = schema.ToQueries() - log.Infof("AtomicCopy used, and schema was normalized via schemadiff. New queries: %v", applyDDLs) + log.Infof("AtomicCopy used, and schema was normalized via schemadiff. %v queries normalized", len(applyDDLs)) } } sql := strings.Join(applyDDLs, ";\n") - log.Infof("materializer.deploySchema: sql=%v", sql) _, err = mz.tmc.ApplySchema(mz.ctx, targetTablet.Tablet, &tmutils.SchemaChange{ SQL: sql, diff --git a/go/vt/wrangler/materializer.go b/go/vt/wrangler/materializer.go index 901b01697a9..01a3a640fb8 100644 --- a/go/vt/wrangler/materializer.go +++ b/go/vt/wrangler/materializer.go @@ -1273,11 +1273,10 @@ func (mz *materializer) deploySchema(ctx context.Context) error { log.Error(vterrors.Wrapf(err, "AtomicCopy: failed to normalize schema via schemadiff")) } else { applyDDLs = schema.ToQueries() - log.Infof("AtomicCopy used, and schema was normalized via schemadiff. New queries: %v", applyDDLs) + log.Infof("AtomicCopy used, and schema was normalized via schemadiff. %v queries normalized", len(applyDDLs)) } } sql := strings.Join(applyDDLs, ";\n") - log.Infof("materializer.deploySchema: sql=%v", sql) _, err = mz.wr.tmc.ApplySchema(ctx, targetTablet.Tablet, &tmutils.SchemaChange{ SQL: sql,