From 8d35dce09d1385b256111186098cc671956a9346 Mon Sep 17 00:00:00 2001 From: FlorentP <35779988+frouioui@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:52:42 +0100 Subject: [PATCH] Trigger OnlineDDL workflow when test data changes (#11827) * trigger OnlineDDL workflow when test data changes Signed-off-by: Florent Poinsard * trigger schemadiff workflows too Signed-off-by: Florent Poinsard Signed-off-by: Florent Poinsard --- .../cluster_endtoend_onlineddl_declarative.yml | 1 + ...luster_endtoend_onlineddl_declarative_mysql57.yml | 1 + .../workflows/cluster_endtoend_onlineddl_ghost.yml | 1 + .../cluster_endtoend_onlineddl_ghost_mysql57.yml | 1 + .../workflows/cluster_endtoend_onlineddl_revert.yml | 1 + .../cluster_endtoend_onlineddl_revert_mysql57.yml | 1 + .../cluster_endtoend_onlineddl_revertible.yml | 1 + ...cluster_endtoend_onlineddl_revertible_mysql57.yml | 1 + .../cluster_endtoend_onlineddl_scheduler.yml | 1 + .../cluster_endtoend_onlineddl_scheduler_mysql57.yml | 1 + .../cluster_endtoend_onlineddl_singleton.yml | 1 + .../cluster_endtoend_onlineddl_singleton_mysql57.yml | 1 + .../workflows/cluster_endtoend_onlineddl_vrepl.yml | 1 + .../cluster_endtoend_onlineddl_vrepl_mysql57.yml | 1 + .../cluster_endtoend_onlineddl_vrepl_stress.yml | 1 + ...uster_endtoend_onlineddl_vrepl_stress_mysql57.yml | 1 + ...cluster_endtoend_onlineddl_vrepl_stress_suite.yml | 1 + ...endtoend_onlineddl_vrepl_stress_suite_mysql57.yml | 1 + .../cluster_endtoend_onlineddl_vrepl_suite.yml | 1 + ...luster_endtoend_onlineddl_vrepl_suite_mysql57.yml | 1 + .../workflows/cluster_endtoend_schemadiff_vrepl.yml | 1 + .../cluster_endtoend_schemadiff_vrepl_mysql57.yml | 1 + ...cluster_endtoend_tabletmanager_throttler_topo.yml | 7 +++++-- test/ci_workflow_gen.go | 12 ++++++++---- test/templates/cluster_endtoend_test.tpl | 3 +++ test/templates/cluster_endtoend_test_mysql57.tpl | 3 +++ 26 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml index 204ee124ee4..96886a93e58 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_declarative.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml index 920e5dcc74e..968cfe9b690 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml index d9224b7e93a..66d37682e3a 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_ghost.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml index e4a20add755..902aadc2042 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 6e793444302..145e68e4545 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_revert.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml index 986f32b3590..b8d355eb3c0 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml index fd88576e47a..ae1789b1e8e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_revertible.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml index cf11ffacfc0..d025230233c 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index d5ca1bce14b..c6f9a44a06e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_scheduler.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml index 5ba00332489..58024902dfc 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml index 41f84f940be..d3f25bd1a22 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_singleton.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml index a2c38b5f8be..f59b9ed2541 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 2e18f4f4a3f..33375df390e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml index 857b3cefd41..26c6d01440e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index b163665eb71..fd42e41ce6d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml index 6ce05d7475b..40517d35936 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index a71aa31bc87..26776f24b28 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml index 04bbee0a5fd..f45d16ac8a5 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 1d0366adf4e..0a079742de5 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml index 587a3e8166a..d3a980b2609 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index df949faab01..838e2adf428 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_schemadiff_vrepl.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml index f0436decc7e..f8870d0760a 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml @@ -57,6 +57,7 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml' + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml index 4c1a296f38c..d98c8048ae7 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml @@ -50,7 +50,8 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' @@ -70,7 +71,9 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 7ef7c532834..15b8ec34178 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -410,8 +410,12 @@ func setupTestDockerFile(test *selfHostedTest) error { return nil } -func writeFileFromTemplate(templateFile, path string, test any) error { - tpl, err := template.ParseFiles(templateFile) +func writeFileFromTemplate(templateFile, filePath string, test any) error { + tpl := template.New(path.Base(templateFile)) + tpl.Funcs(template.FuncMap{ + "contains": strings.Contains, + }) + tpl, err := tpl.ParseFiles(templateFile) if err != nil { return fmt.Errorf("Error: %s\n", err) } @@ -422,7 +426,7 @@ func writeFileFromTemplate(templateFile, path string, test any) error { return fmt.Errorf("Error: %s\n", err) } - f, err := os.Create(path) + f, err := os.Create(filePath) if err != nil { return fmt.Errorf("Error creating file: %s\n", err) } @@ -432,6 +436,6 @@ func writeFileFromTemplate(templateFile, path string, test any) error { if _, err := f.WriteString(mergeBlankLines(buf)); err != nil { return err } - fmt.Printf("Generated %s\n", path) + fmt.Printf("Generated %s\n", filePath) return nil } diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 2a747890bd5..e0bfe3e1a45 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -55,6 +55,9 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/{{.FileName}}' + {{- if or (contains .Name "onlineddl") (contains .Name "schemadiff") }} + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' + {{- end}} - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl index b7c41fdd464..238fab09b11 100644 --- a/test/templates/cluster_endtoend_test_mysql57.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -60,6 +60,9 @@ jobs: - 'config/**' - 'bootstrap.sh' - '.github/workflows/{{.FileName}}' + {{- if or (contains .Name "onlineddl") (contains .Name "schemadiff") }} + - 'go/test/endtoend/onlineddl/vrepl_suite/testdata' + {{- end}} - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'