Trigger OnlineDDL workflow when test data changes (#11827)

* trigger OnlineDDL workflow when test data changes Signed-off-by: Florent Poinsard <[email protected]> * trigger schemadiff workflows too Signed-off-by: Florent Poinsard <[email protected]> Signed-off-by: Florent Poinsard <[email protected]>
vitessio · Nov 28, 2022 · 8d35dce · 8d35dce
1 parent 35216e8
commit 8d35dce
Show file tree

Hide file tree

Showing 26 changed files with 41 additions and 6 deletions.
diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_declarative.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_ghost.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_revert.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_revertible.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_scheduler.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_singleton.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_schemadiff_vrepl.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml
@@ -57,6 +57,7 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml'
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml
@@ -50,7 +50,8 @@ jobs:
             - 'test.go'
             - 'Makefile'
             - 'build.env'
-            - 'go.[sumod]'
+            - 'go.sum'
+            - 'go.mod'
             - 'proto/*.proto'
             - 'tools/**'
             - 'config/**'
@@ -70,7 +71,9 @@ jobs:
     - name: Tune the OS
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
+        # Limit local port range to not use ports that overlap with server side
+        # ports that we listen on.
+        sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
         # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
         echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
         sudo sysctl -p /etc/sysctl.conf

diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go
@@ -410,8 +410,12 @@ func setupTestDockerFile(test *selfHostedTest) error {
 	return nil
 }
 
-func writeFileFromTemplate(templateFile, path string, test any) error {
-	tpl, err := template.ParseFiles(templateFile)
+func writeFileFromTemplate(templateFile, filePath string, test any) error {
+	tpl := template.New(path.Base(templateFile))
+	tpl.Funcs(template.FuncMap{
+		"contains": strings.Contains,
+	})
+	tpl, err := tpl.ParseFiles(templateFile)
 	if err != nil {
 		return fmt.Errorf("Error: %s\n", err)
 	}
@@ -422,7 +426,7 @@ func writeFileFromTemplate(templateFile, path string, test any) error {
 		return fmt.Errorf("Error: %s\n", err)
 	}
 
-	f, err := os.Create(path)
+	f, err := os.Create(filePath)
 	if err != nil {
 		return fmt.Errorf("Error creating file: %s\n", err)
 	}
@@ -432,6 +436,6 @@ func writeFileFromTemplate(templateFile, path string, test any) error {
 	if _, err := f.WriteString(mergeBlankLines(buf)); err != nil {
 		return err
 	}
-	fmt.Printf("Generated %s\n", path)
+	fmt.Printf("Generated %s\n", filePath)
 	return nil
 }
diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl
@@ -55,6 +55,9 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/{{.FileName}}'
+            {{- if or (contains .Name "onlineddl") (contains .Name "schemadiff") }}
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
+            {{- end}}
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl
@@ -60,6 +60,9 @@ jobs:
             - 'config/**'
             - 'bootstrap.sh'
             - '.github/workflows/{{.FileName}}'
+            {{- if or (contains .Name "onlineddl") (contains .Name "schemadiff") }}
+            - 'go/test/endtoend/onlineddl/vrepl_suite/testdata'
+            {{- end}}
 
     - name: Set up Go
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'