diff --git a/.gitignore b/.gitignore index 096ab6a..efe8a28 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,6 @@ go.work .DS_Store # the produced binary +report.xml errors/ vitess-tester \ No newline at end of file diff --git a/go.mod b/go.mod index 4b3a16e..acb2fcd 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/hashicorp/hcl v1.0.1-vault-5 // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/magiconair/properties v1.8.7 // indirect diff --git a/go.sum b/go.sum index 83bcf28..1609518 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -188,6 +189,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= +github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/src/vitess-tester/main.go b/src/vitess-tester/main.go index 7d623e9..5c581c1 100644 --- a/src/vitess-tester/main.go +++ b/src/vitess-tester/main.go @@ -42,6 +42,7 @@ func init() { flag.StringVar(&logLevel, "log-level", "error", "The log level of vitess-tester: info, warn, error, debug.") flag.BoolVar(&sharded, "sharded", false, "run all tests on a sharded keyspace") flag.StringVar(&vschemaFile, "vschema", "", "Disable auto-vschema by providing your own vschema file") + flag.BoolVar(&xunit, "xunit", false, "Get output in an xml file instead of errors directory") } type query struct { @@ -71,16 +72,17 @@ func loadAllTests() (tests []string, err error) { return tests, nil } -func executeTests(fileNames []string) (failed bool) { +func executeTests(fileNames []string, s Suite) (failed bool) { for _, name := range fileNames { - errFileReporter := newFileReporter(name) - vTester := newTester(name, errFileReporter) + errReporter := s.NewReporterForFile(name) + vTester := newTester(name, errReporter) err := vTester.Run() if err != nil { failed = true continue } - failed = errFileReporter.Failed() + failed = failed || errReporter.Failed() + s.CloseReportForFile() } return } @@ -264,8 +266,16 @@ func main() { panic(err.Error()) } - if failed := executeTests(tests); failed { - log.Errorf("some tests failed 😭\nsee errors in errors folder") + var reporterSuite Suite + if xunit { + reporterSuite = newXMLTestSuite() + } else { + reporterSuite = newFileReporterSuite() + } + failed := executeTests(tests, reporterSuite) + outputFile := reporterSuite.Close() + if failed { + log.Errorf("some tests failed 😭\nsee errors in %v", outputFile) os.Exit(1) } println("Great, All tests passed") diff --git a/src/vitess-tester/reporter.go b/src/vitess-tester/reporter.go index aa73728..2d929fb 100644 --- a/src/vitess-tester/reporter.go +++ b/src/vitess-tester/reporter.go @@ -26,6 +26,12 @@ import ( "time" ) +type Suite interface { + NewReporterForFile(name string) Reporter + CloseReportForFile() + Close() string +} + type Reporter interface { AddTestCase(query string, lineNo int) EndTestCase() @@ -34,6 +40,22 @@ type Reporter interface { Failed() bool } +type FileReporterSuite struct{} + +func (frs *FileReporterSuite) NewReporterForFile(name string) Reporter { + return newFileReporter(name) +} + +func (frs *FileReporterSuite) CloseReportForFile() {} + +func (frs *FileReporterSuite) Close() string { + return "errors" +} + +func newFileReporterSuite() *FileReporterSuite { + return &FileReporterSuite{} +} + type FileReporter struct { name string errorFile *os.File diff --git a/src/vitess-tester/xunit.go b/src/vitess-tester/xunit.go new file mode 100644 index 0000000..1dfe9eb --- /dev/null +++ b/src/vitess-tester/xunit.go @@ -0,0 +1,111 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "time" + + "github.com/jstemmer/go-junit-report/v2/junit" +) + +type XMLTestSuite struct { + ts junit.Testsuites + startTime time.Time + currTestSuite junit.Testsuite + currTestCase *junit.Testcase +} + +var _ Suite = (*XMLTestSuite)(nil) + +func newXMLTestSuite() *XMLTestSuite { + return &XMLTestSuite{} +} + +func (xml *XMLTestSuite) NewReporterForFile(name string) Reporter { + xml.startTime = time.Now() + xml.currTestSuite = junit.Testsuite{ + Name: name, + Timestamp: xml.startTime.String(), + } + return xml +} + +func (xml *XMLTestSuite) CloseReportForFile() { + xml.currTestSuite.Time = fmt.Sprintf("%v", time.Since(xml.startTime).Seconds()) + xml.ts.AddSuite(xml.currTestSuite) +} + +func (xml *XMLTestSuite) Close() string { + fileName := "report.xml" + file, err := os.Create(fileName) + if err != nil { + panic(err) + } + defer file.Close() + err = xml.ts.WriteXML(file) + if err != nil { + panic(err) + } + return fileName +} + +func (xml *XMLTestSuite) AddTestCase(query string, lineNo int) { + xml.currTestCase = &junit.Testcase{ + Name: query, + Status: fmt.Sprintf("Line No. - %v", lineNo), + } +} + +func (xml *XMLTestSuite) EndTestCase() { + xml.currTestSuite.AddTestcase(*xml.currTestCase) + xml.currTestCase = nil +} + +func (xml *XMLTestSuite) AddFailure(err error) { + if xml.currTestCase == nil { + xml.AddTestCase("SETUP", 0) + xml.AddFailure(err) + xml.EndTestCase() + return + } + + if xml.currTestCase.Failure != nil { + xml.currTestCase.Failure.Message += "\n" + err.Error() + return + } + xml.currTestCase.Failure = &junit.Result{ + Message: err.Error(), + Type: fmt.Sprintf("%T", err), + } +} + +func (xml *XMLTestSuite) Report() string { + return fmt.Sprintf( + "%s: ok! Ran %d queries, %d successfully and %d failures take time %v s\n", + xml.currTestSuite.Name, + xml.currTestSuite.Tests, + xml.currTestSuite.Tests-xml.currTestSuite.Failures, + xml.currTestSuite.Failures, + time.Since(xml.startTime), + ) +} + +func (xml *XMLTestSuite) Failed() bool { + return xml.currTestSuite.Failures != 0 +}