test-appliance: gce-xfstests: optionally e-mail the junit results file

Signed-off-by: Theodore Ts'o <[email protected]>

run-fstests/gce-xfstests

@@ -804,6 +804,7 @@ SENDGRID_API_KEY="$GCE_SG_API"
 {
     declare -p GCE_REPORT_SENDER
     declare -p GCE_REPORT_EMAIL
+    declare -p GCE_JUNIT_EMAIL
     declare -p SENDGRID_API_KEY
     declare -p GCE_UPLOAD_SUMMARY
     declare -p PRIMARY_FSTYPE
@@ -1235,6 +1236,11 @@ then
     ARG="$ARG report_email=$GCE_REPORT_EMAIL"
 fi
 
+if test -n "$GCE_JUNIT_EMAIL"
+then
+    ARG="$ARG junit_email=$GCE_JUNIT_EMAIL"
+fi
+
 ARG="$ARG orig_cmdline=$ORIG_CMDLINE_B64"
 
 if test -n "$DO_LOCAL_SSD"

run-fstests/util/gce-ltm-funcs

@@ -74,6 +74,9 @@ function send_to_ltm() {
     if [ -n "${GCE_REPORT_EMAIL+x}" ]; then
 	LTM_OPTS="${LTM_OPTS:+$LTM_OPTS, }\"report_email\":\"$GCE_REPORT_EMAIL\""
     fi
+    if [ -n "${GCE_JUNIT_EMAIL+x}" ]; then
+	LTM_OPTS="${LTM_OPTS:+$LTM_OPTS, }\"junit_email\":\"$GCE_JUNIT_EMAIL\""
+    fi
     if [ -n "$COMMIT" ]; then
 	LTM_OPTS="${LTM_OPTS:+$LTM_OPTS, }\"commit_id\":\"$COMMIT\""
     fi

run-fstests/util/parse_cli

@@ -270,6 +270,7 @@ image-family:
 image-project:
 initrd:
 instance-name:
+junit-email:
 kernel:
 kernel-arch:
 kbuild-opts:
@@ -291,6 +292,7 @@ no-archive
 no-collapse
 no-email
 no-insert
+no-junit-email
 no-log
 no-preemptible
 no-spot-fallback
@@ -740,6 +742,14 @@ while (( $# >= 1 )); do
 	    supported_flavors gce
 	    GCE_REPORT_EMAIL=""
 	    ;;
+	--junit-email) shift
+	    supported_flavors gce
+	    GCE_JUNIT_EMAIL="$1"
+	    ;;
+	--no-junit-email)
+	    supported_flavors gce
+	    GCE_JUNIT_EMAIL=""
+	    ;;
 	--no-region-shard)
 	    supported_flavors gce
 	    if test -z "$RUN_ON_LTM"; then

selftests/appliance

@@ -66,7 +66,7 @@ build_appliance ()
 	echo "Launching GCE Test VM $DATECODE-$ARCH"
 	$NO_ACTION $GCE_XFSTESTS --arch $ARCH --kernel "$KERNEL_DEB" \
 		   --testrunid "test-$DATECODE-$ARCH" --note selftest \
-		   --no-email \
+		   --no-email --no-junit-email \
 		   -c ext4/4k,btrfs/4k generic/001 btrfs/001 btrfs/064
 	GCE_TEST_VMS="$GCE_TEST_VMS test-$DATECODE-$ARCH"
     fi

selftests/ltm-kcs

@@ -57,7 +57,7 @@ function test-ltm ()
 	    a="--arch $ARCH"
 	fi
 	echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 " \
-	     --testrunid "test-$DATECODE-$ARCH"	--no-email \
+	     --testrunid "test-$DATECODE-$ARCH"	--no-email --no-junit-email \
 	     "btrfs/064 $a --kernel $k" >> $LTM_TEST_FILE
 	GCE_TEST_VMS="$GCE_TEST_VMS ltm-test-$DATECODE-$ARCH"
     fi
@@ -71,9 +71,10 @@ function test-kcs ()
     if test $ARCH = "arm64" ; then
 	a="--arch $ARCH"
     fi
-    echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 btrfs/064 $a" \
+    echo "gce-xfstests ltm -c ext4/4k,btrfs/4k generic/001 btrfs/001 btrfs/064 $a " \
 	 "--repo https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git " \
-	 --no-email --commit v5.15.79 "--testrunid testkcs-$DATECODE-$ARCH" \
+	 "--no-email --no-junit-email --commit v5.15.79 " \
+	 "--testrunid testkcs-$DATECODE-$ARCH" \
 	 >> $LTM_TEST_FILE
 	GCE_TEST_VMS="$GCE_TEST_VMS ltm-testkcs-$DATECODE-$ARCH"
 }
@@ -100,7 +101,7 @@ echo -n "Waiting for test results..."
 w=60
 while /bin/true
 do
-    n=$($GCE_XFSTESTS ls-results | grep "$DATECODE" | \
+    n=$($GCE_XFSTESTS ls-results | grep "$DATECODE" | grep .tar.xz | \
 	    grep -v "64-a[ab]" | wc -l)
     echo -n "$n"
     sleep "$w"

test-appliance/files/root/runtests.sh

@@ -528,7 +528,8 @@ done
 
 runtests_after_tests
 
-/usr/local/bin/gen_results_summary $RESULTS > $RESULTS/report
+/usr/local/bin/gen_results_summary $RESULTS \
+	--merge_file /tmp/results.xml > $RESULTS/report
 
 echo "-------------------- Summary report"
 

test-appliance/files/usr/local/lib/gce-server/ltm/shard.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime/debug"
+	"strings"
 	"sync"
 	"time"
 
@@ -406,12 +407,26 @@ func (shard *ShardWorker) getResults() string {
 	prefix := fmt.Sprintf("%s/results.%s", shard.sharder.bucketSubdir, shard.resultsName)
 	for attempts := maxAttempts; attempts > 0; attempts-- {
 		resultFiles, err := shard.sharder.gce.GetFileNames(prefix)
+		var resultURL string
 		check.NoError(err, shard.log, "Failed to get GS filenames")
-		if err == nil && len(resultFiles) == 1 {
-			shard.log.WithField("resultURL", resultFiles[0]).Info("Found result file url")
-			return fmt.Sprintf("gs://%s/%s", shard.sharder.gsBucket, resultFiles[0])
+		for _, value := range resultFiles {
+			if !strings.HasSuffix(value, ".tar.xz") {
+				continue
+			}
+			resultURL = fmt.Sprintf("gs://%s/%s", shard.sharder.gsBucket, value)
+			break
+		}
+		if err == nil && resultURL != "" {
+			shard.log.WithField("resultURL", resultURL).Info("Found result file url")
+			return resultURL
+
 		}
-		shard.log.WithField("attemptsLeft", attempts).Debug("No GS file with matching url")
+		shard.log.WithFields(logrus.Fields{
+			"attemptsLeft": attempts,
+			"prefix":       prefix,
+			"resultURL":    resultURL,
+			"err":          err,
+		}).Debug("No GS file with matching url")
 		time.Sleep(gsInterval)
 	}
 	return ""

test-appliance/files/usr/local/lib/gce-server/ltm/sharder.go

@@ -56,6 +56,7 @@ type ShardScheduler struct {
 	kernelArch     string
 	arch           string
 	reportReceiver string
+	junitReceiver  string
 	maxShards      int
 	keepDeadVM     bool
 	monitorTimeout time.Duration
@@ -131,6 +132,7 @@ func NewShardScheduler(c server.TaskRequest, testID string) *ShardScheduler {
 		kernelArch:     "",
 		arch:           c.Options.Arch,
 		reportReceiver: c.Options.ReportEmail,
+		junitReceiver:  c.Options.JunitEmail,
 		maxShards:      0,
 		keepDeadVM:     false,
 		monitorTimeout: defaultMonitorTimeout,
@@ -632,19 +634,34 @@ func (sharder *ShardScheduler) genResultsSummary() {
 func (sharder *ShardScheduler) emailReport() {
 	if sharder.reportReceiver == "" {
 		sharder.log.Info("Skipping e-mail report")
-		return
-	}
-	sharder.log.Info("Sending e-mail report")
-	subject := fmt.Sprintf("xfstests results %s-%s %s", server.LTMUserName, sharder.testID, sharder.kernelVersion)
+	} else {
+		sharder.log.Info("Sending e-mail report")
+		subject := fmt.Sprintf("xfstests results %s-%s %s", server.LTMUserName, sharder.testID, sharder.kernelVersion)
+
+		b, err := ioutil.ReadFile(sharder.aggDir + "report")
+		content := string(b)
+		if !check.NoError(err, sharder.log, "Failed to read the report file") {
+			content = "Unable to generate test summary report"
+		}
 
-	b, err := ioutil.ReadFile(sharder.aggDir + "report")
-	content := string(b)
-	if !check.NoError(err, sharder.log, "Failed to read the report file") {
-		content = "Unable to generate test summary report"
+		err = email.Send(subject, content, sharder.reportReceiver)
+		check.Panic(err, sharder.log, "Failed to send the report email")
 	}
+	if sharder.junitReceiver == "" {
+		sharder.log.Info("Skipping junit file e-mail")
+	} else {
+		sharder.log.Info("Sending junit file e-mail")
+		subject := fmt.Sprintf("xfstests junit results %s-%s %s", server.LTMUserName, sharder.testID, sharder.kernelVersion)
+
+		b, err := ioutil.ReadFile(sharder.aggDir + "results.xml")
+		content := string(b)
+		if !check.NoError(err, sharder.log, "Failed to read the results xml file") {
+			content = "Unable to generate combined junit file"
+		}
 
-	err = email.Send(subject, content, sharder.reportReceiver)
-	check.Panic(err, sharder.log, "Failed to send the email")
+		err = email.Send(subject, content, sharder.junitReceiver)
+		check.Panic(err, sharder.log, "Failed to send the junit email")
+	}
 }
 
 func (sharder *ShardScheduler) sendKCSReport() {

test-appliance/files/usr/local/lib/gce-server/util/parser/parser.go

@@ -7,21 +7,27 @@ package parser
 import (
 	"encoding/base64"
 	"fmt"
-	"thunk.org/gce-server/util/check"
 	"strings"
+	"thunk.org/gce-server/util/check"
 )
 
 const (
 	primaryFS = "ext4"
 	xfsPath   = "/root"
 )
 
-var invalidBools = []string{"ltm", "--no-region-shard", "--no-email"}
+var invalidBools = []string{
+	"ltm",
+	"--no-region-shard",
+	"--no-email",
+	"--no-junit-email",
+}
 var invalidOpts = []string{
 	"--instance-name",
 	"--bucket-subdir",
 	"--gs-bucket",
 	"--email",
+	"--junit-email",
 	"--gce-zone",
 	"--testrunid",
 	"--hooks",
@@ -47,6 +53,7 @@ var invalidOpts = []string{
 Cmd parses a cmdline into validArgs and configs.
 
 Returns:
+
 	validArgs - a slice of cmd args not related to test configurations.
 	Parser removes arguments from the original cmd that don't make sense
 	for LTM (e.g. ltm, --instance-name).
@@ -158,6 +165,7 @@ func defaultConfigs(configs map[string][]string) error {
 /*
 singleConfig parses a single configuration and adds it to the map.
 Possible pattern of configs:
+
 	<fs>/<cfg> (e.g. ext4/4k) - checks /root/fs/<fs>/cfg/<cfg>.list
 	for a list of configurations, and read config lines from each file.
 

test-appliance/files/usr/local/lib/gce-server/util/server/server.go

@@ -2,9 +2,9 @@
 Package server implements server functions shared by LTM and KCS.
 
 Files included in this package:
+
 	server.go: 	Web servers interface and handlers, and functions to send requests.
 	info.go: 	Construct human-friendly status info for multiple modules.
-
 */
 package server
 
@@ -122,13 +122,14 @@ type UserOptions struct {
 	BucketSubdir   string `json:"bucket_subdir"`
 	GsKernel       string `json:"gs_kernel"`
 	ReportEmail    string `json:"report_email"`
+	JunitEmail     string `json:"junit_email"`
 	CommitID       string `json:"commit_id"`
 	GitRepo        string `json:"git_repo"`
 	BranchName     string `json:"branch_name"`
 	UnWatch        string `json:"unwatch"`
 	BadCommit      string `json:"bad_commit"`
 	GoodCommit     string `json:"good_commit"`
-	KConfig	       string `json:"kconfig"`
+	KConfig        string `json:"kconfig"`
 	KConfigOpts    string `json:"kconfig_opts"`
 	KbuildOpts     string `json:"kbuild_opts"`
 	Arch           string `json:"arch"`
@@ -218,7 +219,7 @@ func init() {
 		panic("Failed to get project config in init()")
 	}
 
-	certPath =  "/root/xfstests_bld/run-fstests/.gce_xfstests_cert_" + projID + ".pem"
+	certPath = "/root/xfstests_bld/run-fstests/.gce_xfstests_cert_" + projID + ".pem"
 }
 
 // New sets up a new https server.
@@ -227,7 +228,7 @@ func New(addr string, name string) (*Instance, error) {
 	log.Info("Initiating server")
 
 	server := &Instance{
-		name:	name,
+		name:   name,
 		addr:   addr,
 		router: mux.NewRouter(),
 		store:  sessions.NewCookieStore(key),

test-appliance/files/usr/local/sbin/gce-shutdown

@@ -89,6 +89,7 @@ then
     fi
 
     REPORT_EMAIL=$(gce_attribute report_email)
+    JUNIT_EMAIL=$(gce_attribute junit_email)
     run_hooks send-email
     if test -n "$REPORT_EMAIL" -a -n "$SENDGRID_API_KEY"
     then
@@ -111,9 +112,26 @@ then
 	/usr/local/sbin/send-mail.py --sender "$GCE_REPORT_SENDER" \
 		-s "$subj" "$REPORT_EMAIL" < "$RESULTS_REPORT"
     fi
+    if test -n "$JUNIT_EMAIL" -a -n "$SENDGRID_API_KEY" -a -f /tmp/results.xml
+    then
+	subj="$suite junit results $TESTRUNID - $(uname -r)"
+	if test -f /run/powerbtn
+	then
+	    subj="$subj - test run aborted"
+	fi
+	if test -z "$GCE_REPORT_SENDER"
+	then
+	    GCE_REPORT_SENDER="$USER@$HOSTNAME"
+	fi
+	export SENDGRID_API_KEY
+	/usr/local/sbin/send-mail.py --sender "$GCE_REPORT_SENDER" \
+		-s "$subj" "$JUNIT_EMAIL" < /tmp/results.xml
+    fi
     tar -C /results -cf - . | xz -6e > /tmp/results.tar.xz
     gsutil cp /tmp/results.tar.xz \
 	"gs://$GS_BUCKET/${BUCKET_SUBDIR:-results}/results.$TESTRUNID.$(uname -r).tar.xz"
+    gsutil cp /tmp/results.xml \
+	"gs://$GS_BUCKET/${BUCKET_SUBDIR:-results}/results.$TESTRUNID.$(uname -r).xml"
 fi
 
 if test -f /run/filestore-param ; then