From 22da31cf43e13b6aa57d66d89983ea6c5f5b094e Mon Sep 17 00:00:00 2001
From: Antonio Maiorano <amaiorano@google.com>
Date: Tue, 5 Oct 2021 21:18:10 +0000
Subject: [PATCH] run-cts: add "log" arg to log test results to file

Bug: dawn:1123
Change-Id: Id9ad62dde8f7988cb66e39ca0d543c23d0fad341
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/65781
Commit-Queue: Antonio Maiorano <amaiorano@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
---
 src/dawn_node/tools/src/cmd/run-cts/main.go | 57 ++++++++++++++++++---
 1 file changed, 50 insertions(+), 7 deletions(-)

diff --git a/src/dawn_node/tools/src/cmd/run-cts/main.go b/src/dawn_node/tools/src/cmd/run-cts/main.go
index 872f5cf6e2..d0dd681f4e 100644
--- a/src/dawn_node/tools/src/cmd/run-cts/main.go
+++ b/src/dawn_node/tools/src/cmd/run-cts/main.go
@@ -67,7 +67,7 @@ func run() error {
 		}
 	}
 
-	var dawnNode, cts, node, npx string
+	var dawnNode, cts, node, npx, logFilename string
 	var verbose, build bool
 	var numRunners int
 	flag.StringVar(&dawnNode, "dawn-node", "", "path to dawn.node module")
@@ -78,6 +78,7 @@ func run() error {
 	flag.BoolVar(&build, "build", true, "attempt to build the CTS before running")
 	flag.BoolVar(&colors, "colors", colors, "enable / disable colors")
 	flag.IntVar(&numRunners, "j", runtime.NumCPU(), "number of concurrent runners")
+	flag.StringVar(&logFilename, "log", "", "path to log file of tests run and result")
 	flag.Parse()
 
 	if colors {
@@ -140,6 +141,15 @@ func run() error {
 		  require('./src/common/runtime/cmdline.ts');`,
 	}
 
+	if logFilename != "" {
+		writer, err := os.Create(logFilename)
+		if err != nil {
+			return fmt.Errorf("failed to open log '%v': %w", logFilename, err)
+		}
+		defer writer.Close()
+		r.log = newLogger(writer)
+	}
+
 	if build {
 		// Attempt to build the CTS (instead of using the `setup-ts-in-node` transpiler)
 		if npx != "" {
@@ -163,12 +173,41 @@ func run() error {
 	return r.run()
 }
 
+type logger struct {
+	writer        io.Writer
+	idx           int
+	resultByIndex map[int]result
+}
+
+// newLogger creates a new logger instance.
+func newLogger(writer io.Writer) logger {
+	return logger{writer, 0, map[int]result{}}
+}
+
+// logResult writes the test results to the log file in sequential order.
+// logResult should be called whenever a new test result becomes available.
+func (l *logger) logResults(res result) {
+	if l.writer == nil {
+		return
+	}
+	l.resultByIndex[res.index] = res
+	for {
+		logRes, ok := l.resultByIndex[l.idx]
+		if !ok {
+			break
+		}
+		fmt.Fprintf(l.writer, "%v [%v]\n", logRes.testcase, logRes.status)
+		l.idx++
+	}
+}
+
 type runner struct {
 	numRunners               int
 	verbose                  bool
 	node, npx, dawnNode, cts string
 	evalScript               string
 	testcases                []string
+	log                      logger
 }
 
 // buildCTS calls `npx grunt run:build-out-node` in the CTS directory to compile
@@ -265,9 +304,12 @@ func (r *runner) run() error {
 		// we're not printing endless progress bars.
 		progressUpdateRate = time.Second
 	}
+
 	for res := range results {
-		name := res.testcase
+		r.log.logResults(res)
+
 		numByStatus[res.status] = numByStatus[res.status] + 1
+		name := res.testcase
 		if r.verbose || (res.status != pass && res.status != skip) {
 			fmt.Printf("%v - %v: %v\n", name, res.status, res.message)
 			updateProgress()
@@ -308,6 +350,7 @@ const (
 
 // result holds the information about a completed test
 type result struct {
+	index    int
 	testcase string
 	status   status
 	message  string
@@ -341,15 +384,15 @@ func (r *runner) runTestcase(idx int) result {
 	msg := string(out)
 	switch {
 	case errors.Is(err, context.DeadlineExceeded):
-		return result{testcase: testcase, status: timeout, message: msg}
+		return result{index: idx, testcase: testcase, status: timeout, message: msg}
 	case strings.Contains(msg, "[fail]"):
-		return result{testcase: testcase, status: fail, message: msg}
+		return result{index: idx, testcase: testcase, status: fail, message: msg}
 	case strings.Contains(msg, "[skip]"):
-		return result{testcase: testcase, status: skip, message: msg}
+		return result{index: idx, testcase: testcase, status: skip, message: msg}
 	case strings.Contains(msg, "[pass]"), err == nil:
-		return result{testcase: testcase, status: pass, message: msg}
+		return result{index: idx, testcase: testcase, status: pass, message: msg}
 	}
-	return result{testcase: testcase, status: fail, message: fmt.Sprint(msg, err), error: err}
+	return result{index: idx, testcase: testcase, status: fail, message: fmt.Sprint(msg, err), error: err}
 }
 
 // filterTestcases returns in with empty strings removed