dawn_node: make server mode output stdout of each test in verbose mode

This change also makes sure that all three modes (serial, isolated
parallel, and server) all output stdout of tests only if verbose mode is
enabled.

Bug: dawn:1163
Change-Id: I9e9efcfa4b73571adac15179d2792fb745255fb8
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/68182
Reviewed-by: Ben Clayton <bclayton@google.com>
Commit-Queue: Antonio Maiorano <amaiorano@google.com>

src/dawn_node/README.md

@@ -68,10 +68,10 @@ The `--flag` parameter must be passed in multiple times, once for each flag begi
 For example, on Windows, to use the d3dcompiler_47.dll from a Chromium checkout, and to dump shader output, we could run the following using Git Bash:
 
 ```sh
-./src/dawn_node/tools/run-cts --j=0 --dawn-node=/c/src/dawn/build/Debug/dawn.node --cts=/c/src/gpuweb-cts --flag=dlldir="C:\src\chromium\src\out\Release" --flag=enable-dawn-features=dump_shaders 'webgpu:shader,execution,builtin,abs:integer_builtin_functions,abs_unsigned:storageClass="storage";storageMode="read_write";containerType="vector";isAtomic=false;baseType="u32";type="vec2%3Cu32%3E"'
+./src/dawn_node/tools/run-cts --verbose --dawn-node=/c/src/dawn/build/Debug/dawn.node --cts=/c/src/gpuweb-cts --flag=dlldir="C:\src\chromium\src\out\Release" --flag=enable-dawn-features=dump_shaders 'webgpu:shader,execution,builtin,abs:integer_builtin_functions,abs_unsigned:storageClass="storage";storageMode="read_write";containerType="vector";isAtomic=false;baseType="u32";type="vec2%3Cu32%3E"'
 ```
 
-Note that we pass `--j=0` above so that all output, including the dumped shader, is written to stdout.
+Note that we pass `--verbose` above so that all test output, including the dumped shader, is written to stdout.
 ## Known issues
 
 - Many WebGPU CTS tests are currently known to fail

src/dawn_node/tools/src/cmd/run-cts/main.go

@@ -220,10 +220,13 @@ func run() error {
 			return fmt.Errorf("failed to gather test cases: %w", err)
 		}
 
-		fmt.Printf("Testing %d test cases...\n", len(r.testcases))
 		if isolated {
+			fmt.Println("Running in parallel isolated...")
+			fmt.Printf("Testing %d test cases...\n", len(r.testcases))
 			return r.runParallelIsolated()
 		}
+		fmt.Println("Running in parallel with server...")
+		fmt.Printf("Testing %d test cases...\n", len(r.testcases))
 		return r.runParallelWithServer()
 	}
 
@@ -439,6 +442,10 @@ func (r *runner) runParallelWithServer() error {
 	return nil
 }
 
+type redirectingWriter struct {
+	io.Writer
+}
+
 // runServer starts a test runner server instance, takes case indices from
 // caseIndices, and requests the server run the test with the given index.
 // The result of the test run is written to the results chan.
@@ -446,6 +453,7 @@ func (r *runner) runParallelWithServer() error {
 // runServer returns.
 func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error {
 	var port int
+	var rw redirectingWriter
 
 	stopServer := func() {}
 	startServer := func() error {
@@ -470,12 +478,12 @@ func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error
 		pl := newPortListener()
 
 		cmd.Dir = r.cts
-		cmd.Stdout = io.MultiWriter(serverLog, &pl)
-		cmd.Stderr = serverLog
+		cmd.Stdout = io.MultiWriter(&rw, serverLog, &pl)
+		cmd.Stderr = io.MultiWriter(&rw, serverLog)
 
 		err := cmd.Start()
 		if err != nil {
-			return fmt.Errorf("failed to start test runner server:", err)
+			return fmt.Errorf("failed to start test runner server: %v", err)
 		}
 
 		select {
@@ -495,6 +503,10 @@ func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error
 	}
 
 	for idx := range caseIndices {
+		// Redirect the server log per test case
+		caseServerLog := &bytes.Buffer{}
+		rw.Writer = caseServerLog
+
 		if port == 0 {
 			if err := startServer(); err != nil {
 				return err
@@ -528,16 +540,16 @@ func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error
 			switch resp.Status {
 			case "pass":
 				res.status = pass
-				res.message = resp.Message
+				res.message = resp.Message + caseServerLog.String()
 			case "warn":
 				res.status = warn
-				res.message = resp.Message
+				res.message = resp.Message + caseServerLog.String()
 			case "fail":
 				res.status = fail
-				res.message = resp.Message
+				res.message = resp.Message + caseServerLog.String()
 			case "skip":
 				res.status = skip
-				res.message = resp.Message
+				res.message = resp.Message + caseServerLog.String()
 			default:
 				res.status = fail
 				res.error = fmt.Errorf("unknown status: '%v'", resp.Status)
@@ -581,7 +593,7 @@ func (r *runner) runParallelIsolated() error {
 		go func() {
 			defer wg.Done()
 			for idx := range caseIndices {
-				res := r.runTestcase(r.testcases[idx], false)
+				res := r.runTestcase(r.testcases[idx])
 				res.index = idx
 				results <- res
 			}
@@ -667,11 +679,14 @@ timeout: %v (%v)
 // process.
 func (r *runner) runSerially(query string) error {
 	start := time.Now()
-	result := r.runTestcase(query, true)
+	result := r.runTestcase(query)
 	timeTaken := time.Since(start)
 
-	fmt.Println("Completed in", timeTaken)
+	if r.verbose {
 		fmt.Println(result)
+	}
+	fmt.Println("Status:", result.status)
+	fmt.Println("Completed in", timeTaken)
 	return nil
 }
 
@@ -697,7 +712,7 @@ type result struct {
 
 // runTestcase() runs the CTS testcase with the given query, returning the test
 // result.
-func (r *runner) runTestcase(query string, printToStout bool) result {
+func (r *runner) runTestcase(query string) result {
 	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
 	defer cancel()
 
@@ -721,13 +736,8 @@ func (r *runner) runTestcase(query string, printToStout bool) result {
 	cmd.Dir = r.cts
 
 	var buf bytes.Buffer
-	if printToStout {
-		cmd.Stdout = io.MultiWriter(&buf, os.Stdout)
-		cmd.Stderr = io.MultiWriter(&buf, os.Stderr)
-	} else {
 	cmd.Stdout = &buf
 	cmd.Stderr = &buf
-	}
 
 	err := cmd.Run()
 	msg := buf.String()