dawn-cmake/src/dawn_node/tools/src/cmd/run-cts/main.go

// Copyright 2021 The Dawn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// run-cts is a tool used to run the WebGPU CTS using the Dawn module for NodeJS
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"math"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mattn/go-colorable"
	"github.com/mattn/go-isatty"
)

const (
	testTimeout = time.Minute
)

func main() {
	if err := run(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func showUsage() {
	fmt.Println(`
run-cts is a tool used to run the WebGPU CTS using the Dawn module for NodeJS

Usage:
  run-cts --dawn-node=<path to dawn.node> --cts=<path to WebGPU CTS> [test-query]`)
	os.Exit(1)
}

var colors bool
var stdout io.Writer

func run() error {
	colors = os.Getenv("TERM") != "dumb" ||
		isatty.IsTerminal(os.Stdout.Fd()) ||
		isatty.IsCygwinTerminal(os.Stdout.Fd())
	if colors {
		if _, disable := os.LookupEnv("NO_COLOR"); disable {
			colors = false
		}
	}

	var dawnNode, cts, node, npx, logFilename string
	var verbose, isolated, build bool
	var numRunners int
	flag.StringVar(&dawnNode, "dawn-node", "", "path to dawn.node module")
	flag.StringVar(&cts, "cts", "", "root directory of WebGPU CTS")
	flag.StringVar(&node, "node", "", "path to node executable")
	flag.StringVar(&npx, "npx", "", "path to npx executable")
	flag.BoolVar(&verbose, "verbose", false, "print extra information while testing")
	flag.BoolVar(&build, "build", true, "attempt to build the CTS before running")
	flag.BoolVar(&isolated, "isolate", false, "run each test in an isolated process")
	flag.BoolVar(&colors, "colors", colors, "enable / disable colors")
	flag.IntVar(&numRunners, "j", runtime.NumCPU()/2, "number of concurrent runners. 0 runs serially")
	flag.StringVar(&logFilename, "log", "", "path to log file of tests run and result")
	flag.Parse()

	if colors {
		stdout = colorable.NewColorableStdout()
	} else {
		stdout = colorable.NewNonColorable(os.Stdout)
	}

	// Check mandatory arguments
	if dawnNode == "" || cts == "" {
		showUsage()
	}
	if !isFile(dawnNode) {
		return fmt.Errorf("'%v' is not a file", dawnNode)
	}
	if !isDir(cts) {
		return fmt.Errorf("'%v' is not a directory", cts)
	}

	// Make paths absolute
	for _, path := range []*string{&dawnNode, &cts} {
		abs, err := filepath.Abs(*path)
		if err != nil {
			return fmt.Errorf("unable to get absolute path for '%v'", *path)
		}
		*path = abs
	}

	// The test query is the optional unnamed argument
	query := "webgpu:*"
	switch len(flag.Args()) {
	case 0:
	case 1:
		query = flag.Args()[0]
	default:
		return fmt.Errorf("only a single query can be provided")
	}

	// Find node
	if node == "" {
		var err error
		node, err = exec.LookPath("node")
		if err != nil {
			return fmt.Errorf("add node to PATH or specify with --node")
		}
	}
	// Find npx
	if npx == "" {
		var err error
		npx, err = exec.LookPath("npx")
		if err != nil {
			npx = ""
		}
	}

	r := runner{
		numRunners: numRunners,
		verbose:    verbose,
		node:       node,
		npx:        npx,
		dawnNode:   dawnNode,
		cts:        cts,
		evalScript: func(main string) string {
			return fmt.Sprintf(`require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/%v.ts');`, main)
		},
	}

	if logFilename != "" {
		writer, err := os.Create(logFilename)
		if err != nil {
			return fmt.Errorf("failed to open log '%v': %w", logFilename, err)
		}
		defer writer.Close()
		r.log = newLogger(writer)
	}

	cache := cache{}
	cachePath := dawnNode + ".runcts.cache"
	if err := cache.load(cachePath); err != nil && verbose {
		fmt.Println("failed to load cache from", cachePath, err)
	}
	defer cache.save(cachePath)

	// Scan the CTS source to determine the most recent change to the CTS source
	mostRecentSourceChange, err := r.scanSourceTimestamps(verbose)
	if err != nil {
		return fmt.Errorf("failed to scan source files for modified timestamps: %w", err)
	}

	ctsNeedsRebuild := mostRecentSourceChange.After(cache.BuildTimestamp)
	if build {
		if verbose {
			fmt.Println("CTS needs rebuild:", ctsNeedsRebuild)
		}

		if npx != "" {
			if ctsNeedsRebuild {
				if err := r.buildCTS(verbose); err != nil {
					return fmt.Errorf("failed to build CTS: %w", err)
				}
				cache.BuildTimestamp = mostRecentSourceChange
			}
			// Use the prebuilt CTS (instead of using the `setup-ts-in-node` transpiler)
			r.evalScript = func(main string) string {
				return fmt.Sprintf(`require('./out-node/common/runtime/%v.js');`, main)
			}
		} else {
			fmt.Println("npx not found on PATH. Using runtime TypeScript transpilation (slow)")
		}
	}

	if numRunners > 0 {
		// Find all the test cases that match the given queries.
		if err := r.gatherTestCases(query, verbose); err != nil {
			return fmt.Errorf("failed to gather test cases: %w", err)
		}

		fmt.Printf("Testing %d test cases...\n", len(r.testcases))
		if isolated {
			return r.runParallelIsolated()
		}
		return r.runParallelWithServer()
	}

	fmt.Println("Running serially...")
	return r.runSerially(query)
}

type logger struct {
	writer        io.Writer
	idx           int
	resultByIndex map[int]result
}

// newLogger creates a new logger instance.
func newLogger(writer io.Writer) logger {
	return logger{writer, 0, map[int]result{}}
}

// logResult writes the test results to the log file in sequential order.
// logResult should be called whenever a new test result becomes available.
func (l *logger) logResults(res result) {
	if l.writer == nil {
		return
	}
	l.resultByIndex[res.index] = res
	for {
		logRes, ok := l.resultByIndex[l.idx]
		if !ok {
			break
		}
		fmt.Fprintf(l.writer, "%v [%v]\n", logRes.testcase, logRes.status)
		l.idx++
	}
}

// Cache holds cached information between runs to optimize runs
type cache struct {
	BuildTimestamp time.Time
}

// load loads the cache information from the JSON file at path
func (c *cache) load(path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	return json.NewDecoder(f).Decode(c)
}

// save saves the cache information to the JSON file at path
func (c *cache) save(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(c)
}

type runner struct {
	numRunners               int
	verbose                  bool
	node, npx, dawnNode, cts string
	evalScript               func(string) string
	testcases                []string
	log                      logger
}

// scanSourceTimestamps scans all the .js and .ts files in all subdirectories of
// r.cts, and returns the file with the most recent timestamp.
func (r *runner) scanSourceTimestamps(verbose bool) (time.Time, error) {
	if verbose {
		start := time.Now()
		fmt.Println("Scanning .js / .ts files for changes...")
		defer func() {
			fmt.Println("completed in", time.Since(start))
		}()
	}

	dir := filepath.Join(r.cts, "src")

	mostRecentChange := time.Time{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		switch filepath.Ext(path) {
		case ".ts", ".js":
			if info.ModTime().After(mostRecentChange) {
				mostRecentChange = info.ModTime()
			}
		}
		return nil
	})
	if err != nil {
		return time.Time{}, err
	}
	return mostRecentChange, nil
}

// buildCTS calls `npx grunt run:build-out-node` in the CTS directory to compile
// the TypeScript files down to JavaScript. Doing this once ahead of time can be
// much faster than dynamically transpiling when there are many tests to run.
func (r *runner) buildCTS(verbose bool) error {
	if verbose {
		start := time.Now()
		fmt.Println("Building CTS...")
		defer func() {
			fmt.Println("completed in", time.Since(start))
		}()
	}

	cmd := exec.Command(r.npx, "grunt", "run:build-out-node")
	cmd.Dir = r.cts
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%w: %v", err, string(out))
	}
	return nil
}

// gatherTestCases() queries the CTS for all test cases that match the given
// query. On success, gatherTestCases() populates r.testcases.
func (r *runner) gatherTestCases(query string, verbose bool) error {
	if verbose {
		start := time.Now()
		fmt.Println("Gathering test cases...")
		defer func() {
			fmt.Println("completed in", time.Since(start))
		}()
	}

	args := append([]string{
		"-e", r.evalScript("cmdline"),
		"--", // Start of arguments
		// src/common/runtime/helper/sys.ts expects 'node file.js <args>'
		// and slices away the first two arguments. When running with '-e', args
		// start at 1, so just inject a dummy argument.
		"dummy-arg",
		"--list",
	}, query)

	cmd := exec.Command(r.node, args...)
	cmd.Dir = r.cts
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%w\n%v", err, string(out))
	}

	tests := filterTestcases(strings.Split(string(out), "\n"))
	r.testcases = tests
	return nil
}

type portListener struct {
	buffer strings.Builder
	port   chan int
}

func newPortListener() portListener {
	return portListener{strings.Builder{}, make(chan int)}
}

var portRE = regexp.MustCompile(`\[\[(\d+)\]\]`)

func (p *portListener) Write(data []byte) (n int, err error) {
	if p.port != nil {
		p.buffer.Write(data)
		match := portRE.FindStringSubmatch(p.buffer.String())
		if len(match) == 2 {
			port, err := strconv.Atoi(match[1])
			if err != nil {
				return 0, err
			}
			p.port <- port
			close(p.port)
			p.port = nil
		}
	}
	return len(data), nil
}

// runParallelWithServer() starts r.numRunners instances of the CTS server test
// runner, and issues test run requests to those servers, concurrently.
func (r *runner) runParallelWithServer() error {
	// Create a chan of test indices.
	// This will be read by the test runner goroutines.
	caseIndices := make(chan int, len(r.testcases))
	for i := range r.testcases {
		caseIndices <- i
	}
	close(caseIndices)

	// Create a chan for the test results.
	// This will be written to by the test runner goroutines.
	results := make(chan result, len(r.testcases))

	// Spin up the test runner goroutines
	wg := &sync.WaitGroup{}
	for i := 0; i < r.numRunners; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := r.runServer(caseIndices, results); err != nil {
				results <- result{
					status: fail,
					error:  fmt.Errorf("Test server error: %w", err),
				}
			}
		}()
	}

	r.streamResults(wg, results)
	return nil
}

// runServer starts a test runner server instance, takes case indices from
// caseIndices, and requests the server run the test with the given index.
// The result of the test run is written to the results chan.
// Once the caseIndices chan has been closed, the server is stopped and
// runServer returns.
func (r *runner) runServer(caseIndices <-chan int, results chan<- result) error {
	var port int

	stopServer := func() {}
	startServer := func() error {
		cmd := exec.Command(r.node,
			"-e", r.evalScript("server"), // Evaluate 'eval'.
			"--",
			// src/common/runtime/helper/sys.ts expects 'node file.js <args>'
			// and slices away the first two arguments. When running with '-e', args
			// start at 1, so just inject a dummy argument.
			"dummy-arg",
			// Actual arguments begin here
			"--gpu-provider", r.dawnNode)

		serverLog := &bytes.Buffer{}

		pl := newPortListener()

		cmd.Dir = r.cts
		cmd.Stdout = io.MultiWriter(serverLog, &pl)
		cmd.Stderr = serverLog

		err := cmd.Start()
		if err != nil {
			return fmt.Errorf("failed to start test runner server:", err)
		}

		select {
		case port = <-pl.port:
		case <-time.After(time.Second * 10):
			return fmt.Errorf("timeout waiting for server port:\n%v", serverLog.String())
		}

		return nil
	}
	stopServer = func() {
		if port > 0 {
			go http.Post(fmt.Sprintf("http://localhost:%v/terminate", port), "", &bytes.Buffer{})
			time.Sleep(time.Millisecond * 100)
			port = 0
		}
	}

	for idx := range caseIndices {
		if port == 0 {
			if err := startServer(); err != nil {
				return err
			}
		}

		res := result{index: idx, testcase: r.testcases[idx]}

		type Response struct {
			Status  string
			Message string
		}
		postResp, err := http.Post(fmt.Sprintf("http://localhost:%v/run?%v", port, r.testcases[idx]), "", &bytes.Buffer{})
		if err != nil {
			res.error = fmt.Errorf("server POST failure. Restarting server...")
			res.status = fail
			results <- res
			stopServer()
			continue
		}

		var resp Response
		if err := json.NewDecoder(postResp.Body).Decode(&resp); err != nil {
			res.error = fmt.Errorf("server response decode failure")
			res.status = fail
			results <- res
			continue
		}

		switch resp.Status {
		case "pass":
			res.status = pass
			res.message = resp.Message
		case "warn":
			res.status = warn
			res.message = resp.Message
		case "fail":
			res.status = fail
			res.message = resp.Message
		case "skip":
			res.status = skip
			res.message = resp.Message
		default:
			err = fmt.Errorf("unknown status: '%v'", resp.Status)
		}
		results <- res
	}

	stopServer()
	return nil
}

// runParallelIsolated() calls the CTS command-line test runner to run each
// testcase in a separate process. This reduces possibility of state leakage
// between tests.
// Up to r.numRunners tests will be run concurrently.
func (r *runner) runParallelIsolated() error {
	// Create a chan of test indices.
	// This will be read by the test runner goroutines.
	caseIndices := make(chan int, len(r.testcases))
	for i := range r.testcases {
		caseIndices <- i
	}
	close(caseIndices)

	// Create a chan for the test results.
	// This will be written to by the test runner goroutines.
	results := make(chan result, len(r.testcases))

	// Spin up the test runner goroutines
	wg := &sync.WaitGroup{}
	for i := 0; i < r.numRunners; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for idx := range caseIndices {
				res := r.runTestcase(r.testcases[idx], false)
				res.index = idx
				results <- res
			}
		}()
	}

	r.streamResults(wg, results)
	return nil
}

// streamResults reads from the chan 'results', printing the results in test-id
// sequential order. Once the WaitGroup 'wg' is complete, streamResults() will
// automatically close the 'results' chan.
// Once all the results have been printed, a summary will be printed and the
// function will return.
func (r *runner) streamResults(wg *sync.WaitGroup, results chan result) {
	// Create another goroutine to close the results chan when all the runner
	// goroutines have finished.
	start := time.Now()
	var timeTaken time.Duration
	go func() {
		wg.Wait()
		timeTaken = time.Since(start)
		close(results)
	}()

	// Total number of tests, test counts binned by status
	numTests, numByStatus := len(r.testcases), map[status]int{}

	// Helper function for printing a progress bar.
	lastStatusUpdate, animFrame := time.Now(), 0
	updateProgress := func() {
		printANSIProgressBar(animFrame, numTests, numByStatus)
		animFrame++
		lastStatusUpdate = time.Now()
	}

	// Pull test results as they become available.
	// Update the status counts, and print any failures (or all test results if --verbose)
	progressUpdateRate := time.Millisecond * 10
	if !colors {
		// No colors == no cursor control. Reduce progress updates so that
		// we're not printing endless progress bars.
		progressUpdateRate = time.Second
	}

	for res := range results {
		r.log.logResults(res)

		numByStatus[res.status] = numByStatus[res.status] + 1
		name := res.testcase
		if r.verbose || res.error != nil || (res.status != pass && res.status != skip) {
			fmt.Printf("%v - %v: %v\n", name, res.status, res.message)
			if res.error != nil {
				fmt.Println(res.error)
			}
			updateProgress()
		}
		if time.Since(lastStatusUpdate) > progressUpdateRate {
			updateProgress()
		}
	}
	printANSIProgressBar(animFrame, numTests, numByStatus)

	// All done. Print final stats.
	fmt.Printf(`
Completed in %v

pass:    %v (%v)
fail:    %v (%v)
skip:    %v (%v)
timeout: %v (%v)
`,
		timeTaken,
		numByStatus[pass], percentage(numByStatus[pass], numTests),
		numByStatus[fail], percentage(numByStatus[fail], numTests),
		numByStatus[skip], percentage(numByStatus[skip], numTests),
		numByStatus[timeout], percentage(numByStatus[timeout], numTests),
	)
}

// runSerially() calls the CTS test runner to run the test query in a single
// process.
func (r *runner) runSerially(query string) error {
	start := time.Now()
	result := r.runTestcase(query, true)
	timeTaken := time.Since(start)

	fmt.Println("Completed in", timeTaken)
	fmt.Println(result)
	return nil
}

// status is an enumerator of test result status
type status string

const (
	pass    status = "pass"
	warn    status = "warn"
	fail    status = "fail"
	skip    status = "skip"
	timeout status = "timeout"
)

// result holds the information about a completed test
type result struct {
	index    int
	testcase string
	status   status
	message  string
	error    error
}

// runTestcase() runs the CTS testcase with the given query, returning the test
// result.
func (r *runner) runTestcase(query string, printToStout bool) result {
	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
	defer cancel()

	args := append([]string{
		"-e", r.evalScript("cmdline"), // Evaluate 'eval'.
		"--",
		// src/common/runtime/helper/sys.ts expects 'node file.js <args>'
		// and slices away the first two arguments. When running with '-e', args
		// start at 1, so just inject a dummy argument.
		"dummy-arg",
		// Actual arguments begin here
		"--gpu-provider", r.dawnNode,
		"--verbose",
	}, query)

	cmd := exec.CommandContext(ctx, r.node, args...)
	cmd.Dir = r.cts

	var buf bytes.Buffer
	if printToStout {
		cmd.Stdout = io.MultiWriter(&buf, os.Stdout)
		cmd.Stderr = io.MultiWriter(&buf, os.Stderr)
	} else {
		cmd.Stdout = &buf
		cmd.Stderr = &buf
	}

	err := cmd.Run()
	msg := buf.String()
	switch {
	case errors.Is(err, context.DeadlineExceeded):
		return result{testcase: query, status: timeout, message: msg}
	case strings.Contains(msg, "[fail]"):
		return result{testcase: query, status: fail, message: msg}
	case strings.Contains(msg, "[warn]"):
		return result{testcase: query, status: warn, message: msg}
	case strings.Contains(msg, "[skip]"):
		return result{testcase: query, status: skip, message: msg}
	case strings.Contains(msg, "[pass]"), err == nil:
		return result{testcase: query, status: pass, message: msg}
	}
	return result{testcase: query, status: fail, message: fmt.Sprint(msg, err), error: err}
}

// filterTestcases returns in with empty strings removed
func filterTestcases(in []string) []string {
	out := make([]string, 0, len(in))
	for _, c := range in {
		if c != "" {
			out = append(out, c)
		}
	}
	return out
}

// percentage returns the percentage of n out of total as a string
func percentage(n, total int) string {
	if total == 0 {
		return "-"
	}
	f := float64(n) / float64(total)
	return fmt.Sprintf("%.1f%c", f*100.0, '%')
}

// isDir returns true if the path resolves to a directory
func isDir(path string) bool {
	s, err := os.Stat(path)
	if err != nil {
		return false
	}
	return s.IsDir()
}

// isFile returns true if the path resolves to a file
func isFile(path string) bool {
	s, err := os.Stat(path)
	if err != nil {
		return false
	}
	return !s.IsDir()
}

// printANSIProgressBar prints a colored progress bar, providing realtime
// information about the status of the CTS run.
// Note: We'll want to skip this if !isatty or if we're running on windows.
func printANSIProgressBar(animFrame int, numTests int, numByStatus map[status]int) {
	const (
		barWidth = 50

		escape       = "\u001B["
		positionLeft = escape + "0G"
		red          = escape + "31m"
		green        = escape + "32m"
		yellow       = escape + "33m"
		blue         = escape + "34m"
		magenta      = escape + "35m"
		cyan         = escape + "36m"
		white        = escape + "37m"
		reset        = escape + "0m"
	)

	animSymbols := []rune{'⣾', '⣽', '⣻', '⢿', '⡿', '⣟', '⣯', '⣷'}
	blockSymbols := []rune{'▏', '▎', '▍', '▌', '▋', '▊', '▉'}

	numBlocksPrinted := 0

	fmt.Fprint(stdout, string(animSymbols[animFrame%len(animSymbols)]), " [")
	animFrame++

	numFinished := 0

	for _, ty := range []struct {
		status status
		color  string
	}{{pass, green}, {warn, yellow}, {skip, blue}, {timeout, yellow}, {fail, red}} {
		num := numByStatus[ty.status]
		numFinished += num
		statusFrac := float64(num) / float64(numTests)
		fNumBlocks := barWidth * statusFrac
		fmt.Fprint(stdout, ty.color)
		numBlocks := int(math.Ceil(fNumBlocks))
		if numBlocks > 1 {
			fmt.Print(strings.Repeat(string("▉"), numBlocks))
		}
		if numBlocks > 0 {
			frac := fNumBlocks - math.Floor(fNumBlocks)
			symbol := blockSymbols[int(math.Round(frac*float64(len(blockSymbols)-1)))]
			fmt.Print(string(symbol))
		}
		numBlocksPrinted += numBlocks
	}

	if barWidth > numBlocksPrinted {
		fmt.Print(strings.Repeat(string(" "), barWidth-numBlocksPrinted))
	}
	fmt.Fprint(stdout, reset)
	fmt.Print("] ", percentage(numFinished, numTests))

	if colors {
		// move cursor to start of line so the bar is overridden
		fmt.Fprint(stdout, positionLeft)
	} else {
		// cannot move cursor, so newline
		fmt.Println()
	}
}