From d9250a5a2175fc89d50b7d81c38979af13a92f11 Mon Sep 17 00:00:00 2001
From: Ben Clayton <bclayton@google.com>
Date: Wed, 10 Mar 2021 15:18:39 +0000
Subject: [PATCH] Parallelize cpplint

On my machine this reduces the time taken from 23 seconds -> 2 seconds

Change-Id: I676b89251fc183171cc3d955873960b00cb48bc1
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/44164
Reviewed-by: David Neto <dneto@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
---
 tools/lint                 |  21 ++++++-
 tools/run-parallel/main.go | 122 +++++++++++++++++++++++++++++++++++++
 2 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 tools/run-parallel/main.go

diff --git a/tools/lint b/tools/lint
index 0157fa6c77..1224c0e50c 100755
--- a/tools/lint
+++ b/tools/lint
@@ -13,8 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )"
+ROOT_DIR="$( cd "${SCRIPT_DIR}/.." >/dev/null 2>&1 && pwd )"
+
 set -e  # fail on error
 
 FILTER="-runtime/references"
-./third_party/cpplint/cpplint/cpplint.py --root=. --filter="$FILTER" `find src -type f`
-./third_party/cpplint/cpplint/cpplint.py --root=. --filter="$FILTER" `find samples -type f`
+
+FILES="`find src -type f` `find samples -type f`"
+
+if command -v go &> /dev/null
+then
+    # Go is installed. Run cpplint in parallel for speed wins
+    go run $SCRIPT_DIR/run-parallel/main.go       \
+         --only-print-failures                    \
+         ./third_party/cpplint/cpplint/cpplint.py \
+         --root=$ROOT_DIR                         \
+         --filter="$FILTER"                       \
+         $ -- $FILES
+else
+    ./third_party/cpplint/cpplint/cpplint.py --root=$ROOT_DIR --filter="$FILTER" $FILES
+fi
+
diff --git a/tools/run-parallel/main.go b/tools/run-parallel/main.go
new file mode 100644
index 0000000000..4fd3e48ee1
--- /dev/null
+++ b/tools/run-parallel/main.go
@@ -0,0 +1,122 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// run-parallel is a tool to run an executable with the provided templated
+// arguments across all the hardware threads.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func showUsage() {
+	fmt.Println(`
+run-parallel is a tool to run an executable with the provided templated
+arguments across all the hardware threads.
+
+Usage:
+  run-parallel <executable> [arguments...] -- [per-instance-value...]
+
+  executable         - the path to the executable to run.
+  arguments          - a list of arguments to pass to the executable.
+                       Any occurrance of $ will be substituted with the
+                       per-instance-value for the given invocation.
+  per-instance-value - a list of values. The executable will be invoked for each
+                       value in this list.`)
+	os.Exit(1)
+}
+
+func run() error {
+	onlyPrintFailures := flag.Bool("only-print-failures", false, "Omit output for processes that did not fail")
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) < 2 {
+		showUsage()
+	}
+	exe := args[0]
+	args = args[1:]
+
+	var perInstanceValues []string
+	for i, arg := range args {
+		if arg == "--" {
+			perInstanceValues = args[i+1:]
+			args = args[:i]
+			break
+		}
+	}
+	if perInstanceValues == nil {
+		showUsage()
+	}
+
+	taskIndices := make(chan int, 64)
+	results := make([]string, len(perInstanceValues))
+
+	numCPU := runtime.NumCPU()
+	wg := sync.WaitGroup{}
+	wg.Add(numCPU)
+	for i := 0; i < numCPU; i++ {
+		go func() {
+			defer wg.Done()
+			for idx := range taskIndices {
+				taskArgs := make([]string, len(args))
+				for i, arg := range args {
+					taskArgs[i] = strings.ReplaceAll(arg, "$", perInstanceValues[idx])
+				}
+				success, out := invoke(exe, taskArgs)
+				if !success || !*onlyPrintFailures {
+					results[idx] = out
+				}
+			}
+		}()
+	}
+
+	for i := range perInstanceValues {
+		taskIndices <- i
+	}
+	close(taskIndices)
+
+	wg.Wait()
+
+	for _, output := range results {
+		if output != "" {
+			fmt.Println(output)
+		}
+	}
+
+	return nil
+}
+
+func invoke(exe string, args []string) (ok bool, output string) {
+	cmd := exec.Command(exe, args...)
+	out, err := cmd.CombinedOutput()
+	str := string(out)
+	if err != nil {
+		return false, "\n" + err.Error()
+	}
+	return true, str
+}