tools/run-cts: Optimize coverage collection

Enable coverage collection when using the test server, which is substantially faster than running in separate, isolated processes.

Use clang's `__llvm_profile_*` APIs to reset the counters between each test case run.

Change-Id: I01f8d0c1b3f215f66cfa50ef0fd51f2522c2ea57
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/113880
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Corentin Wallez <cwallez@chromium.org>
Commit-Queue: Ben Clayton <bclayton@google.com>

CMakeLists.txt

@@ -343,6 +343,7 @@ function(common_compile_options TARGET)
   endif()
 
   if (DAWN_EMIT_COVERAGE)
+    target_compile_definitions(${TARGET} PRIVATE "DAWN_EMIT_COVERAGE")
     if(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
         target_compile_options(${TARGET} PRIVATE "--coverage")
         target_link_options(${TARGET} PRIVATE "gcov")

src/dawn/node/Module.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <filesystem>
 #include <string>
 #include <tuple>
 #include <utility>
@@ -22,6 +23,14 @@
 #include "src/dawn/node/binding/GPU.h"
 #include "tint/tint.h"
 
+#ifdef DAWN_EMIT_COVERAGE
+extern "C" {
+void __llvm_profile_reset_counters(void);
+void __llvm_profile_set_filename(const char*);
+int __llvm_profile_write_file(void);
+}
+#endif  // DAWN_EMIT_COVERAGE
+
 namespace {
 Napi::Value CreateGPU(const Napi::CallbackInfo& info) {
     const auto& env = info.Env();
@@ -50,6 +59,32 @@ Napi::Value CreateGPU(const Napi::CallbackInfo& info) {
     return wgpu::interop::GPU::Create<wgpu::binding::GPU>(env, std::move(flags));
 }
 
+#ifdef DAWN_EMIT_COVERAGE
+struct Coverage {
+    Coverage() : output_path_{GetOutputPath()} {
+        __llvm_profile_set_filename(output_path_.c_str());
+    }
+    ~Coverage() { std::filesystem::remove(output_path_); }
+
+    static void Begin(const Napi::CallbackInfo& info) {
+        auto* coverage = static_cast<Coverage*>(info.Data());
+        std::filesystem::remove(coverage->output_path_);
+        __llvm_profile_reset_counters();
+    }
+
+    static Napi::Value End(const Napi::CallbackInfo& info) {
+        __llvm_profile_write_file();
+        auto* coverage = static_cast<Coverage*>(info.Data());
+        return Napi::String::New(info.Env(), coverage->output_path_.c_str());
+    }
+
+  private:
+    static std::filesystem::path GetOutputPath() { return std::tmpnam(nullptr); }
+
+    std::filesystem::path output_path_;
+};
+#endif  // DAWN_EMIT_COVERAGE
+
 }  // namespace
 
 // Initialize() initializes the Dawn node module, registering all the WebGPU
@@ -68,6 +103,15 @@ Napi::Object Initialize(Napi::Env env, Napi::Object exports) {
     // Export function that creates and returns the wgpu::interop::GPU interface
     exports.Set(Napi::String::New(env, "create"), Napi::Function::New<CreateGPU>(env));
 
+#ifdef DAWN_EMIT_COVERAGE
+    Coverage* coverage = new Coverage();
+    auto coverage_provider = Napi::Object::New(env);
+    coverage_provider.Set("begin", Napi::Function::New(env, &Coverage::Begin, nullptr, coverage));
+    coverage_provider.Set("end", Napi::Function::New(env, &Coverage::End, nullptr, coverage));
+    coverage_provider.AddFinalizer([](const Napi::Env&, Coverage* c) { delete c; }, coverage);
+    exports.Set(Napi::String::New(env, "coverage"), coverage_provider);
+#endif  // DAWN_EMIT_COVERAGE
+
     return exports;
 }
 

tools/src/cmd/run-cts/main.go

@@ -153,7 +153,7 @@ func run() error {
 	flag.StringVar(&adapterName, "adapter", "", "name (or substring) of the GPU adapter to use")
 	flag.BoolVar(&dumpShaders, "dump-shaders", false, "dump WGSL shaders. Enables --verbose")
 	flag.BoolVar(&unrollConstEvalLoops, "unroll-const-eval-loops", unrollConstEvalLoopsDefault, "unroll loops in const-eval tests")
-	flag.BoolVar(&genCoverage, "coverage", false, "displays coverage data. Enables --isolated")
+	flag.BoolVar(&genCoverage, "coverage", false, "displays coverage data")
 	flag.StringVar(&coverageFile, "export-coverage", "", "write coverage data to the given path")
 	flag.Parse()
 
@@ -258,7 +258,6 @@ func run() error {
 	}
 
 	if genCoverage {
-		isolated = true
 		llvmCov, err := exec.LookPath("llvm-cov")
 		if err != nil {
 			return fmt.Errorf("failed to find LLVM, required for --coverage")
@@ -657,6 +656,9 @@ func (r *runner) runServer(ctx context.Context, id int, caseIndices <-chan int,
 		if r.colors {
 			args = append(args, "--colors")
 		}
+		if r.covEnv != nil {
+			args = append(args, "--coverage")
+		}
 		if r.verbose {
 			args = append(args,
 				"--verbose",
@@ -720,8 +722,9 @@ func (r *runner) runServer(ctx context.Context, id int, caseIndices <-chan int,
 		res := result{index: idx, testcase: r.testcases[idx]}
 
 		type Response struct {
-			Status  string
+			Status       string
-			Message string
+			Message      string
+			CoverageData string
 		}
 		postResp, err := http.Post(fmt.Sprintf("http://localhost:%v/run?%v", port, r.testcases[idx]), "", &bytes.Buffer{})
 		if err != nil {
@@ -758,6 +761,17 @@ func (r *runner) runServer(ctx context.Context, id int, caseIndices <-chan int,
 				res.status = fail
 				res.error = fmt.Errorf("unknown status: '%v'", resp.Status)
 			}
+
+			if resp.CoverageData != "" {
+				coverage, covErr := r.covEnv.Import(resp.CoverageData)
+				if covErr != nil {
+					if res.message != "" {
+						res.message += "\n"
+					}
+					res.message += fmt.Sprintf("could not import coverage data from '%v': %v", resp.CoverageData, covErr)
+				}
+				res.coverage = coverage
+			}
 		} else {
 			msg, err := ioutil.ReadAll(postResp.Body)
 			if err != nil {
@@ -824,6 +838,11 @@ func (r *runner) runParallelIsolated(ctx context.Context) error {
 // Once all the results have been printed, a summary will be printed and the
 // function will return.
 func (r *runner) streamResults(ctx context.Context, wg *sync.WaitGroup, results chan result) error {
+	// If the context was already cancelled then just return
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
 	// Create another goroutine to close the results chan when all the runner
 	// goroutines have finished.
 	start := time.Now()