From 5c3a7459e6a0054ebf11dca66683602ac16b6992 Mon Sep 17 00:00:00 2001 From: Ben Clayton Date: Tue, 13 Apr 2021 18:48:17 +0000 Subject: [PATCH] Add tools/fix-tests A simple regex based tool for fixing unit tests that fail due to unexpected output Change-Id: I72c47abaff6d6f4ba8cd497240eadc171af0fec3 Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/47629 Commit-Queue: Ben Clayton Reviewed-by: Antonio Maiorano --- tools/fix-tests/fix-tests.go | 277 +++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 tools/fix-tests/fix-tests.go diff --git a/tools/fix-tests/fix-tests.go b/tools/fix-tests/fix-tests.go new file mode 100644 index 0000000000..6b8b860f22 --- /dev/null +++ b/tools/fix-tests/fix-tests.go @@ -0,0 +1,277 @@ +// Copyright 2021 The Tint Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// fix-tests is a tool to update tests with new expected output. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" +) + +func main() { + if err := run(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func showUsage() { + fmt.Println(` +fix-tests is a tool to update tests with new expected output. + +Usage: + fix-tests + + executable - the path to the test executable to run.`) + os.Exit(1) +} + +func run() error { + flag.Parse() + args := flag.Args() + if len(args) < 1 { + showUsage() + } + + exe := args[0] // The path to the test executable + wd := filepath.Dir(exe) // The directory holding the test exe + + // Create a temporary directory to hold the 'test-results.json' file + tmpDir, err := ioutil.TempDir("", "fix-tests") + if err != nil { + return err + } + if err := os.MkdirAll(tmpDir, 0666); err != nil { + return fmt.Errorf("Failed to create temporary directory: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Full path to the 'test-results.json' in the temporary directory + testResultsPath := filepath.Join(tmpDir, "test-results.json") + + // Run the tests + switch err := exec.Command(exe, "--gtest_output=json:"+testResultsPath).Run().(type) { + default: + return err + case nil: + fmt.Println("All tests passed") + case *exec.ExitError: + } + + // Read the 'test-results.json' file + testResultsFile, err := os.Open(testResultsPath) + if err != nil { + return err + } + + var testResults Results + if err := json.NewDecoder(testResultsFile).Decode(&testResults); err != nil { + return err + } + + // For each failing test... + var errs []error + numFixes := 0 + for _, group := range testResults.Groups { + for _, suite := range group.Testsuites { + for _, failure := range suite.Failures { + // .. attempt to fix the problem + test := group.Name + "." + suite.Name + if err := processFailure(test, wd, failure.Failure); err != nil { + errs = append(errs, fmt.Errorf("%v: %w", test, err)) + } else { + numFixes++ + } + } + } + } + + if numFixes > 0 { + fmt.Printf("%v tests fixed\n", numFixes) + } + if n := len(errs); n > 0 { + fmt.Printf("%v tests could not be fixed:\n", n) + for _, err := range errs { + fmt.Println(err) + } + } + return nil +} + +var ( + // Regular expression to match a test declaration + reTests = regexp.MustCompile(`TEST(?:_[FP])?\((\w+),[ \n]*(\w+)\)`) + // Regular expression to match a EXPECT_EQ failure for strings + reExpectEq = regexp.MustCompile(`^([./\\a-z_-]*):(\d+).*\nExpected equality of these values:\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?)[^\\]"\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?)[^\\]"`) +) + +func processFailure(test, wd, failure string) error { + // Start by un-escaping newlines in the failure message + failure = strings.ReplaceAll(failure, "\\n", "\n") + + // Look for a EXPECT_EQ failure pattern + var file, a, b string + if parts := reExpectEq.FindStringSubmatch(failure); len(parts) == 5 { + file, a, b = parts[1], parts[3], parts[4] + } else { + return fmt.Errorf("Cannot fix this type of failure") + } + + // Now un-escape any quotes (the regex is sensitive to these) + a = strings.ReplaceAll(a, `\"`, `"`) + b = strings.ReplaceAll(b, `\"`, `"`) + + // Get the path to the source file containing the test failure + sourcePath := filepath.Join(wd, file) + + // Parse the source file, split into tests + sourceFile, err := parseSourceFile(sourcePath) + if err != nil { + return fmt.Errorf("Couldn't parse tests from file '%v': %w", file, err) + } + + // Find the test + testIdx, ok := sourceFile.tests[test] + if !ok { + return fmt.Errorf("Test '%v' not found in '%v'", test, file) + } + + // Grab the source for the particular test + testSource := sourceFile.parts[testIdx] + + // We don't know if a or b is the expected, so just try flipping the string + // to the other form. + switch { + case strings.Contains(testSource, a): + testSource = strings.Replace(testSource, a, b, -1) + case strings.Contains(testSource, b): + testSource = strings.Replace(testSource, b, a, -1) + default: + // Try escaping for R"(...)" strings + a = strings.ReplaceAll(a, "\n", `\n`) + b = strings.ReplaceAll(b, "\n", `\n`) + a = strings.ReplaceAll(a, "\"", `\"`) + b = strings.ReplaceAll(b, "\"", `\"`) + switch { + case strings.Contains(testSource, a): + testSource = strings.Replace(testSource, a, b, -1) + case strings.Contains(testSource, b): + testSource = strings.Replace(testSource, b, a, -1) + default: + return fmt.Errorf("Could not fix test '%v' in '%v'", test, file) + } + } + + // Replace the part of the source file + sourceFile.parts[testIdx] = testSource + + // Write out the source file + return writeSourceFile(sourcePath, sourceFile) +} + +// parseSourceFile() reads the file at path, splitting the content into chunks +// for each TEST. +func parseSourceFile(path string) (sourceFile, error) { + fileBytes, err := ioutil.ReadFile(path) + if err != nil { + return sourceFile{}, err + } + fileContent := string(fileBytes) + + out := sourceFile{ + tests: map[string]int{}, + } + + pos := 0 + for _, span := range reTests.FindAllStringIndex(fileContent, -1) { + out.parts = append(out.parts, fileContent[pos:span[0]]) + pos = span[0] + + match := reTests.FindStringSubmatch(fileContent[span[0]:span[1]]) + group := match[1] + suite := match[2] + out.tests[group+"."+suite] = len(out.parts) + } + out.parts = append(out.parts, fileContent[pos:]) + + return out, nil +} + +// writeSourceFile() joins the chunks of the file, and writes the content out to +// path. +func writeSourceFile(path string, file sourceFile) error { + body := strings.Join(file.parts, "") + return ioutil.WriteFile(path, []byte(body), 0666) +} + +type sourceFile struct { + parts []string + tests map[string]int // "X.Y" -> part index +} + +// Results is the root JSON structure of the JSON --gtest_output file . +type Results struct { + Tests int `json:"tests"` + Failures int `json:"failures"` + Disabled int `json:"disabled"` + Errors int `json:"errors"` + Timestamp string `json:"timestamp"` + Time string `json:"time"` + Name string `json:"name"` + Groups []TestsuiteGroup `json:"testsuites"` +} + +// TestsuiteGroup is a group of test suites in the JSON --gtest_output file . +type TestsuiteGroup struct { + Name string `json:"name"` + Tests int `json:"tests"` + Failures int `json:"failures"` + Disabled int `json:"disabled"` + Errors int `json:"errors"` + Timestamp string `json:"timestamp"` + Time string `json:"time"` + Testsuites []Testsuite `json:"testsuite"` +} + +// Testsuite is a suite of tests in the JSON --gtest_output file. +type Testsuite struct { + Name string `json:"name"` + ValueParam string `json:"value_param,omitempty"` + Status Status `json:"status"` + Result Result `json:"result"` + Timestamp string `json:"timestamp"` + Time string `json:"time"` + Classname string `json:"classname"` + Failures []Failure `json:"failures,omitempty"` +} + +// Failure is a reported test failure in the JSON --gtest_output file. +type Failure struct { + Failure string `json:"failure"` + Type string `json:"type"` +} + +// Status is a status code in the JSON --gtest_output file. +type Status string + +// Result is a result code in the JSON --gtest_output file. +type Result string