278 lines
7.7 KiB
Go
278 lines
7.7 KiB
Go
|
// Copyright 2021 The Tint Authors.
|
||
|
|
//
|
||
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
// you may not use this file except in compliance with the License.
|
||
|
|
// You may obtain a copy of the License at
|
||
|
|
//
|
||
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
//
|
||
|
|
// Unless required by applicable law or agreed to in writing, software
|
||
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
// See the License for the specific language governing permissions and
|
||
|
|
// limitations under the License.
|
||
|
|
|
||
|
|
// fix-tests is a tool to update tests with new expected output.
|
||
|
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/json"
|
||
|
|
"flag"
|
||
|
|
"fmt"
|
||
|
|
"io/ioutil"
|
||
|
|
"os"
|
||
|
|
"os/exec"
|
||
|
|
"path/filepath"
|
||
|
|
"regexp"
|
||
|
|
"strings"
|
||
|
|
)
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
if err := run(); err != nil {
|
||
|
|
fmt.Println(err)
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func showUsage() {
|
||
|
|
fmt.Println(`
|
||
|
|
fix-tests is a tool to update tests with new expected output.
|
||
|
|
|
||
|
|
Usage:
|
||
|
|
fix-tests <executable>
|
||
|
|
|
||
|
|
executable - the path to the test executable to run.`)
|
||
|
|
os.Exit(1)
|
||
|
|
}
|
||
|
|
|
||
|
|
func run() error {
|
||
|
|
flag.Parse()
|
||
|
|
args := flag.Args()
|
||
|
|
if len(args) < 1 {
|
||
|
|
showUsage()
|
||
|
|
}
|
||
|
|
|
||
|
|
exe := args[0] // The path to the test executable
|
||
|
|
wd := filepath.Dir(exe) // The directory holding the test exe
|
||
|
|
|
||
|
|
// Create a temporary directory to hold the 'test-results.json' file
|
||
|
|
tmpDir, err := ioutil.TempDir("", "fix-tests")
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err := os.MkdirAll(tmpDir, 0666); err != nil {
|
||
|
|
return fmt.Errorf("Failed to create temporary directory: %w", err)
|
||
|
|
}
|
||
|
|
defer os.RemoveAll(tmpDir)
|
||
|
|
|
||
|
|
// Full path to the 'test-results.json' in the temporary directory
|
||
|
|
testResultsPath := filepath.Join(tmpDir, "test-results.json")
|
||
|
|
|
||
|
|
// Run the tests
|
||
|
|
switch err := exec.Command(exe, "--gtest_output=json:"+testResultsPath).Run().(type) {
|
||
|
|
default:
|
||
|
|
return err
|
||
|
|
case nil:
|
||
|
|
fmt.Println("All tests passed")
|
||
|
|
case *exec.ExitError:
|
||
|
|
}
|
||
|
|
|
||
|
|
// Read the 'test-results.json' file
|
||
|
|
testResultsFile, err := os.Open(testResultsPath)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
var testResults Results
|
||
|
|
if err := json.NewDecoder(testResultsFile).Decode(&testResults); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
// For each failing test...
|
||
|
|
var errs []error
|
||
|
|
numFixes := 0
|
||
|
|
for _, group := range testResults.Groups {
|
||
|
|
for _, suite := range group.Testsuites {
|
||
|
|
for _, failure := range suite.Failures {
|
||
|
|
// .. attempt to fix the problem
|
||
|
|
test := group.Name + "." + suite.Name
|
||
|
|
if err := processFailure(test, wd, failure.Failure); err != nil {
|
||
|
|
errs = append(errs, fmt.Errorf("%v: %w", test, err))
|
||
|
|
} else {
|
||
|
|
numFixes++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if numFixes > 0 {
|
||
|
|
fmt.Printf("%v tests fixed\n", numFixes)
|
||
|
|
}
|
||
|
|
if n := len(errs); n > 0 {
|
||
|
|
fmt.Printf("%v tests could not be fixed:\n", n)
|
||
|
|
for _, err := range errs {
|
||
|
|
fmt.Println(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
var (
|
||
|
|
// Regular expression to match a test declaration
|
||
|
|
reTests = regexp.MustCompile(`TEST(?:_[FP])?\((\w+),[ \n]*(\w+)\)`)
|
||
|
|
// Regular expression to match a EXPECT_EQ failure for strings
|
||
|
|
reExpectEq = regexp.MustCompile(`^([./\\a-z_-]*):(\d+).*\nExpected equality of these values:\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?)[^\\]"\n(?:.|\n)*?(?:Which is: | )"((?:.|\n)*?)[^\\]"`)
|
||
|
|
)
|
||
|
|
|
||
|
|
func processFailure(test, wd, failure string) error {
|
||
|
|
// Start by un-escaping newlines in the failure message
|
||
|
|
failure = strings.ReplaceAll(failure, "\\n", "\n")
|
||
|
|
|
||
|
|
// Look for a EXPECT_EQ failure pattern
|
||
|
|
var file, a, b string
|
||
|
|
if parts := reExpectEq.FindStringSubmatch(failure); len(parts) == 5 {
|
||
|
|
file, a, b = parts[1], parts[3], parts[4]
|
||
|
|
} else {
|
||
|
|
return fmt.Errorf("Cannot fix this type of failure")
|
||
|
|
}
|
||
|
|
|
||
|
|
// Now un-escape any quotes (the regex is sensitive to these)
|
||
|
|
a = strings.ReplaceAll(a, `\"`, `"`)
|
||
|
|
b = strings.ReplaceAll(b, `\"`, `"`)
|
||
|
|
|
||
|
|
// Get the path to the source file containing the test failure
|
||
|
|
sourcePath := filepath.Join(wd, file)
|
||
|
|
|
||
|
|
// Parse the source file, split into tests
|
||
|
|
sourceFile, err := parseSourceFile(sourcePath)
|
||
|
|
if err != nil {
|
||
|
|
return fmt.Errorf("Couldn't parse tests from file '%v': %w", file, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Find the test
|
||
|
|
testIdx, ok := sourceFile.tests[test]
|
||
|
|
if !ok {
|
||
|
|
return fmt.Errorf("Test '%v' not found in '%v'", test, file)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Grab the source for the particular test
|
||
|
|
testSource := sourceFile.parts[testIdx]
|
||
|
|
|
||
|
|
// We don't know if a or b is the expected, so just try flipping the string
|
||
|
|
// to the other form.
|
||
|
|
switch {
|
||
|
|
case strings.Contains(testSource, a):
|
||
|
|
testSource = strings.Replace(testSource, a, b, -1)
|
||
|
|
case strings.Contains(testSource, b):
|
||
|
|
testSource = strings.Replace(testSource, b, a, -1)
|
||
|
|
default:
|
||
|
|
// Try escaping for R"(...)" strings
|
||
|
|
a = strings.ReplaceAll(a, "\n", `\n`)
|
||
|
|
b = strings.ReplaceAll(b, "\n", `\n`)
|
||
|
|
a = strings.ReplaceAll(a, "\"", `\"`)
|
||
|
|
b = strings.ReplaceAll(b, "\"", `\"`)
|
||
|
|
switch {
|
||
|
|
case strings.Contains(testSource, a):
|
||
|
|
testSource = strings.Replace(testSource, a, b, -1)
|
||
|
|
case strings.Contains(testSource, b):
|
||
|
|
testSource = strings.Replace(testSource, b, a, -1)
|
||
|
|
default:
|
||
|
|
return fmt.Errorf("Could not fix test '%v' in '%v'", test, file)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Replace the part of the source file
|
||
|
|
sourceFile.parts[testIdx] = testSource
|
||
|
|
|
||
|
|
// Write out the source file
|
||
|
|
return writeSourceFile(sourcePath, sourceFile)
|
||
|
|
}
|
||
|
|
|
||
|
|
// parseSourceFile() reads the file at path, splitting the content into chunks
|
||
|
|
// for each TEST.
|
||
|
|
func parseSourceFile(path string) (sourceFile, error) {
|
||
|
|
fileBytes, err := ioutil.ReadFile(path)
|
||
|
|
if err != nil {
|
||
|
|
return sourceFile{}, err
|
||
|
|
}
|
||
|
|
fileContent := string(fileBytes)
|
||
|
|
|
||
|
|
out := sourceFile{
|
||
|
|
tests: map[string]int{},
|
||
|
|
}
|
||
|
|
|
||
|
|
pos := 0
|
||
|
|
for _, span := range reTests.FindAllStringIndex(fileContent, -1) {
|
||
|
|
out.parts = append(out.parts, fileContent[pos:span[0]])
|
||
|
|
pos = span[0]
|
||
|
|
|
||
|
|
match := reTests.FindStringSubmatch(fileContent[span[0]:span[1]])
|
||
|
|
group := match[1]
|
||
|
|
suite := match[2]
|
||
|
|
out.tests[group+"."+suite] = len(out.parts)
|
||
|
|
}
|
||
|
|
out.parts = append(out.parts, fileContent[pos:])
|
||
|
|
|
||
|
|
return out, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// writeSourceFile() joins the chunks of the file, and writes the content out to
|
||
|
|
// path.
|
||
|
|
func writeSourceFile(path string, file sourceFile) error {
|
||
|
|
body := strings.Join(file.parts, "")
|
||
|
|
return ioutil.WriteFile(path, []byte(body), 0666)
|
||
|
|
}
|
||
|
|
|
||
|
|
type sourceFile struct {
|
||
|
|
parts []string
|
||
|
|
tests map[string]int // "X.Y" -> part index
|
||
|
|
}
|
||
|
|
|
||
|
|
// Results is the root JSON structure of the JSON --gtest_output file .
|
||
|
|
type Results struct {
|
||
|
|
Tests int `json:"tests"`
|
||
|
|
Failures int `json:"failures"`
|
||
|
|
Disabled int `json:"disabled"`
|
||
|
|
Errors int `json:"errors"`
|
||
|
|
Timestamp string `json:"timestamp"`
|
||
|
|
Time string `json:"time"`
|
||
|
|
Name string `json:"name"`
|
||
|
|
Groups []TestsuiteGroup `json:"testsuites"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestsuiteGroup is a group of test suites in the JSON --gtest_output file .
|
||
|
|
type TestsuiteGroup struct {
|
||
|
|
Name string `json:"name"`
|
||
|
|
Tests int `json:"tests"`
|
||
|
|
Failures int `json:"failures"`
|
||
|
|
Disabled int `json:"disabled"`
|
||
|
|
Errors int `json:"errors"`
|
||
|
|
Timestamp string `json:"timestamp"`
|
||
|
|
Time string `json:"time"`
|
||
|
|
Testsuites []Testsuite `json:"testsuite"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// Testsuite is a suite of tests in the JSON --gtest_output file.
|
||
|
|
type Testsuite struct {
|
||
|
|
Name string `json:"name"`
|
||
|
|
ValueParam string `json:"value_param,omitempty"`
|
||
|
|
Status Status `json:"status"`
|
||
|
|
Result Result `json:"result"`
|
||
|
|
Timestamp string `json:"timestamp"`
|
||
|
|
Time string `json:"time"`
|
||
|
|
Classname string `json:"classname"`
|
||
|
|
Failures []Failure `json:"failures,omitempty"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// Failure is a reported test failure in the JSON --gtest_output file.
|
||
|
|
type Failure struct {
|
||
|
|
Failure string `json:"failure"`
|
||
|
|
Type string `json:"type"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// Status is a status code in the JSON --gtest_output file.
|
||
|
|
type Status string
|
||
|
|
|
||
|
|
// Result is a result code in the JSON --gtest_output file.
|
||
|
|
type Result string
|