Re-run fail files from testdata/rapid automatically

Fixes #36

Gregory Petrosyan committed 3 years ago

Commit 4db076b1 parent dab278f5 files · +52 −16Browse files

Unified · Split

5 files

engine.go +29 −14
persist.go +6 −0
persist_test.go +15 −0
shrink_test.go +1 −1
statemachine_test.go +1 −1

 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
 	}
 	start := time.Now()
-	valid, invalid, earlyExit, seed, buf, err1, err2 := doCheck(tb, flags.failfile, deadline, checks, baseSeed(), prop)
+	valid, invalid, earlyExit, seed, failfile, buf, err1, err2 := doCheck(tb, deadline, checks, baseSeed(), flags.failfile, true, prop)
 	dt := time.Since(start)
 	if err1 == nil && err2 == nil {
 			tb.Errorf("[rapid] only generated %v valid tests from %v total (%v)", valid, valid+invalid, dt)
 		}
 	} else {
-		repr := fmt.Sprintf("-rapid.seed=%d", seed)
+		if failfile == "" && !flags.nofailfile {
-		if flags.failfile != "" && seed == 0 {
+			_, failfile = failFileName(tb.Name())
-			repr = fmt.Sprintf("-rapid.failfile=%q", flags.failfile)
-		} else if !flags.nofailfile {
-			_, failfile := failFileName(tb.Name())
 			out := captureTestOutput(tb, prop, buf)
 			err := saveFailFile(failfile, rapidVersion, out, seed, buf)
-			if err == nil {
+			if err != nil {
-				repr = fmt.Sprintf("-rapid.failfile=%q (or -rapid.seed=%d)", failfile, seed)
-			} else {
 				tb.Logf("[rapid] %v", err)
+				failfile = ""
 			}
 		}
+		var repr string
+		switch {
+		case failfile != "" && seed != 0:
+			repr = fmt.Sprintf("-rapid.failfile=%q (or -rapid.seed=%d)", failfile, seed)
+		case failfile != "":
+			repr = fmt.Sprintf("-rapid.failfile=%q", failfile)
+		case seed != 0:
+			repr = fmt.Sprintf("-rapid.seed=%d", seed)
+		}
 		name := regexp.QuoteMeta(tb.Name())
 		if traceback(err1) == traceback(err2) {
 			if err2.isStopTest() {
 	}
 }
-func doCheck(tb tb, failfile string, deadline time.Time, checks int, seed uint64, prop func(*T)) (int, int, bool, uint64, []uint64, *testError, *testError) {
+func doCheck(tb tb, deadline time.Time, checks int, seed uint64, failfile string, globFailFiles bool, prop func(*T)) (int, int, bool, uint64, string, []uint64, *testError, *testError) {
 	tb.Helper()
 	assertf(!tb.Failed(), "check function called with *testing.T which has already failed")
+	var failfiles []string
 	if failfile != "" {
+		failfiles = []string{failfile}
+	}
+	if globFailFiles {
+		matches, _ := filepath.Glob(failFilePattern(tb.Name()))
+		failfiles = append(failfiles, matches...)
+	}
+	for _, failfile := range failfiles {
 		buf, err1, err2 := checkFailFile(tb, failfile, prop)
 		if err1 != nil || err2 != nil {
-			return 0, 0, false, 0, buf, err1, err2
+			return 0, 0, false, 0, failfile, buf, err1, err2
 		}
 	}
 	valid, invalid, earlyExit, seed, err1 := findBug(tb, deadline, checks, seed, prop)
 	if err1 == nil {
-		return valid, invalid, earlyExit, 0, nil, nil, nil
+		return valid, invalid, earlyExit, 0, "", nil, nil, nil
 	}
 	s := newRandomBitStream(seed, true)
 	t.Logf("[rapid] trying to reproduce the failure")
 	err2 := checkOnce(t, prop)
 	if !sameError(err1, err2) {
-		return valid, invalid, false, seed, s.data, err1, err2
+		return valid, invalid, false, seed, "", s.data, err1, err2
 	}
 	t.Logf("[rapid] trying to minimize the failing test case")
 	buf, err3 := shrink(tb, shrinkDeadline(deadline), s.recordedBits, err2, prop)
-	return valid, invalid, false, seed, buf, err2, err3
+	return valid, invalid, false, seed, "", buf, err2, err3
 }
 func checkFailFile(tb tb, failfile string, prop func(*T)) ([]uint64, *testError, *testError) {

persist.go+6−0

 	return dirName, filepath.Join(dirName, fileName)
 }
+func failFilePattern(testName string) string {
+	fileName := fmt.Sprintf("%s-*.fail", kindaSafeFilename(testName))
+	dirName := filepath.Join("testdata", "rapid", kindaSafeFilename(testName))
+	return filepath.Join(dirName, fileName)
+}
 func saveFailFile(filename string, version string, output []byte, seed uint64, buf []uint64) error {
 	dir := filepath.Dir(filename)
 	err := os.MkdirAll(dir, persistDirMode)

persist_test.go+15−0

@@ -8,9 +8,24 @@ package rapid · 7 unmodified lines
8		8
9	import (	9	import (
10	"os"	10	"os"
		11	"path/filepath"
11	"testing"	12	"testing"
12	)	13	)
13		14
		15	func TestFailFilePattern(t *testing.T) {
		16	t.Parallel()
		17
		18	Check(t, func(t *T) {
		19	testName := String().Draw(t, "testName")
		20	_, fileName := failFileName(testName)
		21	pattern := failFilePattern(testName)
		22	match, err := filepath.Match(pattern, fileName)
		23	if !match \|\| err != nil {
		24	t.Fatalf("pattern %q does not match %q; err %v", pattern, fileName, err)
		25	}
		26	})
		27	}
		28
14	func TestFailFileRoundtrip(t *testing.T) {	29	func TestFailFileRoundtrip(t *testing.T) {
15	t.Parallel()	30	t.Parallel()
16		31

shrink_test.go+1−1

@@ -210,7 +210,7 @@ func checkShrink(t testing.T, prop func(T), draws ...any) { · 209 unmodified lines
210	t.Run(strconv.Itoa(i), func(t *testing.T) {	210	t.Run(strconv.Itoa(i), func(t *testing.T) {
211	t.Helper()	211	t.Helper()
212		212
213	_, _, _, seed, buf, err1, err2 := doCheck(t, "", checkDeadline(nil), 100, baseSeed(), prop)	213	_, _, _, seed, _, buf, err1, err2 := doCheck(t, checkDeadline(nil), 100, baseSeed(), "", false, prop)
214	if seed != 0 && err1 == nil && err2 == nil {	214	if seed != 0 && err1 == nil && err2 == nil {
215	t.Fatalf("shrink test did not fail (seed %v)", seed)	215	t.Fatalf("shrink test did not fail (seed %v)", seed)
216	}	216	}

statemachine_test.go+1−1

@@ -235,6 +235,6 @@ func TestStateMachine_DiscardGarbage(t *testing.T) { · 234 unmodified lines
235		235
236	func BenchmarkCheckQueue(b *testing.B) {	236	func BenchmarkCheckQueue(b *testing.B) {
237	for i := 0; i < b.N; i++ {	237	for i := 0; i < b.N; i++ {
238	_, _, _, _, _, _, _ = doCheck(b, "", checkDeadline(nil), 100, baseSeed(), queueTest)	238	_, _, _, _, _, _, _, _ = doCheck(b, checkDeadline(nil), 100, baseSeed(), "", false, queueTest)
239	}	239	}
240	}	240	}

Re-run fail files from testdata/rapid automatically

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code