Re-run fail files from testdata/rapid automatically

Fixes #36

Gregory Petrosyan committed 3 years ago

Commit 4db076b1 parent dab278f5 files · +52 −16Browse files

Unified · Split

5 files

engine.go +29 −14
persist.go +6 −0
persist_test.go +15 −0
shrink_test.go +1 −1
statemachine_test.go +1 −1

@@ -13,6 +13,7 @@ import ( · 12 unmodified lines
13	13	"fmt"
14	14	"log"
15	15	"os"
	16	+ "path/filepath"
16	17	"regexp"
17	18	"runtime"
18	19	"strings"
@@ -186,7 +187,7 @@ func checkTB(tb tb, deadline time.Time, prop func(*T)) { · 167 unmodified lines
186	187	}
187	188
188	189	start := time.Now()
189		− valid, invalid, earlyExit, seed, buf, err1, err2 := doCheck(tb, flags.failfile, deadline, checks, baseSeed(), prop)
	190	+ valid, invalid, earlyExit, seed, failfile, buf, err1, err2 := doCheck(tb, deadline, checks, baseSeed(), flags.failfile, true, prop)
190	191	dt := time.Since(start)
191	192
192	193	if err1 == nil && err2 == nil {
@@ -196,20 +197,26 @@ func checkTB(tb tb, deadline time.Time, prop func(*T)) { · 3 unmodified lines
196	197	tb.Errorf("[rapid] only generated %v valid tests from %v total (%v)", valid, valid+invalid, dt)
197	198	}
198	199	} else {
199		− repr := fmt.Sprintf("-rapid.seed=%d", seed)
200		− if flags.failfile != "" && seed == 0 {
201		− repr = fmt.Sprintf("-rapid.failfile=%q", flags.failfile)
202		− } else if !flags.nofailfile {
203		− _, failfile := failFileName(tb.Name())
	200	+ if failfile == "" && !flags.nofailfile {
	201	+ _, failfile = failFileName(tb.Name())
204	202	out := captureTestOutput(tb, prop, buf)
205	203	err := saveFailFile(failfile, rapidVersion, out, seed, buf)
206		− if err == nil {
207		− repr = fmt.Sprintf("-rapid.failfile=%q (or -rapid.seed=%d)", failfile, seed)
208		− } else {
	204	+ if err != nil {
209	205	tb.Logf("[rapid] %v", err)
	206	+ failfile = ""
210	207	}
211	208	}
212	209
	210	+ var repr string
	211	+ switch {
	212	+ case failfile != "" && seed != 0:
	213	+ repr = fmt.Sprintf("-rapid.failfile=%q (or -rapid.seed=%d)", failfile, seed)
	214	+ case failfile != "":
	215	+ repr = fmt.Sprintf("-rapid.failfile=%q", failfile)
	216	+ case seed != 0:
	217	+ repr = fmt.Sprintf("-rapid.seed=%d", seed)
	218	+ }
	219	+
213	220	name := regexp.QuoteMeta(tb.Name())
214	221	if traceback(err1) == traceback(err2) {
215	222	if err2.isStopTest() {
@@ -229,21 +236,29 @@ func checkTB(tb tb, deadline time.Time, prop func(*T)) { · 13 unmodified lines
229	236	}
230	237	}
231	238
232		−func doCheck(tb tb, failfile string, deadline time.Time, checks int, seed uint64, prop func(T)) (int, int, bool, uint64, []uint64, testError, *testError) {
	239	+func doCheck(tb tb, deadline time.Time, checks int, seed uint64, failfile string, globFailFiles bool, prop func(T)) (int, int, bool, uint64, string, []uint64, testError, *testError) {
233	240	tb.Helper()
234	241
235	242	assertf(!tb.Failed(), "check function called with *testing.T which has already failed")
236	243
	244	+ var failfiles []string
237	245	if failfile != "" {
	246	+ failfiles = []string{failfile}
	247	+ }
	248	+ if globFailFiles {
	249	+ matches, _ := filepath.Glob(failFilePattern(tb.Name()))
	250	+ failfiles = append(failfiles, matches...)
	251	+ }
	252	+ for _, failfile := range failfiles {
238	253	buf, err1, err2 := checkFailFile(tb, failfile, prop)
239	254	if err1 != nil \|\| err2 != nil {
240		− return 0, 0, false, 0, buf, err1, err2
	255	+ return 0, 0, false, 0, failfile, buf, err1, err2
241	256	}
242	257	}
243	258
244	259	valid, invalid, earlyExit, seed, err1 := findBug(tb, deadline, checks, seed, prop)
245	260	if err1 == nil {
246		− return valid, invalid, earlyExit, 0, nil, nil, nil
	261	+ return valid, invalid, earlyExit, 0, "", nil, nil, nil
247	262	}
248	263
249	264	s := newRandomBitStream(seed, true)
@@ -251,13 +266,13 @@ func doCheck(tb tb, failfile string, deadline time.Time, checks int, seed uint64 · 1 unmodified line
251	266	t.Logf("[rapid] trying to reproduce the failure")
252	267	err2 := checkOnce(t, prop)
253	268	if !sameError(err1, err2) {
254		− return valid, invalid, false, seed, s.data, err1, err2
	269	+ return valid, invalid, false, seed, "", s.data, err1, err2
255	270	}
256	271
257	272	t.Logf("[rapid] trying to minimize the failing test case")
258	273	buf, err3 := shrink(tb, shrinkDeadline(deadline), s.recordedBits, err2, prop)
259	274
260		− return valid, invalid, false, seed, buf, err2, err3
	275	+ return valid, invalid, false, seed, "", buf, err2, err3
261	276	}
262	277
263	278	func checkFailFile(tb tb, failfile string, prop func(T)) ([]uint64, testError, *testError) {

persist.go+6−0

@@ -43,6 +43,12 @@ func failFileName(testName string) (string, string) { · 42 unmodified lines
43	43	return dirName, filepath.Join(dirName, fileName)
44	44	}
45	45
	46	+func failFilePattern(testName string) string {
	47	+ fileName := fmt.Sprintf("%s-*.fail", kindaSafeFilename(testName))
	48	+ dirName := filepath.Join("testdata", "rapid", kindaSafeFilename(testName))
	49	+ return filepath.Join(dirName, fileName)
	50	+}
	51	+
46	52	func saveFailFile(filename string, version string, output []byte, seed uint64, buf []uint64) error {
47	53	dir := filepath.Dir(filename)
48	54	err := os.MkdirAll(dir, persistDirMode)

persist_test.go+15−0

@@ -8,9 +8,24 @@ package rapid · 7 unmodified lines
8	8
9	9	import (
10	10	"os"
	11	+ "path/filepath"
11	12	"testing"
12	13	)
13	14
	15	+func TestFailFilePattern(t *testing.T) {
	16	+ t.Parallel()
	17	+
	18	+ Check(t, func(t *T) {
	19	+ testName := String().Draw(t, "testName")
	20	+ _, fileName := failFileName(testName)
	21	+ pattern := failFilePattern(testName)
	22	+ match, err := filepath.Match(pattern, fileName)
	23	+ if !match \|\| err != nil {
	24	+ t.Fatalf("pattern %q does not match %q; err %v", pattern, fileName, err)
	25	+ }
	26	+ })
	27	+}
	28	+
14	29	func TestFailFileRoundtrip(t *testing.T) {
15	30	t.Parallel()
16	31

shrink_test.go+1−1

@@ -210,7 +210,7 @@ func checkShrink(t testing.T, prop func(T), draws ...any) { · 209 unmodified lines
210	210	t.Run(strconv.Itoa(i), func(t *testing.T) {
211	211	t.Helper()
212	212
213		− _, _, _, seed, buf, err1, err2 := doCheck(t, "", checkDeadline(nil), 100, baseSeed(), prop)
	213	+ _, _, _, seed, _, buf, err1, err2 := doCheck(t, checkDeadline(nil), 100, baseSeed(), "", false, prop)
214	214	if seed != 0 && err1 == nil && err2 == nil {
215	215	t.Fatalf("shrink test did not fail (seed %v)", seed)
216	216	}

statemachine_test.go+1−1

@@ -235,6 +235,6 @@ func TestStateMachine_DiscardGarbage(t *testing.T) { · 234 unmodified lines
235	235
236	236	func BenchmarkCheckQueue(b *testing.B) {
237	237	for i := 0; i < b.N; i++ {
238		− _, _, _, _, _, _, _ = doCheck(b, "", checkDeadline(nil), 100, baseSeed(), queueTest)
	238	+ _, _, _, _, _, _, _, _ = doCheck(b, checkDeadline(nil), 100, baseSeed(), "", false, queueTest)
239	239	}
240	240	}

Re-run fail files from testdata/rapid automatically

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code