// TestBisync is a test engine for bisync test cases.// See https://rclone.org/bisync/#testing for documentation.// Test cases are organized in subdirs beneath ./testdata// Results are compared against golden listings and log file.package bisync_test import (	"bytes"	"context"	"errors"	"flag"	"fmt"	"log"	"os"	"path"	"path/filepath"	"regexp"	"runtime"	"sort"	"strconv"	"strings"	"testing"	"time" 	"github.com/rclone/rclone/cmd/bisync"	"github.com/rclone/rclone/cmd/bisync/bilib"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/accounting"	"github.com/rclone/rclone/fs/cache"	"github.com/rclone/rclone/fs/filter"	"github.com/rclone/rclone/fs/fspath"	"github.com/rclone/rclone/fs/object"	"github.com/rclone/rclone/fs/operations"	"github.com/rclone/rclone/fs/sync"	"github.com/rclone/rclone/fstest"	"github.com/rclone/rclone/lib/atexit"	"github.com/rclone/rclone/lib/random" 	"github.com/pmezard/go-difflib/difflib"	"github.com/stretchr/testify/assert"	"github.com/stretchr/testify/require" 	_ "github.com/rclone/rclone/backend/all" // for integration tests) const (	touchDateFormat = "2006-01-02"	goldenCanonBase = "_testdir_"	logFileName     = "test.log"	dropMe          = "*** [DROP THIS LINE] ***"	eol             = "\n"	slash           = string(os.PathSeparator)	fixSlash        = (runtime.GOOS == "windows")) // logReplacements make modern test logs comparable with golden dir.// It is a string slice of even length with this structure:////	{`matching regular expression`, "mangled result string", ...}var logReplacements = []string{	// skip syslog facility markers	`^(<[1-9]>)(INFO  |ERROR |NOTICE|DEBUG ):(.*)$`, "$2:$3",	// skip log prefixes	`^\d+/\d\d/\d\d \d\d:\d\d:\d\d(?:\.\d{6})? `, "",	// ignore rclone info messages	`^INFO  : .*?: (Deleted|Copied |Moved |Updated ).*$`, dropMe,	`^NOTICE: .*?: Replacing invalid UTF-8 characters in "[^"]*"$`, dropMe,	// ignore rclone debug messages	`^DEBUG : .*$`, dropMe,	// ignore dropbox info messages	`^NOTICE: too_many_(requests|write_operations)/\.*: Too many requests or write operations.*$`, dropMe,	`^NOTICE: Dropbox root .*?: Forced to upload files to set modification times on this backend.$`, dropMe,	`^INFO  : .*?: src and dst identical but can't set mod time without deleting and re-uploading$`, dropMe,} // Some dry-run messages differ depending on the particular remote.var dryrunReplacements = []string{	`^(NOTICE: file5.txt: Skipped) (copy|update modification time) (as --dry-run is set [(]size \d+[)])$`,	`$1 copy (or update modification time) $3`,} // Some groups of log lines may appear unordered because rclone applies// many operations in parallel to boost performance.var logHoppers = []string{	// Test case `dry-run` produced log mismatches due to non-deterministic	// order of captured dry-run info messages.	`NOTICE: \S+?: Skipped (?:copy|move|delete|copy \(or [^)]+\)|update modification time) as --dry-run is set \(size \d+\)`, 	// Test case `extended-filenames` detected difference in order of files	// with extended unicode names between Windows and Unix or GDrive,	// but the order is in fact not important for success.	`(?:INFO  |NOTICE): - Path[12] +File (?:was deleted|is new|is newer|is OLDER) +- .*`, 	// Test case `check-access-filters` detected listing miscompares due	// to indeterminate order of rclone operations in presence of multiple	// subdirectories. The order inconsistency initially showed up in the	// listings and triggered reordering of log messages, but the actual	// files will in fact match.	`ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`, 	// Test case `resync` suffered from the order of queued copies.	`(?:INFO  |NOTICE): - Path2    Resync will copy to Path1 +- .*`,} // Some log lines can contain Windows path separator that must be// converted to "/" in every matching token to match golden logs.var logLinesWithSlash = []string{	`\(\d\d\)  : (touch-glob|touch-copy|copy-file|copy-as|copy-dir|delete-file) `,	`INFO  : - Path[12] +Queue copy to Path[12] `,	`INFO  : Synching Path1 .*? with Path2 `,	`INFO  : Validating listings for `,}var regexFixSlash = regexp.MustCompile("^(" + strings.Join(logLinesWithSlash, "|") + ")") // Command line flags for bisync testvar (	argTestCase  = flag.String("case", "", "Bisync test case to run")	argRemote2   = flag.String("remote2", "", "Path2 for bisync tests")	argNoCompare = flag.Bool("no-compare", false, "Do not compare test results with golden")	argNoCleanup = flag.Bool("no-cleanup", false, "Keep test files")	argGolden    = flag.Bool("golden", false, "Store results as golden")	argDebug     = flag.Bool("debug", false, "Print debug messages")	argStopAt    = flag.Int("stop-at", 0, "Stop after given test step")	// Flag -refresh-times helps with Dropbox tests failing with message	// "src and dst identical but can't set mod time without deleting and re-uploading"	argRefreshTimes = flag.Bool("refresh-times", false, "Force refreshing the target modtime, useful for Dropbox (default: false)")) // bisyncTest keeps all test data in a single placetype bisyncTest struct {	// per-test state	t           *testing.T	step        int	stopped     bool	stepStr     string	testCase    string	sessionName string	// test dirs	testDir    string	dataDir    string	initDir    string	goldenDir  string	workDir    string	fs1        fs.Fs	path1      string	canonPath1 string	fs2        fs.Fs	path2      string	canonPath2 string	// test log	logDir  string	logPath string	logFile *os.File	// global state	dataRoot string	randName string	tempDir  string	parent1  fs.Fs	parent2  fs.Fs	// global flags	argRemote1 string	argRemote2 string	noCompare  bool	noCleanup  bool	golden     bool	debug      bool	stopAt     int} // TestBisync is a test engine for bisync test cases.func TestBisync(t *testing.T) {	ctx := context.Background()	fstest.Initialise() 	ci := fs.GetConfig(ctx)	ciSave := *ci	defer func() {		*ci = ciSave	}()	if *argRefreshTimes {		ci.RefreshTimes = true	} 	baseDir, err := os.Getwd()	require.NoError(t, err, "get current directory")	randName := "bisync." + time.Now().Format("150405-") + random.String(5)	tempDir := filepath.Join(os.TempDir(), randName)	workDir := filepath.Join(tempDir, "workdir") 	b := &bisyncTest{		// per-test state		t: t,		// global state		tempDir:  tempDir,		randName: randName,		workDir:  workDir,		dataRoot: filepath.Join(baseDir, "testdata"),		logDir:   filepath.Join(tempDir, "logs"),		logPath:  filepath.Join(workDir, logFileName),		// global flags		argRemote1: *fstest.RemoteName,		argRemote2: *argRemote2,		noCompare:  *argNoCompare,		noCleanup:  *argNoCleanup,		golden:     *argGolden,		debug:      *argDebug,		stopAt:     *argStopAt,	} 	b.mkdir(b.tempDir)	b.mkdir(b.logDir) 	fnHandle := atexit.Register(func() {		if atexit.Signalled() {			b.cleanupAll()		}	})	defer func() {		b.cleanupAll()		atexit.Unregister(fnHandle)	}() 	argCase := *argTestCase	if argCase == "" {		argCase = "all"		if testing.Short() {			// remote tests can be long, help with "go test -short"			argCase = "basic"		}	} 	testList := strings.Split(argCase, ",")	if strings.ToLower(argCase) == "all" {		testList = nil		for _, testCase := range b.listDir(b.dataRoot) {			if strings.HasPrefix(testCase, "test_") {				testList = append(testList, testCase)			}		}	}	require.False(t, b.stopAt > 0 && len(testList) > 1, "-stop-at is meaningful only for a single test") 	for _, testCase := range testList {		testCase = strings.ReplaceAll(testCase, "-", "_")		testCase = strings.TrimPrefix(testCase, "test_")		t.Run(testCase, func(childTest *testing.T) {			b.runTestCase(ctx, childTest, testCase)		})	}} func (b *bisyncTest) cleanupAll() {	if b.noCleanup {		return	}	ctx := context.Background()	if b.parent1 != nil {		_ = operations.Purge(ctx, b.parent1, "")	}	if b.parent2 != nil {		_ = operations.Purge(ctx, b.parent2, "")	}	_ = os.RemoveAll(b.tempDir)} func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase string) {	b.t = t	b.testCase = testCase	var err error 	b.fs1, b.parent1, b.path1, b.canonPath1 = b.makeTempRemote(ctx, b.argRemote1, "path1")	b.fs2, b.parent2, b.path2, b.canonPath2 = b.makeTempRemote(ctx, b.argRemote2, "path2") 	b.sessionName = bilib.SessionName(b.fs1, b.fs2)	b.testDir = b.ensureDir(b.dataRoot, "test_"+b.testCase, false)	b.initDir = b.ensureDir(b.testDir, "initial", false)	b.goldenDir = b.ensureDir(b.testDir, "golden", false)	b.dataDir = b.ensureDir(b.testDir, "modfiles", true) // optional 	// For test stability, jam initial dates to a fixed past date.	// Test cases that change files will touch specific files to fixed new dates.	initDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)	err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error {		if err == nil && !info.IsDir() {			return os.Chtimes(path, initDate, initDate)		}		return err	})	require.NoError(b.t, err, "jamming initial dates") 	// Prepare initial content	b.cleanupCase(ctx)	initFs, err := fs.NewFs(ctx, b.initDir)	require.NoError(b.t, err)	require.NoError(b.t, sync.CopyDir(ctx, b.fs1, initFs, true), "setting up path1")	require.NoError(b.t, sync.CopyDir(ctx, b.fs2, initFs, true), "setting up path2") 	// Create log file	b.mkdir(b.workDir)	b.logFile, err = os.Create(b.logPath)	require.NoError(b.t, err, "creating log file") 	// Execute test scenario	scenFile := filepath.Join(b.testDir, "scenario.txt")	scenBuf, err := os.ReadFile(scenFile)	scenReplacer := b.newReplacer(false)	require.NoError(b.t, err)	b.step = 0	b.stopped = false	for _, line := range strings.Split(string(scenBuf), "\n") {		comment := strings.Index(line, "#")		if comment != -1 {			line = line[:comment]		}		line = strings.TrimSpace(line)		if line == "" {			if b.golden {				// Keep empty lines in golden logs				_, _ = b.logFile.WriteString("\n")			}			continue		} 		b.step++		b.stepStr = fmt.Sprintf("(%02d)  :", b.step)		line = scenReplacer.Replace(line)		if err = b.runTestStep(ctx, line); err != nil {			require.Failf(b.t, "test step failed", "step %d failed: %v", b.step, err)			return		}		if b.stopAt > 0 && b.step >= b.stopAt {			comment := ""			if b.golden {				comment = " (ignoring -golden)"			}			b.logPrintf("Stopping after step %d%s", b.step, comment)			b.stopped = true			b.noCleanup = true			b.noCompare = true			break		}	} 	// Perform post-run activities	require.NoError(b.t, b.logFile.Close(), "flushing test log")	b.logFile = nil 	savedLog := b.testCase + ".log"	err = bilib.CopyFile(b.logPath, filepath.Join(b.logDir, savedLog))	require.NoError(b.t, err, "saving log file %s", savedLog) 	if b.golden && !b.stopped {		log.Printf("Store results to golden directory")		b.storeGolden()		return	} 	errorCount := 0	if b.noCompare {		log.Printf("Skip comparing results with golden directory")		errorCount = -2	} else {		errorCount = b.compareResults()	} 	if b.noCleanup {		log.Printf("Skip cleanup")	} else {		b.cleanupCase(ctx)	} 	var msg string	var passed bool	switch errorCount {	case 0:		msg = fmt.Sprintf("TEST %s PASSED", b.testCase)		passed = true	case -2:		msg = fmt.Sprintf("TEST %s SKIPPED", b.testCase)		passed = true	case -1:		msg = fmt.Sprintf("TEST %s FAILED - WRONG NUMBER OF FILES", b.testCase)		passed = false	default:		msg = fmt.Sprintf("TEST %s FAILED - %d MISCOMPARED FILES", b.testCase, errorCount)		buckets := b.fs1.Features().BucketBased || b.fs2.Features().BucketBased		passed = false		if b.testCase == "rmdirs" && buckets {			msg += " (expected failure on bucket remotes)"			passed = true		}	}	b.t.Log(msg)	if !passed {		b.t.FailNow()	}} // makeTempRemote creates temporary folder and makes a filesystem// if a local path is provided, it's ignored (the test will run under system temp)func (b *bisyncTest) makeTempRemote(ctx context.Context, remote, subdir string) (f, parent fs.Fs, path, canon string) {	var err error	if bilib.IsLocalPath(remote) {		if remote != "" && remote != "local" {			b.t.Fatalf(`Missing ":" in remote %q. Use "local" to test with local filesystem.`, remote)		}		parent, err = fs.NewFs(ctx, b.tempDir)		require.NoError(b.t, err, "parsing %s", b.tempDir) 		path = filepath.Join(b.tempDir, b.testCase)		canon = bilib.CanonicalPath(path) + "_"		path = filepath.Join(path, subdir)	} else {		last := remote[len(remote)-1]		if last != ':' && last != '/' {			remote += "/"		}		remote += b.randName		parent, err = fs.NewFs(ctx, remote)		require.NoError(b.t, err, "parsing %s", remote) 		path = remote + "/" + b.testCase		canon = bilib.CanonicalPath(path) + "_"		path += "/" + subdir	} 	f, err = fs.NewFs(ctx, path)	require.NoError(b.t, err, "parsing %s/%s", remote, subdir)	path = bilib.FsPath(f) // Make it canonical 	if f.Precision() == fs.ModTimeNotSupported {		b.t.Skipf("modification time support is missing on %s", subdir)	}	return} func (b *bisyncTest) cleanupCase(ctx context.Context) {	// Silence "directory not found" errors from the ftp backend	_ = bilib.CaptureOutput(func() {		_ = operations.Purge(ctx, b.fs1, "")	})	_ = bilib.CaptureOutput(func() {		_ = operations.Purge(ctx, b.fs2, "")	})	_ = os.RemoveAll(b.workDir)	accounting.Stats(ctx).ResetCounters()} func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {	var fsrc, fdst fs.Fs	accounting.Stats(ctx).ResetErrors()	b.logPrintf("%s %s", b.stepStr, line) 	ci := fs.GetConfig(ctx)	ciSave := *ci	defer func() {		*ci = ciSave	}()	ci.LogLevel = fs.LogLevelInfo	if b.debug {		ci.LogLevel = fs.LogLevelDebug	} 	args := splitLine(line)	switch args[0] {	case "test":		b.checkArgs(args, 1, 0)		return nil	case "copy-listings":		b.checkArgs(args, 1, 1)		return b.saveTestListings(args[1], true)	case "move-listings":		b.checkArgs(args, 1, 1)		return b.saveTestListings(args[1], false)	case "purge-children":		b.checkArgs(args, 1, 1)		if fsrc, err = fs.NewFs(ctx, args[1]); err != nil {			return err		}		return purgeChildren(ctx, fsrc, "")	case "delete-file":		b.checkArgs(args, 1, 1)		dir, file := filepath.Split(args[1])		if fsrc, err = fs.NewFs(ctx, dir); err != nil {			return err		}		var obj fs.Object		if obj, err = fsrc.NewObject(ctx, file); err != nil {			return err		}		return operations.DeleteFile(ctx, obj)	case "delete-glob":		b.checkArgs(args, 2, 2)		if fsrc, err = fs.NewFs(ctx, args[1]); err != nil {			return err		}		return deleteFiles(ctx, fsrc, args[2])	case "touch-glob":		b.checkArgs(args, 3, 3)		date, src, glob := args[1], args[2], args[3]

Lines 1–500 of 1250. Load more