// TestBisync is a test engine for bisync test cases.// See https://rclone.org/bisync/#testing for documentation.// Test cases are organized in subdirs beneath ./testdata// Results are compared against golden listings and log file.package bisync_test import ( "bytes" "context" "errors" "flag" "fmt" "os" "path" "path/filepath" "regexp" "runtime" "slices" "sort" "strconv" "strings" "testing" "time" "unicode/utf8" "github.com/rclone/rclone/cmd/bisync" "github.com/rclone/rclone/cmd/bisync/bilib" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/terminal" "golang.org/x/text/unicode/norm" "github.com/pmezard/go-difflib/difflib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" _ "github.com/rclone/rclone/backend/all" // for integration tests) const ( touchDateFormat = "2006-01-02" goldenCanonBase = "_testdir_" logFileName = "test.log" dropMe = "*** [DROP THIS LINE] ***" eol = "\n" slash = string(os.PathSeparator) fixSlash = (runtime.GOOS == "windows")) var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ) /* Useful Command Shortcuts */// go test ./cmd/bisync -remote local -race// go test ./cmd/bisync -remote local -golden// go test ./cmd/bisync -remote local -case extended_filenames// go run ./fstest/test_all -run '^TestBisync.*$' -timeout 3h -verbose -maxtries 5// go run ./fstest/test_all -remotes local,TestCrypt:,TestDrive:,TestOneDrive:,TestOneDriveBusiness:,TestDropbox:,TestCryptDrive:,TestOpenDrive:,TestChunker:,:memory:,TestCryptNoEncryption:,TestCombine:DirA,TestFTPRclone:,TestWebdavRclone:,TestS3Rclone:,TestSFTPRclone:,TestSFTPRcloneSSH:,TestNextcloud:,TestChunkerNometaLocal:,TestChunkerChunk3bLocal:,TestChunkerLocal:,TestChunkerChunk3bNometaLocal:,TestStorj: -run '^TestBisync.*$' -timeout 3h -verbose -maxtries 5// go test -timeout 3h -run '^TestBisync.*$' github.com/rclone/rclone/cmd/bisync -remote TestDrive:Bisync -v// go test -timeout 3h -run '^TestBisyncRemoteRemote/basic$' github.com/rclone/rclone/cmd/bisync -remote TestDropbox:Bisync -v// TestFTPProftpd:,TestFTPPureftpd:,TestFTPRclone:,TestFTPVsftpd:,TestHdfs:,TestS3Minio:,TestS3MinioEdge:,TestS3Rclone:,TestSeafile:,TestSeafileEncrypted:,TestSeafileV6:,TestSFTPOpenssh:,TestSFTPRclone:,TestSFTPRcloneSSH:,TestSia:,TestSwiftAIO:,TestWebdavNextcloud:,TestWebdavOwncloud:,TestWebdavRclone: // logReplacements make modern test logs comparable with golden dir.// It is a string slice of even length with this structure://// {`matching regular expression`, "mangled result string", ...}var logReplacements = []string{ // skip syslog facility markers `^(<[1-9]>)(INFO |ERROR |NOTICE|DEBUG ):(.*)$`, "$2:$3", // skip log prefixes `^\d+/\d\d/\d\d \d\d:\d\d:\d\d(?:\.\d{6})? `, "", // ignore rclone info messages `^INFO : .*?: (Deleted|Copied |Moved |Updated ).*$`, dropMe, `^NOTICE: .*?: Replacing invalid UTF-8 characters in "[^"]*"$`, dropMe, // ignore rclone debug messages `^DEBUG : .*$`, dropMe, // ignore SFTP host key messages `^NOTICE: .*?No host key validation is being performed.*$`, dropMe, // ignore dropbox info messages `^NOTICE: too_many_(requests|write_operations)/\.*: Too many requests or write operations.*$`, dropMe, `^NOTICE: .*?: Forced to upload files to set modification times on this backend.$`, dropMe, `^INFO : .*? Committing uploads - please wait...$`, dropMe, `^INFO : .*?: src and dst identical but can't set mod time without deleting and re-uploading$`, dropMe, `^INFO : .*?: src and dst identical but can't set mod time without re-uploading$`, dropMe, // ignore crypt info messages `^INFO : .*?: Crypt detected! Using cryptcheck instead of check. \(Use --size-only or --ignore-checksum to disable\)$`, dropMe, // ignore drive info messages `^NOTICE:.*?Files of unknown size \(such as Google Docs\) do not sync reliably with --checksum or --size-only\. Consider using modtime instead \(the default\) or --drive-skip-gdocs.*?$`, dropMe, // ignore cache backend cache expired messages `^INFO : .*cache expired.*$`, dropMe, // ignore "Implicitly create directory" messages (TestnStorage:) `^INFO : .*Implicitly create directory.*$`, dropMe, // ignore differences in backend features `^.*?"HashType1":.*?$`, dropMe, `^.*?"HashType2":.*?$`, dropMe, `^.*?"SlowHashDetected":.*?$`, dropMe, `^.*? for same-side diffs on .*?$`, dropMe, `^.*?Downloading hashes.*?$`, dropMe, `^.*?Can't compare hashes, so using check --download.*?$`, dropMe, // ignore timestamps in directory time updates `^(INFO : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe, // ignore equivalent log for backends lacking dir modtime support `^(INFO : .*?: Making directory).*$`, dropMe, // ignore sizes in directory time updates `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe, // ignore sizes in directory metadata updates `^(NOTICE: .*?: Skipped update directory metadata as --dry-run is set).*$`, dropMe,} // Some dry-run messages differ depending on the particular remote.var dryrunReplacements = []string{ `^(NOTICE: file5.txt: Skipped) (copy|update modification time) (as --dry-run is set [(]size \d+[)])$`, `$1 copy (or update modification time) $3`,} // Some groups of log lines may appear unordered because rclone applies// many operations in parallel to boost performance.var logHoppers = []string{ // Test case `dry-run` produced log mismatches due to non-deterministic // order of captured dry-run info messages. `NOTICE: \S+?: Skipped (?:copy|move|delete|copy \(or [^)]+\)|update modification time) as --dry-run is set \(size \d+\)`, // Test case `extended-filenames` detected difference in order of files // with extended unicode names between Windows and Unix or GDrive, // but the order is in fact not important for success. `(?:INFO |NOTICE): - Path[12] +File (?:was deleted|is new|is newer|is OLDER) +- .*`, // Test case `check-access-filters` detected listing miscompares due // to indeterminate order of rclone operations in presence of multiple // subdirectories. The order inconsistency initially showed up in the // listings and triggered reordering of log messages, but the actual // files will in fact match. `.* +.....Access test failed: Path[12] file not found in Path[12].*`, // Test case `resync` suffered from the order of queued copies. `(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`, // Test case `normalization` can have random order of fix-case files. `(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`, // order of files re-checked prior to a conflict rename `ERROR : .*: {hashtype} differ.*`, // Directory modification time setting can happen in any order `INFO : .*: (Set directory modification time|Made directory with metadata).*`,} // Some log lines can contain Windows path separator that must be// converted to "/" in every matching token to match golden logs.var logLinesWithSlash = []string{ `.*\(\d\d\) :.*(fix-names|touch-glob|touch-copy|copy-file|copy-as|copy-dir|delete-file) `, `INFO : - .*Path[12].* +.*Queue copy to.* Path[12].*`, `INFO : Synching Path1 .*? with Path2 `, `INFO : Validating listings for `,}var regexFixSlash = regexp.MustCompile("^(" + strings.Join(logLinesWithSlash, "|") + ")") // Command line flags for bisync testvar ( argTestCase = flag.String("case", "", "Bisync test case to run") argRemote2 = flag.String("remote2", "", "Path2 for bisync tests") argNoCompare = flag.Bool("no-compare", false, "Do not compare test results with golden") argNoCleanup = flag.Bool("no-cleanup", false, "Keep test files") argGolden = flag.Bool("golden", false, "Store results as golden") argDebug = flag.Bool("debug", false, "Print debug messages") argStopAt = flag.Int("stop-at", 0, "Stop after given test step") // Flag -refresh-times helps with Dropbox tests failing with message // "src and dst identical but can't set mod time without deleting and re-uploading" argRefreshTimes = flag.Bool("refresh-times", false, "Force refreshing the target modtime, useful for Dropbox (default: false)") ignoreLogs = flag.Bool("ignore-logs", false, "skip comparing log lines but still compare listings") argPCount = flag.Int("pcount", 2, "number of parallel subtests to run for TestBisyncConcurrent") // go test ./cmd/bisync -race -pcount 10) // bisyncTest keeps all test data in a single placetype bisyncTest struct { // per-test state t *testing.T step int stopped bool stepStr string testCase string sessionName string // test dirs testDir string dataDir string initDir string goldenDir string workDir string fs1 fs.Fs path1 string canonPath1 string fs2 fs.Fs path2 string canonPath2 string // test log logDir string logPath string logFile *os.File // global state dataRoot string randName string tempDir string parent1 fs.Fs parent2 fs.Fs // global flags argRemote1 string argRemote2 string noCompare bool noCleanup bool golden bool debug bool stopAt int TestFn bisync.TestFunc ignoreModtime bool // ignore modtimes when comparing final listings, for backends without support ignoreBlankHash bool // ignore blank hashes for backends where we allow them to be blank} var color = bisync.Color // TestMain drives the testsfunc TestMain(m *testing.M) { bisync.LogTZ = time.UTC ci := fs.GetConfig(context.TODO()) ciSave := *ci defer func() { *ci = ciSave }() // need to set context.TODO() here as we cannot pass a ctx to fs.LogLevelPrintf ci.LogLevel = fs.LogLevelInfo if *argDebug { ci.LogLevel = fs.LogLevelDebug } fstest.Initialise() fstest.TestMain(m)} // Path1 is remote, Path2 is localfunc TestBisyncRemoteLocal(t *testing.T) { if *fstest.RemoteName == *argRemote2 { t.Skip("path1 and path2 are the same remote") } _, remote, cleanup, err := fstest.RandomRemote() fs.Logf(nil, "remote: %v", remote) require.NoError(t, err) defer cleanup() ctx, _ := fs.AddConfig(context.TODO()) testBisync(ctx, t, remote, *argRemote2)} // Path1 is local, Path2 is remotefunc TestBisyncLocalRemote(t *testing.T) { if *fstest.RemoteName == *argRemote2 { t.Skip("path1 and path2 are the same remote") } _, remote, cleanup, err := fstest.RandomRemote() fs.Logf(nil, "remote: %v", remote) require.NoError(t, err) defer cleanup() ctx, _ := fs.AddConfig(context.TODO()) testBisync(ctx, t, *argRemote2, remote)} // Path1 and Path2 are both different directories on remote// (useful for testing server-side copy/move)func TestBisyncRemoteRemote(t *testing.T) { _, remote, cleanup, err := fstest.RandomRemote() fs.Logf(nil, "remote: %v", remote) require.NoError(t, err) defer cleanup() ctx, _ := fs.AddConfig(context.TODO()) testBisync(ctx, t, remote, remote)} // make sure rc can cope with running concurrent jobsfunc TestBisyncConcurrent(t *testing.T) { if !isLocal(*fstest.RemoteName) { t.Skip("TestBisyncConcurrent is skipped on non-local") } if *argTestCase != "" && *argTestCase != "basic" { t.Skip("TestBisyncConcurrent only tests 'basic'") } if *argPCount < 2 { t.Skip("TestBisyncConcurrent is pointless with -pcount < 2") } if *argGolden { t.Skip("skip TestBisyncConcurrent when goldenizing") } oldArgTestCase := argTestCase *argTestCase = "basic" *ignoreLogs = true // not useful to compare logs here because both runs will be logging at once t.Cleanup(func() { argTestCase = oldArgTestCase *ignoreLogs = false }) for i := 0; i < *argPCount; i++ { t.Run(fmt.Sprintf("test%v", i), testParallel) }} func testParallel(t *testing.T) { t.Parallel() TestBisyncRemoteRemote(t)} // TestBisync is a test engine for bisync test cases.func testBisync(ctx context.Context, t *testing.T, path1, path2 string) { ci := fs.GetConfig(ctx) ciSave := *ci defer func() { *ci = ciSave }() if *argRefreshTimes { ci.RefreshTimes = true } bisync.ColorsLock.Lock() bisync.Colors = true bisync.ColorsLock.Unlock() ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour) baseDir, err := os.Getwd() require.NoError(t, err, "get current directory") randName := time.Now().Format("150405") + random.String(8) // some bucket backends don't like dots, keep this short to avoid linux errors tempDir := filepath.Join(os.TempDir(), randName) workDir := filepath.Join(tempDir, "workdir") b := &bisyncTest{ // per-test state t: t, // global state tempDir: tempDir, randName: randName, workDir: workDir, dataRoot: filepath.Join(baseDir, "testdata"), logDir: filepath.Join(tempDir, "logs"), logPath: filepath.Join(workDir, logFileName), // global flags argRemote1: path1, argRemote2: path2, noCompare: *argNoCompare, noCleanup: *argNoCleanup, golden: *argGolden, debug: *argDebug, stopAt: *argStopAt, } b.mkdir(b.tempDir) b.mkdir(b.logDir) fnHandle := atexit.Register(func() { if atexit.Signalled() { b.cleanupAll() } }) defer func() { b.cleanupAll() atexit.Unregister(fnHandle) }() argCase := *argTestCase if argCase == "" { argCase = "all" if testing.Short() { // remote tests can be long, help with "go test -short" argCase = "basic" } } testList := strings.Split(argCase, ",") if strings.ToLower(argCase) == "all" { testList = nil for _, testCase := range b.listDir(b.dataRoot) { if strings.HasPrefix(testCase, "test_") { // if dir is empty, skip it (can happen due to gitignored files/dirs when checking out branch) if len(b.listDir(filepath.Join(b.dataRoot, testCase))) == 0 { continue } testList = append(testList, testCase) } } } require.False(t, b.stopAt > 0 && len(testList) > 1, "-stop-at is meaningful only for a single test") deadline, hasDeadline := t.Deadline() var maxRunDuration time.Duration for _, testCase := range testList { testCase = strings.ReplaceAll(testCase, "-", "_") testCase = strings.TrimPrefix(testCase, "test_") t.Run(testCase, func(childTest *testing.T) { startTime := time.Now() remaining := time.Until(deadline) if hasDeadline && (remaining < maxRunDuration || remaining < 10*time.Second) { // avoid starting tests we don't have time to finish childTest.Fatalf("test %v timed out - not enough time to start test (%v remaining, need %v for test)", testCase, remaining, maxRunDuration) } bCopy := *b bCopy.runTestCase(ctx, childTest, testCase) if time.Since(startTime) > maxRunDuration { maxRunDuration = time.Since(startTime) } }) }} func (b *bisyncTest) cleanupAll() { if b.noCleanup { return } ctx := context.Background() if b.parent1 != nil { _ = operations.Purge(ctx, b.parent1, "") } if b.parent2 != nil { _ = operations.Purge(ctx, b.parent2, "") } _ = os.RemoveAll(b.tempDir)} func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase string) { b.t = t b.testCase = testCase var err error b.fs1, b.parent1, b.path1, b.canonPath1 = b.makeTempRemote(ctx, b.argRemote1, "path1") b.fs2, b.parent2, b.path2, b.canonPath2 = b.makeTempRemote(ctx, b.argRemote2, "path2") if strings.Contains(b.replaceHex(b.path1), " ") || strings.Contains(b.replaceHex(b.path2), " ") { b.t.Skip("skipping as tests can't handle spaces config string") } b.sessionName = bilib.SessionName(b.fs1, b.fs2) b.testDir = b.ensureDir(b.dataRoot, "test_"+b.testCase, false) b.initDir = b.ensureDir(b.testDir, "initial", false) b.goldenDir = b.ensureDir(b.testDir, "golden", false) b.dataDir = b.ensureDir(b.testDir, "modfiles", true) // optional // normalize unicode so tets are runnable on macOS b.sessionName = norm.NFC.String(b.sessionName) b.goldenDir = norm.NFC.String(b.goldenDir) // For test stability, jam initial dates to a fixed past date. // Test cases that change files will touch specific files to fixed new dates. err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error { if err == nil && !info.IsDir() { return os.Chtimes(path, initDate, initDate) } return err }) require.NoError(b.t, err, "jamming initial dates") // copy to a new unique initdir and datadir so concurrent tests don't interfere with each other ctxNoDsStore, _ := ctxNoDsStore(ctx, b.t) makeUnique := func(label, oldPath string) (newPath string) { newPath = oldPath info, err := os.Stat(oldPath) if err == nil && info.IsDir() { // datadir is optional oldFs, err := cache.Get(ctx, oldPath) require.NoError(b.t, err) newPath = b.tempDir + "/" + label + "/" + "test_" + b.testCase + "-" + random.String(8) newFs, err := cache.Get(ctx, newPath) require.NoError(b.t, err) require.NoError(b.t, sync.CopyDir(ctxNoDsStore, newFs, oldFs, true), "setting up "+label) } return newPath } b.initDir = makeUnique("initdir", b.initDir) b.dataDir = makeUnique("datadir", b.dataDir) // Prepare initial content b.cleanupCase(ctx) ctx = accounting.WithStatsGroup(ctx, random.String(8)) fstest.CheckListingWithPrecision(b.t, b.fs1, []fstest.Item{}, []string{}, b.fs1.Precision()) // verify starting from empty fstest.CheckListingWithPrecision(b.t, b.fs2, []fstest.Item{}, []string{}, b.fs2.Precision()) initFs, err := cache.Get(ctx, b.initDir) require.NoError(b.t, err) // verify pre-test equality (garbage in, garbage out!) srcObjs, srcDirs, err := walk.GetAll(ctxNoDsStore, initFs, "", false, -1) assert.NoError(b.t, err) items := []fstest.Item{} for _, obj := range srcObjs { require.False(b.t, strings.Contains(obj.Remote(), ".partial")) rc, err := operations.Open(ctxNoDsStore, obj) assert.NoError(b.t, err) bytes := make([]byte, obj.Size()) _, err = rc.Read(bytes) assert.NoError(b.t, err) assert.NoError(b.t, rc.Close()) item := fstest.NewItem(norm.NFC.String(obj.Remote()), string(bytes), obj.ModTime(ctxNoDsStore)) items = append(items, item)Lines 1–500 of 2026. Load more
// TestBisync is a test engine for bisync test cases. // See https://rclone.org/bisync/#testing for documentation. // Test cases are organized in subdirs beneath ./testdata // Results are compared against golden listings and log file. package bisync_test import ( "bytes" "context" "errors" "flag" "fmt" "os" "path" "path/filepath" "regexp" "runtime" "slices" "sort" "strconv" "strings" "testing" "time" "unicode/utf8" "github.com/rclone/rclone/cmd/bisync" "github.com/rclone/rclone/cmd/bisync/bilib" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/terminal" "golang.org/x/text/unicode/norm" "github.com/pmezard/go-difflib/difflib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" _ "github.com/rclone/rclone/backend/all" // for integration tests ) const ( touchDateFormat = "2006-01-02" goldenCanonBase = "_testdir_" logFileName = "test.log" dropMe = "*** [DROP THIS LINE] ***" eol = "\n" slash = string(os.PathSeparator) fixSlash = (runtime.GOOS == "windows") ) var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ) /* Useful Command Shortcuts */ // go test ./cmd/bisync -remote local -race // go test ./cmd/bisync -remote local -golden // go test ./cmd/bisync -remote local -case extended_filenames // go run ./fstest/test_all -run '^TestBisync.*$' -timeout 3h -verbose -maxtries 5 // go run ./fstest/test_all -remotes local,TestCrypt:,TestDrive:,TestOneDrive:,TestOneDriveBusiness:,TestDropbox:,TestCryptDrive:,TestOpenDrive:,TestChunker:,:memory:,TestCryptNoEncryption:,TestCombine:DirA,TestFTPRclone:,TestWebdavRclone:,TestS3Rclone:,TestSFTPRclone:,TestSFTPRcloneSSH:,TestNextcloud:,TestChunkerNometaLocal:,TestChunkerChunk3bLocal:,TestChunkerLocal:,TestChunkerChunk3bNometaLocal:,TestStorj: -run '^TestBisync.*$' -timeout 3h -verbose -maxtries 5 // go test -timeout 3h -run '^TestBisync.*$' github.com/rclone/rclone/cmd/bisync -remote TestDrive:Bisync -v // go test -timeout 3h -run '^TestBisyncRemoteRemote/basic$' github.com/rclone/rclone/cmd/bisync -remote TestDropbox:Bisync -v // TestFTPProftpd:,TestFTPPureftpd:,TestFTPRclone:,TestFTPVsftpd:,TestHdfs:,TestS3Minio:,TestS3MinioEdge:,TestS3Rclone:,TestSeafile:,TestSeafileEncrypted:,TestSeafileV6:,TestSFTPOpenssh:,TestSFTPRclone:,TestSFTPRcloneSSH:,TestSia:,TestSwiftAIO:,TestWebdavNextcloud:,TestWebdavOwncloud:,TestWebdavRclone: // logReplacements make modern test logs comparable with golden dir. // It is a string slice of even length with this structure: // // {`matching regular expression`, "mangled result string", ...} var logReplacements = []string{ // skip syslog facility markers `^(<[1-9]>)(INFO |ERROR |NOTICE|DEBUG ):(.*)$`, "$2:$3", // skip log prefixes `^\d+/\d\d/\d\d \d\d:\d\d:\d\d(?:\.\d{6})? `, "", // ignore rclone info messages `^INFO : .*?: (Deleted|Copied |Moved |Updated ).*$`, dropMe, `^NOTICE: .*?: Replacing invalid UTF-8 characters in "[^"]*"$`, dropMe, // ignore rclone debug messages `^DEBUG : .*$`, dropMe, // ignore SFTP host key messages `^NOTICE: .*?No host key validation is being performed.*$`, dropMe, // ignore dropbox info messages `^NOTICE: too_many_(requests|write_operations)/\.*: Too many requests or write operations.*$`, dropMe, `^NOTICE: .*?: Forced to upload files to set modification times on this backend.$`, dropMe, `^INFO : .*? Committing uploads - please wait...$`, dropMe, `^INFO : .*?: src and dst identical but can't set mod time without deleting and re-uploading$`, dropMe, `^INFO : .*?: src and dst identical but can't set mod time without re-uploading$`, dropMe, // ignore crypt info messages `^INFO : .*?: Crypt detected! Using cryptcheck instead of check. \(Use --size-only or --ignore-checksum to disable\)$`, dropMe, // ignore drive info messages `^NOTICE:.*?Files of unknown size \(such as Google Docs\) do not sync reliably with --checksum or --size-only\. Consider using modtime instead \(the default\) or --drive-skip-gdocs.*?$`, dropMe, // ignore cache backend cache expired messages `^INFO : .*cache expired.*$`, dropMe, // ignore "Implicitly create directory" messages (TestnStorage:) `^INFO : .*Implicitly create directory.*$`, dropMe, // ignore differences in backend features `^.*?"HashType1":.*?$`, dropMe, `^.*?"HashType2":.*?$`, dropMe, `^.*?"SlowHashDetected":.*?$`, dropMe, `^.*? for same-side diffs on .*?$`, dropMe, `^.*?Downloading hashes.*?$`, dropMe, `^.*?Can't compare hashes, so using check --download.*?$`, dropMe, // ignore timestamps in directory time updates `^(INFO : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe, // ignore equivalent log for backends lacking dir modtime support `^(INFO : .*?: Making directory).*$`, dropMe, // ignore sizes in directory time updates `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe, // ignore sizes in directory metadata updates `^(NOTICE: .*?: Skipped update directory metadata as --dry-run is set).*$`, dropMe, } // Some dry-run messages differ depending on the particular remote. var dryrunReplacements = []string{ `^(NOTICE: file5.txt: Skipped) (copy|update modification time) (as --dry-run is set [(]size \d+[)])$`, `$1 copy (or update modification time) $3`, } // Some groups of log lines may appear unordered because rclone applies // many operations in parallel to boost performance. var logHoppers = []string{ // Test case `dry-run` produced log mismatches due to non-deterministic // order of captured dry-run info messages. `NOTICE: \S+?: Skipped (?:copy|move|delete|copy \(or [^)]+\)|update modification time) as --dry-run is set \(size \d+\)`, // Test case `extended-filenames` detected difference in order of files // with extended unicode names between Windows and Unix or GDrive, // but the order is in fact not important for success. `(?:INFO |NOTICE): - Path[12] +File (?:was deleted|is new|is newer|is OLDER) +- .*`, // Test case `check-access-filters` detected listing miscompares due // to indeterminate order of rclone operations in presence of multiple // subdirectories. The order inconsistency initially showed up in the // listings and triggered reordering of log messages, but the actual // files will in fact match. `.* +.....Access test failed: Path[12] file not found in Path[12].*`, // Test case `resync` suffered from the order of queued copies. `(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`, // Test case `normalization` can have random order of fix-case files. `(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`, // order of files re-checked prior to a conflict rename `ERROR : .*: {hashtype} differ.*`, // Directory modification time setting can happen in any order `INFO : .*: (Set directory modification time|Made directory with metadata).*`, } // Some log lines can contain Windows path separator that must be // converted to "/" in every matching token to match golden logs. var logLinesWithSlash = []string{ `.*\(\d\d\) :.*(fix-names|touch-glob|touch-copy|copy-file|copy-as|copy-dir|delete-file) `, `INFO : - .*Path[12].* +.*Queue copy to.* Path[12].*`, `INFO : Synching Path1 .*? with Path2 `, `INFO : Validating listings for `, } var regexFixSlash = regexp.MustCompile("^(" + strings.Join(logLinesWithSlash, "|") + ")") // Command line flags for bisync test var ( argTestCase = flag.String("case", "", "Bisync test case to run") argRemote2 = flag.String("remote2", "", "Path2 for bisync tests") argNoCompare = flag.Bool("no-compare", false, "Do not compare test results with golden") argNoCleanup = flag.Bool("no-cleanup", false, "Keep test files") argGolden = flag.Bool("golden", false, "Store results as golden") argDebug = flag.Bool("debug", false, "Print debug messages") argStopAt = flag.Int("stop-at", 0, "Stop after given test step") // Flag -refresh-times helps with Dropbox tests failing with message // "src and dst identical but can't set mod time without deleting and re-uploading" argRefreshTimes = flag.Bool("refresh-times", false, "Force refreshing the target modtime, useful for Dropbox (default: false)") ignoreLogs = flag.Bool("ignore-logs", false, "skip comparing log lines but still compare listings") argPCount = flag.Int("pcount", 2, "number of parallel subtests to run for TestBisyncConcurrent") // go test ./cmd/bisync -race -pcount 10 ) // bisyncTest keeps all test data in a single place type bisyncTest struct { // per-test state t *testing.T step int stopped bool stepStr string testCase string sessionName string // test dirs testDir string dataDir string initDir string goldenDir string workDir string fs1 fs.Fs path1 string canonPath1 string fs2 fs.Fs path2 string canonPath2 string // test log logDir string logPath string logFile *os.File // global state dataRoot string randName string tempDir string parent1 fs.Fs parent2 fs.Fs // global flags argRemote1 string argRemote2 string noCompare bool noCleanup bool golden bool debug bool stopAt int TestFn bisync.TestFunc ignoreModtime bool // ignore modtimes when comparing final listings, for backends without support ignoreBlankHash bool // ignore blank hashes for backends where we allow them to be blank } var color = bisync.Color // TestMain drives the tests func TestMain(m *testing.M) { bisync.LogTZ = time.UTC ci := fs.GetConfig(context.TODO()) ciSave := *ci defer func() { *ci = ciSave }() // need to set context.TODO() here as we cannot pass a ctx to fs.LogLevelPrintf ci.LogLevel = fs.LogLevelInfo if *argDebug { ci.LogLevel = fs.LogLevelDebug } fstest.Initialise() fstest.TestMain(m) } // Path1 is remote, Path2 is local func TestBisyncRemoteLocal(t *testing.T) { if *fstest.RemoteName == *argRemote2 { t.Skip("path1 and path2 are the same remote") } _, remote, cleanup, err := fstest.RandomRemote() fs.Logf(nil, "remote: %v", remote) require.NoError(t, err) defer cleanup() ctx, _ := fs.AddConfig(context.TODO()) testBisync(ctx, t, remote, *argRemote2) } // Path1 is local, Path2 is remote func TestBisyncLocalRemote(t *testing.T) { if *fstest.RemoteName == *argRemote2 { t.Skip("path1 and path2 are the same remote") } _, remote, cleanup, err := fstest.RandomRemote() fs.Logf(nil, "remote: %v", remote) require.NoError(t, err) defer cleanup() ctx, _ := fs.AddConfig(context.TODO()) testBisync(ctx, t, *argRemote2, remote) } // Path1 and Path2 are both different directories on remote // (useful for testing server-side copy/move) func TestBisyncRemoteRemote(t *testing.T) { _, remote, cleanup, err := fstest.RandomRemote() fs.Logf(nil, "remote: %v", remote) require.NoError(t, err) defer cleanup() ctx, _ := fs.AddConfig(context.TODO()) testBisync(ctx, t, remote, remote) } // make sure rc can cope with running concurrent jobs func TestBisyncConcurrent(t *testing.T) { if !isLocal(*fstest.RemoteName) { t.Skip("TestBisyncConcurrent is skipped on non-local") } if *argTestCase != "" && *argTestCase != "basic" { t.Skip("TestBisyncConcurrent only tests 'basic'") } if *argPCount < 2 { t.Skip("TestBisyncConcurrent is pointless with -pcount < 2") } if *argGolden { t.Skip("skip TestBisyncConcurrent when goldenizing") } oldArgTestCase := argTestCase *argTestCase = "basic" *ignoreLogs = true // not useful to compare logs here because both runs will be logging at once t.Cleanup(func() { argTestCase = oldArgTestCase *ignoreLogs = false }) for i := 0; i < *argPCount; i++ { t.Run(fmt.Sprintf("test%v", i), testParallel) } } func testParallel(t *testing.T) { t.Parallel() TestBisyncRemoteRemote(t) } // TestBisync is a test engine for bisync test cases. func testBisync(ctx context.Context, t *testing.T, path1, path2 string) { ci := fs.GetConfig(ctx) ciSave := *ci defer func() { *ci = ciSave }() if *argRefreshTimes { ci.RefreshTimes = true } bisync.ColorsLock.Lock() bisync.Colors = true bisync.ColorsLock.Unlock() ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour) baseDir, err := os.Getwd() require.NoError(t, err, "get current directory") randName := time.Now().Format("150405") + random.String(8) // some bucket backends don't like dots, keep this short to avoid linux errors tempDir := filepath.Join(os.TempDir(), randName) workDir := filepath.Join(tempDir, "workdir") b := &bisyncTest{ // per-test state t: t, // global state tempDir: tempDir, randName: randName, workDir: workDir, dataRoot: filepath.Join(baseDir, "testdata"), logDir: filepath.Join(tempDir, "logs"), logPath: filepath.Join(workDir, logFileName), // global flags argRemote1: path1, argRemote2: path2, noCompare: *argNoCompare, noCleanup: *argNoCleanup, golden: *argGolden, debug: *argDebug, stopAt: *argStopAt, } b.mkdir(b.tempDir) b.mkdir(b.logDir) fnHandle := atexit.Register(func() { if atexit.Signalled() { b.cleanupAll() } }) defer func() { b.cleanupAll() atexit.Unregister(fnHandle) }() argCase := *argTestCase if argCase == "" { argCase = "all" if testing.Short() { // remote tests can be long, help with "go test -short" argCase = "basic" } } testList := strings.Split(argCase, ",") if strings.ToLower(argCase) == "all" { testList = nil for _, testCase := range b.listDir(b.dataRoot) { if strings.HasPrefix(testCase, "test_") { // if dir is empty, skip it (can happen due to gitignored files/dirs when checking out branch) if len(b.listDir(filepath.Join(b.dataRoot, testCase))) == 0 { continue } testList = append(testList, testCase) } } } require.False(t, b.stopAt > 0 && len(testList) > 1, "-stop-at is meaningful only for a single test") deadline, hasDeadline := t.Deadline() var maxRunDuration time.Duration for _, testCase := range testList { testCase = strings.ReplaceAll(testCase, "-", "_") testCase = strings.TrimPrefix(testCase, "test_") t.Run(testCase, func(childTest *testing.T) { startTime := time.Now() remaining := time.Until(deadline) if hasDeadline && (remaining < maxRunDuration || remaining < 10*time.Second) { // avoid starting tests we don't have time to finish childTest.Fatalf("test %v timed out - not enough time to start test (%v remaining, need %v for test)", testCase, remaining, maxRunDuration) } bCopy := *b bCopy.runTestCase(ctx, childTest, testCase) if time.Since(startTime) > maxRunDuration { maxRunDuration = time.Since(startTime) } }) } } func (b *bisyncTest) cleanupAll() { if b.noCleanup { return } ctx := context.Background() if b.parent1 != nil { _ = operations.Purge(ctx, b.parent1, "") } if b.parent2 != nil { _ = operations.Purge(ctx, b.parent2, "") } _ = os.RemoveAll(b.tempDir) } func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase string) { b.t = t b.testCase = testCase var err error b.fs1, b.parent1, b.path1, b.canonPath1 = b.makeTempRemote(ctx, b.argRemote1, "path1") b.fs2, b.parent2, b.path2, b.canonPath2 = b.makeTempRemote(ctx, b.argRemote2, "path2") if strings.Contains(b.replaceHex(b.path1), " ") || strings.Contains(b.replaceHex(b.path2), " ") { b.t.Skip("skipping as tests can't handle spaces config string") } b.sessionName = bilib.SessionName(b.fs1, b.fs2) b.testDir = b.ensureDir(b.dataRoot, "test_"+b.testCase, false) b.initDir = b.ensureDir(b.testDir, "initial", false) b.goldenDir = b.ensureDir(b.testDir, "golden", false) b.dataDir = b.ensureDir(b.testDir, "modfiles", true) // optional // normalize unicode so tets are runnable on macOS b.sessionName = norm.NFC.String(b.sessionName) b.goldenDir = norm.NFC.String(b.goldenDir) // For test stability, jam initial dates to a fixed past date. // Test cases that change files will touch specific files to fixed new dates. err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error { if err == nil && !info.IsDir() { return os.Chtimes(path, initDate, initDate) } return err }) require.NoError(b.t, err, "jamming initial dates") // copy to a new unique initdir and datadir so concurrent tests don't interfere with each other ctxNoDsStore, _ := ctxNoDsStore(ctx, b.t) makeUnique := func(label, oldPath string) (newPath string) { newPath = oldPath info, err := os.Stat(oldPath) if err == nil && info.IsDir() { // datadir is optional oldFs, err := cache.Get(ctx, oldPath) require.NoError(b.t, err) newPath = b.tempDir + "/" + label + "/" + "test_" + b.testCase + "-" + random.String(8) newFs, err := cache.Get(ctx, newPath) require.NoError(b.t, err) require.NoError(b.t, sync.CopyDir(ctxNoDsStore, newFs, oldFs, true), "setting up "+label) } return newPath } b.initDir = makeUnique("initdir", b.initDir) b.dataDir = makeUnique("datadir", b.dataDir) // Prepare initial content b.cleanupCase(ctx) ctx = accounting.WithStatsGroup(ctx, random.String(8)) fstest.CheckListingWithPrecision(b.t, b.fs1, []fstest.Item{}, []string{}, b.fs1.Precision()) // verify starting from empty fstest.CheckListingWithPrecision(b.t, b.fs2, []fstest.Item{}, []string{}, b.fs2.Precision()) initFs, err := cache.Get(ctx, b.initDir) require.NoError(b.t, err) // verify pre-test equality (garbage in, garbage out!) srcObjs, srcDirs, err := walk.GetAll(ctxNoDsStore, initFs, "", false, -1) assert.NoError(b.t, err) items := []fstest.Item{} for _, obj := range srcObjs { require.False(b.t, strings.Contains(obj.Remote(), ".partial")) rc, err := operations.Open(ctxNoDsStore, obj) assert.NoError(b.t, err) bytes := make([]byte, obj.Size()) _, err = rc.Read(bytes) assert.NoError(b.t, err) assert.NoError(b.t, rc.Close()) item := fstest.NewItem(norm.NFC.String(obj.Remote()), string(bytes), obj.ModTime(ctxNoDsStore)) items = append(items, item)