// Test sync/copy/move package sync import (	"context"	"fmt"	"runtime"	"strings"	"testing"	"time" 	"github.com/pkg/errors"	_ "github.com/rclone/rclone/backend/all" // import all backends	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/accounting"	"github.com/rclone/rclone/fs/filter"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/fs/operations"	"github.com/rclone/rclone/fstest"	"github.com/stretchr/testify/assert"	"github.com/stretchr/testify/require"	"golang.org/x/text/unicode/norm") // Some times used in the testsvar (	t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")	t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")	t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")) // TestMain drives the testsfunc TestMain(m *testing.M) {	fstest.TestMain(m)} // Check dry run is workingfunc TestCopyWithDryRun(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	r.Mkdir(context.Background(), r.Fremote) 	fs.Config.DryRun = true	err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)	fs.Config.DryRun = false	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote)} // Now without dry runfunc TestCopy(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	r.Mkdir(context.Background(), r.Fremote) 	err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote, file1)} func TestCopyMissingDirectory(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	r.Mkdir(context.Background(), r.Fremote) 	nonExistingFs, err := fs.NewFs("/non-existing")	if err != nil {		t.Fatal(err)	} 	err = CopyDir(context.Background(), r.Fremote, nonExistingFs, false)	require.Error(t, err)} // Now with --no-traversefunc TestCopyNoTraverse(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise() 	fs.Config.NoTraverse = true	defer func() { fs.Config.NoTraverse = false }() 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 	err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote, file1)} // Now with --check-firstfunc TestCopyCheckFirst(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise() 	fs.Config.CheckFirst = true	defer func() { fs.Config.CheckFirst = false }() 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 	err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote, file1)} // Now with --no-traversefunc TestSyncNoTraverse(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise() 	fs.Config.NoTraverse = true	defer func() { fs.Config.NoTraverse = false }() 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 	accounting.GlobalStats().ResetCounters()	err := Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote, file1)} // Test copy with depthfunc TestCopyWithDepth(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	file2 := r.WriteFile("hello world2", "hello world2", t2) 	// Check the MaxDepth too	fs.Config.MaxDepth = 1	defer func() { fs.Config.MaxDepth = -1 }() 	err := CopyDir(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal, file1, file2)	fstest.CheckItems(t, r.Fremote, file2)} // Test copy with files fromfunc testCopyWithFilesFrom(t *testing.T, noTraverse bool) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("potato2", "hello world", t1)	file2 := r.WriteFile("hello world2", "hello world2", t2) 	// Set the --files-from equivalent	f, err := filter.NewFilter(nil)	require.NoError(t, err)	require.NoError(t, f.AddFile("potato2"))	require.NoError(t, f.AddFile("notfound")) 	// Monkey patch the active filter	oldFilter := filter.Active	oldNoTraverse := fs.Config.NoTraverse	filter.Active = f	fs.Config.NoTraverse = noTraverse	unpatch := func() {		filter.Active = oldFilter		fs.Config.NoTraverse = oldNoTraverse	}	defer unpatch() 	err = CopyDir(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err)	unpatch() 	fstest.CheckItems(t, r.Flocal, file1, file2)	fstest.CheckItems(t, r.Fremote, file1)}func TestCopyWithFilesFrom(t *testing.T)              { testCopyWithFilesFrom(t, false) }func TestCopyWithFilesFromAndNoTraverse(t *testing.T) { testCopyWithFilesFrom(t, true) } // Test copy empty directoriesfunc TestCopyEmptyDirectories(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	err := operations.Mkdir(context.Background(), r.Flocal, "sub dir2")	require.NoError(t, err)	r.Mkdir(context.Background(), r.Fremote) 	err = CopyDir(context.Background(), r.Fremote, r.Flocal, true)	require.NoError(t, err) 	fstest.CheckListingWithPrecision(		t,		r.Fremote,		[]fstest.Item{			file1,		},		[]string{			"sub dir",			"sub dir2",		},		fs.GetModifyWindow(r.Fremote),	)} // Test move empty directoriesfunc TestMoveEmptyDirectories(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	err := operations.Mkdir(context.Background(), r.Flocal, "sub dir2")	require.NoError(t, err)	r.Mkdir(context.Background(), r.Fremote) 	err = MoveDir(context.Background(), r.Fremote, r.Flocal, false, true)	require.NoError(t, err) 	fstest.CheckListingWithPrecision(		t,		r.Fremote,		[]fstest.Item{			file1,		},		[]string{			"sub dir",			"sub dir2",		},		fs.GetModifyWindow(r.Fremote),	)} // Test sync empty directoriesfunc TestSyncEmptyDirectories(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	err := operations.Mkdir(context.Background(), r.Flocal, "sub dir2")	require.NoError(t, err)	r.Mkdir(context.Background(), r.Fremote) 	err = Sync(context.Background(), r.Fremote, r.Flocal, true)	require.NoError(t, err) 	fstest.CheckListingWithPrecision(		t,		r.Fremote,		[]fstest.Item{			file1,		},		[]string{			"sub dir",			"sub dir2",		},		fs.GetModifyWindow(r.Fremote),	)} // Test a server side copy if possible, or the backup path if notfunc TestServerSideCopy(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)	fstest.CheckItems(t, r.Fremote, file1) 	FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote()	require.NoError(t, err)	defer finaliseCopy()	t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy) 	err = CopyDir(context.Background(), FremoteCopy, r.Fremote, false)	require.NoError(t, err) 	fstest.CheckItems(t, FremoteCopy, file1)} // Check that if the local file doesn't exist when we copy it up,// nothing happens to the remote filefunc TestCopyAfterDelete(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)	fstest.CheckItems(t, r.Flocal)	fstest.CheckItems(t, r.Fremote, file1) 	err := operations.Mkdir(context.Background(), r.Flocal, "")	require.NoError(t, err) 	err = CopyDir(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	fstest.CheckItems(t, r.Flocal)	fstest.CheckItems(t, r.Fremote, file1)} // Check the copy downloading a filefunc TestCopyRedownload(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1)	fstest.CheckItems(t, r.Fremote, file1) 	err := CopyDir(context.Background(), r.Flocal, r.Fremote, false)	require.NoError(t, err) 	// Test with combined precision of local and remote as we copied it there and back	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1}, nil, fs.GetModifyWindow(r.Flocal, r.Fremote))} // Create a file and sync it. Change the last modified date and resync.// If we're only doing sync by size and checksum, we expect nothing to// to be transferred on the second sync.func TestSyncBasedOnCheckSum(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	fs.Config.CheckSum = true	defer func() { fs.Config.CheckSum = false }() 	file1 := r.WriteFile("check sum", "-", t1)	fstest.CheckItems(t, r.Flocal, file1) 	accounting.GlobalStats().ResetCounters()	err := Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred exactly one file.	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())	fstest.CheckItems(t, r.Fremote, file1) 	// Change last modified date only	file2 := r.WriteFile("check sum", "-", t2)	fstest.CheckItems(t, r.Flocal, file2) 	accounting.GlobalStats().ResetCounters()	err = Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred no files	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())	fstest.CheckItems(t, r.Flocal, file2)	fstest.CheckItems(t, r.Fremote, file1)} // Create a file and sync it. Change the last modified date and the// file contents but not the size.  If we're only doing sync by size// only, we expect nothing to to be transferred on the second sync.func TestSyncSizeOnly(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	fs.Config.SizeOnly = true	defer func() { fs.Config.SizeOnly = false }() 	file1 := r.WriteFile("sizeonly", "potato", t1)	fstest.CheckItems(t, r.Flocal, file1) 	accounting.GlobalStats().ResetCounters()	err := Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred exactly one file.	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())	fstest.CheckItems(t, r.Fremote, file1) 	// Update mtime, md5sum but not length of file	file2 := r.WriteFile("sizeonly", "POTATO", t2)	fstest.CheckItems(t, r.Flocal, file2) 	accounting.GlobalStats().ResetCounters()	err = Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred no files	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())	fstest.CheckItems(t, r.Flocal, file2)	fstest.CheckItems(t, r.Fremote, file1)} // Create a file and sync it. Keep the last modified date but change// the size.  With --ignore-size we expect nothing to to be// transferred on the second sync.func TestSyncIgnoreSize(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	fs.Config.IgnoreSize = true	defer func() { fs.Config.IgnoreSize = false }() 	file1 := r.WriteFile("ignore-size", "contents", t1)	fstest.CheckItems(t, r.Flocal, file1) 	accounting.GlobalStats().ResetCounters()	err := Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred exactly one file.	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers())	fstest.CheckItems(t, r.Fremote, file1) 	// Update size but not date of file	file2 := r.WriteFile("ignore-size", "longer contents but same date", t1)	fstest.CheckItems(t, r.Flocal, file2) 	accounting.GlobalStats().ResetCounters()	err = Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred no files	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())	fstest.CheckItems(t, r.Flocal, file2)	fstest.CheckItems(t, r.Fremote, file1)} func TestSyncIgnoreTimes(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteBoth(context.Background(), "existing", "potato", t1)	fstest.CheckItems(t, r.Fremote, file1) 	accounting.GlobalStats().ResetCounters()	err := Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred exactly 0 files because the	// files were identical.	assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) 	fs.Config.IgnoreTimes = true	defer func() { fs.Config.IgnoreTimes = false }() 	accounting.GlobalStats().ResetCounters()	err = Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err) 	// We should have transferred exactly one file even though the	// files were identical.	assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) 	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote, file1)} func TestSyncIgnoreExisting(t *testing.T) {	r := fstest.NewRun(t)	defer r.Finalise()	file1 := r.WriteFile("existing", "potato", t1) 	fs.Config.IgnoreExisting = true	defer func() { fs.Config.IgnoreExisting = false }() 	accounting.GlobalStats().ResetCounters()	err := Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err)	fstest.CheckItems(t, r.Flocal, file1)	fstest.CheckItems(t, r.Fremote, file1) 	// Change everything	r.WriteFile("existing", "newpotatoes", t2)	accounting.GlobalStats().ResetCounters()	err = Sync(context.Background(), r.Fremote, r.Flocal, false)	require.NoError(t, err)	// Items should not change	fstest.CheckItems(t, r.Fremote, file1)} func TestSyncIgnoreErrors(t *testing.T) {	r := fstest.NewRun(t)	fs.Config.IgnoreErrors = true	defer func() {		fs.Config.IgnoreErrors = false		r.Finalise()	}()	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)	file2 := r.WriteObject(context.Background(), "b/potato", "SMALLER BUT SAME DATE", t2)	file3 := r.WriteBoth(context.Background(), "c/non empty space", "AhHa!", t2)	require.NoError(t, operations.Mkdir(context.Background(), r.Fremote, "d")) 	fstest.CheckListingWithPrecision(		t,		r.Flocal,		[]fstest.Item{			file1,			file3,		},		[]string{			"a",			"c",		},		fs.GetModifyWindow(r.Fremote),	)	fstest.CheckListingWithPrecision(		t,		r.Fremote,		[]fstest.Item{			file2,			file3,

Lines 1–500 of 1852. Load more