// Test sync/copy/move package sync import (	"bytes"	"context"	"errors"	"fmt"	"io"	"os"	"os/exec"	"runtime"	"slices"	"sort"	"strings"	"testing"	"time" 	mutex "sync" // renamed as "sync" already in use 	_ "github.com/rclone/rclone/backend/all" // import all backends	"github.com/rclone/rclone/cmd/bisync/bilib"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/accounting"	"github.com/rclone/rclone/fs/filter"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/fs/operations"	"github.com/rclone/rclone/fstest"	"github.com/rclone/rclone/lib/transform"	"github.com/stretchr/testify/assert"	"github.com/stretchr/testify/require"	"golang.org/x/text/unicode/norm") // Some times used in the testsvar (	t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")	t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")	t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")) // TestMain drives the testsfunc TestMain(m *testing.M) {	fstest.TestMain(m)} // Check dry run is workingfunc TestCopyWithDryRun(t *testing.T) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	r.Mkdir(ctx, r.Fremote) 	ci.DryRun = true	ctx = predictDstFromLogger(ctx)	err := CopyDir(ctx, r.Fremote, r.Flocal, false)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // error expected here because dry-run	require.NoError(t, err) 	r.CheckLocalItems(t, file1)	r.CheckRemoteItems(t)} // Now without dry runfunc TestCopy(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	_, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)	if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {		require.NoError(t, err)	}	r.Mkdir(ctx, r.Fremote) 	ctx = predictDstFromLogger(ctx)	err = CopyDir(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1)	r.CheckRemoteItems(t, file1) 	// Check that the modtimes of the directories are as expected	r.CheckDirectoryModTimes(t, "sub dir")} func testCopyMetadata(t *testing.T, createEmptySrcDirs bool) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	ci.Metadata = true	r := fstest.NewRun(t)	features := r.Fremote.Features() 	if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata &&		!features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata {		t.Skip("Skipping as metadata not supported")	} 	if createEmptySrcDirs && !features.CanHaveEmptyDirectories {		t.Skip("Skipping as can't have empty directories")	} 	const content = "hello metadata world!"	const dirPath = "metadata sub dir"	const emptyDirPath = "empty metadata sub dir"	const filePath = dirPath + "/hello metadata world" 	fileMetadata := fs.Metadata{		// System metadata supported by all backends		"mtime": t1.Format(time.RFC3339Nano),		// User metadata		"potato": "jersey",	} 	dirMetadata := fs.Metadata{		// System metadata supported by all backends		"mtime": t2.Format(time.RFC3339Nano),		// User metadata		"potato": "king edward",	} 	// Make the directory with metadata - may fall back to Mkdir	_, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata)	require.NoError(t, err) 	// Make the empty directory with metadata - may fall back to Mkdir	_, err = operations.MkdirMetadata(ctx, r.Flocal, emptyDirPath, dirMetadata)	require.NoError(t, err) 	// Upload the file with metadata	in := io.NopCloser(bytes.NewBufferString(content))	_, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata)	require.NoError(t, err)	file1 := fstest.NewItem(filePath, content, t1) 	// Reset the time of the directory	_, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2)	if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {		require.NoError(t, err)	} 	ctx = predictDstFromLogger(ctx)	err = CopyDir(ctx, r.Fremote, r.Flocal, createEmptySrcDirs)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1)	r.CheckRemoteItems(t, file1) 	// Check that the modtimes of the directories are as expected	r.CheckDirectoryModTimes(t, dirPath) 	// Check that the metadata on the directory and file is correct	if features.WriteMetadata && features.ReadMetadata {		fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata)	}	if features.WriteDirMetadata && features.ReadDirMetadata {		fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata)	}	if !createEmptySrcDirs {		// dir must not exist		_, err := fstest.NewDirectoryRetries(ctx, t, r.Fremote, emptyDirPath, 1)		assert.Error(t, err, "Not expecting to find empty directory")		assert.True(t, errors.Is(err, fs.ErrorDirNotFound), fmt.Sprintf("expecting wrapped %#v not: %#v", fs.ErrorDirNotFound, err))	} else {		// dir must exist		dir := fstest.NewDirectory(ctx, t, r.Fremote, emptyDirPath)		if features.ReadDirMetadata {			fstest.CheckEntryMetadata(ctx, t, r.Fremote, dir, dirMetadata)		}	}} func TestCopyMetadata(t *testing.T) {	testCopyMetadata(t, true)} func TestCopyMetadataNoEmptyDirs(t *testing.T) {	testCopyMetadata(t, false)} func TestCopyMissingDirectory(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)	r.Mkdir(ctx, r.Fremote) 	nonExistingFs, err := fs.NewFs(ctx, "/non-existing")	if err != nil {		t.Fatal(err)	} 	ctx = predictDstFromLogger(ctx)	err = CopyDir(ctx, r.Fremote, nonExistingFs, false)	require.Error(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t)} // Now with --no-traversefunc TestCopyNoTraverse(t *testing.T) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	r := fstest.NewRun(t) 	ci.NoTraverse = true 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 	ctx = predictDstFromLogger(ctx)	err := CopyDir(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1)	r.CheckRemoteItems(t, file1)} func TestCopyNoTraverseDeadlock(t *testing.T) {	r := fstest.NewRun(t)	if !r.Fremote.Features().IsLocal {		t.Skip("Only runs on local")	}	const nFiles = 200	t1 := fstest.Time("2001-02-03T04:05:06.499999999Z") 	// Create lots of source files.	items := make([]fstest.Item, nFiles)	for i := range items {		name := fmt.Sprintf("file%d.txt", i)		items[i] = r.WriteFile(name, fmt.Sprintf("content%d", i), t1)	}	r.CheckLocalItems(t, items...) 	// Set --no-traverse	ctx, ci := fs.AddConfig(context.Background())	ci.NoTraverse = true 	// Initial copy to establish destination.	require.NoError(t, CopyDir(ctx, r.Fremote, r.Flocal, false))	r.CheckRemoteItems(t, items...) 	// Second copy which shouldn't deadlock	require.NoError(t, CopyDir(ctx, r.Flocal, r.Fremote, false))	r.CheckRemoteItems(t, items...)} // Now with --check-firstfunc TestCopyCheckFirst(t *testing.T) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	r := fstest.NewRun(t) 	ci.CheckFirst = true 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 	ctx = predictDstFromLogger(ctx)	err := CopyDir(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1)	r.CheckRemoteItems(t, file1)} // Now with --no-traversefunc TestSyncNoTraverse(t *testing.T) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	r := fstest.NewRun(t) 	ci.NoTraverse = true 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1) 	accounting.GlobalStats().ResetCounters()	ctx = predictDstFromLogger(ctx)	err := Sync(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1)	r.CheckRemoteItems(t, file1)} // Test copy with depthfunc TestCopyWithDepth(t *testing.T) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	file2 := r.WriteFile("hello world2", "hello world2", t2) 	// Check the MaxDepth too	ci.MaxDepth = 1 	ctx = predictDstFromLogger(ctx)	err := CopyDir(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1, file2)	r.CheckRemoteItems(t, file2)} // Test copy with files fromfunc testCopyWithFilesFrom(t *testing.T, noTraverse bool) {	ctx := context.Background()	ctx, ci := fs.AddConfig(ctx)	r := fstest.NewRun(t)	file1 := r.WriteFile("potato2", "hello world", t1)	file2 := r.WriteFile("hello world2", "hello world2", t2) 	// Set the --files-from equivalent	f, err := filter.NewFilter(nil)	require.NoError(t, err)	require.NoError(t, f.AddFile("potato2"))	require.NoError(t, f.AddFile("notfound")) 	// Change the active filter	ctx = filter.ReplaceConfig(ctx, f) 	ci.NoTraverse = noTraverse 	ctx = predictDstFromLogger(ctx)	err = CopyDir(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckLocalItems(t, file1, file2)	r.CheckRemoteItems(t, file1)}func TestCopyWithFilesFrom(t *testing.T)              { testCopyWithFilesFrom(t, false) }func TestCopyWithFilesFromAndNoTraverse(t *testing.T) { testCopyWithFilesFrom(t, true) } // Test copy empty directoriesfunc TestCopyEmptyDirectories(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2/sub sub dir2", t2)	require.NoError(t, err)	_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir2", t2)	require.NoError(t, err)	r.Mkdir(ctx, r.Fremote) 	// Set the modtime on "sub dir" to something specific	// Without this it fails on the CI and in VirtualBox with variances of up to 10mS	_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)	require.NoError(t, err) 	ctx = predictDstFromLogger(ctx)	err = CopyDir(ctx, r.Fremote, r.Flocal, true)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckRemoteListing(		t,		[]fstest.Item{			file1,		},		[]string{			"sub dir",			"sub dir2",			"sub dir2/sub sub dir2",		},	) 	// Check that the modtimes of the directories are as expected	r.CheckDirectoryModTimes(t, "sub dir", "sub dir2", "sub dir2/sub sub dir2")} // Test copy empty directories when we are configured not to create themfunc TestCopyNoEmptyDirectories(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	err := operations.Mkdir(ctx, r.Flocal, "sub dir2")	require.NoError(t, err)	_, err = operations.MkdirModTime(ctx, r.Flocal, "sub dir2/sub sub dir2", t2)	require.NoError(t, err)	r.Mkdir(ctx, r.Fremote) 	err = CopyDir(ctx, r.Fremote, r.Flocal, false)	require.NoError(t, err) 	r.CheckRemoteListing(		t,		[]fstest.Item{			file1,		},		[]string{			"sub dir",		},	)} // Test move empty directoriesfunc TestMoveEmptyDirectories(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)	require.NoError(t, err)	subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir")	subDirT := subDir.ModTime(ctx)	r.Mkdir(ctx, r.Fremote) 	ctx = predictDstFromLogger(ctx)	err = MoveDir(ctx, r.Fremote, r.Flocal, false, true)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckRemoteListing(		t,		[]fstest.Item{			file1,		},		[]string{			"sub dir",			"sub dir2",		},	) 	// Check that the modtimes of the directories are as expected	r.CheckDirectoryModTimes(t, "sub dir2")	// Note that "sub dir" mod time is updated when file1 is deleted from it	// So check it more manually	got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir")	fstest.CheckDirModTime(ctx, t, r.Fremote, got, subDirT)} // Test that --no-update-dir-modtime is workingfunc TestSyncNoUpdateDirModtime(t *testing.T) {	r := fstest.NewRun(t)	if r.Fremote.Features().DirSetModTime == nil {		t.Skip("Skipping test as backend does not support DirSetModTime")	} 	ctx, ci := fs.AddConfig(context.Background())	ci.NoUpdateDirModTime = true	const name = "sub dir no update dir modtime" 	// Set the modtime on name to something specific	_, err := operations.MkdirModTime(ctx, r.Flocal, name, t1)	require.NoError(t, err) 	// Create the remote directory with the current time	require.NoError(t, r.Fremote.Mkdir(ctx, name)) 	// Read its modification time	wantT := fstest.NewDirectory(ctx, t, r.Fremote, name).ModTime(ctx) 	ctx = predictDstFromLogger(ctx)	err = Sync(ctx, r.Fremote, r.Flocal, true)	require.NoError(t, err)	testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) 	r.CheckRemoteListing(		t,		[]fstest.Item{},		[]string{			name,		},	) 	// Read the new directory modification time - it should not have changed	gotT := fstest.NewDirectory(ctx, t, r.Fremote, name).ModTime(ctx)	fstest.AssertTimeEqualWithPrecision(t, name, wantT, gotT, r.Fremote.Precision())} // Test move empty directories when we are not configured to create themfunc TestMoveNoEmptyDirectories(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)	err := operations.Mkdir(ctx, r.Flocal, "sub dir2")	require.NoError(t, err)	r.Mkdir(ctx, r.Fremote) 	err = MoveDir(ctx, r.Fremote, r.Flocal, false, false)	require.NoError(t, err) 	r.CheckRemoteListing(		t,		[]fstest.Item{			file1,		},		[]string{			"sub dir",		},	)} // Test sync empty directoriesfunc TestSyncEmptyDirectories(t *testing.T) {	ctx := context.Background()	r := fstest.NewRun(t)

Lines 1–500 of 3169. Load more