// Package sync is the implementation of sync/copy/movepackage sync import (	"context"	"errors"	"fmt"	"path"	"slices"	"sort"	"strings"	"sync"	"time" 	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/accounting"	"github.com/rclone/rclone/fs/filter"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/fs/march"	"github.com/rclone/rclone/fs/operations"	"github.com/rclone/rclone/lib/errcount"	"github.com/rclone/rclone/lib/transform"	"golang.org/x/sync/errgroup") // ErrorMaxDurationReached defines error when transfer duration is reached// Used for checking on exit and matching to correct exit code.var ErrorMaxDurationReached = errors.New("max transfer duration reached as set by --max-duration") // ErrorMaxDurationReachedFatal is returned from when the max// duration limit is reached.var ErrorMaxDurationReachedFatal = fserrors.FatalError(ErrorMaxDurationReached) type syncCopyMove struct {	// parameters	fdst               fs.Fs	fsrc               fs.Fs	deleteMode         fs.DeleteMode // how we are doing deletions	DoMove             bool	copyEmptySrcDirs   bool	deleteEmptySrcDirs bool	dir                string	// internal state	ci                     *fs.ConfigInfo         // global config	fi                     *filter.Filter         // filter config	ctx                    context.Context        // internal context for controlling go-routines	cancel                 func()                 // cancel the context	inCtx                  context.Context        // internal context for controlling march	inCancel               func()                 // cancel the march context	noTraverse             bool                   // if set don't traverse the dst	noCheckDest            bool                   // if set transfer all objects regardless without checking dst	noUnicodeNormalization bool                   // don't normalize unicode characters in filenames	deletersWg             sync.WaitGroup         // for delete before go routine	deleteFilesCh          chan fs.Object         // channel to receive deletes if delete before	trackRenames           bool                   // set if we should do server-side renames	trackRenamesStrategy   trackRenamesStrategy   // strategies used for tracking renames	dstFilesMu             sync.Mutex             // protect dstFiles	dstFiles               map[string]fs.Object   // dst files, always filled	srcFiles               map[string]fs.Object   // src files, only used if deleteBefore	srcFilesChan           chan fs.Object         // passes src objects	srcFilesResult         chan error             // error result of src listing	dstFilesResult         chan error             // error result of dst listing	dstEmptyDirsMu         sync.Mutex             // protect dstEmptyDirs	dstEmptyDirs           map[string]fs.DirEntry // potentially empty directories	srcEmptyDirsMu         sync.Mutex             // protect srcEmptyDirs	srcEmptyDirs           map[string]fs.DirEntry // potentially empty directories	srcMoveEmptyDirs       map[string]fs.DirEntry // potentially empty directories when moving files out of them	checkerWg              sync.WaitGroup         // wait for checkers	toBeChecked            *pipe                  // checkers channel	transfersWg            sync.WaitGroup         // wait for transfers	toBeUploaded           *pipe                  // copiers channel	errorMu                sync.Mutex             // Mutex covering the errors variables	err                    error                  // normal error from copy process	noRetryErr             error                  // error with NoRetry set	fatalErr               error                  // fatal error	commonHash             hash.Type              // common hash type between src and dst	modifyWindow           time.Duration          // modify window between fsrc, fdst	renameMapMu            sync.Mutex             // mutex to protect the below	renameMap              map[string][]fs.Object // dst files by hash - only used by trackRenames	renamerWg              sync.WaitGroup         // wait for renamers	toBeRenamed            *pipe                  // renamers channel	trackRenamesWg         sync.WaitGroup         // wg for background track renames	trackRenamesCh         chan fs.Object         // objects are pumped in here	renameCheck            []fs.Object            // accumulate files to check for rename here	compareCopyDest        []fs.Fs                // place to check for files to server side copy	backupDir              fs.Fs                  // place to store overwrites/deletes	checkFirst             bool                   // if set run all the checkers before starting transfers	maxDurationEndTime     time.Time              // end time if --max-duration is set	logger                 operations.LoggerFn    // LoggerFn used to report the results of a sync (or bisync) to an io.Writer	usingLogger            bool                   // whether we are using logger	setDirMetadata         bool                   // if set we set the directory metadata	setDirModTime          bool                   // if set we set the directory modtimes	setDirModTimeAfter     bool                   // if set we set the directory modtimes at the end of the sync	setDirModTimeMu        sync.Mutex             // protect setDirModTimes and modifiedDirs	setDirModTimes         []setDirModTime        // directories that need their modtime set	setDirModTimesMaxLevel int                    // max level of the directories to set	modifiedDirs           map[string]struct{}    // dirs with changed contents (if s.setDirModTimeAfter)	allowOverlap           bool                   // whether we allow src and dst to overlap (i.e. for convmv)} // For keeping track of delayed modtime setstype setDirModTime struct {	src     fs.Directory	dst     fs.Directory	dir     string	modTime time.Time	level   int // the level of the directory, 0 is root} type trackRenamesStrategy byte const (	trackRenamesStrategyHash trackRenamesStrategy = 1 << iota	trackRenamesStrategyModtime	trackRenamesStrategyLeaf) func (strategy trackRenamesStrategy) hash() bool {	return (strategy & trackRenamesStrategyHash) != 0} func (strategy trackRenamesStrategy) modTime() bool {	return (strategy & trackRenamesStrategyModtime) != 0} func (strategy trackRenamesStrategy) leaf() bool {	return (strategy & trackRenamesStrategyLeaf) != 0} func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) (*syncCopyMove, error) {	if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) && !allowOverlap {		return nil, fserrors.FatalError(fs.ErrorOverlapping)	}	ci := fs.GetConfig(ctx)	fi := filter.GetConfig(ctx)	s := &syncCopyMove{		ci:                     ci,		fi:                     fi,		fdst:                   fdst,		fsrc:                   fsrc,		deleteMode:             deleteMode,		DoMove:                 DoMove,		copyEmptySrcDirs:       copyEmptySrcDirs,		deleteEmptySrcDirs:     deleteEmptySrcDirs,		dir:                    "",		srcFilesChan:           make(chan fs.Object, ci.Checkers+ci.Transfers),		srcFilesResult:         make(chan error, 1),		dstFilesResult:         make(chan error, 1),		dstEmptyDirs:           make(map[string]fs.DirEntry),		srcEmptyDirs:           make(map[string]fs.DirEntry),		srcMoveEmptyDirs:       make(map[string]fs.DirEntry),		noTraverse:             ci.NoTraverse,		noCheckDest:            ci.NoCheckDest,		noUnicodeNormalization: ci.NoUnicodeNormalization,		deleteFilesCh:          make(chan fs.Object, ci.Checkers),		trackRenames:           ci.TrackRenames,		commonHash:             fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),		modifyWindow:           fs.GetModifyWindow(ctx, fsrc, fdst),		trackRenamesCh:         make(chan fs.Object, ci.Checkers),		checkFirst:             ci.CheckFirst,		setDirMetadata:         ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,		setDirModTime:          (!ci.NoUpdateDirModTime && fsrc.Features().CanHaveEmptyDirectories) && (fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil),		setDirModTimeAfter:     !ci.NoUpdateDirModTime && (!copyEmptySrcDirs || fsrc.Features().CanHaveEmptyDirectories && fdst.Features().DirModTimeUpdatesOnWrite),		modifiedDirs:           make(map[string]struct{}),		allowOverlap:           allowOverlap,	} 	s.logger, s.usingLogger = operations.GetLogger(ctx) 	if deleteMode == fs.DeleteModeOff {		loggerOpt := operations.GetLoggerOpt(ctx)		loggerOpt.DeleteModeOff = true		loggerOpt.LoggerFn = s.logger		ctx = operations.WithLoggerOpt(ctx, loggerOpt)	} 	backlog := ci.MaxBacklog	if s.checkFirst {		fs.Infof(s.fdst, "Running all checks before starting transfers")		backlog = -1	}	var err error	s.toBeChecked, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetCheckQueue, backlog)	if err != nil {		return nil, err	}	s.toBeUploaded, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetTransferQueue, backlog)	if err != nil {		return nil, err	}	s.toBeRenamed, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetRenameQueue, backlog)	if err != nil {		return nil, err	}	if ci.MaxDuration > 0 {		s.maxDurationEndTime = time.Now().Add(time.Duration(ci.MaxDuration))		fs.Infof(s.fdst, "Transfer session %v deadline: %s", ci.CutoffMode, s.maxDurationEndTime.Format("2006/01/02 15:04:05"))	}	// If a max session duration has been defined add a deadline	// to the main context if cutoff mode is hard. This will cut	// the transfers off.	if !s.maxDurationEndTime.IsZero() && ci.CutoffMode == fs.CutoffModeHard {		s.ctx, s.cancel = context.WithDeadline(ctx, s.maxDurationEndTime)	} else {		s.ctx, s.cancel = context.WithCancel(ctx)	}	// Input context - cancel this for graceful stop.	//	// If a max session duration has been defined add a deadline	// to the input context if cutoff mode is graceful or soft.	// This won't stop the transfers but will cut the	// list/check/transfer pipelines.	if !s.maxDurationEndTime.IsZero() && ci.CutoffMode != fs.CutoffModeHard {		s.inCtx, s.inCancel = context.WithDeadline(s.ctx, s.maxDurationEndTime)	} else {		s.inCtx, s.inCancel = context.WithCancel(s.ctx)	}	if s.noTraverse && s.deleteMode != fs.DeleteModeOff {		if !fi.HaveFilesFrom() {			fs.Errorf(nil, "Ignoring --no-traverse with sync")		}		s.noTraverse = false	}	s.trackRenamesStrategy, err = parseTrackRenamesStrategy(ci.TrackRenamesStrategy)	if err != nil {		return nil, err	}	if s.noCheckDest {		if s.deleteMode != fs.DeleteModeOff {			return nil, errors.New("can't use --no-check-dest with sync: use copy instead")		}		if ci.Immutable {			return nil, errors.New("can't use --no-check-dest with --immutable")		}		if s.backupDir != nil {			return nil, errors.New("can't use --no-check-dest with --backup-dir")		}	}	if s.trackRenames {		// Don't track renames for remotes without server-side move support.		if !operations.CanServerSideMove(fdst) {			fs.Errorf(fdst, "Ignoring --track-renames as the destination does not support server-side move or copy")			s.trackRenames = false		}		if s.trackRenamesStrategy.hash() && s.commonHash == hash.None {			fs.Errorf(fdst, "Ignoring --track-renames as the source and destination do not have a common hash")			s.trackRenames = false		} 		if s.trackRenamesStrategy.modTime() && s.modifyWindow == fs.ModTimeNotSupported {			fs.Errorf(fdst, "Ignoring --track-renames as either the source or destination do not support modtime")			s.trackRenames = false		} 		if s.deleteMode == fs.DeleteModeOff {			fs.Errorf(fdst, "Ignoring --track-renames as it doesn't work with copy or move, only sync")			s.trackRenames = false		}	}	if s.trackRenames {		// track renames needs delete after		if s.deleteMode != fs.DeleteModeOff {			s.deleteMode = fs.DeleteModeAfter		}		if s.noTraverse {			fs.Errorf(nil, "Ignoring --no-traverse with --track-renames")			s.noTraverse = false		}	}	// Make Fs for --backup-dir if required	if ci.BackupDir != "" || ci.Suffix != "" {		var err error		s.backupDir, err = operations.BackupDir(ctx, fdst, fsrc, "")		if err != nil {			return nil, err		}	}	if len(ci.CompareDest) > 0 {		var err error		s.compareCopyDest, err = operations.GetCompareDest(ctx)		if err != nil {			return nil, err		}	} else if len(ci.CopyDest) > 0 {		var err error		s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst)		if err != nil {			return nil, err		}	}	return s, nil} // Check to see if the context has been cancelledfunc (s *syncCopyMove) aborting() bool {	return s.ctx.Err() != nil} // This reads the map and pumps it into the channel passed in, closing// the channel at the endfunc (s *syncCopyMove) pumpMapToChan(files map[string]fs.Object, out chan<- fs.Object) {outer:	for _, o := range files {		if s.aborting() {			break outer		}		select {		case out <- o:		case <-s.ctx.Done():			break outer		}	}	close(out)	s.srcFilesResult <- nil} // This checks the types of errors returned while copying filesfunc (s *syncCopyMove) processError(err error) {	if err == nil {		return	}	if err == context.DeadlineExceeded {		err = fserrors.NoRetryError(err)	} else if err == accounting.ErrorMaxTransferLimitReachedGraceful {		if s.inCtx.Err() == nil {			fs.Logf(nil, "%v - stopping transfers", err)			// Cancel the march and stop the pipes			s.inCancel()		}	} else if err == context.Canceled && s.inCtx.Err() != nil {		// Ignore context Canceled if we have called s.inCancel()		return	}	s.errorMu.Lock()	defer s.errorMu.Unlock()	switch {	case fserrors.IsFatalError(err):		if !s.aborting() {			fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err)			s.cancel()		}		s.fatalErr = err	case fserrors.IsNoRetryError(err):		s.noRetryErr = err	default:		s.err = err	}} // Returns the current error (if any) in the order of precedence////	fatalErr//	normal error//	noRetryErrfunc (s *syncCopyMove) currentError() error {	s.errorMu.Lock()	defer s.errorMu.Unlock()	if s.fatalErr != nil {		return s.fatalErr	}	if s.err != nil {		return s.err	}	return s.noRetryErr} // pairChecker reads Objects~s on in send to out if they need transferring.//// FIXME potentially doing lots of hashes at oncefunc (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.WaitGroup) {	defer wg.Done()	for {		pair, ok := in.GetMax(s.inCtx, fraction)		if !ok {			return		}		src := pair.Src		var err error		tr := accounting.Stats(s.ctx).NewCheckingTransfer(src, "checking")		// Check to see if can store this		if src.Storable() {			needTransfer := operations.NeedTransfer(s.ctx, pair.Dst, pair.Src)			if needTransfer {				NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir)				if err != nil {					s.processError(err)					s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, err)				}				if NoNeedTransfer {					needTransfer = false				}			}			// Fix case for case insensitive filesystems			if s.ci.FixCase && !s.ci.Immutable && pair.Dst != nil && src.Remote() != pair.Dst.Remote() {				// NeedTransfer's equality check may have deleted pair.Dst as a precursor				// to re-uploading it (the way modtime updates are done on backends like				// Dropbox that can't set modtime in place). If so, there is nothing to				// rename - nil out pair.Dst so the upload below recreates the file at				// src.Remote() (the correctly-cased name).				if needTransfer {					if _, statErr := s.fdst.NewObject(s.ctx, pair.Dst.Remote()); errors.Is(statErr, fs.ErrorObjectNotFound) {						fs.Debugf(pair.Dst, "Skipping fix-case rename: destination removed for re-upload, will recreate at %s", src.Remote())						pair.Dst = nil					}				}				if pair.Dst != nil {					if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil {						fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err)						s.processError(err)					} else {						fs.Infof(pair.Dst, "Fixed case by renaming to: %s", src.Remote())						pair.Dst = newDst					}				}			}			if needTransfer {				// If files are treated as immutable, fail if destination exists and does not match				if s.ci.Immutable && pair.Dst != nil {					err := fs.CountError(s.ctx, fserrors.NoRetryError(fs.ErrorImmutableModified))					fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err)					s.processError(err)				} else {					if pair.Dst != nil {						s.markDirModifiedObject(pair.Dst)					} else {						s.markDirModifiedObject(src)					}					// If destination already exists, then we must move it into --backup-dir if required					if pair.Dst != nil && s.backupDir != nil {						err := operations.MoveBackupDir(s.ctx, s.backupDir, pair.Dst)						if err != nil {							s.processError(err)							s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, err)						} else {							// If successful zero out the dst as it is no longer there and copy the file							pair.Dst = nil							ok = out.Put(s.inCtx, pair)							if !ok {								return							}						}					} else {						ok = out.Put(s.inCtx, pair)						if !ok {							return						}					}				}			} else {				// If moving need to delete the files we don't need to copy				if s.DoMove {					// Delete src if no error on copy					if operations.SameObject(src, pair.Dst) {						fs.Logf(src, "Not removing source file as it is the same file as the destination")					} else if s.ci.IgnoreExisting {						fs.Debugf(src, "Not removing source file as destination file exists and --ignore-existing is set")					} else if s.checkFirst && s.ci.OrderBy != "" {						// If we want perfect ordering then use the transfers to delete the file						//						// We send src == dst, to say we want the src deleted						ok = out.Put(s.inCtx, fs.ObjectPair{Src: src, Dst: src})						if !ok {							return						}					} else {						deleteFileErr := operations.DeleteFile(s.ctx, src)						s.processError(deleteFileErr)						s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, deleteFileErr)					}				}			}		}		tr.Done(s.ctx, err)	}} // pairRenamer reads Objects~s on in and attempts to rename them,// otherwise it sends them out if they need transferring.func (s *syncCopyMove) pairRenamer(in *pipe, out *pipe, fraction int, wg *sync.WaitGroup) {	defer wg.Done()	for {		pair, ok := in.GetMax(s.inCtx, fraction)		if !ok {			return		}		src := pair.Src		if !s.tryRename(src) {			// pass on if not renamed			fs.Debugf(src, "Need to transfer - No matching file found at Destination")			ok = out.Put(s.inCtx, pair)			if !ok {				return			}		}	}} // pairCopyOrMove reads Objects on in and moves or copies them.func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs, fraction int, wg *sync.WaitGroup) {

Lines 1–500 of 1445. Load more