// Package mega provides an interface to the Mega// object storage system.package mega /*Open questions* Does mega support a content hash - what exactly are the mega hashes?* Can mega support setting modification times? Improvements:* Uploads could be done in parallel* Downloads would be more efficient done in one go* Uploads would be more efficient with bigger chunks* Looks like mega can support server-side copy, but it isn't implemented in go-mega* Upload can set modtime... - set as int64_t - can set ctime and mtime?*/ import (	"context"	"fmt"	"io"	"path"	"strings"	"sync"	"time" 	"github.com/pkg/errors"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/config/obscure"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/fshttp"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/pacer"	"github.com/rclone/rclone/lib/readers"	mega "github.com/t3rm1n4l/go-mega") const (	minSleep      = 10 * time.Millisecond	maxSleep      = 2 * time.Second	eventWaitTime = 500 * time.Millisecond	decayConstant = 2 // bigger for slower decay, exponential) var (	megaCacheMu sync.Mutex                // mutex for the below	megaCache   = map[string]*mega.Mega{} // cache logged in Mega's by user) // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "mega",		Description: "Mega",		NewFs:       NewFs,		Options: []fs.Option{{			Name:     "user",			Help:     "User name",			Required: true,		}, {			Name:       "pass",			Help:       "Password.",			Required:   true,			IsPassword: true,		}, {			Name: "debug",			Help: `Output more debug from Mega. If this flag is set (along with -vv) it will print further debugginginformation from the mega backend.`,			Default:  false,			Advanced: true,		}, {			Name: "hard_delete",			Help: `Delete files permanently rather than putting them into the trash. Normally the mega backend will put all deletions into the trash ratherthan permanently deleting them.  If you specify this then rclone willpermanently delete objects instead.`,			Default:  false,			Advanced: true,		}, {			Name:     config.ConfigEncoding,			Help:     config.ConfigEncodingHelp,			Advanced: true,			// Encode invalid UTF-8 bytes as json doesn't handle them properly.			Default: (encoder.Base |				encoder.EncodeInvalidUtf8),		}},	})} // Options defines the configuration for this backendtype Options struct {	User       string               `config:"user"`	Pass       string               `config:"pass"`	Debug      bool                 `config:"debug"`	HardDelete bool                 `config:"hard_delete"`	Enc        encoder.MultiEncoder `config:"encoding"`} // Fs represents a remote megatype Fs struct {	name       string       // name of this remote	root       string       // the path we are working on	opt        Options      // parsed config options	features   *fs.Features // optional features	srv        *mega.Mega   // the connection to the server	pacer      *fs.Pacer    // pacer for API calls	rootNodeMu sync.Mutex   // mutex for _rootNode	_rootNode  *mega.Node   // root node - call findRoot to use this	mkdirMu    sync.Mutex   // used to serialize calls to mkdir / rmdir} // Object describes a mega object//// Will definitely have info but maybe not meta//// Normally rclone would just store an ID here but go-mega and mega.nz// expect you to build an entire tree of all the objects in memory.// In this case we just store a pointer to the object.type Object struct {	fs     *Fs        // what this object is part of	remote string     // The remote path	info   *mega.Node // pointer to the mega node} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	return fmt.Sprintf("mega root '%s'", f.root)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // parsePath parses a mega 'url'func parsePath(path string) (root string) {	root = strings.Trim(path, "/")	return} // shouldRetry returns a boolean as to whether this err deserves to be// retried.  It returns the err as a conveniencefunc shouldRetry(ctx context.Context, err error) (bool, error) {	if fserrors.ContextError(ctx, &err) {		return false, err	}	// Let the mega library handle the low level retries	return false, err	/*		switch errors.Cause(err) {		case mega.EAGAIN, mega.ERATELIMIT, mega.ETEMPUNAVAIL:			return true, err		}		return fserrors.ShouldRetry(err), err	*/} // readMetaDataForPath reads the metadata from the pathfunc (f *Fs) readMetaDataForPath(ctx context.Context, remote string) (info *mega.Node, err error) {	rootNode, err := f.findRoot(ctx, false)	if err != nil {		return nil, err	}	return f.findObject(rootNode, remote)} // NewFs constructs an Fs from the path, container:pathfunc NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {	// Parse config into Options struct	opt := new(Options)	err := configstruct.Set(m, opt)	if err != nil {		return nil, err	}	if opt.Pass != "" {		var err error		opt.Pass, err = obscure.Reveal(opt.Pass)		if err != nil {			return nil, errors.Wrap(err, "couldn't decrypt password")		}	}	ci := fs.GetConfig(ctx) 	// cache *mega.Mega on username so we can re-use and share	// them between remotes.  They are expensive to make as they	// contain all the objects and sharing the objects makes the	// move code easier as we don't have to worry about mixing	// them up between different remotes.	megaCacheMu.Lock()	defer megaCacheMu.Unlock()	srv := megaCache[opt.User]	if srv == nil {		srv = mega.New().SetClient(fshttp.NewClient(ctx))		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries		srv.SetLogger(func(format string, v ...interface{}) {			fs.Infof("*go-mega*", format, v...)		})		if opt.Debug {			srv.SetDebugger(func(format string, v ...interface{}) {				fs.Debugf("*go-mega*", format, v...)			})		} 		err := srv.Login(opt.User, opt.Pass)		if err != nil {			return nil, errors.Wrap(err, "couldn't login")		}		megaCache[opt.User] = srv	} 	root = parsePath(root)	f := &Fs{		name:  name,		root:  root,		opt:   *opt,		srv:   srv,		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),	}	f.features = (&fs.Features{		DuplicateFiles:          true,		CanHaveEmptyDirectories: true,	}).Fill(ctx, f) 	// Find the root node and check if it is a file or not	_, err = f.findRoot(ctx, false)	switch err {	case nil:		// root node found and is a directory	case fs.ErrorDirNotFound:		// root node not found, so can't be a file	case fs.ErrorIsFile:		// root node is a file so point to parent directory		root = path.Dir(root)		if root == "." {			root = ""		}		f.root = root		return f, err	}	return f, nil} // splitNodePath splits nodePath into / separated parts, returning nil if it// should refer to the root.// It also encodes the parts into backend specific encodingfunc (f *Fs) splitNodePath(nodePath string) (parts []string) {	nodePath = path.Clean(nodePath)	if nodePath == "." || nodePath == "/" {		return nil	}	nodePath = f.opt.Enc.FromStandardPath(nodePath)	return strings.Split(nodePath, "/")} // findNode looks up the node for the path of the name given from the root given//// It returns mega.ENOENT if it wasn't foundfunc (f *Fs) findNode(rootNode *mega.Node, nodePath string) (*mega.Node, error) {	parts := f.splitNodePath(nodePath)	if parts == nil {		return rootNode, nil	}	nodes, err := f.srv.FS.PathLookup(rootNode, parts)	if err != nil {		return nil, err	}	return nodes[len(nodes)-1], nil} // findDir finds the directory rooted from the node passed infunc (f *Fs) findDir(rootNode *mega.Node, dir string) (node *mega.Node, err error) {	node, err = f.findNode(rootNode, dir)	if err == mega.ENOENT {		return nil, fs.ErrorDirNotFound	} else if err == nil && node.GetType() == mega.FILE {		return nil, fs.ErrorIsFile	}	return node, err} // findObject looks up the node for the object of the name givenfunc (f *Fs) findObject(rootNode *mega.Node, file string) (node *mega.Node, err error) {	node, err = f.findNode(rootNode, file)	if err == mega.ENOENT {		return nil, fs.ErrorObjectNotFound	} else if err == nil && node.GetType() != mega.FILE {		return nil, fs.ErrorNotAFile	}	return node, err} // lookupDir looks up the node for the directory of the name given//// if create is true it tries to create the root directory if not foundfunc (f *Fs) lookupDir(ctx context.Context, dir string) (*mega.Node, error) {	rootNode, err := f.findRoot(ctx, false)	if err != nil {		return nil, err	}	return f.findDir(rootNode, dir)} // lookupParentDir finds the parent node for the remote passed infunc (f *Fs) lookupParentDir(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {	parent, leaf := path.Split(remote)	dirNode, err = f.lookupDir(ctx, parent)	return dirNode, leaf, err} // mkdir makes the directory and any parent directories for the// directory of the name givenfunc (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *mega.Node, err error) {	f.mkdirMu.Lock()	defer f.mkdirMu.Unlock() 	parts := f.splitNodePath(dir)	if parts == nil {		return rootNode, nil	}	var i int	// look up until we find a directory which exists	for i = 0; i <= len(parts); i++ {		var nodes []*mega.Node		nodes, err = f.srv.FS.PathLookup(rootNode, parts[:len(parts)-i])		if err == nil {			if len(nodes) == 0 {				node = rootNode			} else {				node = nodes[len(nodes)-1]			}			break		}		if err != mega.ENOENT {			return nil, errors.Wrap(err, "mkdir lookup failed")		}	}	if err != nil {		return nil, errors.Wrap(err, "internal error: mkdir called with non existent root node")	}	// i is number of directories to create (may be 0)	// node is directory to create them from	for _, name := range parts[len(parts)-i:] {		// create directory called name in node		err = f.pacer.Call(func() (bool, error) {			node, err = f.srv.CreateDir(name, node)			return shouldRetry(ctx, err)		})		if err != nil {			return nil, errors.Wrap(err, "mkdir create node failed")		}	}	return node, nil} // mkdirParent creates the parent directory of remotefunc (f *Fs) mkdirParent(ctx context.Context, remote string) (dirNode *mega.Node, leaf string, err error) {	rootNode, err := f.findRoot(ctx, true)	if err != nil {		return nil, "", err	}	parent, leaf := path.Split(remote)	dirNode, err = f.mkdir(ctx, rootNode, parent)	return dirNode, leaf, err} // findRoot looks up the root directory node and returns it.//// if create is true it tries to create the root directory if not foundfunc (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {	f.rootNodeMu.Lock()	defer f.rootNodeMu.Unlock() 	// Check if we haven't found it already	if f._rootNode != nil {		return f._rootNode, nil	} 	// Check for pre-existing root	absRoot := f.srv.FS.GetRoot()	node, err := f.findDir(absRoot, f.root)	//log.Printf("findRoot findDir %p %v", node, err)	if err == nil {		f._rootNode = node		return node, nil	}	if !create || err != fs.ErrorDirNotFound {		return nil, err	} 	//..not found so create the root directory	f._rootNode, err = f.mkdir(ctx, absRoot, f.root)	return f._rootNode, err} // clearRoot unsets the root directoryfunc (f *Fs) clearRoot() {	f.rootNodeMu.Lock()	f._rootNode = nil	f.rootNodeMu.Unlock()	//log.Printf("cleared root directory")} // CleanUp deletes all files currently in trashfunc (f *Fs) CleanUp(ctx context.Context) (err error) {	trash := f.srv.FS.GetTrash()	items := []*mega.Node{}	_, err = f.list(ctx, trash, func(item *mega.Node) bool {		items = append(items, item)		return false	})	if err != nil {		return errors.Wrap(err, "CleanUp failed to list items in trash")	}	fs.Infof(f, "Deleting %d items from the trash", len(items))	errors := 0	// similar to f.deleteNode(trash) but with HardDelete as true	for _, item := range items {		fs.Debugf(f, "Deleting trash %q", f.opt.Enc.ToStandardName(item.GetName()))		deleteErr := f.pacer.Call(func() (bool, error) {			err := f.srv.Delete(item, true)			return shouldRetry(ctx, err)		})		if deleteErr != nil {			err = deleteErr			errors++		}	}	fs.Infof(f, "Deleted %d items from the trash with %d errors", len(items), errors)	return err} // Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *mega.Node) (fs.Object, error) {	o := &Object{		fs:     f,		remote: remote,	}	var err error	if info != nil {		// Set info		err = o.setMetaData(info)	} else {		err = o.readMetaData(ctx) // reads info and meta, returning an error	}	if err != nil {		return nil, err	}	return o, nil} // NewObject finds the Object at remote.  If it can't be found// it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {	return f.newObjectWithInfo(ctx, remote, nil)} // list the objects into the function supplied//// If directories is set it only sends directories// User function to process a File item from listAll//// Should return true to finish processingtype listFn func(*mega.Node) bool // Lists the directory required calling the user function on each item found//// If the user fn ever returns true then it early exits with found = truefunc (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, err error) {	nodes, err := f.srv.FS.GetChildren(dir)	if err != nil {		return false, errors.Wrapf(err, "list failed")	}	for _, item := range nodes {		if fn(item) {			found = true			break		}	}	return

Lines 1–500 of 1219. Load more