1159 lines · 31.7 KB · Go
RawDownload
// Package pcloud provides an interface to the Pcloud// object storage system.package pcloud // FIXME implement ListR? /listfolder can do recursive lists // FIXME cleanup returns login required? // FIXME mime type? Fix overview if implement. import (	"context"	"fmt"	"io"	"log"	"net/http"	"net/url"	"path"	"strings"	"time" 	"github.com/pkg/errors"	"github.com/rclone/rclone/backend/pcloud/api"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/config/obscure"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/lib/dircache"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/oauthutil"	"github.com/rclone/rclone/lib/pacer"	"github.com/rclone/rclone/lib/rest"	"golang.org/x/oauth2") const (	rcloneClientID              = "DnONSzyJXpm"	rcloneEncryptedClientSecret = "ej1OIF39VOQQ0PXaSdK9ztkLw3tdLNscW2157TKNQdQKkICR4uU7aFg4eFM"	minSleep                    = 10 * time.Millisecond	maxSleep                    = 2 * time.Second	decayConstant               = 2 // bigger for slower decay, exponential	rootURL                     = "https://api.pcloud.com") // Globalsvar (	// Description of how to auth for this app	oauthConfig = &oauth2.Config{		Scopes: nil,		Endpoint: oauth2.Endpoint{			AuthURL:  "https://my.pcloud.com/oauth2/authorize",			TokenURL: "https://api.pcloud.com/oauth2_token",		},		ClientID:     rcloneClientID,		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),		RedirectURL:  oauthutil.RedirectLocalhostURL,	}) // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "pcloud",		Description: "Pcloud",		NewFs:       NewFs,		Config: func(name string, m configmap.Mapper) {			opt := oauthutil.Options{				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210			}			err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)			if err != nil {				log.Fatalf("Failed to configure token: %v", err)			}		},		Options: []fs.Option{{			Name: config.ConfigClientID,			Help: "Pcloud App Client Id\nLeave blank normally.",		}, {			Name: config.ConfigClientSecret,			Help: "Pcloud App Client Secret\nLeave blank normally.",		}, {			Name:     config.ConfigEncoding,			Help:     config.ConfigEncodingHelp,			Advanced: true,			// Encode invalid UTF-8 bytes as json doesn't handle them properly.			//			// TODO: Investigate Unicode simplification (\ gets converted to \ server-side)			Default: (encoder.Display |				encoder.EncodeBackSlash |				encoder.EncodeInvalidUtf8),		}, {			Name:     "root_folder_id",			Help:     "Fill in for rclone to use a non root folder as its starting point.",			Default:  "d0",			Advanced: true,		}},	})} // Options defines the configuration for this backendtype Options struct {	Enc          encoder.MultiEncoder `config:"encoding"`	RootFolderID string               `config:"root_folder_id"`} // Fs represents a remote pcloudtype Fs struct {	name         string             // name of this remote	root         string             // the path we are working on	opt          Options            // parsed options	features     *fs.Features       // optional features	srv          *rest.Client       // the connection to the server	dirCache     *dircache.DirCache // Map of directory path to directory id	pacer        *fs.Pacer          // pacer for API calls	tokenRenewer *oauthutil.Renew   // renew the token on expiry} // Object describes a pcloud object//// Will definitely have info but maybe not metatype Object struct {	fs          *Fs       // what this object is part of	remote      string    // The remote path	hasMetaData bool      // whether info below has been set	size        int64     // size of the object	modTime     time.Time // modification time of the object	id          string    // ID of the object	md5         string    // MD5 if known	sha1        string    // SHA1 if known	link        *api.GetFileLinkResult} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	return fmt.Sprintf("pcloud root '%s'", f.root)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // parsePath parses a pcloud 'url'func parsePath(path string) (root string) {	root = strings.Trim(path, "/")	return} // retryErrorCodes is a slice of error codes that we will retryvar retryErrorCodes = []int{	429, // Too Many Requests.	500, // Internal Server Error	502, // Bad Gateway	503, // Service Unavailable	504, // Gateway Timeout	509, // Bandwidth Limit Exceeded} // shouldRetry returns a boolean as to whether this resp and err// deserve to be retried.  It returns the err as a conveniencefunc shouldRetry(resp *http.Response, err error) (bool, error) {	doRetry := false 	// Check if it is an api.Error	if apiErr, ok := err.(*api.Error); ok {		// See https://docs.pcloud.com/errors/ for error treatment		// Errors are classified as 1xxx, 2xxx etc		switch apiErr.Result / 1000 {		case 4: // 4xxx: rate limiting			doRetry = true		case 5: // 5xxx: internal errors			doRetry = true		}	} 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {		doRetry = true		fs.Debugf(nil, "Should retry: %v", err)	}	return doRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err} // readMetaDataForPath reads the metadata from the pathfunc (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)	if err != nil {		if err == fs.ErrorDirNotFound {			return nil, fs.ErrorObjectNotFound		}		return nil, err	} 	found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {		if item.Name == leaf {			info = item			return true		}		return false	})	if err != nil {		return nil, err	}	if !found {		return nil, fs.ErrorObjectNotFound	}	return info, nil} // errorHandler parses a non 2xx error response into an errorfunc errorHandler(resp *http.Response) error {	// Decode error response	errResponse := new(api.Error)	err := rest.DecodeJSON(resp, &errResponse)	if err != nil {		fs.Debugf(nil, "Couldn't decode error response: %v", err)	}	if errResponse.ErrorString == "" {		errResponse.ErrorString = resp.Status	}	if errResponse.Result == 0 {		errResponse.Result = resp.StatusCode	}	return errResponse} // NewFs constructs an Fs from the path, container:pathfunc NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {	ctx := context.Background()	// Parse config into Options struct	opt := new(Options)	err := configstruct.Set(m, opt)	if err != nil {		return nil, err	}	root = parsePath(root)	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)	if err != nil {		return nil, errors.Wrap(err, "failed to configure Pcloud")	} 	f := &Fs{		name:  name,		root:  root,		opt:   *opt,		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),	}	f.features = (&fs.Features{		CaseInsensitive:         false,		CanHaveEmptyDirectories: true,	}).Fill(f)	f.srv.SetErrorHandler(errorHandler) 	// Renew the token in the background	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {		_, err := f.readMetaDataForPath(ctx, "")		return err	}) 	// Get rootFolderID	rootID := f.opt.RootFolderID	f.dirCache = dircache.New(root, rootID, f) 	// Find the current root	err = f.dirCache.FindRoot(ctx, false)	if err != nil {		// Assume it is a file		newRoot, remote := dircache.SplitPath(root)		tempF := *f		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)		tempF.root = newRoot		// Make new Fs which is the parent		err = tempF.dirCache.FindRoot(ctx, false)		if err != nil {			// No root so return old f			return f, nil		}		_, err := tempF.newObjectWithInfo(ctx, remote, nil)		if err != nil {			if err == fs.ErrorObjectNotFound {				// File doesn't exist so return old f				return f, nil			}			return nil, err		}		// XXX: update the old f here instead of returning tempF, since		// `features` were already filled with functions having *f as a receiver.		// See https://github.com/rclone/rclone/issues/2182		f.dirCache = tempF.dirCache		f.root = tempF.root		// return an error with an fs which points to the parent		return f, fs.ErrorIsFile	}	return f, nil} // Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {	o := &Object{		fs:     f,		remote: remote,	}	var err error	if info != nil {		// Set info		err = o.setMetaData(info)	} else {		err = o.readMetaData(ctx) // reads info and meta, returning an error	}	if err != nil {		return nil, err	}	return o, nil} // NewObject finds the Object at remote.  If it can't be found// it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {	return f.newObjectWithInfo(ctx, remote, nil)} // FindLeaf finds a directory of name leaf in the folder with ID pathIDfunc (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {	// Find the leaf in pathID	found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {		if item.Name == leaf {			pathIDOut = item.ID			return true		}		return false	})	return pathIDOut, found, err} // CreateDir makes a directory with pathID as parent and name leaffunc (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {	// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)	var resp *http.Response	var result api.ItemResult	opts := rest.Opts{		Method:     "POST",		Path:       "/createfolder",		Parameters: url.Values{},	}	opts.Parameters.Set("name", f.opt.Enc.FromStandardName(leaf))	opts.Parameters.Set("folderid", dirIDtoNumber(pathID))	err = f.pacer.Call(func() (bool, error) {		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)		err = result.Error.Update(err)		return shouldRetry(resp, err)	})	if err != nil {		//fmt.Printf("...Error %v\n", err)		return "", err	}	// fmt.Printf("...Id %q\n", *info.Id)	return result.Metadata.ID, nil} // Converts a dirID which is usually 'd' followed by digits into just// the digitsfunc dirIDtoNumber(dirID string) string {	if len(dirID) > 0 && dirID[0] == 'd' {		return dirID[1:]	}	fs.Debugf(nil, "Invalid directory id %q", dirID)	return dirID} // Converts a fileID which is usually 'f' followed by digits into just// the digitsfunc fileIDtoNumber(fileID string) string {	if len(fileID) > 0 && fileID[0] == 'f' {		return fileID[1:]	}	fs.Debugf(nil, "Invalid file id %q", fileID)	return fileID} // list the objects into the function supplied//// If directories is set it only sends directories// User function to process a File item from listAll//// Should return true to finish processingtype listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found//// If the user fn ever returns true then it early exits with found = truefunc (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {	opts := rest.Opts{		Method:     "GET",		Path:       "/listfolder",		Parameters: url.Values{},	}	opts.Parameters.Set("folderid", dirIDtoNumber(dirID))	// FIXME can do recursive 	var result api.ItemResult	var resp *http.Response	err = f.pacer.Call(func() (bool, error) {		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)		err = result.Error.Update(err)		return shouldRetry(resp, err)	})	if err != nil {		return found, errors.Wrap(err, "couldn't list files")	}	for i := range result.Metadata.Contents {		item := &result.Metadata.Contents[i]		if item.IsFolder {			if filesOnly {				continue			}		} else {			if directoriesOnly {				continue			}		}		item.Name = f.opt.Enc.ToStandardName(item.Name)		if fn(item) {			found = true			break		}	}	return} // List the objects and directories in dir into entries.  The// entries can be returned in any order but should be for a// complete directory.//// dir should be "" to list the root, and should not have// trailing slashes.//// This should return ErrDirNotFound if the directory isn't// found.func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {	directoryID, err := f.dirCache.FindDir(ctx, dir, false)	if err != nil {		return nil, err	}	var iErr error	_, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {		remote := path.Join(dir, info.Name)		if info.IsFolder {			// cache the directory ID for later lookups			f.dirCache.Put(remote, info.ID)			d := fs.NewDir(remote, info.ModTime()).SetID(info.ID)			// FIXME more info from dir?			entries = append(entries, d)		} else {			o, err := f.newObjectWithInfo(ctx, remote, info)			if err != nil {				iErr = err				return true			}			entries = append(entries, o)		}		return false	})	if err != nil {		return nil, err	}	if iErr != nil {		return nil, iErr	}	return entries, nil} // Creates from the parameters passed in a half finished Object which// must have setMetaData called on it//// Returns the object, leaf, directoryID and error//// Used to create new objectsfunc (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {	// Create the directory for the object if it doesn't exist	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)	if err != nil {		return

Lines 1–500 of 1159.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t