// Package pcloud provides an interface to the Pcloud// object storage system.package pcloud // FIXME implement ListR? /listfolder can do recursive lists // FIXME cleanup returns login required? // FIXME mime type? Fix overview if implement. import (	"context"	"fmt"	"io"	"log"	"net/http"	"net/url"	"path"	"strings"	"time" 	"github.com/pkg/errors"	"github.com/rclone/rclone/backend/pcloud/api"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/config/obscure"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/lib/dircache"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/oauthutil"	"github.com/rclone/rclone/lib/pacer"	"github.com/rclone/rclone/lib/rest"	"golang.org/x/oauth2") const (	rcloneClientID              = "DnONSzyJXpm"	rcloneEncryptedClientSecret = "ej1OIF39VOQQ0PXaSdK9ztkLw3tdLNscW2157TKNQdQKkICR4uU7aFg4eFM"	minSleep                    = 10 * time.Millisecond	maxSleep                    = 2 * time.Second	decayConstant               = 2 // bigger for slower decay, exponential	defaultHostname             = "api.pcloud.com") // Globalsvar (	// Description of how to auth for this app	oauthConfig = &oauth2.Config{		Scopes: nil,		Endpoint: oauth2.Endpoint{			AuthURL: "https://my.pcloud.com/oauth2/authorize",			// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL		},		ClientID:     rcloneClientID,		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),		RedirectURL:  oauthutil.RedirectLocalhostURL,	}) // Update the TokenURL with the actual hostnamefunc updateTokenURL(oauthConfig *oauth2.Config, hostname string) {	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"} // Register with Fsfunc init() {	updateTokenURL(oauthConfig, defaultHostname)	fs.Register(&fs.RegInfo{		Name:        "pcloud",		Description: "Pcloud",		NewFs:       NewFs,		Config: func(ctx context.Context, name string, m configmap.Mapper) {			optc := new(Options)			err := configstruct.Set(m, optc)			if err != nil {				fs.Errorf(nil, "Failed to read config: %v", err)			}			updateTokenURL(oauthConfig, optc.Hostname)			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {				if auth == nil || auth.Form == nil {					return errors.New("form not found in response")				}				hostname := auth.Form.Get("hostname")				if hostname == "" {					hostname = defaultHostname				}				// Save the hostname in the config				m.Set("hostname", hostname)				// Update the token URL				updateTokenURL(oauthConfig, hostname)				fs.Debugf(nil, "pcloud: got hostname %q", hostname)				return nil			}			opt := oauthutil.Options{				CheckAuth:    checkAuth,				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210			}			err = oauthutil.Config(ctx, "pcloud", name, m, oauthConfig, &opt)			if err != nil {				log.Fatalf("Failed to configure token: %v", err)			}		},		Options: append(oauthutil.SharedOptions, []fs.Option{{			Name:     config.ConfigEncoding,			Help:     config.ConfigEncodingHelp,			Advanced: true,			// Encode invalid UTF-8 bytes as json doesn't handle them properly.			//			// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)			Default: (encoder.Display |				encoder.EncodeBackSlash |				encoder.EncodeInvalidUtf8),		}, {			Name:     "root_folder_id",			Help:     "Fill in for rclone to use a non root folder as its starting point.",			Default:  "d0",			Advanced: true,		}, {			Name: "hostname",			Help: `Hostname to connect to. This is normally set when rclone initially does the oauth connection,however you will need to set it by hand if you are using remote configwith rclone authorize.`,			Default:  defaultHostname,			Advanced: true,			Examples: []fs.OptionExample{{				Value: defaultHostname,				Help:  "Original/US region",			}, {				Value: "eapi.pcloud.com",				Help:  "EU region",			}},		}}...),	})} // Options defines the configuration for this backendtype Options struct {	Enc          encoder.MultiEncoder `config:"encoding"`	RootFolderID string               `config:"root_folder_id"`	Hostname     string               `config:"hostname"`} // Fs represents a remote pcloudtype Fs struct {	name         string             // name of this remote	root         string             // the path we are working on	opt          Options            // parsed options	features     *fs.Features       // optional features	srv          *rest.Client       // the connection to the server	dirCache     *dircache.DirCache // Map of directory path to directory id	pacer        *fs.Pacer          // pacer for API calls	tokenRenewer *oauthutil.Renew   // renew the token on expiry} // Object describes a pcloud object//// Will definitely have info but maybe not metatype Object struct {	fs          *Fs       // what this object is part of	remote      string    // The remote path	hasMetaData bool      // whether info below has been set	size        int64     // size of the object	modTime     time.Time // modification time of the object	id          string    // ID of the object	md5         string    // MD5 if known	sha1        string    // SHA1 if known	link        *api.GetFileLinkResult} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	return fmt.Sprintf("pcloud root '%s'", f.root)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // parsePath parses a pcloud 'url'func parsePath(path string) (root string) {	root = strings.Trim(path, "/")	return} // retryErrorCodes is a slice of error codes that we will retryvar retryErrorCodes = []int{	429, // Too Many Requests.	500, // Internal Server Error	502, // Bad Gateway	503, // Service Unavailable	504, // Gateway Timeout	509, // Bandwidth Limit Exceeded} // shouldRetry returns a boolean as to whether this resp and err// deserve to be retried.  It returns the err as a conveniencefunc shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {	if fserrors.ContextError(ctx, &err) {		return false, err	}	doRetry := false 	// Check if it is an api.Error	if apiErr, ok := err.(*api.Error); ok {		// See https://docs.pcloud.com/errors/ for error treatment		// Errors are classified as 1xxx, 2xxx, etc.		switch apiErr.Result / 1000 {		case 4: // 4xxx: rate limiting			doRetry = true		case 5: // 5xxx: internal errors			doRetry = true		}	} 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {		doRetry = true		fs.Debugf(nil, "Should retry: %v", err)	}	return doRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err} // readMetaDataForPath reads the metadata from the pathfunc (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)	if err != nil {		if err == fs.ErrorDirNotFound {			return nil, fs.ErrorObjectNotFound		}		return nil, err	} 	found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {		if item.Name == leaf {			info = item			return true		}		return false	})	if err != nil {		return nil, err	}	if !found {		return nil, fs.ErrorObjectNotFound	}	return info, nil} // errorHandler parses a non 2xx error response into an errorfunc errorHandler(resp *http.Response) error {	// Decode error response	errResponse := new(api.Error)	err := rest.DecodeJSON(resp, &errResponse)	if err != nil {		fs.Debugf(nil, "Couldn't decode error response: %v", err)	}	if errResponse.ErrorString == "" {		errResponse.ErrorString = resp.Status	}	if errResponse.Result == 0 {		errResponse.Result = resp.StatusCode	}	return errResponse} // NewFs constructs an Fs from the path, container:pathfunc NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {	// Parse config into Options struct	opt := new(Options)	err := configstruct.Set(m, opt)	if err != nil {		return nil, err	}	root = parsePath(root)	oAuthClient, ts, err := oauthutil.NewClient(ctx, name, m, oauthConfig)	if err != nil {		return nil, errors.Wrap(err, "failed to configure Pcloud")	}	updateTokenURL(oauthConfig, opt.Hostname) 	f := &Fs{		name:  name,		root:  root,		opt:   *opt,		srv:   rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),	}	f.features = (&fs.Features{		CaseInsensitive:         false,		CanHaveEmptyDirectories: true,	}).Fill(ctx, f)	f.srv.SetErrorHandler(errorHandler) 	// Renew the token in the background	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {		_, err := f.readMetaDataForPath(ctx, "")		return err	}) 	// Get rootFolderID	rootID := f.opt.RootFolderID	f.dirCache = dircache.New(root, rootID, f) 	// Find the current root	err = f.dirCache.FindRoot(ctx, false)	if err != nil {		// Assume it is a file		newRoot, remote := dircache.SplitPath(root)		tempF := *f		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)		tempF.root = newRoot		// Make new Fs which is the parent		err = tempF.dirCache.FindRoot(ctx, false)		if err != nil {			// No root so return old f			return f, nil		}		_, err := tempF.newObjectWithInfo(ctx, remote, nil)		if err != nil {			if err == fs.ErrorObjectNotFound {				// File doesn't exist so return old f				return f, nil			}			return nil, err		}		// XXX: update the old f here instead of returning tempF, since		// `features` were already filled with functions having *f as a receiver.		// See https://github.com/rclone/rclone/issues/2182		f.dirCache = tempF.dirCache		f.root = tempF.root		// return an error with an fs which points to the parent		return f, fs.ErrorIsFile	}	return f, nil} // Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {	o := &Object{		fs:     f,		remote: remote,	}	var err error	if info != nil {		// Set info		err = o.setMetaData(info)	} else {		err = o.readMetaData(ctx) // reads info and meta, returning an error	}	if err != nil {		return nil, err	}	return o, nil} // NewObject finds the Object at remote.  If it can't be found// it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {	return f.newObjectWithInfo(ctx, remote, nil)} // FindLeaf finds a directory of name leaf in the folder with ID pathIDfunc (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {	// Find the leaf in pathID	found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {		if item.Name == leaf {			pathIDOut = item.ID			return true		}		return false	})	return pathIDOut, found, err} // CreateDir makes a directory with pathID as parent and name leaffunc (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {	// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)	var resp *http.Response	var result api.ItemResult	opts := rest.Opts{		Method:     "POST",		Path:       "/createfolder",		Parameters: url.Values{},	}	opts.Parameters.Set("name", f.opt.Enc.FromStandardName(leaf))	opts.Parameters.Set("folderid", dirIDtoNumber(pathID))	err = f.pacer.Call(func() (bool, error) {		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)		err = result.Error.Update(err)		return shouldRetry(ctx, resp, err)	})	if err != nil {		//fmt.Printf("...Error %v\n", err)		return "", err	}	// fmt.Printf("...Id %q\n", *info.Id)	return result.Metadata.ID, nil} // Converts a dirID which is usually 'd' followed by digits into just// the digitsfunc dirIDtoNumber(dirID string) string {	if len(dirID) > 0 && dirID[0] == 'd' {		return dirID[1:]	}	fs.Debugf(nil, "Invalid directory id %q", dirID)	return dirID} // Converts a fileID which is usually 'f' followed by digits into just// the digitsfunc fileIDtoNumber(fileID string) string {	if len(fileID) > 0 && fileID[0] == 'f' {		return fileID[1:]	}	fs.Debugf(nil, "Invalid file id %q", fileID)	return fileID} // list the objects into the function supplied//// If directories is set it only sends directories// User function to process a File item from listAll//// Should return true to finish processingtype listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found//// If the user fn ever returns true then it early exits with found = truefunc (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {	opts := rest.Opts{		Method:     "GET",		Path:       "/listfolder",		Parameters: url.Values{},	}	opts.Parameters.Set("folderid", dirIDtoNumber(dirID))	// FIXME can do recursive 	var result api.ItemResult	var resp *http.Response	err = f.pacer.Call(func() (bool, error) {		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)		err = result.Error.Update(err)		return shouldRetry(ctx, resp, err)	})	if err != nil {		return found, errors.Wrap(err, "couldn't list files")	}	for i := range result.Metadata.Contents {		item := &result.Metadata.Contents[i]		if item.IsFolder {			if filesOnly {				continue			}		} else {			if directoriesOnly {				continue			}		}		item.Name = f.opt.Enc.ToStandardName(item.Name)		if fn(item) {			found = true			break		}	}	return} // List the objects and directories in dir into entries.  The// entries can be returned in any order but should be for a// complete directory.//// dir should be "" to list the root, and should not have// trailing slashes.//// This should return ErrDirNotFound if the directory isn't// found.func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {

Lines 1–500 of 1209. Load more