// Package pcloud provides an interface to the Pcloud// object storage system.package pcloud // FIXME implement ListR? /listfolder can do recursive lists // FIXME cleanup returns login required? // FIXME mime type? Fix overview if implement. import ( "context" "fmt" "io" "log" "net/http" "net/url" "path" "strings" "time" "github.com/pkg/errors" "github.com/rclone/rclone/backend/pcloud/api" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/oauthutil" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/rest" "golang.org/x/oauth2") const ( rcloneClientID = "DnONSzyJXpm" rcloneEncryptedClientSecret = "ej1OIF39VOQQ0PXaSdK9ztkLw3tdLNscW2157TKNQdQKkICR4uU7aFg4eFM" minSleep = 10 * time.Millisecond maxSleep = 2 * time.Second decayConstant = 2 // bigger for slower decay, exponential rootURL = "https://api.pcloud.com") // Globalsvar ( // Description of how to auth for this app oauthConfig = &oauth2.Config{ Scopes: nil, Endpoint: oauth2.Endpoint{ AuthURL: "https://my.pcloud.com/oauth2/authorize", TokenURL: "https://api.pcloud.com/oauth2_token", }, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, }) // Register with Fsfunc init() { fs.Register(&fs.RegInfo{ Name: "pcloud", Description: "Pcloud", NewFs: NewFs, Config: func(name string, m configmap.Mapper) { opt := oauthutil.Options{ StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210 } err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt) if err != nil { log.Fatalf("Failed to configure token: %v", err) } }, Options: []fs.Option{{ Name: config.ConfigClientID, Help: "Pcloud App Client Id\nLeave blank normally.", }, { Name: config.ConfigClientSecret, Help: "Pcloud App Client Secret\nLeave blank normally.", }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, // Encode invalid UTF-8 bytes as json doesn't handle them properly. // // TODO: Investigate Unicode simplification (\ gets converted to \ server-side) Default: (encoder.Display | encoder.EncodeBackSlash | encoder.EncodeInvalidUtf8), }, { Name: "root_folder_id", Help: "Fill in for rclone to use a non root folder as its starting point.", Default: "d0", Advanced: true, }}, })} // Options defines the configuration for this backendtype Options struct { Enc encoder.MultiEncoder `config:"encoding"` RootFolderID string `config:"root_folder_id"`} // Fs represents a remote pcloudtype Fs struct { name string // name of this remote root string // the path we are working on opt Options // parsed options features *fs.Features // optional features srv *rest.Client // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls tokenRenewer *oauthutil.Renew // renew the token on expiry} // Object describes a pcloud object//// Will definitely have info but maybe not metatype Object struct { fs *Fs // what this object is part of remote string // The remote path hasMetaData bool // whether info below has been set size int64 // size of the object modTime time.Time // modification time of the object id string // ID of the object md5 string // MD5 if known sha1 string // SHA1 if known link *api.GetFileLinkResult} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string { return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string { return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string { return fmt.Sprintf("pcloud root '%s'", f.root)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features { return f.features} // parsePath parses a pcloud 'url'func parsePath(path string) (root string) { root = strings.Trim(path, "/") return} // retryErrorCodes is a slice of error codes that we will retryvar retryErrorCodes = []int{ 429, // Too Many Requests. 500, // Internal Server Error 502, // Bad Gateway 503, // Service Unavailable 504, // Gateway Timeout 509, // Bandwidth Limit Exceeded} // shouldRetry returns a boolean as to whether this resp and err// deserve to be retried. It returns the err as a conveniencefunc shouldRetry(resp *http.Response, err error) (bool, error) { doRetry := false // Check if it is an api.Error if apiErr, ok := err.(*api.Error); ok { // See https://docs.pcloud.com/errors/ for error treatment // Errors are classified as 1xxx, 2xxx etc switch apiErr.Result / 1000 { case 4: // 4xxx: rate limiting doRetry = true case 5: // 5xxx: internal errors doRetry = true } } if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { doRetry = true fs.Debugf(nil, "Should retry: %v", err) } return doRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err} // readMetaDataForPath reads the metadata from the pathfunc (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) { // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) if err != nil { if err == fs.ErrorDirNotFound { return nil, fs.ErrorObjectNotFound } return nil, err } found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool { if item.Name == leaf { info = item return true } return false }) if err != nil { return nil, err } if !found { return nil, fs.ErrorObjectNotFound } return info, nil} // errorHandler parses a non 2xx error response into an errorfunc errorHandler(resp *http.Response) error { // Decode error response errResponse := new(api.Error) err := rest.DecodeJSON(resp, &errResponse) if err != nil { fs.Debugf(nil, "Couldn't decode error response: %v", err) } if errResponse.ErrorString == "" { errResponse.ErrorString = resp.Status } if errResponse.Result == 0 { errResponse.Result = resp.StatusCode } return errResponse} // NewFs constructs an Fs from the path, container:pathfunc NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { ctx := context.Background() // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) if err != nil { return nil, err } root = parsePath(root) oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig) if err != nil { return nil, errors.Wrap(err, "failed to configure Pcloud") } f := &Fs{ name: name, root: root, opt: *opt, srv: rest.NewClient(oAuthClient).SetRoot(rootURL), pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), } f.features = (&fs.Features{ CaseInsensitive: false, CanHaveEmptyDirectories: true, }).Fill(f) f.srv.SetErrorHandler(errorHandler) // Renew the token in the background f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { _, err := f.readMetaDataForPath(ctx, "") return err }) // Get rootFolderID rootID := f.opt.RootFolderID f.dirCache = dircache.New(root, rootID, f) // Find the current root err = f.dirCache.FindRoot(ctx, false) if err != nil { // Assume it is a file newRoot, remote := dircache.SplitPath(root) tempF := *f tempF.dirCache = dircache.New(newRoot, rootID, &tempF) tempF.root = newRoot // Make new Fs which is the parent err = tempF.dirCache.FindRoot(ctx, false) if err != nil { // No root so return old f return f, nil } _, err := tempF.newObjectWithInfo(ctx, remote, nil) if err != nil { if err == fs.ErrorObjectNotFound { // File doesn't exist so return old f return f, nil } return nil, err } // XXX: update the old f here instead of returning tempF, since // `features` were already filled with functions having *f as a receiver. // See https://github.com/rclone/rclone/issues/2182 f.dirCache = tempF.dirCache f.root = tempF.root // return an error with an fs which points to the parent return f, fs.ErrorIsFile } return f, nil} // Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { o := &Object{ fs: f, remote: remote, } var err error if info != nil { // Set info err = o.setMetaData(info) } else { err = o.readMetaData(ctx) // reads info and meta, returning an error } if err != nil { return nil, err } return o, nil} // NewObject finds the Object at remote. If it can't be found// it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return f.newObjectWithInfo(ctx, remote, nil)} // FindLeaf finds a directory of name leaf in the folder with ID pathIDfunc (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { // Find the leaf in pathID found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { if item.Name == leaf { pathIDOut = item.ID return true } return false }) return pathIDOut, found, err} // CreateDir makes a directory with pathID as parent and name leaffunc (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) var resp *http.Response var result api.ItemResult opts := rest.Opts{ Method: "POST", Path: "/createfolder", Parameters: url.Values{}, } opts.Parameters.Set("name", f.opt.Enc.FromStandardName(leaf)) opts.Parameters.Set("folderid", dirIDtoNumber(pathID)) err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) return "", err } // fmt.Printf("...Id %q\n", *info.Id) return result.Metadata.ID, nil} // Converts a dirID which is usually 'd' followed by digits into just// the digitsfunc dirIDtoNumber(dirID string) string { if len(dirID) > 0 && dirID[0] == 'd' { return dirID[1:] } fs.Debugf(nil, "Invalid directory id %q", dirID) return dirID} // Converts a fileID which is usually 'f' followed by digits into just// the digitsfunc fileIDtoNumber(fileID string) string { if len(fileID) > 0 && fileID[0] == 'f' { return fileID[1:] } fs.Debugf(nil, "Invalid file id %q", fileID) return fileID} // list the objects into the function supplied//// If directories is set it only sends directories// User function to process a File item from listAll//// Should return true to finish processingtype listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found//// If the user fn ever returns true then it early exits with found = truefunc (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { opts := rest.Opts{ Method: "GET", Path: "/listfolder", Parameters: url.Values{}, } opts.Parameters.Set("folderid", dirIDtoNumber(dirID)) // FIXME can do recursive var result api.ItemResult var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") } for i := range result.Metadata.Contents { item := &result.Metadata.Contents[i] if item.IsFolder { if filesOnly { continue } } else { if directoriesOnly { continue } } item.Name = f.opt.Enc.ToStandardName(item.Name) if fn(item) { found = true break } } return} // List the objects and directories in dir into entries. The// entries can be returned in any order but should be for a// complete directory.//// dir should be "" to list the root, and should not have// trailing slashes.//// This should return ErrDirNotFound if the directory isn't// found.func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { directoryID, err := f.dirCache.FindDir(ctx, dir, false) if err != nil { return nil, err } var iErr error _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { remote := path.Join(dir, info.Name) if info.IsFolder { // cache the directory ID for later lookups f.dirCache.Put(remote, info.ID) d := fs.NewDir(remote, info.ModTime()).SetID(info.ID) // FIXME more info from dir? entries = append(entries, d) } else { o, err := f.newObjectWithInfo(ctx, remote, info) if err != nil { iErr = err return true } entries = append(entries, o) } return false }) if err != nil { return nil, err } if iErr != nil { return nil, iErr } return entries, nil} // Creates from the parameters passed in a half finished Object which// must have setMetaData called on it//// Returns the object, leaf, directoryID and error//// Used to create new objectsfunc (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { // Create the directory for the object if it doesn't exist leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) if err != nil { returnLines 1–500 of 1159. Load more
// Package pcloud provides an interface to the Pcloud // object storage system. package pcloud // FIXME implement ListR? /listfolder can do recursive lists // FIXME cleanup returns login required? // FIXME mime type? Fix overview if implement. import ( "context" "fmt" "io" "log" "net/http" "net/url" "path" "strings" "time" "github.com/pkg/errors" "github.com/rclone/rclone/backend/pcloud/api" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/oauthutil" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/rest" "golang.org/x/oauth2" ) const ( rcloneClientID = "DnONSzyJXpm" rcloneEncryptedClientSecret = "ej1OIF39VOQQ0PXaSdK9ztkLw3tdLNscW2157TKNQdQKkICR4uU7aFg4eFM" minSleep = 10 * time.Millisecond maxSleep = 2 * time.Second decayConstant = 2 // bigger for slower decay, exponential rootURL = "https://api.pcloud.com" ) // Globals var ( // Description of how to auth for this app oauthConfig = &oauth2.Config{ Scopes: nil, Endpoint: oauth2.Endpoint{ AuthURL: "https://my.pcloud.com/oauth2/authorize", TokenURL: "https://api.pcloud.com/oauth2_token", }, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, } ) // Register with Fs func init() { fs.Register(&fs.RegInfo{ Name: "pcloud", Description: "Pcloud", NewFs: NewFs, Config: func(name string, m configmap.Mapper) { opt := oauthutil.Options{ StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210 } err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt) if err != nil { log.Fatalf("Failed to configure token: %v", err) } }, Options: []fs.Option{{ Name: config.ConfigClientID, Help: "Pcloud App Client Id\nLeave blank normally.", }, { Name: config.ConfigClientSecret, Help: "Pcloud App Client Secret\nLeave blank normally.", }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, Advanced: true, // Encode invalid UTF-8 bytes as json doesn't handle them properly. // // TODO: Investigate Unicode simplification (\ gets converted to \ server-side) Default: (encoder.Display | encoder.EncodeBackSlash | encoder.EncodeInvalidUtf8), }, { Name: "root_folder_id", Help: "Fill in for rclone to use a non root folder as its starting point.", Default: "d0", Advanced: true, }}, }) } // Options defines the configuration for this backend type Options struct { Enc encoder.MultiEncoder `config:"encoding"` RootFolderID string `config:"root_folder_id"` } // Fs represents a remote pcloud type Fs struct { name string // name of this remote root string // the path we are working on opt Options // parsed options features *fs.Features // optional features srv *rest.Client // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls tokenRenewer *oauthutil.Renew // renew the token on expiry } // Object describes a pcloud object // // Will definitely have info but maybe not meta type Object struct { fs *Fs // what this object is part of remote string // The remote path hasMetaData bool // whether info below has been set size int64 // size of the object modTime time.Time // modification time of the object id string // ID of the object md5 string // MD5 if known sha1 string // SHA1 if known link *api.GetFileLinkResult } // ------------------------------------------------------------ // Name of the remote (as passed into NewFs) func (f *Fs) Name() string { return f.name } // Root of the remote (as passed into NewFs) func (f *Fs) Root() string { return f.root } // String converts this Fs to a string func (f *Fs) String() string { return fmt.Sprintf("pcloud root '%s'", f.root) } // Features returns the optional features of this Fs func (f *Fs) Features() *fs.Features { return f.features } // parsePath parses a pcloud 'url' func parsePath(path string) (root string) { root = strings.Trim(path, "/") return } // retryErrorCodes is a slice of error codes that we will retry var retryErrorCodes = []int{ 429, // Too Many Requests. 500, // Internal Server Error 502, // Bad Gateway 503, // Service Unavailable 504, // Gateway Timeout 509, // Bandwidth Limit Exceeded } // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience func shouldRetry(resp *http.Response, err error) (bool, error) { doRetry := false // Check if it is an api.Error if apiErr, ok := err.(*api.Error); ok { // See https://docs.pcloud.com/errors/ for error treatment // Errors are classified as 1xxx, 2xxx etc switch apiErr.Result / 1000 { case 4: // 4xxx: rate limiting doRetry = true case 5: // 5xxx: internal errors doRetry = true } } if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { doRetry = true fs.Debugf(nil, "Should retry: %v", err) } return doRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } // readMetaDataForPath reads the metadata from the path func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) { // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) if err != nil { if err == fs.ErrorDirNotFound { return nil, fs.ErrorObjectNotFound } return nil, err } found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool { if item.Name == leaf { info = item return true } return false }) if err != nil { return nil, err } if !found { return nil, fs.ErrorObjectNotFound } return info, nil } // errorHandler parses a non 2xx error response into an error func errorHandler(resp *http.Response) error { // Decode error response errResponse := new(api.Error) err := rest.DecodeJSON(resp, &errResponse) if err != nil { fs.Debugf(nil, "Couldn't decode error response: %v", err) } if errResponse.ErrorString == "" { errResponse.ErrorString = resp.Status } if errResponse.Result == 0 { errResponse.Result = resp.StatusCode } return errResponse } // NewFs constructs an Fs from the path, container:path func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { ctx := context.Background() // Parse config into Options struct opt := new(Options) err := configstruct.Set(m, opt) if err != nil { return nil, err } root = parsePath(root) oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig) if err != nil { return nil, errors.Wrap(err, "failed to configure Pcloud") } f := &Fs{ name: name, root: root, opt: *opt, srv: rest.NewClient(oAuthClient).SetRoot(rootURL), pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), } f.features = (&fs.Features{ CaseInsensitive: false, CanHaveEmptyDirectories: true, }).Fill(f) f.srv.SetErrorHandler(errorHandler) // Renew the token in the background f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { _, err := f.readMetaDataForPath(ctx, "") return err }) // Get rootFolderID rootID := f.opt.RootFolderID f.dirCache = dircache.New(root, rootID, f) // Find the current root err = f.dirCache.FindRoot(ctx, false) if err != nil { // Assume it is a file newRoot, remote := dircache.SplitPath(root) tempF := *f tempF.dirCache = dircache.New(newRoot, rootID, &tempF) tempF.root = newRoot // Make new Fs which is the parent err = tempF.dirCache.FindRoot(ctx, false) if err != nil { // No root so return old f return f, nil } _, err := tempF.newObjectWithInfo(ctx, remote, nil) if err != nil { if err == fs.ErrorObjectNotFound { // File doesn't exist so return old f return f, nil } return nil, err } // XXX: update the old f here instead of returning tempF, since // `features` were already filled with functions having *f as a receiver. // See https://github.com/rclone/rclone/issues/2182 f.dirCache = tempF.dirCache f.root = tempF.root // return an error with an fs which points to the parent return f, fs.ErrorIsFile } return f, nil } // Return an Object from a path // // If it can't be found it returns the error fs.ErrorObjectNotFound. func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { o := &Object{ fs: f, remote: remote, } var err error if info != nil { // Set info err = o.setMetaData(info) } else { err = o.readMetaData(ctx) // reads info and meta, returning an error } if err != nil { return nil, err } return o, nil } // NewObject finds the Object at remote. If it can't be found // it returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return f.newObjectWithInfo(ctx, remote, nil) } // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { // Find the leaf in pathID found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { if item.Name == leaf { pathIDOut = item.ID return true } return false }) return pathIDOut, found, err } // CreateDir makes a directory with pathID as parent and name leaf func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) var resp *http.Response var result api.ItemResult opts := rest.Opts{ Method: "POST", Path: "/createfolder", Parameters: url.Values{}, } opts.Parameters.Set("name", f.opt.Enc.FromStandardName(leaf)) opts.Parameters.Set("folderid", dirIDtoNumber(pathID)) err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) if err != nil { //fmt.Printf("...Error %v\n", err) return "", err } // fmt.Printf("...Id %q\n", *info.Id) return result.Metadata.ID, nil } // Converts a dirID which is usually 'd' followed by digits into just // the digits func dirIDtoNumber(dirID string) string { if len(dirID) > 0 && dirID[0] == 'd' { return dirID[1:] } fs.Debugf(nil, "Invalid directory id %q", dirID) return dirID } // Converts a fileID which is usually 'f' followed by digits into just // the digits func fileIDtoNumber(fileID string) string { if len(fileID) > 0 && fileID[0] == 'f' { return fileID[1:] } fs.Debugf(nil, "Invalid file id %q", fileID) return fileID } // list the objects into the function supplied // // If directories is set it only sends directories // User function to process a File item from listAll // // Should return true to finish processing type listAllFn func(*api.Item) bool // Lists the directory required calling the user function on each item found // // If the user fn ever returns true then it early exits with found = true func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { opts := rest.Opts{ Method: "GET", Path: "/listfolder", Parameters: url.Values{}, } opts.Parameters.Set("folderid", dirIDtoNumber(dirID)) // FIXME can do recursive var result api.ItemResult var resp *http.Response err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) err = result.Error.Update(err) return shouldRetry(resp, err) }) if err != nil { return found, errors.Wrap(err, "couldn't list files") } for i := range result.Metadata.Contents { item := &result.Metadata.Contents[i] if item.IsFolder { if filesOnly { continue } } else { if directoriesOnly { continue } } item.Name = f.opt.Enc.ToStandardName(item.Name) if fn(item) { found = true break } } return } // List the objects and directories in dir into entries. The // entries can be returned in any order but should be for a // complete directory. // // dir should be "" to list the root, and should not have // trailing slashes. // // This should return ErrDirNotFound if the directory isn't // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { directoryID, err := f.dirCache.FindDir(ctx, dir, false) if err != nil { return nil, err } var iErr error _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { remote := path.Join(dir, info.Name) if info.IsFolder { // cache the directory ID for later lookups f.dirCache.Put(remote, info.ID) d := fs.NewDir(remote, info.ModTime()).SetID(info.ID) // FIXME more info from dir? entries = append(entries, d) } else { o, err := f.newObjectWithInfo(ctx, remote, info) if err != nil { iErr = err return true } entries = append(entries, o) } return false }) if err != nil { return nil, err } if iErr != nil { return nil, iErr } return entries, nil } // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // // Returns the object, leaf, directoryID and error // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { // Create the directory for the object if it doesn't exist leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) if err != nil { return