// Package googlecloudstorage provides an interface to Google Cloud Storagepackage googlecloudstorage /*Notes Can't set Updated but can set Metadata on object creation Patch needs full_control not just read_write FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 error- https://code.google.com/p/google-api-go-client/issues/detail?id=64*/ import (	"encoding/base64"	"encoding/hex"	"fmt"	"io"	"io/ioutil"	"log"	"net/http"	"os"	"path"	"regexp"	"strings"	"sync"	"time" 	"github.com/ncw/rclone/fs"	"github.com/ncw/rclone/fs/config"	"github.com/ncw/rclone/fs/config/flags"	"github.com/ncw/rclone/fs/config/obscure"	"github.com/ncw/rclone/fs/fserrors"	"github.com/ncw/rclone/fs/fshttp"	"github.com/ncw/rclone/fs/hash"	"github.com/ncw/rclone/fs/walk"	"github.com/ncw/rclone/lib/oauthutil"	"github.com/ncw/rclone/lib/pacer"	"github.com/pkg/errors"	"golang.org/x/oauth2"	"golang.org/x/oauth2/google"	"google.golang.org/api/googleapi"	storage "google.golang.org/api/storage/v1") const (	rcloneClientID              = "202264815644.apps.googleusercontent.com"	rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"	timeFormatIn                = time.RFC3339	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"	metaMtime                   = "mtime" // key to store mtime under in metadata	listChunks                  = 1000    // chunk size to read directory listings	minSleep                    = 10 * time.Millisecond) var (	gcsLocation     = flags.StringP("gcs-location", "", "", "Default location for buckets (us|eu|asia|us-central1|us-east1|us-east4|us-west1|asia-east1|asia-noetheast1|asia-southeast1|australia-southeast1|europe-west1|europe-west2).")	gcsStorageClass = flags.StringP("gcs-storage-class", "", "", "Default storage class for buckets (MULTI_REGIONAL|REGIONAL|STANDARD|NEARLINE|COLDLINE|DURABLE_REDUCED_AVAILABILITY).")	// Description of how to auth for this app	storageConfig = &oauth2.Config{		Scopes:       []string{storage.DevstorageFullControlScope},		Endpoint:     google.Endpoint,		ClientID:     rcloneClientID,		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),		RedirectURL:  oauthutil.TitleBarRedirectURL,	}) // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "google cloud storage",		Description: "Google Cloud Storage (this is not Google Drive)",		NewFs:       NewFs,		Config: func(name string) {			if config.FileGet(name, "service_account_file") != "" {				return			}			err := oauthutil.Config("google cloud storage", name, storageConfig)			if err != nil {				log.Fatalf("Failed to configure token: %v", err)			}		},		Options: []fs.Option{{			Name: config.ConfigClientID,			Help: "Google Application Client Id - leave blank normally.",		}, {			Name: config.ConfigClientSecret,			Help: "Google Application Client Secret - leave blank normally.",		}, {			Name: "project_number",			Help: "Project number optional - needed only for list/create/delete buckets - see your developer console.",		}, {			Name: "service_account_file",			Help: "Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login.",		}, {			Name: "object_acl",			Help: "Access Control List for new objects.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Object owner gets OWNER access, and all Authenticated Users get READER access.",			}, {				Value: "bucketOwnerFullControl",				Help:  "Object owner gets OWNER access, and project team owners get OWNER access.",			}, {				Value: "bucketOwnerRead",				Help:  "Object owner gets OWNER access, and project team owners get READER access.",			}, {				Value: "private",				Help:  "Object owner gets OWNER access [default if left blank].",			}, {				Value: "projectPrivate",				Help:  "Object owner gets OWNER access, and project team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Object owner gets OWNER access, and all Users get READER access.",			}},		}, {			Name: "bucket_acl",			Help: "Access Control List for new buckets.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Project team owners get OWNER access, and all Authenticated Users get READER access.",			}, {				Value: "private",				Help:  "Project team owners get OWNER access [default if left blank].",			}, {				Value: "projectPrivate",				Help:  "Project team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Project team owners get OWNER access, and all Users get READER access.",			}, {				Value: "publicReadWrite",				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",			}},		}, {			Name: "location",			Help: "Location for the newly created buckets.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Empty for default location (US).",			}, {				Value: "asia",				Help:  "Multi-regional location for Asia.",			}, {				Value: "eu",				Help:  "Multi-regional location for Europe.",			}, {				Value: "us",				Help:  "Multi-regional location for United States.",			}, {				Value: "asia-east1",				Help:  "Taiwan.",			}, {				Value: "asia-northeast1",				Help:  "Tokyo.",			}, {				Value: "asia-southeast1",				Help:  "Singapore.",			}, {				Value: "australia-southeast1",				Help:  "Sydney.",			}, {				Value: "europe-west1",				Help:  "Belgium.",			}, {				Value: "europe-west2",				Help:  "London.",			}, {				Value: "us-central1",				Help:  "Iowa.",			}, {				Value: "us-east1",				Help:  "South Carolina.",			}, {				Value: "us-east4",				Help:  "Northern Virginia.",			}, {				Value: "us-west1",				Help:  "Oregon.",			}},		}, {			Name: "storage_class",			Help: "The storage class to use when storing objects in Google Cloud Storage.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Default",			}, {				Value: "MULTI_REGIONAL",				Help:  "Multi-regional storage class",			}, {				Value: "REGIONAL",				Help:  "Regional storage class",			}, {				Value: "NEARLINE",				Help:  "Nearline storage class",			}, {				Value: "COLDLINE",				Help:  "Coldline storage class",			}, {				Value: "DURABLE_REDUCED_AVAILABILITY",				Help:  "Durable reduced availability storage class",			}},		}},	})} // Fs represents a remote storage servertype Fs struct {	name          string           // name of this remote	root          string           // the path we are working on if any	features      *fs.Features     // optional features	svc           *storage.Service // the connection to the storage server	client        *http.Client     // authorized client	bucket        string           // the bucket we are working on	bucketOKMu    sync.Mutex       // mutex to protect bucket OK	bucketOK      bool             // true if we have created the bucket	projectNumber string           // used for finding buckets	objectACL     string           // used when creating new objects	bucketACL     string           // used when creating new buckets	location      string           // location of new buckets	storageClass  string           // storage class of new buckets	pacer         *pacer.Pacer     // To pace the API calls} // Object describes a storage object//// Will definitely have info but maybe not metatype Object struct {	fs       *Fs       // what this object is part of	remote   string    // The remote path	url      string    // download path	md5sum   string    // The MD5Sum of the object	bytes    int64     // Bytes in the object	modTime  time.Time // Modified time of the object	mimeType string} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	if f.root == "" {		return f.bucket	}	return f.bucket + "/" + f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	if f.root == "" {		return fmt.Sprintf("Storage bucket %s", f.bucket)	}	return fmt.Sprintf("Storage bucket %s path %s", f.bucket, f.root)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // shouldRetry determines whehter a given err rates being retriedfunc shouldRetry(err error) (again bool, errOut error) {	again = false	if err != nil {		if fserrors.ShouldRetry(err) {			again = true		} else {			switch gerr := err.(type) {			case *googleapi.Error:				if gerr.Code >= 500 && gerr.Code < 600 {					// All 5xx errors should be retried					again = true				} else if len(gerr.Errors) > 0 {					reason := gerr.Errors[0].Reason					if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {						again = true					}				}			}		}	}	return again, err} // Pattern to match a storage pathvar matcher = regexp.MustCompile(`^([^/]*)(.*)$`) // parseParse parses a storage 'url'func parsePath(path string) (bucket, directory string, err error) {	parts := matcher.FindStringSubmatch(path)	if parts == nil {		err = errors.Errorf("couldn't find bucket in storage path %q", path)	} else {		bucket, directory = parts[1], parts[2]		directory = strings.Trim(directory, "/")	}	return} func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {	conf, err := google.JWTConfigFromJSON(credentialsData, storageConfig.Scopes...)	if err != nil {		return nil, errors.Wrap(err, "error processing credentials")	}	ctxWithSpecialClient := oauthutil.Context(fshttp.NewClient(fs.Config))	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil} // NewFs contstructs an Fs from the path, bucket:pathfunc NewFs(name, root string) (fs.Fs, error) {	var oAuthClient *http.Client	var err error 	// try loading service account credentials from env variable, then from a file	serviceAccountCreds := []byte(config.FileGet(name, "service_account_credentials"))	serviceAccountPath := config.FileGet(name, "service_account_file")	if len(serviceAccountCreds) == 0 && serviceAccountPath != "" {		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(serviceAccountPath))		if err != nil {			return nil, errors.Wrap(err, "error opening service account credentials file")		}		serviceAccountCreds = loadedCreds	}	if len(serviceAccountCreds) > 0 {		oAuthClient, err = getServiceAccountClient(serviceAccountCreds)		if err != nil {			return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")		}	} else {		oAuthClient, _, err = oauthutil.NewClient(name, storageConfig)		if err != nil {			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")		}	} 	bucket, directory, err := parsePath(root)	if err != nil {		return nil, err	} 	f := &Fs{		name:          name,		bucket:        bucket,		root:          directory,		projectNumber: config.FileGet(name, "project_number"),		objectACL:     config.FileGet(name, "object_acl"),		bucketACL:     config.FileGet(name, "bucket_acl"),		location:      config.FileGet(name, "location"),		storageClass:  config.FileGet(name, "storage_class"),		pacer:         pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),	}	f.features = (&fs.Features{		ReadMimeType:  true,		WriteMimeType: true,		BucketBased:   true,	}).Fill(f)	if f.objectACL == "" {		f.objectACL = "private"	}	if f.bucketACL == "" {		f.bucketACL = "private"	}	if *gcsLocation != "" {		f.location = *gcsLocation	}	if *gcsStorageClass != "" {		f.storageClass = *gcsStorageClass	} 	// Create a new authorized Drive client.	f.client = oAuthClient	f.svc, err = storage.New(f.client)	if err != nil {		return nil, errors.Wrap(err, "couldn't create Google Cloud Storage client")	} 	if f.root != "" {		f.root += "/"		// Check to see if the object exists		err = f.pacer.Call(func() (bool, error) {			_, err = f.svc.Objects.Get(bucket, directory).Do()			return shouldRetry(err)		})		if err == nil {			f.root = path.Dir(directory)			if f.root == "." {				f.root = ""			} else {				f.root += "/"			}			// return an error with an fs which points to the parent			return f, fs.ErrorIsFile		}	}	return f, nil} // Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object, error) {	o := &Object{		fs:     f,		remote: remote,	}	if info != nil {		o.setMetaData(info)	} else {		err := o.readMetaData() // reads info and meta, returning an error		if err != nil {			return nil, err		}	}	return o, nil} // NewObject finds the Object at remote.  If it can't be found// it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(remote string) (fs.Object, error) {	return f.newObjectWithInfo(remote, nil)} // listFn is called from list to handle an object.type listFn func(remote string, object *storage.Object, isDirectory bool) error // list the objects into the function supplied//// dir is the starting directory, "" for root//// Set recurse to read sub directoriesfunc (f *Fs) list(dir string, recurse bool, fn listFn) (err error) {	root := f.root	rootLength := len(root)	if dir != "" {		root += dir + "/"	}	list := f.svc.Objects.List(f.bucket).Prefix(root).MaxResults(listChunks)	if !recurse {		list = list.Delimiter("/")	}	for {		var objects *storage.Objects		err = f.pacer.Call(func() (bool, error) {			objects, err = list.Do()			return shouldRetry(err)		})		if err != nil {			if gErr, ok := err.(*googleapi.Error); ok {				if gErr.Code == http.StatusNotFound {					err = fs.ErrorDirNotFound				}			}			return err		}		if !recurse {			var object storage.Object			for _, prefix := range objects.Prefixes {				if !strings.HasSuffix(prefix, "/") {					continue				}				err = fn(prefix[rootLength:len(prefix)-1], &object, true)				if err != nil {					return err				}			}		}		for _, object := range objects.Items {			if !strings.HasPrefix(object.Name, root) {				fs.Logf(f, "Odd name received %q", object.Name)				continue			}			remote := object.Name[rootLength:]			// is this a directory marker?			if (strings.HasSuffix(remote, "/") || remote == "") && object.Size == 0 {				if recurse && remote != "" {					// add a directory in if --fast-list since will have no prefixes					err = fn(remote[:len(remote)-1], object, true)					if err != nil {						return err					}				}				continue // skip directory marker			}			err = fn(remote, object, false)			if err != nil {				return err			}		}		if objects.NextPageToken == "" {			break		}		list.PageToken(objects.NextPageToken)

Lines 1–500 of 984. Load more