// Package googlecloudstorage provides an interface to Google Cloud Storagepackage googlecloudstorage /*Notes Can't set Updated but can set Metadata on object creation Patch needs full_control not just read_write FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 error- https://code.google.com/p/google-api-go-client/issues/detail?id=64*/ import (	"context"	"encoding/base64"	"encoding/hex"	"fmt"	"io"	"io/ioutil"	"log"	"net/http"	"path"	"strings"	"time" 	"github.com/pkg/errors"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/config/obscure"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/fshttp"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/fs/walk"	"github.com/rclone/rclone/lib/bucket"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/env"	"github.com/rclone/rclone/lib/oauthutil"	"github.com/rclone/rclone/lib/pacer"	"golang.org/x/oauth2"	"golang.org/x/oauth2/google"	"google.golang.org/api/googleapi" 	// NOTE: This API is deprecated	storage "google.golang.org/api/storage/v1") const (	rcloneClientID              = "202264815644.apps.googleusercontent.com"	rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"	timeFormatIn                = time.RFC3339	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"	metaMtime                   = "mtime" // key to store mtime under in metadata	listChunks                  = 1000    // chunk size to read directory listings	minSleep                    = 10 * time.Millisecond) var (	// Description of how to auth for this app	storageConfig = &oauth2.Config{		Scopes:       []string{storage.DevstorageReadWriteScope},		Endpoint:     google.Endpoint,		ClientID:     rcloneClientID,		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),		RedirectURL:  oauthutil.TitleBarRedirectURL,	}) // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "google cloud storage",		Prefix:      "gcs",		Description: "Google Cloud Storage (this is not Google Drive)",		NewFs:       NewFs,		Config: func(ctx context.Context, name string, m configmap.Mapper) {			saFile, _ := m.Get("service_account_file")			saCreds, _ := m.Get("service_account_credentials")			anonymous, _ := m.Get("anonymous")			if saFile != "" || saCreds != "" || anonymous == "true" {				return			}			err := oauthutil.Config(ctx, "google cloud storage", name, m, storageConfig, nil)			if err != nil {				log.Fatalf("Failed to configure token: %v", err)			}		},		Options: append(oauthutil.SharedOptions, []fs.Option{{			Name: "project_number",			Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",		}, {			Name: "service_account_file",			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,		}, {			Name: "service_account_credentials",			Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",			Hide: fs.OptionHideBoth,		}, {			Name:    "anonymous",			Help:    "Access public buckets and objects without credentials\nSet to 'true' if you just want to download files and don't configure credentials.",			Default: false,		}, {			Name: "object_acl",			Help: "Access Control List for new objects.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Object owner gets OWNER access, and all Authenticated Users get READER access.",			}, {				Value: "bucketOwnerFullControl",				Help:  "Object owner gets OWNER access, and project team owners get OWNER access.",			}, {				Value: "bucketOwnerRead",				Help:  "Object owner gets OWNER access, and project team owners get READER access.",			}, {				Value: "private",				Help:  "Object owner gets OWNER access [default if left blank].",			}, {				Value: "projectPrivate",				Help:  "Object owner gets OWNER access, and project team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Object owner gets OWNER access, and all Users get READER access.",			}},		}, {			Name: "bucket_acl",			Help: "Access Control List for new buckets.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Project team owners get OWNER access, and all Authenticated Users get READER access.",			}, {				Value: "private",				Help:  "Project team owners get OWNER access [default if left blank].",			}, {				Value: "projectPrivate",				Help:  "Project team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Project team owners get OWNER access, and all Users get READER access.",			}, {				Value: "publicReadWrite",				Help:  "Project team owners get OWNER access, and all Users get WRITER access.",			}},		}, {			Name: "bucket_policy_only",			Help: `Access checks should use bucket-level IAM policies. If you want to upload objects to a bucket with Bucket Policy Only setthen you will need to set this. When it is set, rclone: - ignores ACLs set on buckets- ignores ACLs set on objects- creates buckets with Bucket Policy Only set Docs: https://cloud.google.com/storage/docs/bucket-policy-only`,			Default: false,		}, {			Name: "location",			Help: "Location for the newly created buckets.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Empty for default location (US).",			}, {				Value: "asia",				Help:  "Multi-regional location for Asia.",			}, {				Value: "eu",				Help:  "Multi-regional location for Europe.",			}, {				Value: "us",				Help:  "Multi-regional location for United States.",			}, {				Value: "asia-east1",				Help:  "Taiwan.",			}, {				Value: "asia-east2",				Help:  "Hong Kong.",			}, {				Value: "asia-northeast1",				Help:  "Tokyo.",			}, {				Value: "asia-south1",				Help:  "Mumbai.",			}, {				Value: "asia-southeast1",				Help:  "Singapore.",			}, {				Value: "australia-southeast1",				Help:  "Sydney.",			}, {				Value: "europe-north1",				Help:  "Finland.",			}, {				Value: "europe-west1",				Help:  "Belgium.",			}, {				Value: "europe-west2",				Help:  "London.",			}, {				Value: "europe-west3",				Help:  "Frankfurt.",			}, {				Value: "europe-west4",				Help:  "Netherlands.",			}, {				Value: "us-central1",				Help:  "Iowa.",			}, {				Value: "us-east1",				Help:  "South Carolina.",			}, {				Value: "us-east4",				Help:  "Northern Virginia.",			}, {				Value: "us-west1",				Help:  "Oregon.",			}, {				Value: "us-west2",				Help:  "California.",			}},		}, {			Name: "storage_class",			Help: "The storage class to use when storing objects in Google Cloud Storage.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Default",			}, {				Value: "MULTI_REGIONAL",				Help:  "Multi-regional storage class",			}, {				Value: "REGIONAL",				Help:  "Regional storage class",			}, {				Value: "NEARLINE",				Help:  "Nearline storage class",			}, {				Value: "COLDLINE",				Help:  "Coldline storage class",			}, {				Value: "ARCHIVE",				Help:  "Archive storage class",			}, {				Value: "DURABLE_REDUCED_AVAILABILITY",				Help:  "Durable reduced availability storage class",			}},		}, {			Name:     config.ConfigEncoding,			Help:     config.ConfigEncodingHelp,			Advanced: true,			Default: (encoder.Base |				encoder.EncodeCrLf |				encoder.EncodeInvalidUtf8),		}}...),	})} // Options defines the configuration for this backendtype Options struct {	ProjectNumber             string               `config:"project_number"`	ServiceAccountFile        string               `config:"service_account_file"`	ServiceAccountCredentials string               `config:"service_account_credentials"`	Anonymous                 bool                 `config:"anonymous"`	ObjectACL                 string               `config:"object_acl"`	BucketACL                 string               `config:"bucket_acl"`	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`	Location                  string               `config:"location"`	StorageClass              string               `config:"storage_class"`	Enc                       encoder.MultiEncoder `config:"encoding"`} // Fs represents a remote storage servertype Fs struct {	name          string           // name of this remote	root          string           // the path we are working on if any	opt           Options          // parsed options	features      *fs.Features     // optional features	svc           *storage.Service // the connection to the storage server	client        *http.Client     // authorized client	rootBucket    string           // bucket part of root (if any)	rootDirectory string           // directory part of root (if any)	cache         *bucket.Cache    // cache of bucket status	pacer         *fs.Pacer        // To pace the API calls} // Object describes a storage object//// Will definitely have info but maybe not metatype Object struct {	fs       *Fs       // what this object is part of	remote   string    // The remote path	url      string    // download path	md5sum   string    // The MD5Sum of the object	bytes    int64     // Bytes in the object	modTime  time.Time // Modified time of the object	mimeType string} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	if f.rootBucket == "" {		return fmt.Sprintf("GCS root")	}	if f.rootDirectory == "" {		return fmt.Sprintf("GCS bucket %s", f.rootBucket)	}	return fmt.Sprintf("GCS bucket %s path %s", f.rootBucket, f.rootDirectory)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // shouldRetry determines whether a given err rates being retriedfunc shouldRetry(ctx context.Context, err error) (again bool, errOut error) {	if fserrors.ContextError(ctx, &err) {		return false, err	}	again = false	if err != nil {		if fserrors.ShouldRetry(err) {			again = true		} else {			switch gerr := err.(type) {			case *googleapi.Error:				if gerr.Code >= 500 && gerr.Code < 600 {					// All 5xx errors should be retried					again = true				} else if len(gerr.Errors) > 0 {					reason := gerr.Errors[0].Reason					if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {						again = true					}				}			}		}	}	return again, err} // parsePath parses a remote 'url'func parsePath(path string) (root string) {	root = strings.Trim(path, "/")	return} // split returns bucket and bucketPath from the rootRelativePath// relative to f.rootfunc (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)} // split returns bucket and bucketPath from the objectfunc (o *Object) split() (bucket, bucketPath string) {	return o.fs.split(o.remote)} func getServiceAccountClient(ctx context.Context, credentialsData []byte) (*http.Client, error) {	conf, err := google.JWTConfigFromJSON(credentialsData, storageConfig.Scopes...)	if err != nil {		return nil, errors.Wrap(err, "error processing credentials")	}	ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(ctx))	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil} // setRoot changes the root of the Fsfunc (f *Fs) setRoot(root string) {	f.root = parsePath(root)	f.rootBucket, f.rootDirectory = bucket.Split(f.root)} // NewFs constructs an Fs from the path, bucket:pathfunc NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {	var oAuthClient *http.Client 	// Parse config into Options struct	opt := new(Options)	err := configstruct.Set(m, opt)	if err != nil {		return nil, err	}	if opt.ObjectACL == "" {		opt.ObjectACL = "private"	}	if opt.BucketACL == "" {		opt.BucketACL = "private"	} 	// try loading service account credentials from env variable, then from a file	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))		if err != nil {			return nil, errors.Wrap(err, "error opening service account credentials file")		}		opt.ServiceAccountCredentials = string(loadedCreds)	}	if opt.Anonymous {		oAuthClient = fshttp.NewClient(ctx)	} else if opt.ServiceAccountCredentials != "" {		oAuthClient, err = getServiceAccountClient(ctx, []byte(opt.ServiceAccountCredentials))		if err != nil {			return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")		}	} else {		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)		if err != nil {			ctx := context.Background()			oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)			if err != nil {				return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")			}		}	} 	f := &Fs{		name:  name,		root:  root,		opt:   *opt,		pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),		cache: bucket.NewCache(),	}	f.setRoot(root)	f.features = (&fs.Features{		ReadMimeType:      true,		WriteMimeType:     true,		BucketBased:       true,		BucketBasedRootOK: true,	}).Fill(ctx, f) 	// Create a new authorized Drive client.	f.client = oAuthClient	f.svc, err = storage.New(f.client)	if err != nil {		return nil, errors.Wrap(err, "couldn't create Google Cloud Storage client")	} 	if f.rootBucket != "" && f.rootDirectory != "" {		// Check to see if the object exists		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)		err = f.pacer.Call(func() (bool, error) {			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()			return shouldRetry(ctx, err)		})		if err == nil {			newRoot := path.Dir(f.root)			if newRoot == "." {				newRoot = ""			}			f.setRoot(newRoot)			// return an error with an fs which points to the parent			return f, fs.ErrorIsFile		}	}	return f, nil} // Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *storage.Object) (fs.Object, error) {	o := &Object{		fs:     f,		remote: remote,	}	if info != nil {		o.setMetaData(info)	} else {		err := o.readMetaData(ctx) // reads info and meta, returning an error		if err != nil {			return nil, err		}	}	return o, nil} // NewObject finds the Object at remote.  If it can't be found// it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {	return f.newObjectWithInfo(ctx, remote, nil)} 

Lines 1–500 of 1148. Load more