// Package googlecloudstorage provides an interface to Google Cloud Storagepackage googlecloudstorage /*Notes Can't set Updated but can set Metadata on object creation Patch needs full_control not just read_write FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 error- https://code.google.com/p/google-api-go-client/issues/detail?id=64*/ import (	"context"	"encoding/base64"	"encoding/hex"	"errors"	"fmt"	"io"	"net/http"	"os"	"path"	"strconv"	"strings"	"sync"	"time" 	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/config/obscure"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/fshttp"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/fs/walk"	"github.com/rclone/rclone/lib/bucket"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/env"	"github.com/rclone/rclone/lib/oauthutil"	"github.com/rclone/rclone/lib/pacer"	"golang.org/x/oauth2"	"golang.org/x/oauth2/google"	"google.golang.org/api/googleapi"	option "google.golang.org/api/option" 	// NOTE: This API is deprecated	storage "google.golang.org/api/storage/v1") const (	rcloneClientID              = "202264815644.apps.googleusercontent.com"	rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"	timeFormat                  = time.RFC3339Nano	metaMtime                   = "mtime"                    // key to store mtime in metadata	metaMtimeGsutil             = "goog-reserved-file-mtime" // key used by GSUtil to store mtime in metadata	listChunks                  = 1000                       // chunk size to read directory listings	minSleep                    = 10 * time.Millisecond) var (	// Description of how to auth for this app	storageConfig = &oauth2.Config{		Scopes:       []string{storage.DevstorageReadWriteScope},		Endpoint:     google.Endpoint,		ClientID:     rcloneClientID,		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),		RedirectURL:  oauthutil.RedirectURL,	}) // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "google cloud storage",		Prefix:      "gcs",		Description: "Google Cloud Storage (this is not Google Drive)",		NewFs:       NewFs,		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {			saFile, _ := m.Get("service_account_file")			saCreds, _ := m.Get("service_account_credentials")			anonymous, _ := m.Get("anonymous")			envAuth, _ := m.Get("env_auth")			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {				return nil, nil			}			return oauthutil.ConfigOut("", &oauthutil.Options{				OAuth2Config: storageConfig,			})		},		Options: append(oauthutil.SharedOptions, []fs.Option{{			Name:      "project_number",			Help:      "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",			Sensitive: true,		}, {			Name:      "user_project",			Help:      "User project.\n\nOptional - needed only for requester pays.",			Sensitive: true,		}, {			Name: "service_account_file",			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,		}, {			Name:      "service_account_credentials",			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",			Hide:      fs.OptionHideBoth,			Sensitive: true,		}, {			Name:    "anonymous",			Help:    "Access public buckets and objects without credentials.\n\nSet to 'true' if you just want to download files and don't configure credentials.",			Default: false,		}, {			Name: "object_acl",			Help: "Access Control List for new objects.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Object owner gets OWNER access.\nAll Authenticated Users get READER access.",			}, {				Value: "bucketOwnerFullControl",				Help:  "Object owner gets OWNER access.\nProject team owners get OWNER access.",			}, {				Value: "bucketOwnerRead",				Help:  "Object owner gets OWNER access.\nProject team owners get READER access.",			}, {				Value: "private",				Help:  "Object owner gets OWNER access.\nDefault if left blank.",			}, {				Value: "projectPrivate",				Help:  "Object owner gets OWNER access.\nProject team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Object owner gets OWNER access.\nAll Users get READER access.",			}},		}, {			Name: "bucket_acl",			Help: "Access Control List for new buckets.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Project team owners get OWNER access.\nAll Authenticated Users get READER access.",			}, {				Value: "private",				Help:  "Project team owners get OWNER access.\nDefault if left blank.",			}, {				Value: "projectPrivate",				Help:  "Project team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Project team owners get OWNER access.\nAll Users get READER access.",			}, {				Value: "publicReadWrite",				Help:  "Project team owners get OWNER access.\nAll Users get WRITER access.",			}},		}, {			Name: "bucket_policy_only",			Help: `Access checks should use bucket-level IAM policies. If you want to upload objects to a bucket with Bucket Policy Only setthen you will need to set this. When it is set, rclone: - ignores ACLs set on buckets- ignores ACLs set on objects- creates buckets with Bucket Policy Only set Docs: https://cloud.google.com/storage/docs/bucket-policy-only`,			Default: false,		}, {			Name: "location",			Help: "Location for the newly created buckets.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Empty for default location (US)",			}, {				Value: "asia",				Help:  "Multi-regional location for Asia",			}, {				Value: "eu",				Help:  "Multi-regional location for Europe",			}, {				Value: "us",				Help:  "Multi-regional location for United States",			}, {				Value: "asia-east1",				Help:  "Taiwan",			}, {				Value: "asia-east2",				Help:  "Hong Kong",			}, {				Value: "asia-northeast1",				Help:  "Tokyo",			}, {				Value: "asia-northeast2",				Help:  "Osaka",			}, {				Value: "asia-northeast3",				Help:  "Seoul",			}, {				Value: "asia-south1",				Help:  "Mumbai",			}, {				Value: "asia-south2",				Help:  "Delhi",			}, {				Value: "asia-southeast1",				Help:  "Singapore",			}, {				Value: "asia-southeast2",				Help:  "Jakarta",			}, {				Value: "australia-southeast1",				Help:  "Sydney",			}, {				Value: "australia-southeast2",				Help:  "Melbourne",			}, {				Value: "europe-north1",				Help:  "Finland",			}, {				Value: "europe-west1",				Help:  "Belgium",			}, {				Value: "europe-west2",				Help:  "London",			}, {				Value: "europe-west3",				Help:  "Frankfurt",			}, {				Value: "europe-west4",				Help:  "Netherlands",			}, {				Value: "europe-west6",				Help:  "Zürich",			}, {				Value: "europe-central2",				Help:  "Warsaw",			}, {				Value: "us-central1",				Help:  "Iowa",			}, {				Value: "us-east1",				Help:  "South Carolina",			}, {				Value: "us-east4",				Help:  "Northern Virginia",			}, {				Value: "us-west1",				Help:  "Oregon",			}, {				Value: "us-west2",				Help:  "California",			}, {				Value: "us-west3",				Help:  "Salt Lake City",			}, {				Value: "us-west4",				Help:  "Las Vegas",			}, {				Value: "northamerica-northeast1",				Help:  "Montréal",			}, {				Value: "northamerica-northeast2",				Help:  "Toronto",			}, {				Value: "southamerica-east1",				Help:  "São Paulo",			}, {				Value: "southamerica-west1",				Help:  "Santiago",			}, {				Value: "asia1",				Help:  "Dual region: asia-northeast1 and asia-northeast2.",			}, {				Value: "eur4",				Help:  "Dual region: europe-north1 and europe-west4.",			}, {				Value: "nam4",				Help:  "Dual region: us-central1 and us-east1.",			}},		}, {			Name: "storage_class",			Help: "The storage class to use when storing objects in Google Cloud Storage.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Default",			}, {				Value: "MULTI_REGIONAL",				Help:  "Multi-regional storage class",			}, {				Value: "REGIONAL",				Help:  "Regional storage class",			}, {				Value: "NEARLINE",				Help:  "Nearline storage class",			}, {				Value: "COLDLINE",				Help:  "Coldline storage class",			}, {				Value: "ARCHIVE",				Help:  "Archive storage class",			}, {				Value: "DURABLE_REDUCED_AVAILABILITY",				Help:  "Durable reduced availability storage class",			}},		}, {			Name:     "directory_markers",			Default:  false,			Advanced: true,			Help: `Upload an empty object with a trailing slash when a new directory is created Empty folders are unsupported for bucket based remotes, this option creates an emptyobject ending with "/", to persist the folder.`,		}, {			Name: "no_check_bucket",			Help: `If set, don't attempt to check the bucket exists or create it. This can be useful when trying to minimise the number of transactionsrclone does if you know the bucket exists already.`,			Default:  false,			Advanced: true,		}, {			Name: "decompress",			Help: `If set this will decompress gzip encoded objects. It is possible to upload objects to GCS with "Content-Encoding: gzip"set. Normally rclone will download these files as compressed objects. If this flag is set then rclone will decompress these files with"Content-Encoding: gzip" as they are received. This means that rclonecan't check the size and hash but the file contents will be decompressed.`,			Advanced: true,			Default:  false,		}, {			Name:     "endpoint",			Help:     "Endpoint for the service.\n\nLeave blank normally.",			Advanced: true,		}, {			Name:     config.ConfigEncoding,			Help:     config.ConfigEncodingHelp,			Advanced: true,			Default: (encoder.Base |				encoder.EncodeCrLf |				encoder.EncodeInvalidUtf8),		}, {			Name:    "env_auth",			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",			Default: false,			Examples: []fs.OptionExample{{				Value: "false",				Help:  "Enter credentials in the next step.",			}, {				Value: "true",				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",			}},		}}...),	})} // Options defines the configuration for this backendtype Options struct {	ProjectNumber             string               `config:"project_number"`	UserProject               string               `config:"user_project"`	ServiceAccountFile        string               `config:"service_account_file"`	ServiceAccountCredentials string               `config:"service_account_credentials"`	Anonymous                 bool                 `config:"anonymous"`	ObjectACL                 string               `config:"object_acl"`	BucketACL                 string               `config:"bucket_acl"`	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`	Location                  string               `config:"location"`	StorageClass              string               `config:"storage_class"`	NoCheckBucket             bool                 `config:"no_check_bucket"`	Decompress                bool                 `config:"decompress"`	Endpoint                  string               `config:"endpoint"`	Enc                       encoder.MultiEncoder `config:"encoding"`	EnvAuth                   bool                 `config:"env_auth"`	DirectoryMarkers          bool                 `config:"directory_markers"`} // Fs represents a remote storage servertype Fs struct {	name           string           // name of this remote	root           string           // the path we are working on if any	opt            Options          // parsed options	features       *fs.Features     // optional features	svc            *storage.Service // the connection to the storage server	client         *http.Client     // authorized client	rootBucket     string           // bucket part of root (if any)	rootDirectory  string           // directory part of root (if any)	cache          *bucket.Cache    // cache of bucket status	pacer          *fs.Pacer        // To pace the API calls	warnCompressed sync.Once        // warn once about compressed files} // Object describes a storage object//// Will definitely have info but maybe not metatype Object struct {	fs       *Fs       // what this object is part of	remote   string    // The remote path	url      string    // download path	md5sum   string    // The MD5Sum of the object	bytes    int64     // Bytes in the object	modTime  time.Time // Modified time of the object	mimeType string	gzipped  bool // set if object has Content-Encoding: gzip} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	if f.rootBucket == "" {		return "GCS root"	}	if f.rootDirectory == "" {		return fmt.Sprintf("GCS bucket %s", f.rootBucket)	}	return fmt.Sprintf("GCS bucket %s path %s", f.rootBucket, f.rootDirectory)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // shouldRetry determines whether a given err rates being retriedfunc shouldRetry(ctx context.Context, err error) (again bool, errOut error) {	if fserrors.ContextError(ctx, &err) {		return false, err	}	again = false	if err != nil {		if fserrors.ShouldRetry(err) {			again = true		} else {			switch gerr := err.(type) {			case *googleapi.Error:				if gerr.Code >= 500 && gerr.Code < 600 {					// All 5xx errors should be retried					again = true				} else if len(gerr.Errors) > 0 {					reason := gerr.Errors[0].Reason					if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {						again = true					}				}			}		}	}	return again, err} // parsePath parses a remote 'url'func parsePath(path string) (root string) {	root = strings.Trim(path, "/")	return} // split returns bucket and bucketPath from the rootRelativePath// relative to f.rootfunc (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)} // split returns bucket and bucketPath from the objectfunc (o *Object) split() (bucket, bucketPath string) {	return o.fs.split(o.remote)} func getServiceAccountClient(ctx context.Context, credentialsData []byte) (*http.Client, error) {	conf, err := google.JWTConfigFromJSON(credentialsData, storageConfig.Scopes...)	if err != nil {		return nil, fmt.Errorf("error processing credentials: %w", err)	}	ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(ctx))	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil} // setRoot changes the root of the Fsfunc (f *Fs) setRoot(root string) {	f.root = parsePath(root)	f.rootBucket, f.rootDirectory = bucket.Split(f.root)} 

Lines 1–500 of 1453. Load more