// Package googlecloudstorage provides an interface to Google Cloud Storagepackage googlecloudstorage /*Notes Can't set Updated but can set Metadata on object creation Patch needs full_control not just read_write FIXME Patch/Delete/Get isn't working with files with spaces in - giving 404 error- https://code.google.com/p/google-api-go-client/issues/detail?id=64*/ import (	"context"	"encoding/base64"	"encoding/hex"	"errors"	"fmt"	"io"	"net/http"	"os"	"path"	"strconv"	"strings"	"sync"	"time" 	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/config/obscure"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/fshttp"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/fs/list"	"github.com/rclone/rclone/lib/bucket"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/env"	"github.com/rclone/rclone/lib/oauthutil"	"github.com/rclone/rclone/lib/pacer"	"golang.org/x/oauth2"	"golang.org/x/oauth2/google"	"google.golang.org/api/googleapi"	option "google.golang.org/api/option" 	// NOTE: This API is deprecated	storage "google.golang.org/api/storage/v1") const (	rcloneClientID              = "202264815644.apps.googleusercontent.com"	rcloneEncryptedClientSecret = "Uj7C9jGfb9gmeaV70Lh058cNkWvepr-Es9sBm0zdgil7JaOWF1VySw"	timeFormat                  = time.RFC3339Nano	metaMtime                   = "mtime"                    // key to store mtime in metadata	metaMtimeGsutil             = "goog-reserved-file-mtime" // key used by GSUtil to store mtime in metadata	listChunks                  = 1000                       // chunk size to read directory listings	minSleep                    = 10 * time.Millisecond) var (	// Description of how to auth for this app	storageConfig = &oauthutil.Config{		Scopes:       []string{storage.DevstorageReadWriteScope},		AuthURL:      google.Endpoint.AuthURL,		TokenURL:     google.Endpoint.TokenURL,		ClientID:     rcloneClientID,		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),		RedirectURL:  oauthutil.RedirectURL,	}) // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "google cloud storage",		Prefix:      "gcs",		Description: "Google Cloud Storage (this is not Google Drive)",		NewFs:       NewFs,		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {			saFile, _ := m.Get("service_account_file")			saCreds, _ := m.Get("service_account_credentials")			anonymous, _ := m.Get("anonymous")			envAuth, _ := m.Get("env_auth")			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {				return nil, nil			}			return oauthutil.ConfigOut("", &oauthutil.Options{				OAuth2Config: storageConfig,			})		},		Options: append(oauthutil.SharedOptions, []fs.Option{{			Name:      "project_number",			Help:      "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",			Sensitive: true,		}, {			Name:      "user_project",			Help:      "User project.\n\nOptional - needed only for requester pays.",			Sensitive: true,		}, {			Name: "service_account_file",			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,		}, {			Name:      "service_account_credentials",			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",			Hide:      fs.OptionHideBoth,			Sensitive: true,		}, {			Name:      "access_token",			Help:      "Short-lived access token.\n\nLeave blank normally.\nNeeded only if you want use short-lived access token instead of interactive login.",			Hide:      fs.OptionHideConfigurator,			Sensitive: true,			Advanced:  true,		}, {			Name:    "anonymous",			Help:    "Access public buckets and objects without credentials.\n\nSet to 'true' if you just want to download files and don't configure credentials.",			Default: false,		}, {			Name: "object_acl",			Help: "Access Control List for new objects.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Object owner gets OWNER access.\nAll Authenticated Users get READER access.",			}, {				Value: "bucketOwnerFullControl",				Help:  "Object owner gets OWNER access.\nProject team owners get OWNER access.",			}, {				Value: "bucketOwnerRead",				Help:  "Object owner gets OWNER access.\nProject team owners get READER access.",			}, {				Value: "private",				Help:  "Object owner gets OWNER access.\nDefault if left blank.",			}, {				Value: "projectPrivate",				Help:  "Object owner gets OWNER access.\nProject team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Object owner gets OWNER access.\nAll Users get READER access.",			}},		}, {			Name: "bucket_acl",			Help: "Access Control List for new buckets.",			Examples: []fs.OptionExample{{				Value: "authenticatedRead",				Help:  "Project team owners get OWNER access.\nAll Authenticated Users get READER access.",			}, {				Value: "private",				Help:  "Project team owners get OWNER access.\nDefault if left blank.",			}, {				Value: "projectPrivate",				Help:  "Project team members get access according to their roles.",			}, {				Value: "publicRead",				Help:  "Project team owners get OWNER access.\nAll Users get READER access.",			}, {				Value: "publicReadWrite",				Help:  "Project team owners get OWNER access.\nAll Users get WRITER access.",			}},		}, {			Name: "bucket_policy_only",			Help: `Access checks should use bucket-level IAM policies. If you want to upload objects to a bucket with Bucket Policy Only setthen you will need to set this. When it is set, rclone: - ignores ACLs set on buckets- ignores ACLs set on objects- creates buckets with Bucket Policy Only set Docs: https://cloud.google.com/storage/docs/bucket-policy-only`,			Default: false,		}, {			Name: "location",			Help: "Location for the newly created buckets.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Empty for default location (US)",			}, {				Value: "asia",				Help:  "Multi-regional location for Asia",			}, {				Value: "eu",				Help:  "Multi-regional location for Europe",			}, {				Value: "us",				Help:  "Multi-regional location for United States",			}, {				Value: "asia-east1",				Help:  "Taiwan",			}, {				Value: "asia-east2",				Help:  "Hong Kong",			}, {				Value: "asia-northeast1",				Help:  "Tokyo",			}, {				Value: "asia-northeast2",				Help:  "Osaka",			}, {				Value: "asia-northeast3",				Help:  "Seoul",			}, {				Value: "asia-south1",				Help:  "Mumbai",			}, {				Value: "asia-south2",				Help:  "Delhi",			}, {				Value: "asia-southeast1",				Help:  "Singapore",			}, {				Value: "asia-southeast2",				Help:  "Jakarta",			}, {				Value: "australia-southeast1",				Help:  "Sydney",			}, {				Value: "australia-southeast2",				Help:  "Melbourne",			}, {				Value: "europe-north1",				Help:  "Finland",			}, {				Value: "europe-west1",				Help:  "Belgium",			}, {				Value: "europe-west2",				Help:  "London",			}, {				Value: "europe-west3",				Help:  "Frankfurt",			}, {				Value: "europe-west4",				Help:  "Netherlands",			}, {				Value: "europe-west6",				Help:  "Zürich",			}, {				Value: "europe-central2",				Help:  "Warsaw",			}, {				Value: "us-central1",				Help:  "Iowa",			}, {				Value: "us-east1",				Help:  "South Carolina",			}, {				Value: "us-east4",				Help:  "Northern Virginia",			}, {				Value: "us-east5",				Help:  "Ohio",			}, {				Value: "us-west1",				Help:  "Oregon",			}, {				Value: "us-west2",				Help:  "California",			}, {				Value: "us-west3",				Help:  "Salt Lake City",			}, {				Value: "us-west4",				Help:  "Las Vegas",			}, {				Value: "northamerica-northeast1",				Help:  "Montréal",			}, {				Value: "northamerica-northeast2",				Help:  "Toronto",			}, {				Value: "southamerica-east1",				Help:  "São Paulo",			}, {				Value: "southamerica-west1",				Help:  "Santiago",			}, {				Value: "asia1",				Help:  "Dual region: asia-northeast1 and asia-northeast2.",			}, {				Value: "eur4",				Help:  "Dual region: europe-north1 and europe-west4.",			}, {				Value: "nam4",				Help:  "Dual region: us-central1 and us-east1.",			}},		}, {			Name: "storage_class",			Help: "The storage class to use when storing objects in Google Cloud Storage.",			Examples: []fs.OptionExample{{				Value: "",				Help:  "Default",			}, {				Value: "MULTI_REGIONAL",				Help:  "Multi-regional storage class",			}, {				Value: "REGIONAL",				Help:  "Regional storage class",			}, {				Value: "NEARLINE",				Help:  "Nearline storage class",			}, {				Value: "COLDLINE",				Help:  "Coldline storage class",			}, {				Value: "ARCHIVE",				Help:  "Archive storage class",			}, {				Value: "DURABLE_REDUCED_AVAILABILITY",				Help:  "Durable reduced availability storage class",			}},		}, {			Name:     "directory_markers",			Default:  false,			Advanced: true,			Help: `Upload an empty object with a trailing slash when a new directory is created Empty folders are unsupported for bucket based remotes, this option creates an emptyobject ending with "/", to persist the folder.`,		}, {			Name: "no_check_bucket",			Help: `If set, don't attempt to check the bucket exists or create it. This can be useful when trying to minimise the number of transactionsrclone does if you know the bucket exists already.`,			Default:  false,			Advanced: true,		}, {			Name: "decompress",			Help: `If set this will decompress gzip encoded objects. It is possible to upload objects to GCS with "Content-Encoding: gzip"set. Normally rclone will download these files as compressed objects. If this flag is set then rclone will decompress these files with"Content-Encoding: gzip" as they are received. This means that rclonecan't check the size and hash but the file contents will be decompressed.`,			Advanced: true,			Default:  false,		}, {			Name: "endpoint",			Help: `Custom endpoint for the storage API. Leave blank to use the provider default. When using a custom endpoint that includes a subpath (e.g. example.org/custom/endpoint),the subpath will be ignored during upload operations due to a limitation in theunderlying Google API Go client library.Download and listing operations will work correctly with the full endpoint path.If you require subpath support for uploads, avoid using subpaths in your customendpoint configuration.`,			Advanced: true,			Examples: []fs.OptionExample{{				Value: "storage.example.org",				Help:  "Specify a custom endpoint",			}, {				Value: "storage.example.org:4443",				Help:  "Specifying a custom endpoint with port",			}, {				Value: "storage.example.org:4443/gcs/api",				Help:  "Specifying a subpath, see the note, uploads won't use the custom path!",			}},		}, {			Name:     config.ConfigEncoding,			Help:     config.ConfigEncodingHelp,			Advanced: true,			Default: (encoder.Base |				encoder.EncodeCrLf |				encoder.EncodeInvalidUtf8),		}, {			Name:    "env_auth",			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",			Default: false,			Examples: []fs.OptionExample{{				Value: "false",				Help:  "Enter credentials in the next step.",			}, {				Value: "true",				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",			}},		}}...),	})} // Options defines the configuration for this backendtype Options struct {	ProjectNumber             string               `config:"project_number"`	UserProject               string               `config:"user_project"`	ServiceAccountFile        string               `config:"service_account_file"`	ServiceAccountCredentials string               `config:"service_account_credentials"`	Anonymous                 bool                 `config:"anonymous"`	ObjectACL                 string               `config:"object_acl"`	BucketACL                 string               `config:"bucket_acl"`	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`	Location                  string               `config:"location"`	StorageClass              string               `config:"storage_class"`	NoCheckBucket             bool                 `config:"no_check_bucket"`	Decompress                bool                 `config:"decompress"`	Endpoint                  string               `config:"endpoint"`	Enc                       encoder.MultiEncoder `config:"encoding"`	EnvAuth                   bool                 `config:"env_auth"`	DirectoryMarkers          bool                 `config:"directory_markers"`	AccessToken               string               `config:"access_token"`} // Fs represents a remote storage servertype Fs struct {	name           string           // name of this remote	root           string           // the path we are working on if any	opt            Options          // parsed options	features       *fs.Features     // optional features	svc            *storage.Service // the connection to the storage server	client         *http.Client     // authorized client	rootBucket     string           // bucket part of root (if any)	rootDirectory  string           // directory part of root (if any)	cache          *bucket.Cache    // cache of bucket status	pacer          *fs.Pacer        // To pace the API calls	warnCompressed sync.Once        // warn once about compressed files} // Object describes a storage object//// Will definitely have info but maybe not metatype Object struct {	fs       *Fs       // what this object is part of	remote   string    // The remote path	url      string    // download path	md5sum   string    // The MD5Sum of the object	bytes    int64     // Bytes in the object	modTime  time.Time // Modified time of the object	mimeType string	gzipped  bool // set if object has Content-Encoding: gzip} // ------------------------------------------------------------ // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // String converts this Fs to a stringfunc (f *Fs) String() string {	if f.rootBucket == "" {		return "GCS root"	}	if f.rootDirectory == "" {		return fmt.Sprintf("GCS bucket %s", f.rootBucket)	}	return fmt.Sprintf("GCS bucket %s path %s", f.rootBucket, f.rootDirectory)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // shouldRetry determines whether a given err rates being retriedfunc shouldRetry(ctx context.Context, err error) (again bool, errOut error) {	if fserrors.ContextError(ctx, &err) {		return false, err	}	again = false	if err != nil {		if fserrors.ShouldRetry(err) {			again = true		} else {			switch gerr := err.(type) {			case *googleapi.Error:				if gerr.Code >= 500 && gerr.Code < 600 {					// All 5xx errors should be retried					again = true				} else if len(gerr.Errors) > 0 {					reason := gerr.Errors[0].Reason					if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {						again = true					}				}			}		}	}	return again, err} // parsePath parses a remote 'url'func parsePath(path string) (root string) {	root = strings.Trim(path, "/")	return}

Lines 1–500 of 1532. Load more