// Package cloudinary provides an interface to the Cloudinary DAMpackage cloudinary import (	"context"	"encoding/hex"	"errors"	"fmt"	"io"	"net/http"	"net/url"	"path"	"slices"	"strconv"	"strings"	"time" 	"github.com/cloudinary/cloudinary-go/v2"	SDKApi "github.com/cloudinary/cloudinary-go/v2/api"	"github.com/cloudinary/cloudinary-go/v2/api/admin"	"github.com/cloudinary/cloudinary-go/v2/api/admin/search"	"github.com/cloudinary/cloudinary-go/v2/api/uploader"	"github.com/rclone/rclone/backend/cloudinary/api"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/config/configstruct"	"github.com/rclone/rclone/fs/fserrors"	"github.com/rclone/rclone/fs/fshttp"	"github.com/rclone/rclone/fs/hash"	"github.com/rclone/rclone/lib/encoder"	"github.com/rclone/rclone/lib/pacer"	"github.com/rclone/rclone/lib/rest"	"github.com/zeebo/blake3") // Pacer settings - tuned to match other rclone backendsconst (	minSleep      = 10 * time.Millisecond	maxSleep      = 2 * time.Second	decayConstant = 2 // bigger for slower decay, exponential) // Cloudinary shouldn't have a trailing dot if there is no pathfunc cldPathDir(somePath string) string {	if somePath == "" || somePath == "." {		return somePath	}	dir := path.Dir(somePath)	if dir == "." {		return ""	}	return dir} // Register with Fsfunc init() {	fs.Register(&fs.RegInfo{		Name:        "cloudinary",		Description: "Cloudinary",		NewFs:       NewFs,		Options: []fs.Option{			{				Name:      "cloud_name",				Help:      "Cloudinary Environment Name",				Required:  true,				Sensitive: true,			},			{				Name:      "api_key",				Help:      "Cloudinary API Key",				Required:  true,				Sensitive: true,			},			{				Name:      "api_secret",				Help:      "Cloudinary API Secret",				Required:  true,				Sensitive: true,			},			{				Name: "upload_prefix",				Help: "Specify the API endpoint for environments out of the US",			},			{				Name: "upload_preset",				Help: "Upload Preset to select asset manipulation on upload",			},			{				Name:     config.ConfigEncoding,				Help:     config.ConfigEncodingHelp,				Advanced: true,				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot					encoder.EncodeSlash |					encoder.EncodeLtGt |					encoder.EncodeDoubleQuote |					encoder.EncodeQuestion |					encoder.EncodeAsterisk |					encoder.EncodePipe |					encoder.EncodeHash |					encoder.EncodePercent |					encoder.EncodeBackSlash |					encoder.EncodeDel |					encoder.EncodeCtl |					encoder.EncodeRightSpace |					encoder.EncodeInvalidUtf8 |					encoder.EncodeDot),			},			{				Name:     "eventually_consistent_delay",				Default:  fs.Duration(0),				Advanced: true,				Help:     "Wait N seconds for eventual consistency of the databases that support the backend operation",			},			{				Name:     "adjust_media_files_extensions",				Default:  true,				Advanced: true,				Help:     "Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems",			},			{				Name: "media_extensions",				Default: []string{					"3ds", "3g2", "3gp", "ai", "arw", "avi", "avif", "bmp", "bw",					"cr2", "cr3", "djvu", "dng", "eps3", "fbx", "flif", "flv", "gif",					"glb", "gltf", "hdp", "heic", "heif", "ico", "indd", "jp2", "jpe",					"jpeg", "jpg", "jxl", "jxr", "m2ts", "mov", "mp4", "mpeg", "mts",					"mxf", "obj", "ogv", "pdf", "ply", "png", "psd", "svg", "tga",					"tif", "tiff", "ts", "u3ma", "usdz", "wdp", "webm", "webp", "wmv"},				Advanced: true,				Help:     "Cloudinary supported media extensions",			},		},	})} // Options defines the configuration for this backendtype Options struct {	CloudName                  string               `config:"cloud_name"`	APIKey                     string               `config:"api_key"`	APISecret                  string               `config:"api_secret"`	UploadPrefix               string               `config:"upload_prefix"`	UploadPreset               string               `config:"upload_preset"`	Enc                        encoder.MultiEncoder `config:"encoding"`	EventuallyConsistentDelay  fs.Duration          `config:"eventually_consistent_delay"`	MediaExtensions            []string             `config:"media_extensions"`	AdjustMediaFilesExtensions bool                 `config:"adjust_media_files_extensions"`} // Fs represents a remote cloudinary servertype Fs struct {	name     string	root     string	opt      Options	features *fs.Features	pacer    *fs.Pacer	srv      *rest.Client           // For downloading assets via the Cloudinary CDN	cld      *cloudinary.Cloudinary // API calls are going through the Cloudinary SDK	lastCRUD time.Time} // Object describes a cloudinary objecttype Object struct {	fs           *Fs	remote       string	size         int64	modTime      time.Time	url          string	md5sum       string	publicID     string	resourceType string	deliveryType string} // NewFs constructs an Fs from the path, bucket:pathfunc NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {	opt := new(Options)	err := configstruct.Set(m, opt)	if err != nil {		return nil, err	} 	// Initialize the Cloudinary client	cld, err := cloudinary.NewFromParams(opt.CloudName, opt.APIKey, opt.APISecret)	if err != nil {		return nil, fmt.Errorf("failed to create Cloudinary client: %w", err)	}	cld.Admin.Client = *fshttp.NewClient(ctx)	cld.Upload.Client = *fshttp.NewClient(ctx)	if opt.UploadPrefix != "" {		cld.Config.API.UploadPrefix = opt.UploadPrefix	}	client := fshttp.NewClient(ctx)	f := &Fs{		name:  name,		root:  root,		opt:   *opt,		cld:   cld,		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),		srv:   rest.NewClient(client),	} 	f.features = (&fs.Features{		CanHaveEmptyDirectories: true,	}).Fill(ctx, f) 	if root != "" {		// Check to see if the root actually an existing file		remote := path.Base(root)		f.root = cldPathDir(root)		_, err := f.NewObject(ctx, remote)		if err != nil {			if err == fs.ErrorObjectNotFound || errors.Is(err, fs.ErrorNotAFile) {				// File doesn't exist so return the previous root				f.root = root				return f, nil			}			return nil, err		}		// return an error with an fs which points to the parent		return f, fs.ErrorIsFile	}	return f, nil} // ------------------------------------------------------------ // FromStandardPath implementation of the api.CloudinaryEncoderfunc (f *Fs) FromStandardPath(s string) string {	return strings.ReplaceAll(f.opt.Enc.FromStandardPath(s), "&", "\uFF06")} // FromStandardName implementation of the api.CloudinaryEncoderfunc (f *Fs) FromStandardName(s string) string {	if f.opt.AdjustMediaFilesExtensions {		parsedURL, err := url.Parse(s)		ext := ""		if err != nil {			fs.Logf(nil, "Error parsing URL: %v", err)		} else {			ext = path.Ext(parsedURL.Path)			if slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) {				s = strings.TrimSuffix(parsedURL.Path, ext)			}		}	}	return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")} // ToStandardPath implementation of the api.CloudinaryEncoderfunc (f *Fs) ToStandardPath(s string) string {	return strings.ReplaceAll(f.opt.Enc.ToStandardPath(s), "\uFF06", "&")} // ToStandardName implementation of the api.CloudinaryEncoderfunc (f *Fs) ToStandardName(s string, assetURL string) string {	ext := ""	if f.opt.AdjustMediaFilesExtensions {		parsedURL, err := url.Parse(assetURL)		if err != nil {			fs.Logf(nil, "Error parsing URL: %v", err)		} else {			ext = path.Ext(parsedURL.Path)			if !slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) {				ext = ""			}		}	}	return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&") + ext} // FromStandardFullPath encodes a full path to Cloudinary standardfunc (f *Fs) FromStandardFullPath(dir string) string {	return path.Join(api.CloudinaryEncoder.FromStandardPath(f, f.root), api.CloudinaryEncoder.FromStandardPath(f, dir))} // ToAssetFolderAPI encodes folders as expected by the Cloudinary SDKfunc (f *Fs) ToAssetFolderAPI(dir string) string {	return strings.ReplaceAll(dir, "%", "%25")} // ToDisplayNameElastic encodes a special case of elasticsearchfunc (f *Fs) ToDisplayNameElastic(dir string) string {	return strings.ReplaceAll(dir, "!", "\\!")} // Name of the remote (as passed into NewFs)func (f *Fs) Name() string {	return f.name} // Root of the remote (as passed into NewFs)func (f *Fs) Root() string {	return f.root} // WaitEventuallyConsistent waits till the FS is eventually consistentfunc (f *Fs) WaitEventuallyConsistent() {	if f.opt.EventuallyConsistentDelay == fs.Duration(0) {		return	}	delay := time.Duration(f.opt.EventuallyConsistentDelay)	timeSinceLastCRUD := time.Since(f.lastCRUD)	if timeSinceLastCRUD < delay {		time.Sleep(delay - timeSinceLastCRUD)	}} // String converts this Fs to a stringfunc (f *Fs) String() string {	return fmt.Sprintf("Cloudinary root '%s'", f.root)} // Features returns the optional features of this Fsfunc (f *Fs) Features() *fs.Features {	return f.features} // List the objects and directories in dir into entriesfunc (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {	remotePrefix := f.FromStandardFullPath(dir)	if remotePrefix != "" && !strings.HasSuffix(remotePrefix, "/") {		remotePrefix += "/"	} 	var entries fs.DirEntries	dirs := make(map[string]struct{})	nextCursor := ""	f.WaitEventuallyConsistent()	for {		// user the folders api to list folders.		folderParams := admin.SubFoldersParams{			Folder:     f.ToAssetFolderAPI(remotePrefix),			MaxResults: 500,		}		if nextCursor != "" {			folderParams.NextCursor = nextCursor		} 		results, err := f.cld.Admin.SubFolders(ctx, folderParams)		if err != nil {			return nil, fmt.Errorf("failed to list sub-folders: %w", err)		}		if results.Error.Message != "" {			if strings.HasPrefix(results.Error.Message, "Can't find folder with path") {				return nil, fs.ErrorDirNotFound			} 			return nil, fmt.Errorf("failed to list sub-folders: %s", results.Error.Message)		} 		for _, folder := range results.Folders {			relativePath := api.CloudinaryEncoder.ToStandardPath(f, strings.TrimPrefix(folder.Path, remotePrefix))			parts := strings.Split(relativePath, "/") 			// It's a directory			dirName := parts[len(parts)-1]			if _, found := dirs[dirName]; !found {				d := fs.NewDir(path.Join(dir, dirName), time.Time{})				entries = append(entries, d)				dirs[dirName] = struct{}{}			}		}		// Break if there are no more results		if results.NextCursor == "" {			break		}		nextCursor = results.NextCursor	} 	for {		// Use the assets.AssetsByAssetFolder API to list assets		assetsParams := admin.AssetsByAssetFolderParams{			AssetFolder: remotePrefix,			MaxResults:  500,		}		if nextCursor != "" {			assetsParams.NextCursor = nextCursor		} 		results, err := f.cld.Admin.AssetsByAssetFolder(ctx, assetsParams)		if err != nil {			return nil, fmt.Errorf("failed to list assets: %w", err)		} 		for _, asset := range results.Assets {			remote := path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName, asset.SecureURL))			o := &Object{				fs:           f,				remote:       remote,				size:         int64(asset.Bytes),				modTime:      asset.CreatedAt,				url:          asset.SecureURL,				publicID:     asset.PublicID,				resourceType: asset.AssetType,				deliveryType: asset.Type,			}			entries = append(entries, o)		} 		// Break if there are no more results		if results.NextCursor == "" {			break		}		nextCursor = results.NextCursor	} 	return entries, nil} // NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {	searchParams := search.Query{		Expression: fmt.Sprintf("asset_folder:\"%s\" AND display_name:\"%s\"",			f.FromStandardFullPath(cldPathDir(remote)),			f.ToDisplayNameElastic(api.CloudinaryEncoder.FromStandardName(f, path.Base(remote)))),		SortBy:     []search.SortByField{{"uploaded_at": "desc"}},		MaxResults: 2,	}	var results *admin.SearchResult	f.WaitEventuallyConsistent()	err := f.pacer.Call(func() (bool, error) {		var err1 error		results, err1 = f.cld.Admin.Search(ctx, searchParams)		if err1 == nil && results.TotalCount != len(results.Assets) {			err1 = errors.New("partial response so waiting for eventual consistency")		}		return shouldRetry(ctx, nil, err1)	})	if err != nil {		return nil, fs.ErrorObjectNotFound	}	if results.TotalCount == 0 || len(results.Assets) == 0 {		return nil, fs.ErrorObjectNotFound	}	asset := results.Assets[0] 	o := &Object{		fs:           f,		remote:       remote,		size:         int64(asset.Bytes),		modTime:      asset.UploadedAt,		url:          asset.SecureURL,		md5sum:       asset.Etag,		publicID:     asset.PublicID,		resourceType: asset.ResourceType,		deliveryType: asset.Type,	} 	return o, nil} func (f *Fs) getSuggestedPublicID(assetFolder string, displayName string, modTime time.Time) string {	payload := []byte(path.Join(assetFolder, displayName))	hash := blake3.Sum256(payload)	return hex.EncodeToString(hash[:])} // Put uploads content to Cloudinaryfunc (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {	if src.Size() == 0 {		return nil, fs.ErrorCantUploadEmptyFiles	} 	params := uploader.UploadParams{		UploadPreset: f.opt.UploadPreset,	} 	updateObject := false	var modTime time.Time	for _, option := range options {		if updateOptions, ok := option.(*api.UpdateOptions); ok {			if updateOptions.PublicID != "" {				updateObject = true				params.Overwrite = new(true)				params.Invalidate = new(true)				params.PublicID = updateOptions.PublicID				params.ResourceType = updateOptions.ResourceType				params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)				params.AssetFolder = updateOptions.AssetFolder				params.DisplayName = updateOptions.DisplayName				modTime = src.ModTime(ctx)			}		}	}	if !updateObject {		params.AssetFolder = f.FromStandardFullPath(cldPathDir(src.Remote()))		params.DisplayName = api.CloudinaryEncoder.FromStandardName(f, path.Base(src.Remote()))		// We want to conform to the unique asset ID of rclone, which is (asset_folder,display_name,last_modified).		// We also want to enable customers to choose their own public_id, in case duplicate names are not a crucial use case.		// Upload_presets that apply randomness to the public ID would not work well with rclone duplicate assets support.		params.FilenameOverride = f.getSuggestedPublicID(params.AssetFolder, params.DisplayName, src.ModTime(ctx))	}	uploadResult, err := f.cld.Upload.Upload(ctx, in, params)	f.lastCRUD = time.Now()	if err != nil {		return nil, fmt.Errorf("failed to upload to Cloudinary: %w", err)	}	if !updateObject {		modTime = uploadResult.CreatedAt

Lines 1–500 of 758. Load more