// +build !plan9// +build !race package cache_test import (	"bytes"	"context"	"encoding/base64"	goflag "flag"	"fmt"	"io"	"io/ioutil"	"log"	"math/rand"	"os"	"path"	"path/filepath"	"runtime"	"runtime/debug"	"strings"	"testing"	"time" 	"github.com/pkg/errors"	"github.com/rclone/rclone/backend/cache"	"github.com/rclone/rclone/backend/crypt"	_ "github.com/rclone/rclone/backend/drive"	"github.com/rclone/rclone/backend/local"	"github.com/rclone/rclone/fs"	"github.com/rclone/rclone/fs/config"	"github.com/rclone/rclone/fs/config/configmap"	"github.com/rclone/rclone/fs/object"	"github.com/rclone/rclone/fs/rc"	"github.com/rclone/rclone/fstest"	"github.com/rclone/rclone/fstest/testy"	"github.com/rclone/rclone/lib/random"	"github.com/rclone/rclone/vfs"	"github.com/rclone/rclone/vfs/vfsflags"	"github.com/stretchr/testify/assert"	"github.com/stretchr/testify/require") const (	// these 2 passwords are test random	cryptPassword1     = "3XcvMMdsV3d-HGAReTMdNH-5FcX5q32_lUeA"                                                     // oGJdUbQc7s8	cryptPassword2     = "NlgTBEIe-qibA7v-FoMfuX6Cw8KlLai_aMvV"                                                     // mv4mZW572HM	cryptedTextBase64  = "UkNMT05FAAC320i2xIee0BiNyknSPBn+Qcw3q9FhIFp3tvq6qlqvbsno3PnxmEFeJG3jDBnR/wku2gHWeQ=="     // one content	cryptedText2Base64 = "UkNMT05FAAATcQkVsgjBh8KafCKcr0wdTa1fMmV0U8hsCLGFoqcvxKVmvv7wx3Hf5EXxFcki2FFV4sdpmSrb9Q==" // updated content	cryptedText3Base64 = "UkNMT05FAAB/f7YtYKbPfmk9+OX/ffN3qG3OEdWT+z74kxCX9V/YZwJ4X2DN3HOnUC3gKQ4Gcoud5UtNvQ=="     // test content	letterBytes        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") var (	remoteName                  string	mountDir                    string	uploadDir                   string	useMount                    bool	runInstance                 *run	errNotSupported             = errors.New("not supported")	decryptedToEncryptedRemotes = map[string]string{		"one":                  "lm4u7jjt3c85bf56vjqgeenuno",		"second":               "qvt1ochrkcfbptp5mu9ugb2l14",		"test":                 "jn4tegjtpqro30t3o11thb4b5s",		"test2":                "qakvqnh8ttei89e0gc76crpql4",		"data.bin":             "0q2847tfko6mhj3dag3r809qbc",		"ticw/data.bin":        "5mv97b0ule6pht33srae5pice8/0q2847tfko6mhj3dag3r809qbc",		"tiuufo/test/one":      "vi6u1olqhirqv14cd8qlej1mgo/jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno",		"tiuufo/test/second":   "vi6u1olqhirqv14cd8qlej1mgo/jn4tegjtpqro30t3o11thb4b5s/qvt1ochrkcfbptp5mu9ugb2l14",		"tiutfo/test/one":      "legd371aa8ol36tjfklt347qnc/jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno",		"tiutfo/second/one":    "legd371aa8ol36tjfklt347qnc/qvt1ochrkcfbptp5mu9ugb2l14/lm4u7jjt3c85bf56vjqgeenuno",		"second/one":           "qvt1ochrkcfbptp5mu9ugb2l14/lm4u7jjt3c85bf56vjqgeenuno",		"test/one":             "jn4tegjtpqro30t3o11thb4b5s/lm4u7jjt3c85bf56vjqgeenuno",		"test/second":          "jn4tegjtpqro30t3o11thb4b5s/qvt1ochrkcfbptp5mu9ugb2l14",		"one/test":             "lm4u7jjt3c85bf56vjqgeenuno/jn4tegjtpqro30t3o11thb4b5s",		"one/test/data.bin":    "lm4u7jjt3c85bf56vjqgeenuno/jn4tegjtpqro30t3o11thb4b5s/0q2847tfko6mhj3dag3r809qbc",		"second/test/data.bin": "qvt1ochrkcfbptp5mu9ugb2l14/jn4tegjtpqro30t3o11thb4b5s/0q2847tfko6mhj3dag3r809qbc",		"test/third":           "jn4tegjtpqro30t3o11thb4b5s/2nd7fjiop5h3ihfj1vl953aa5g",		"test/0.bin":           "jn4tegjtpqro30t3o11thb4b5s/e6frddt058b6kvbpmlstlndmtk",		"test/1.bin":           "jn4tegjtpqro30t3o11thb4b5s/kck472nt1k7qbmob0mt1p1crgc",		"test/2.bin":           "jn4tegjtpqro30t3o11thb4b5s/744oe9ven2rmak4u27if51qk24",		"test/3.bin":           "jn4tegjtpqro30t3o11thb4b5s/2bjd8kef0u5lmsu6qhqll34bcs",		"test/4.bin":           "jn4tegjtpqro30t3o11thb4b5s/cvjs73iv0a82v0c7r67avllh7s",		"test/5.bin":           "jn4tegjtpqro30t3o11thb4b5s/0plkdo790b6bnmt33qsdqmhv9c",		"test/6.bin":           "jn4tegjtpqro30t3o11thb4b5s/s5r633srnjtbh83893jovjt5d0",		"test/7.bin":           "jn4tegjtpqro30t3o11thb4b5s/6rq45tr9bjsammku622flmqsu4",		"test/8.bin":           "jn4tegjtpqro30t3o11thb4b5s/37bc6tcl3e31qb8cadvjb749vk",		"test/9.bin":           "jn4tegjtpqro30t3o11thb4b5s/t4pr35hnls32789o8fk0chk1ec",	}) func init() {	goflag.StringVar(&remoteName, "remote-internal", "TestInternalCache", "Remote to test with, defaults to local filesystem")	goflag.StringVar(&mountDir, "mount-dir-internal", "", "")	goflag.StringVar(&uploadDir, "upload-dir-internal", "", "")	goflag.BoolVar(&useMount, "cache-use-mount", false, "Test only with mount")} // TestMain drives the testsfunc TestMain(m *testing.M) {	goflag.Parse()	var rc int 	log.Printf("Running with the following params: \n remote: %v, \n mount: %v", remoteName, useMount)	runInstance = newRun()	rc = m.Run()	os.Exit(rc)} func TestInternalListRootAndInnerRemotes(t *testing.T) {	id := fmt.Sprintf("tilrair%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	// Instantiate inner fs	innerFolder := "inner"	runInstance.mkdir(t, rootFs, innerFolder)	rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs2, boltDb2) 	runInstance.writeObjectString(t, rootFs2, "one", "content")	listRoot, err := runInstance.list(t, rootFs, "")	require.NoError(t, err)	listRootInner, err := runInstance.list(t, rootFs, innerFolder)	require.NoError(t, err)	listInner, err := rootFs2.List(context.Background(), "")	require.NoError(t, err) 	require.Len(t, listRoot, 1)	require.Len(t, listRootInner, 1)	require.Len(t, listInner, 1)} /* TODO: is this testing something?func TestInternalVfsCache(t *testing.T) {	vfsflags.Opt.DirCacheTime = time.Second * 30	testSize := int64(524288000) 	vfsflags.Opt.CacheMode = vfs.CacheModeWrites	id := "tiuufo"	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})	defer runInstance.cleanupFs(t, rootFs, boltDb) 	err := rootFs.Mkdir(context.Background(), "test")	require.NoError(t, err)	runInstance.writeObjectString(t, rootFs, "test/second", "content")	_, err = rootFs.List(context.Background(), "test")	require.NoError(t, err) 	testReader := runInstance.randomReader(t, testSize)	writeCh := make(chan interface{})	//write2Ch := make(chan interface{})	readCh := make(chan interface{})	cacheCh := make(chan interface{})	// write the main file	go func() {		defer func() {			writeCh <- true		}() 		log.Printf("========== started writing file 'test/one'")		runInstance.writeRemoteReader(t, rootFs, "test/one", testReader)		log.Printf("========== done writing file 'test/one'")	}()	// routine to check which cache has what, autostarts	go func() {		for {			select {			case <-cacheCh:				log.Printf("========== finished checking caches")				return			default:			}			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}			for _, r := range li2 {				var err error				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))				if err != nil || len(ci) == 0 {					log.Printf("========== '%v' not in cache", r)				} else {					log.Printf("========== '%v' IN CACHE", r)				}				_, err = os.Stat(path.Join(runInstance.vfsCachePath, id, r))				if err != nil {					log.Printf("========== '%v' not in vfs", r)				} else {					log.Printf("========== '%v' IN VFS", r)				}			}			time.Sleep(time.Second * 10)		}	}()	// routine to list, autostarts	go func() {		for {			select {			case <-readCh:				log.Printf("========== finished checking listings and readings")				return			default:			}			li, err := runInstance.list(t, rootFs, "test")			if err != nil {				log.Printf("========== error listing 'test' folder: %v", err)			} else {				log.Printf("========== list 'test' folder count: %v", len(li))			} 			time.Sleep(time.Second * 10)		}	}() 	// wait for main file to be written	<-writeCh	log.Printf("========== waiting for VFS to expire")	time.Sleep(time.Second * 120) 	// try a final read	li2 := [2]string{"test/one", "test/second"}	for _, r := range li2 {		_, err := runInstance.readDataFromRemote(t, rootFs, r, int64(0), int64(2), false)		if err != nil {			log.Printf("========== error reading '%v': %v", r, err)		} else {			log.Printf("========== read '%v'", r)		}	}	// close the cache and list checkers	cacheCh <- true	readCh <- true}*/ func TestInternalObjWrapFsFound(t *testing.T) {	id := fmt.Sprintf("tiowff%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err)	wrappedFs := cfs.UnWrap() 	var testData []byte	if runInstance.rootIsCrypt {		testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64)		require.NoError(t, err)	} else {		testData = []byte("test content")	} 	runInstance.writeObjectBytes(t, wrappedFs, runInstance.encryptRemoteIfNeeded(t, "test"), testData)	listRoot, err := runInstance.list(t, rootFs, "")	require.NoError(t, err)	require.Len(t, listRoot, 1) 	cachedData, err := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false)	require.NoError(t, err)	require.Equal(t, "test content", string(cachedData)) 	err = runInstance.rm(t, rootFs, "test")	require.NoError(t, err)	listRoot, err = runInstance.list(t, rootFs, "")	require.NoError(t, err)	require.Len(t, listRoot, 0)} func TestInternalObjNotFound(t *testing.T) {	id := fmt.Sprintf("tionf%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	obj, err := rootFs.NewObject(context.Background(), "404")	require.Error(t, err)	require.Nil(t, obj)} func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {	if !runInstance.useMount {		t.Skip("test needs mount mode")	}	id := fmt.Sprintf("tirwffim%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err) 	var testData []byte	if runInstance.rootIsCrypt {		testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64)		require.NoError(t, err)	} else {		testData = []byte("test content")	} 	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test"), testData)	data, err := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false)	require.NoError(t, err)	require.Equal(t, "test content", string(data))} func TestInternalCachedWrittenContentMatches(t *testing.T) {	testy.SkipUnreliable(t)	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testData := randStringBytes(int(chunkSize*4 + chunkSize/2)) 	// write the object	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) 	// check sample of data from in-file	sampleStart := chunkSize / 2	sampleEnd := chunkSize	testSample := testData[sampleStart:sampleEnd]	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", sampleStart, sampleEnd, false)	require.NoError(t, err)	require.Equal(t, int64(len(checkSample)), sampleEnd-sampleStart)	require.Equal(t, checkSample, testSample)} func TestInternalDoubleWrittenContentMatches(t *testing.T) {	id := fmt.Sprintf("tidwcm%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	// write the object	runInstance.writeRemoteString(t, rootFs, "one", "one content")	err := runInstance.updateData(t, rootFs, "one", "one content", " updated")	require.NoError(t, err)	err = runInstance.updateData(t, rootFs, "one", "one content updated", " double")	require.NoError(t, err) 	// check sample of data from in-file	data, err := runInstance.readDataFromRemote(t, rootFs, "one", int64(0), int64(len("one content updated double")), true)	require.NoError(t, err)	require.Equal(t, "one content updated double", string(data))} func TestInternalCachedUpdatedContentMatches(t *testing.T) {	testy.SkipUnreliable(t)	id := fmt.Sprintf("ticucm%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb)	var err error 	// create some rand test data	var testData1 []byte	var testData2 []byte	if runInstance.rootIsCrypt {		testData1, err = base64.StdEncoding.DecodeString(cryptedTextBase64)		require.NoError(t, err)		testData2, err = base64.StdEncoding.DecodeString(cryptedText2Base64)		require.NoError(t, err)	} else {		testData1 = []byte(random.String(100))		testData2 = []byte(random.String(200))	} 	// write the object	o := runInstance.updateObjectRemote(t, rootFs, "data.bin", testData1, testData2)	require.Equal(t, o.Size(), int64(len(testData2))) 	// check data from in-file	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(len(testData2)), false)	require.NoError(t, err)	require.Equal(t, checkSample, testData2)} func TestInternalWrappedWrittenContentMatches(t *testing.T) {	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())	vfsflags.Opt.DirCacheTime = time.Second	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb)	if runInstance.rootIsCrypt {		t.Skip("test skipped with crypt remote")	} 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testSize := chunkSize*4 + chunkSize/2	testData := randStringBytes(int(testSize)) 	// write the object	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)	require.Equal(t, o.Size(), testSize)	time.Sleep(time.Second * 3) 	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)	require.NoError(t, err)	require.Equal(t, int64(len(checkSample)), o.Size()) 	for i := 0; i < len(checkSample); i++ {		require.Equal(t, testData[i], checkSample[i])	}} func TestInternalLargeWrittenContentMatches(t *testing.T) {	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())	vfsflags.Opt.DirCacheTime = time.Second	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb)	if runInstance.rootIsCrypt {		t.Skip("test skipped with crypt remote")	} 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testSize := chunkSize*10 + chunkSize/2	testData := randStringBytes(int(testSize)) 	// write the object	runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)	time.Sleep(time.Second * 3) 	readData, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)	require.NoError(t, err)	for i := 0; i < len(readData); i++ {		require.Equalf(t, testData[i], readData[i], "at byte %v", i)	}} func TestInternalWrappedFsChangeNotSeen(t *testing.T) {	id := fmt.Sprintf("tiwfcns%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb) 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData) 	// update in the wrapped fs	originalSize, err := runInstance.size(t, rootFs, "data.bin")	require.NoError(t, err)	log.Printf("original size: %v", originalSize) 	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))	require.NoError(t, err)	expectedSize := int64(len([]byte("test content")))	var data2 []byte	if runInstance.rootIsCrypt {		data2, err = base64.StdEncoding.DecodeString(cryptedText3Base64)		require.NoError(t, err)		expectedSize = expectedSize + 1 // FIXME newline gets in, likely test data issue	} else {		data2 = []byte("test content")	}	objInfo := object.NewStaticObjectInfo(runInstance.encryptRemoteIfNeeded(t, "data.bin"), time.Now(), int64(len(data2)), true, nil, cfs.UnWrap())	err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)	require.NoError(t, err)	require.Equal(t, int64(len(data2)), o.Size())	log.Printf("updated size: %v", len(data2)) 	// get a new instance from the cache	if runInstance.wrappedIsExternal {		err = runInstance.retryBlock(func() error {			coSize, err := runInstance.size(t, rootFs, "data.bin")			if err != nil {				return err			}			if coSize != expectedSize {				return errors.Errorf("%v <> %v", coSize, expectedSize)			}			return nil		}, 12, time.Second*10)		require.NoError(t, err)	} else {		coSize, err := runInstance.size(t, rootFs, "data.bin")		require.NoError(t, err)		require.NotEqual(t, coSize, expectedSize)	}} func TestInternalMoveWithNotify(t *testing.T) {	id := fmt.Sprintf("timwn%v", time.Now().Unix())	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)	defer runInstance.cleanupFs(t, rootFs, boltDb)	if !runInstance.wrappedIsExternal {		t.Skipf("Not external")	} 	cfs, err := runInstance.getCacheFs(rootFs)	require.NoError(t, err) 

Lines 1–500 of 1632. Load more