758 lines · 22.0 KB · Go
RawDownload
// +build !plan9,go1.7 package cache_test import (	"bytes"	"fmt"	"io"	"io/ioutil"	"math/rand"	"path"	"path/filepath"	"runtime"	"strconv"	"testing"	"time" 	//"os"	"os/exec"	//"strings" 	"github.com/ncw/rclone/cache"	//"github.com/ncw/rclone/cmd/mount"	//_ "github.com/ncw/rclone/cmd/cmount"	//"github.com/ncw/rclone/cmd/mountlib"	_ "github.com/ncw/rclone/drive"	"github.com/ncw/rclone/fs"	"github.com/ncw/rclone/fstest"	"github.com/ncw/rclone/local"	flag "github.com/spf13/pflag"	"github.com/stretchr/testify/require") var (	infoAge    = time.Second * 10	chunkClean = time.Second	okDiff     = time.Second * 9 // really big diff here but the build machines seem to be slow. need a different way for this	workers    = 2) func TestInternalListRootAndInnerRemotes(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "tilrair-local", "tilrair-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	// Instantiate inner fs	innerFolder := "inner"	err := rootFs.Mkdir(innerFolder)	require.NoError(t, err)	innerFs, err := fs.NewFs("tilrair-cache:" + innerFolder)	require.NoError(t, err) 	obj := writeObjectString(t, innerFs, "one", "content") 	listRoot, err := rootFs.List("")	require.NoError(t, err)	listRootInner, err := rootFs.List(innerFolder)	require.NoError(t, err)	listInner, err := innerFs.List("")	require.NoError(t, err) 	require.Lenf(t, listRoot, 1, "remote %v should have 1 entry", rootFs.Root())	require.Lenf(t, listRootInner, 1, "remote %v should have 1 entry in %v", rootFs.Root(), innerFolder)	require.Lenf(t, listInner, 1, "remote %v should have 1 entry", innerFs.Root()) 	err = obj.Remove()	require.NoError(t, err)} func TestInternalObjWrapFsFound(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "tiowff-local", "tiowff-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	wrappedFs := cfs.UnWrap()	data := "content"	writeObjectString(t, wrappedFs, "second", data) 	listRoot, err := rootFs.List("")	require.NoError(t, err)	require.Lenf(t, listRoot, 1, "remote %v should have 1 entry", rootFs.Root()) 	co, err := rootFs.NewObject("second")	require.NoError(t, err)	r, err := co.Open()	require.NoError(t, err)	cachedData, err := ioutil.ReadAll(r)	require.NoError(t, err)	err = r.Close()	require.NoError(t, err) 	strCached := string(cachedData)	require.Equal(t, data, strCached) 	err = co.Remove()	require.NoError(t, err) 	listRoot, err = wrappedFs.List("")	require.NoError(t, err)	require.Lenf(t, listRoot, 0, "remote %v should have 0 entries: %v", wrappedFs.Root(), listRoot)} func TestInternalObjNotFound(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "tionf-local", "tionf-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	obj, err := rootFs.NewObject("404")	require.Error(t, err)	require.Nil(t, obj)} func TestInternalCachedWrittenContentMatches(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "ticwcm-local", "ticwcm-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testData := make([]byte, (chunkSize*4 + chunkSize/2))	testSize, err := rand.Read(testData)	require.Equal(t, len(testData), testSize, "data size doesn't match")	require.NoError(t, err) 	// write the object	o := writeObjectBytes(t, rootFs, "data.bin", testData)	require.Equal(t, o.Size(), int64(testSize)) 	// check sample of data from in-file	sampleStart := chunkSize / 2	sampleEnd := chunkSize	testSample := testData[sampleStart:sampleEnd]	checkSample := readDataFromObj(t, o, sampleStart, sampleEnd, false)	require.Equal(t, int64(len(checkSample)), sampleEnd-sampleStart)	require.Equal(t, checkSample, testSample)} func TestInternalCachedUpdatedContentMatches(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "ticucm-local", "ticucm-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	// create some rand test data	testData1 := []byte(fstest.RandomString(100))	testData2 := []byte(fstest.RandomString(200)) 	// write the object	o := updateObjectBytes(t, rootFs, "data.bin", testData1, testData2)	require.Equal(t, o.Size(), int64(len(testData2))) 	// check data from in-file	reader, err := o.Open()	require.NoError(t, err)	checkSample, err := ioutil.ReadAll(reader)	_ = reader.Close()	require.NoError(t, err)	require.Equal(t, checkSample, testData2)} func TestInternalWrappedWrittenContentMatches(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "tiwwcm-local", "tiwwcm-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testData := make([]byte, (chunkSize*4 + chunkSize/2))	testSize, err := rand.Read(testData)	require.Equal(t, len(testData), testSize)	require.NoError(t, err) 	// write the object	o := writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)	require.Equal(t, o.Size(), int64(testSize)) 	o2, err := rootFs.NewObject("data.bin")	require.NoError(t, err)	require.Equal(t, o2.Size(), o.Size()) 	// check sample of data from in-file	sampleStart := chunkSize / 2	sampleEnd := chunkSize	testSample := testData[sampleStart:sampleEnd]	checkSample := readDataFromObj(t, o2, sampleStart, sampleEnd, false)	require.Equal(t, len(checkSample), len(testSample)) 	for i := 0; i < len(checkSample); i++ {		require.Equal(t, testSample[i], checkSample[i])	}} func TestInternalLargeWrittenContentMatches(t *testing.T) {	t.Skip("FIXME disabled because it is unreliable")	rootFs, boltDb := newLocalCacheFs(t, "tilwcm-local", "tilwcm-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	testData := make([]byte, (chunkSize*10 + chunkSize/2))	testSize, err := rand.Read(testData)	require.Equal(t, len(testData), testSize)	require.NoError(t, err) 	// write the object	o := writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)	require.Equal(t, o.Size(), int64(testSize)) 	o2, err := rootFs.NewObject("data.bin")	require.NoError(t, err)	require.Equal(t, o2.Size(), o.Size()) 	// check data from in-file	checkSample := readDataFromObj(t, o2, int64(0), int64(testSize), false)	require.Equal(t, len(checkSample), len(testData)) 	for i := 0; i < len(checkSample); i++ {		require.Equal(t, testData[i], checkSample[i], "byte: %d (%d), chunk: %d", int64(i)%chunkSize, i, int64(i)/chunkSize)	}} func TestInternalLargeWrittenContentMatches2(t *testing.T) {	cryptFs, boltDb := newLocalCacheCryptFs(t, "tilwcm2-local", "tilwcm2-cache", "tilwcm2-crypt", true, nil)	defer cleanupFs(t, cryptFs, boltDb) 	cfs, err := getCacheFs(cryptFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize()	fileSize := 87197196	readOffset := 87195648 	// create some rand test data	testData := make([]byte, fileSize)	testSize, err := rand.Read(testData)	require.Equal(t, len(testData), testSize)	require.NoError(t, err) 	// write the object	o := writeObjectBytes(t, cryptFs, "data.bin", testData)	require.Equal(t, o.Size(), int64(testSize)) 	o2, err := cryptFs.NewObject("data.bin")	require.NoError(t, err)	require.Equal(t, o2.Size(), o.Size()) 	// check data from in-file	reader, err := o2.Open(&fs.SeekOption{Offset: int64(readOffset)})	require.NoError(t, err)	rs, ok := reader.(io.Seeker)	require.True(t, ok)	checkOffset, err := rs.Seek(int64(readOffset), 0)	require.NoError(t, err)	require.Equal(t, checkOffset, int64(readOffset))	checkSample, err := ioutil.ReadAll(reader)	require.NoError(t, err)	_ = reader.Close() 	require.Equal(t, len(checkSample), fileSize-readOffset)	for i := 0; i < fileSize-readOffset; i++ {		require.Equal(t, testData[readOffset+i], checkSample[i], "byte: %d (%d), chunk: %d", int64(i)%chunkSize, i, int64(i)/chunkSize)	}} func TestInternalWrappedFsChangeNotSeen(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "tiwfcns-local", "tiwfcns-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	co := writeObjectRandomBytes(t, rootFs, (chunkSize*4 + chunkSize/2)) 	// update in the wrapped fs	o, err := cfs.UnWrap().NewObject(co.Remote())	require.NoError(t, err)	err = o.SetModTime(co.ModTime().Truncate(time.Hour))	require.NoError(t, err) 	// get a new instance from the cache	co2, err := rootFs.NewObject(o.Remote())	require.NoError(t, err) 	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())	require.NotEqual(t, o.ModTime().String(), co2.ModTime().String())	require.Equal(t, co.ModTime().String(), co2.ModTime().String())} func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "ticsadcf-local", "ticsadcf-cache", nil)	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	co := writeObjectRandomBytes(t, rootFs, (chunkSize*4 + chunkSize/2)) 	// update in the wrapped fs	o, err := cfs.UnWrap().NewObject(co.Remote())	require.NoError(t, err)	err = o.SetModTime(co.ModTime().Add(-1 * time.Hour))	require.NoError(t, err) 	// get a new instance from the cache	co2, err := rootFs.NewObject(o.Remote())	require.NoError(t, err) 	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())	require.NotEqual(t, o.ModTime().String(), co2.ModTime().String())	require.Equal(t, co.ModTime().String(), co2.ModTime().String()) 	cfs.DirCacheFlush() // flush the cache 	l, err := cfs.UnWrap().List("")	require.NoError(t, err)	require.Len(t, l, 1)	o2 := l[0] 	// get a new instance from the cache	co, err = rootFs.NewObject(o.Remote())	require.NoError(t, err)	require.Equal(t, o2.ModTime().String(), co.ModTime().String())} func TestInternalCacheWrites(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "ticw-local", "ticw-cache", map[string]string{"cache-writes": "true"})	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize() 	// create some rand test data	co := writeObjectRandomBytes(t, rootFs, (chunkSize*4 + chunkSize/2))	expectedTs := time.Now()	ts, err := boltDb.GetChunkTs(path.Join(rootFs.Root(), co.Remote()), 0)	require.NoError(t, err)	require.WithinDuration(t, expectedTs, ts, okDiff)} func TestInternalMaxChunkSizeRespected(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "timcsr-local", "timcsr-cache", map[string]string{"cache-workers": "1"})	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err)	chunkSize := cfs.ChunkSize()	totalChunks := 20 	// create some rand test data	obj := writeObjectRandomBytes(t, cfs, (int64(totalChunks-1)*chunkSize + chunkSize/2))	o, err := rootFs.NewObject(obj.Remote())	require.NoError(t, err)	co, ok := o.(*cache.Object)	require.True(t, ok) 	for i := 0; i < 4; i++ { // read first 4		_ = readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false)	}	cfs.CleanUpCache(true)	// the last 2 **must** be in the cache	require.True(t, boltDb.HasChunk(co, chunkSize*2))	require.True(t, boltDb.HasChunk(co, chunkSize*3)) 	for i := 4; i < 6; i++ { // read next 2		_ = readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false)	}	cfs.CleanUpCache(true)	// the last 2 **must** be in the cache	require.True(t, boltDb.HasChunk(co, chunkSize*4))	require.True(t, boltDb.HasChunk(co, chunkSize*5))} func TestInternalExpiredEntriesRemoved(t *testing.T) {	rootFs, boltDb := newLocalCacheFs(t, "tieer-local", "tieer-cache", map[string]string{"info_age": "5s"})	defer cleanupFs(t, rootFs, boltDb) 	cfs, err := getCacheFs(rootFs)	require.NoError(t, err) 	// create some rand test data	_ = writeObjectString(t, cfs, "one", "one content")	err = cfs.Mkdir("test")	require.NoError(t, err)	_ = writeObjectString(t, cfs, "test/second", "second content") 	l, err := cfs.List("test")	require.NoError(t, err)	require.Len(t, l, 1) 	err = cfs.UnWrap().Mkdir("test/test2")	require.NoError(t, err) 	l, err = cfs.List("test")	require.NoError(t, err)	require.Len(t, l, 1) 	waitTime := time.Second * 5	t.Logf("Waiting %v seconds for cache to expire\n", waitTime)	time.Sleep(waitTime) 	l, err = cfs.List("test")	require.NoError(t, err)	require.Len(t, l, 2)} // FIXME, enable this when mount is sorted out//func TestInternalFilesMissingInMount1904(t *testing.T) {//	t.Skip("Not yet")//	if runtime.GOOS == "windows" {//		t.Skip("Not yet")//	}//	id := "tifm1904"//	rootFs, _ := newLocalCacheCryptFs(t, "test-local", "test-cache", "test-crypt", false,//		map[string]string{"chunk_size": "5M", "info_age": "1m", "chunk_total_size": "500M", "cache-writes": "true"})//	mntPoint := path.Join("/tmp", "tifm1904-mnt")//	testPoint := path.Join(mntPoint, id)//	checkOutput := "1 10 100 11 12 13 14 15 16 17 18 19 2 20 21 22 23 24 25 26 27 28 29 3 30 31 32 33 34 35 36 37 38 39 4 40 41 42 43 44 45 46 47 48 49 5 50 51 52 53 54 55 56 57 58 59 6 60 61 62 63 64 65 66 67 68 69 7 70 71 72 73 74 75 76 77 78 79 8 80 81 82 83 84 85 86 87 88 89 9 90 91 92 93 94 95 96 97 98 99 "////	_ = os.MkdirAll(mntPoint, os.ModePerm)////	list, err := rootFs.List("")//	require.NoError(t, err)//	found := false//	list.ForDir(func(d fs.Directory) {//		if strings.Contains(d.Remote(), id) {//			found = true//		}//	})////	if !found {//		t.Skip("Test folder '%v' doesn't exist", id)//	}////	mountFs(t, rootFs, mntPoint)//	defer unmountFs(t, mntPoint)////	for i := 1; i <= 2; i++ {//		out, err := exec.Command("ls", testPoint).Output()//		require.NoError(t, err)//		require.Equal(t, checkOutput, strings.Replace(string(out), "\n", " ", -1))//		t.Logf("root path has all files")//		_ = writeObjectString(t, rootFs, path.Join(id, strconv.Itoa(i), strconv.Itoa(i), "one_file"), "one content")////		for j := 1; j <= 100; j++ {//			out, err := exec.Command("ls", path.Join(testPoint, strconv.Itoa(j))).Output()//			require.NoError(t, err)//			require.Equal(t, checkOutput, strings.Replace(string(out), "\n", " ", -1), "'%v' doesn't match", j)//		}//		obj, err := rootFs.NewObject(path.Join(id, strconv.Itoa(i), strconv.Itoa(i), "one_file"))//		require.NoError(t, err)//		err = obj.Remove()//		require.NoError(t, err)//		t.Logf("folders contain all the files")////		out, err = exec.Command("date").Output()//		require.NoError(t, err)//		t.Logf("check #%v date: '%v'", i, strings.Replace(string(out), "\n", " ", -1))////		if i < 2 {//			time.Sleep(time.Second * 60)//		}//	}//} func writeObjectRandomBytes(t *testing.T, f fs.Fs, size int64) fs.Object {	remote := strconv.Itoa(rand.Int()) + ".bin"	// create some rand test data	testData := make([]byte, size)	testSize, err := rand.Read(testData)	require.Equal(t, size, int64(len(testData)))	require.Equal(t, size, int64(testSize))	require.NoError(t, err) 	o := writeObjectBytes(t, f, remote, testData)	require.Equal(t, size, o.Size()) 	return o} func writeObjectString(t *testing.T, f fs.Fs, remote, content string) fs.Object {	return writeObjectBytes(t, f, remote, []byte(content))} func writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object {	in := bytes.NewReader(data)	modTime := time.Now()	objInfo := fs.NewStaticObjectInfo(remote, modTime, int64(len(data)), true, nil, f) 	obj, err := f.Put(in, objInfo)	require.NoError(t, err) 	return obj

Lines 1–500 of 758.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t