cmd/gitannex: Port unit tests to fstest

This enables the unit tests to run on any given backend, via the `-remote` flag, e.g. `go test -v ./cmd/gitannex/... -remote dropbox:`. We should also port the gitannex e2e tests at some point.

Dan McArdle committed 2 years ago

Commit be33e281 parent 00100903 files · +347 −190Browse files

Unified · Split

3 files

cmd/gitannex/gitannex.go +26 −13
cmd/gitannex/gitannex_test.go +311 −172
cmd/gitannex/layout.go +10 −5

cmd/gitannex/gitannex.go+26−13

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	7	// (Tracked in [issue #7625].)
8	8	//
9	9	// 1. ✅ Minimal support for the [external special remote protocol]. Tested on
10		−// "local" and "drive" backends.
	10	+// "local", "drive", and "dropbox" backends.
11	11	// 2. Add support for the ASYNC protocol extension. This may improve performance.
12	12	// 3. Support the [simple export interface]. This will enable `git-annex
13	13	// export` functionality.
@@ -33,6 +33,7 @@ import ( · 19 unmodified lines
33	33	"github.com/rclone/rclone/cmd"
34	34	"github.com/rclone/rclone/fs"
35	35	"github.com/rclone/rclone/fs/cache"
	36	+ "github.com/rclone/rclone/fs/config"
36	37	"github.com/rclone/rclone/fs/operations"
37	38	"github.com/spf13/cobra"
38	39	)
@@ -267,22 +268,34 @@ func (s *server) handleInitRemote() error { · 228 unmodified lines
267	268	return fmt.Errorf("failed to get configs: %w", err)
268	269	}
269	270
270		− remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))
271		− if err != nil {
272		− s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")
273		− return fmt.Errorf("failed to open root directory of rclone remote: %w", err)
274		− }
	271	+ // Explicitly check that a remote with the given name exists. If we just
	272	+ // relied on `cache.Get()` to return `fs.ErrorNotFoundInConfigFile`, this
	273	+ // function would incorrectly succeed when the given remote name is actually
	274	+ // a file path.
	275	+ //
	276	+ // The :local: backend does not correspond to a remote named by the rclone
	277	+ // config, but is permitted to enable testing. Technically, a user might hit
	278	+ // this code path, but it would be a strange choice because git-annex
	279	+ // natively supports a "directory" special remote.
275	280
276		− if !remoteRootFs.Features().CanHaveEmptyDirectories {
277		− s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")
278		− return fmt.Errorf("rclone remote does not support empty directories")
279		− }
	281	+ trimmedName := strings.TrimSuffix(s.configRcloneRemoteName, ":")
280	282
281		− if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {
282		− s.sendMsg("INITREMOTE-FAILURE failed to mkdir")
283		− return fmt.Errorf("failed to mkdir: %w", err)
	283	+ if s.configRcloneRemoteName != ":local" {
	284	+ var remoteExists bool
	285	+ for _, remoteName := range config.FileSections() {
	286	+ if remoteName == trimmedName {
	287	+ remoteExists = true
	288	+ break
	289	+ }
	290	+ }
	291	+ if !remoteExists {
	292	+ s.sendMsg("INITREMOTE-FAILURE remote does not exist: " + s.configRcloneRemoteName)
	293	+ return fmt.Errorf("remote does not exist: %s", s.configRcloneRemoteName)
	294	+ }
284	295	}
285	296
	297	+ s.configRcloneRemoteName = trimmedName + ":"
	298	+
286	299	s.sendMsg("INITREMOTE-SUCCESS")
287	300	return nil
288	301	}

cmd/gitannex/gitannex_test.go+311−172

@@ -2,25 +2,24 @@ package gitannex · 1 unmodified line
2	2
3	3	import (
4	4	"bufio"
5		− "crypto/sha256"
	5	+ "context"
6	6	"fmt"
7	7	"io"
8	8	"os"
9	9	"path/filepath"
10	10	"regexp"
	11	+ "runtime"
11	12	"strings"
12	13	"sync"
13	14	"testing"
	15	+ "time"
14	16
15		− // Without this import, the local filesystem backend would be unavailable.
16		− // It looks unused, but the act of importing it runs its `init()` function.
17		− _ "github.com/rclone/rclone/backend/local"
	17	+ // Without this import, the various backends would be unavailable. It looks
	18	+ // unused, but the act of importing runs the package's `init()` function.
	19	+ _ "github.com/rclone/rclone/backend/all"
18	20
19		− "github.com/rclone/rclone/fs"
20		− "github.com/rclone/rclone/fs/cache"
21		− "github.com/rclone/rclone/fs/config"
22		− "github.com/rclone/rclone/fs/config/configfile"
23		− "github.com/rclone/rclone/fstest/mockfs"
	21	+ "github.com/rclone/rclone/fs/fspath"
	22	+ "github.com/rclone/rclone/fstest"
24	23
25	24	"github.com/stretchr/testify/assert"
26	25	"github.com/stretchr/testify/require"
@@ -256,9 +255,9 @@ type testState struct { · 229 unmodified lines
256	255	mockStdinW *io.PipeWriter
257	256	mockStdoutReader *bufio.Reader
258	257
259		− localFsDir string
260		− configPath string
261		− remoteName string
	258	+ fstestRun *fstest.Run
	259	+ remoteName string
	260	+ remotePrefix string
262	261	}
263	262
264	263	func makeTestState(t *testing.T) testState {
@@ -276,6 +275,10 @@ func makeTestState(t *testing.T) testState { · 11 unmodified lines
276	275	}
277	276	}
278	277
	278	+func (h *testState) requireRemoteIsEmpty() {
	279	+ h.fstestRun.CheckRemoteItems(h.t)
	280	+}
	281	+
279	282	func (h *testState) requireReadLineExact(line string) {
280	283	receivedLine, err := h.mockStdoutReader.ReadString('\n')
281	284	require.NoError(h.t, err)
@@ -296,21 +299,106 @@ func (h *testState) requireWriteLine(line string) { · 14 unmodified lines
296	299	// Preconfigure the handle. This enables the calling test to skip the PREPARE
297	300	// handshake.
298	301	func (h *testState) preconfigureServer() {
299		− h.server.configPrefix = h.localFsDir
300	302	h.server.configRcloneRemoteName = h.remoteName
	303	+ h.server.configPrefix = h.remotePrefix
301	304	h.server.configRcloneLayout = string(layoutModeNodir)
302	305	h.server.configsDone = true
303	306	}
304	307
305		−// getUniqueRemoteName returns a valid remote name derived from the given test's
306		−// name. This is necessary because when a test registers a second remote with
307		−// the same name, the original remote appears to take precedence. This function
308		−// is injective, so each test gets a unique remote name. Returned strings
309		−// contain no spaces.
310		−func getUniqueRemoteName(t *testing.T) string {
311		− // Using sha256 as a hack to ensure injectivity without adding a global
312		− // variable.
313		− return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))
	308	+// Drop-in replacement for `filepath.Rel()` that works around a Windows-specific
	309	+// quirk when one of the paths begins with `\\?\` or `//?/`. It seems that
	310	+// fstest gives us paths with this prefix on Windows, which throws a wrench in
	311	+// the gitannex tests that need to construct relative paths from absolute paths.
	312	+// For a demonstration, see `TestWindowsFilepathRelQuirk` below.
	313	+//
	314	+// The `\\?\` prefix tells Windows APIs to pass strings unmodified to the
	315	+// filesystem without additional parsing [1]. Our workaround is roughly to add
	316	+// the prefix to whichever parameter doesn't have it (when the OS is Windows).
	317	+// I'm not sure this generalizes, but it works for the the kinds of inputs we're
	318	+// throwing at it.
	319	+//
	320	+// [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#win32-file-namespaces
	321	+func relativeFilepathWorkaround(basepath, targpath string) (string, error) {
	322	+ if runtime.GOOS != "windows" {
	323	+ return filepath.Rel(basepath, targpath)
	324	+ }
	325	+ // Canonicalize paths to use backslashes.
	326	+ basepath = filepath.Clean(basepath)
	327	+ targpath = filepath.Clean(targpath)
	328	+
	329	+ const winFilePrefixDisableStringParsing = `\\?\`
	330	+ baseHasPrefix := strings.HasPrefix(basepath, winFilePrefixDisableStringParsing)
	331	+ targHasPrefix := strings.HasPrefix(targpath, winFilePrefixDisableStringParsing)
	332	+
	333	+ if baseHasPrefix && !targHasPrefix {
	334	+ targpath = winFilePrefixDisableStringParsing + targpath
	335	+ }
	336	+ if !baseHasPrefix && targHasPrefix {
	337	+ basepath = winFilePrefixDisableStringParsing + basepath
	338	+ }
	339	+ return filepath.Rel(basepath, targpath)
	340	+}
	341	+
	342	+func TestWindowsFilepathRelQuirk(t *testing.T) {
	343	+ if runtime.GOOS != "windows" {
	344	+ t.Skip()
	345	+ }
	346	+
	347	+ t.Run("filepathRelQuirk", func(t *testing.T) {
	348	+ var err error
	349	+
	350	+ _, err = filepath.Rel(`C:\foo`, `\\?\C:\foo\bar`)
	351	+ require.Error(t, err)
	352	+
	353	+ _, err = filepath.Rel(`C:/foo`, `//?/C:/foo/bar`)
	354	+ require.Error(t, err)
	355	+
	356	+ _, err = filepath.Rel(`\\?\C:\foo`, `C:\foo\bar`)
	357	+ require.Error(t, err)
	358	+
	359	+ _, err = filepath.Rel(`//?/C:/foo`, `C:/foo/bar`)
	360	+ require.Error(t, err)
	361	+
	362	+ path, err := filepath.Rel(`\\?\C:\foo`, `\\?\C:\foo\bar`)
	363	+ require.NoError(t, err)
	364	+ require.Equal(t, path, `bar`)
	365	+
	366	+ path, err = filepath.Rel(`//?/C:/foo`, `//?/C:/foo/bar`)
	367	+ require.NoError(t, err)
	368	+ require.Equal(t, path, `bar`)
	369	+ })
	370	+
	371	+ t.Run("fstestAndTempDirHaveDifferentPrefixes", func(t *testing.T) {
	372	+ r := fstest.NewRun(t)
	373	+ p := r.Flocal.Root()
	374	+ require.True(t, strings.HasPrefix(p, `//?/`))
	375	+
	376	+ tempDir := t.TempDir()
	377	+ require.False(t, strings.HasPrefix(tempDir, `//?/`))
	378	+ require.False(t, strings.HasPrefix(tempDir, `\\?\`))
	379	+ })
	380	+
	381	+ t.Run("workaroundWorks", func(t *testing.T) {
	382	+ path, err := relativeFilepathWorkaround(`C:\foo`, `\\?\C:\foo\bar`)
	383	+ require.NoError(t, err)
	384	+ require.Equal(t, path, "bar")
	385	+
	386	+ path, err = relativeFilepathWorkaround(`C:/foo`, `//?/C:/foo/bar`)
	387	+ require.NoError(t, err)
	388	+ require.Equal(t, path, "bar")
	389	+
	390	+ path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `C:\foo\bar`)
	391	+ require.NoError(t, err)
	392	+ require.Equal(t, path, `bar`)
	393	+
	394	+ path, err = relativeFilepathWorkaround(`//?/C:/foo`, `C:/foo/bar`)
	395	+ require.NoError(t, err)
	396	+ require.Equal(t, path, `bar`)
	397	+
	398	+ path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `\\?\C:\foo\bar`)
	399	+ require.NoError(t, err)
	400	+ require.Equal(t, path, `bar`)
	401	+ })
314	402	}
315	403
316	404	type testCase struct {
@@ -319,8 +407,8 @@ type testCase struct { · 2 unmodified lines
319	407	expectedError string
320	408	}
321	409
322		−// These test cases run against the "local" backend.
323		−var localBackendTestCases = []testCase{
	410	+// These test cases run against a backend selected by the `-remote` flag.
	411	+var fstestTestCases = []testCase{
324	412	{
325	413	label: "HandlesInit",
326	414	testProtocolFunc: func(t testing.T, h testState) {
@@ -377,17 +465,85 @@ var localBackendTestCases = []testCase{ · 50 unmodified lines
377	465	h.requireReadLineExact("GETCONFIG rcloneremotename")
378	466	h.requireWriteLine("VALUE " + h.remoteName)
379	467	h.requireReadLineExact("GETCONFIG rcloneprefix")
380		− h.requireWriteLine("VALUE " + h.localFsDir)
	468	+ h.requireWriteLine("VALUE " + h.remotePrefix)
381	469	h.requireReadLineExact("GETCONFIG rclonelayout")
382	470	h.requireWriteLine("VALUE foo")
383	471	h.requireReadLineExact("PREPARE-SUCCESS")
384	472
385	473	require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
386		− require.Equal(t, h.server.configPrefix, h.localFsDir)
	474	+ require.Equal(t, h.server.configPrefix, h.remotePrefix)
	475	+ require.True(t, h.server.configsDone)
	476	+
	477	+ require.NoError(t, h.mockStdinW.Close())
	478	+ },
	479	+ },
	480	+ {
	481	+ label: "HandlesPrepareWithNonexistentRemote",
	482	+ testProtocolFunc: func(t testing.T, h testState) {
	483	+ h.requireReadLineExact("VERSION 1")
	484	+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
	485	+ h.requireReadLineExact("EXTENSIONS")
	486	+
	487	+ if !h.server.extensionInfo {
	488	+ t.Errorf("expected INFO extension to be enabled")
	489	+ return
	490	+ }
	491	+
	492	+ h.requireWriteLine("PREPARE")
	493	+ h.requireReadLineExact("GETCONFIG rcloneremotename")
	494	+ h.requireWriteLine("VALUE thisRemoteDoesNotExist")
	495	+ h.requireReadLineExact("GETCONFIG rcloneprefix")
	496	+ h.requireWriteLine("VALUE " + h.remotePrefix)
	497	+ h.requireReadLineExact("GETCONFIG rclonelayout")
	498	+ h.requireWriteLine("VALUE foo")
	499	+ h.requireReadLineExact("PREPARE-SUCCESS")
	500	+
	501	+ require.Equal(t, h.server.configRcloneRemoteName, "thisRemoteDoesNotExist")
	502	+ require.Equal(t, h.server.configPrefix, h.remotePrefix)
	503	+ require.True(t, h.server.configsDone)
	504	+
	505	+ h.requireWriteLine("INITREMOTE")
	506	+ h.requireReadLineExact("INITREMOTE-FAILURE remote does not exist: thisRemoteDoesNotExist")
	507	+
	508	+ require.NoError(t, h.mockStdinW.Close())
	509	+ },
	510	+ expectedError: "remote does not exist: thisRemoteDoesNotExist",
	511	+ },
	512	+ {
	513	+ label: "HandlesPrepareWithPathAsRemote",
	514	+ testProtocolFunc: func(t testing.T, h testState) {
	515	+ h.requireReadLineExact("VERSION 1")
	516	+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
	517	+ h.requireReadLineExact("EXTENSIONS")
	518	+
	519	+ if !h.server.extensionInfo {
	520	+ t.Errorf("expected INFO extension to be enabled")
	521	+ return
	522	+ }
	523	+
	524	+ h.requireWriteLine("PREPARE")
	525	+ h.requireReadLineExact("GETCONFIG rcloneremotename")
	526	+ h.requireWriteLine("VALUE " + h.remotePrefix)
	527	+ h.requireReadLineExact("GETCONFIG rcloneprefix")
	528	+ h.requireWriteLine("VALUE /foo")
	529	+ h.requireReadLineExact("GETCONFIG rclonelayout")
	530	+ h.requireWriteLine("VALUE foo")
	531	+ h.requireReadLineExact("PREPARE-SUCCESS")
	532	+
	533	+ require.Equal(t, h.server.configRcloneRemoteName, h.remotePrefix)
	534	+ require.Equal(t, h.server.configPrefix, "/foo")
387	535	require.True(t, h.server.configsDone)
388	536
	537	+ h.requireWriteLine("INITREMOTE")
	538	+
	539	+ require.Regexp(t,
	540	+ regexp.MustCompile("^INITREMOTE-FAILURE remote does not exist: "),
	541	+ h.requireReadLine(),
	542	+ )
	543	+
389	544	require.NoError(t, h.mockStdinW.Close())
390	545	},
	546	+ expectedError: "remote does not exist:",
391	547	},
392	548	{
393	549	label: "HandlesPrepareWithSynonyms",
@@ -409,13 +565,13 @@ var localBackendTestCases = []testCase{ · 15 unmodified lines
409	565	h.requireWriteLine("VALUE " + h.remoteName)
410	566
411	567	h.requireReadLineExact("GETCONFIG rcloneprefix")
412		− h.requireWriteLine("VALUE " + h.localFsDir)
	568	+ h.requireWriteLine("VALUE " + h.remotePrefix)
413	569	h.requireReadLineExact("GETCONFIG rclonelayout")
414	570	h.requireWriteLine("VALUE foo")
415	571	h.requireReadLineExact("PREPARE-SUCCESS")
416	572
417	573	require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
418		− require.Equal(t, h.server.configPrefix, h.localFsDir)
	574	+ require.Equal(t, h.server.configPrefix, h.remotePrefix)
419	575	require.True(t, h.server.configsDone)
420	576
421	577	require.NoError(t, h.mockStdinW.Close())
@@ -437,12 +593,12 @@ var localBackendTestCases = []testCase{ · 15 unmodified lines
437	593	h.requireReadLineExact("GETCONFIG rcloneremotename")
438	594
439	595	remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)
440		− localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
	596	+ prefixWithWhitespace := fmt.Sprintf(" %s\t", h.remotePrefix)
441	597
442	598	h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
443	599
444	600	h.requireReadLineExact("GETCONFIG rcloneprefix")
445		− h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
	601	+ h.requireWriteLine(fmt.Sprintf("VALUE %s", prefixWithWhitespace))
446	602
447	603	h.requireReadLineExact("GETCONFIG rclonelayout")
448	604	h.requireWriteLine("VALUE")
@@ -452,7 +608,7 @@ var localBackendTestCases = []testCase{ · 3 unmodified lines
452	608	h.requireReadLineExact("PREPARE-SUCCESS")
453	609
454	610	require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
455		− require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)
	611	+ require.Equal(t, h.server.configPrefix, prefixWithWhitespace)
456	612	require.True(t, h.server.configsDone)
457	613
458	614	require.NoError(t, h.mockStdinW.Close())
@@ -639,20 +795,25 @@ var localBackendTestCases = []testCase{ · 180 unmodified lines
639	795	h.requireReadLineExact("INITREMOTE-SUCCESS")
640	796
641	797	// Create temp file for transfer with an absolute path.
642		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
643		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
644		− require.FileExists(t, fileToTransfer)
645		− require.True(t, filepath.IsAbs(fileToTransfer))
	798	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	799	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	800	+ require.True(t, filepath.IsAbs(absPath))
646	801
647	802	// Specify an absolute path to transfer.
648		− h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
	803	+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)
649	804	h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
650		− require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
	805	+
	806	+ // Check that the file was transferred.
	807	+ remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)
	808	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
651	809
652	810	// Transfer the same absolute path a second time, but with a different key.
653		− h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)
	811	+ h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + absPath)
654	812	h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")
655		− require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))
	813	+
	814	+ // Check that the same file was transferred to a new name.
	815	+ remoteItem2 := fstest.NewItem("KeyAbsolute2", "HELLO", item.ModTime)
	816	+ h.fstestRun.CheckRemoteItems(t, remoteItem, remoteItem2)
656	817
657	818	h.requireWriteLine("CHECKPRESENT KeyAbsolute2")
658	819	h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")
@@ -668,30 +829,36 @@ var localBackendTestCases = []testCase{ · 9 unmodified lines
668	829	{
669	830	label: "TransferStoreRelative",
670	831	testProtocolFunc: func(t testing.T, h testState) {
671		− h.preconfigureServer()
672		−
673	832	// Save the current working directory so we can restore it when this
674	833	// test ends.
675	834	cwd, err := os.Getwd()
676	835	require.NoError(t, err)
677	836
678		− require.NoError(t, os.Chdir(t.TempDir()))
	837	+ tempDir := t.TempDir()
	838	+
	839	+ require.NoError(t, os.Chdir(tempDir))
679	840	t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
680	841
	842	+ h.preconfigureServer()
	843	+
681	844	h.requireReadLineExact("VERSION 1")
682	845	h.requireWriteLine("INITREMOTE")
683	846	h.requireReadLineExact("INITREMOTE-SUCCESS")
684	847
685		− // Create temp file for transfer with a relative path.
686		− fileToTransfer := "file.txt"
687		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
688		− require.FileExists(t, fileToTransfer)
689		− require.False(t, filepath.IsAbs(fileToTransfer))
	848	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	849	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	850	+
	851	+ relativePath, err := relativeFilepathWorkaround(tempDir, absPath)
	852	+ require.NoError(t, err)
	853	+ require.False(t, filepath.IsAbs(relativePath))
	854	+ require.FileExists(t, relativePath)
690	855
691	856	// Specify a relative path to transfer.
692		− h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
	857	+ h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)
693	858	h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
694		− require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
	859	+
	860	+ remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)
	861	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
695	862
696	863	h.requireWriteLine("CHECKPRESENT KeyRelative")
697	864	h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
@@ -710,7 +877,8 @@ var localBackendTestCases = []testCase{ · 12 unmodified lines
710	877	cwd, err := os.Getwd()
711	878	require.NoError(t, err)
712	879
713		− require.NoError(t, os.Chdir(t.TempDir()))
	880	+ tempDir := t.TempDir()
	881	+ require.NoError(t, os.Chdir(tempDir))
714	882	t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
715	883
716	884	h.preconfigureServer()
@@ -720,15 +888,19 @@ var localBackendTestCases = []testCase{ · 3 unmodified lines
720	888	h.requireReadLineExact("INITREMOTE-SUCCESS")
721	889
722	890	// Create temp file for transfer.
723		− fileToTransfer := "filename with spaces.txt"
724		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
725		− require.FileExists(t, fileToTransfer)
726		− require.False(t, filepath.IsAbs(fileToTransfer))
	891	+ item := h.fstestRun.WriteFile("filename with spaces.txt", "HELLO", time.Now())
	892	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	893	+ relativePath, err := relativeFilepathWorkaround(tempDir, absPath)
	894	+ require.NoError(t, err)
	895	+ require.False(t, filepath.IsAbs(relativePath))
	896	+ require.FileExists(t, relativePath)
727	897
728	898	// Specify a relative path to transfer.
729		− h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
	899	+ h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)
730	900	h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
731		− require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
	901	+
	902	+ remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)
	903	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
732	904
733	905	h.requireWriteLine("CHECKPRESENT KeyRelative")
734	906	h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
@@ -745,8 +917,9 @@ var localBackendTestCases = []testCase{ · 10 unmodified lines
745	917	h.preconfigureServer()
746	918
747	919	// Create temp file for transfer.
748		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
749		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
	920	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	921	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	922	+ require.True(t, filepath.IsAbs(absPath))
750	923
751	924	h.requireReadLineExact("VERSION 1")
752	925	h.requireWriteLine("INITREMOTE")
@@ -756,10 +929,11 @@ var localBackendTestCases = []testCase{ · 3 unmodified lines
756	929	h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
757	930
758	931	// Specify an absolute path to transfer.
759		− require.True(t, filepath.IsAbs(fileToTransfer))
760		− h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
	932	+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)
761	933	h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
762		− require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
	934	+
	935	+ remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)
	936	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
763	937
764	938	require.NoError(t, h.mockStdinW.Close())
765	939	},
@@ -781,8 +955,9 @@ var localBackendTestCases = []testCase{ · 15 unmodified lines
781	955	h.preconfigureServer()
782	956
783	957	// Create temp file for transfer.
784		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
785		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
	958	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	959	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	960	+ require.True(t, filepath.IsAbs(absPath))
786	961
787	962	h.requireReadLineExact("VERSION 1")
788	963	h.requireWriteLine("INITREMOTE")
@@ -791,9 +966,11 @@ var localBackendTestCases = []testCase{ · 2 unmodified lines
791	966	h.requireWriteLine("CHECKPRESENT foo")
792	967	h.requireReadLineExact("CHECKPRESENT-FAILURE foo")
793	968
794		− h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)
	969	+ h.requireWriteLine("TRANSFER STORE foo " + absPath)
795	970	h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")
796		− require.FileExists(t, filepath.Join(h.localFsDir, "foo"))
	971	+
	972	+ remoteItem := fstest.NewItem("foo", "HELLO", item.ModTime)
	973	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
797	974
798	975	h.requireWriteLine("CHECKPRESENT foo")
799	976	h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")
@@ -807,8 +984,9 @@ var localBackendTestCases = []testCase{ · 7 unmodified lines
807	984	h.preconfigureServer()
808	985
809	986	// Create temp file for transfer.
810		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
811		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
	987	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	988	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	989	+ require.True(t, filepath.IsAbs(absPath))
812	990
813	991	h.requireReadLineExact("VERSION 1")
814	992	h.requireWriteLine("INITREMOTE")
@@ -817,10 +995,11 @@ var localBackendTestCases = []testCase{ · 2 unmodified lines
817	995	realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"
818	996
819	997	// Specify an absolute path to transfer.
820		− require.True(t, filepath.IsAbs(fileToTransfer))
821		− h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))
	998	+ h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, absPath))
822	999	h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)
823		− require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))
	1000	+
	1001	+ remoteItem := fstest.NewItem(realisticKey, "HELLO", item.ModTime)
	1002	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
824	1003
825	1004	h.requireWriteLine("CHECKPRESENT " + realisticKey)
826	1005	h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)
@@ -849,27 +1028,36 @@ var localBackendTestCases = []testCase{ · 22 unmodified lines
849	1028	h.preconfigureServer()
850	1029
851	1030	// Create temp file for transfer.
852		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
853		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
	1031	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	1032	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	1033	+ require.True(t, filepath.IsAbs(absPath))
854	1034
855	1035	h.requireReadLineExact("VERSION 1")
856	1036	h.requireWriteLine("INITREMOTE")
857	1037	h.requireReadLineExact("INITREMOTE-SUCCESS")
858	1038
859	1039	// Specify an absolute path to transfer.
860		− require.True(t, filepath.IsAbs(fileToTransfer))
861		− h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
	1040	+ h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)
862	1041	h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
863		− require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
	1042	+
	1043	+ remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)
	1044	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
864	1045
865	1046	h.requireWriteLine("CHECKPRESENT SomeKey")
866	1047	h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
867	1048
868		− retrievedFilePath := fileToTransfer + ".retrieved"
869		− require.NoFileExists(t, retrievedFilePath)
	1049	+ h.fstestRun.CheckLocalItems(t,
	1050	+ fstest.NewItem("file.txt", "HELLO", item.ModTime),
	1051	+ )
	1052	+
	1053	+ retrievedFilePath := absPath + ".retrieved"
870	1054	h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)
871	1055	h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")
872		− require.FileExists(t, retrievedFilePath)
	1056	+
	1057	+ h.fstestRun.CheckLocalItems(t,
	1058	+ fstest.NewItem("file.txt", "HELLO", item.ModTime),
	1059	+ fstest.NewItem("file.txt.retrieved", "HELLO", item.ModTime),
	1060	+ )
873	1061
874	1062	require.NoError(t, h.mockStdinW.Close())
875	1063	},
@@ -879,11 +1067,13 @@ var localBackendTestCases = []testCase{ · 3 unmodified lines
879	1067	testProtocolFunc: func(t testing.T, h testState) {
880	1068	h.preconfigureServer()
881	1069
	1070	+ ctx := context.WithoutCancel(context.Background())
	1071	+
882	1072	// Write a file into the remote without using the git-annex
883	1073	// protocol.
884		− remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")
885		− require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))
886		− require.FileExists(t, remoteFilePath)
	1074	+ remoteItem := h.fstestRun.WriteObject(ctx, "SomeKey", "HELLO", time.Now())
	1075	+
	1076	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
887	1077
888	1078	h.requireReadLineExact("VERSION 1")
889	1079	h.requireWriteLine("INITREMOTE")
@@ -891,15 +1081,18 @@ var localBackendTestCases = []testCase{ · 1 unmodified line
891	1081
892	1082	h.requireWriteLine("CHECKPRESENT SomeKey")
893	1083	h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
894		− require.FileExists(t, remoteFilePath)
	1084	+
	1085	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
895	1086
896	1087	h.requireWriteLine("REMOVE SomeKey")
897	1088	h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
898		− require.NoFileExists(t, remoteFilePath)
	1089	+
	1090	+ h.requireRemoteIsEmpty()
899	1091
900	1092	h.requireWriteLine("CHECKPRESENT SomeKey")
901	1093	h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
902		− require.NoFileExists(t, remoteFilePath)
	1094	+
	1095	+ h.requireRemoteIsEmpty()
903	1096
904	1097	require.NoError(t, h.mockStdinW.Close())
905	1098	},
@@ -910,8 +1103,9 @@ var localBackendTestCases = []testCase{ · 4 unmodified lines
910	1103	h.preconfigureServer()
911	1104
912	1105	// Create temp file for transfer.
913		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
914		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
	1106	+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
	1107	+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
	1108	+ require.True(t, filepath.IsAbs(absPath))
915	1109
916	1110	h.requireReadLineExact("VERSION 1")
917	1111	h.requireWriteLine("INITREMOTE")
@@ -921,17 +1115,19 @@ var localBackendTestCases = []testCase{ · 3 unmodified lines
921	1115	h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
922	1116
923	1117	// Specify an absolute path to transfer.
924		− require.True(t, filepath.IsAbs(fileToTransfer))
925		− h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
	1118	+ h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)
926	1119	h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
927		− require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
	1120	+
	1121	+ remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)
	1122	+ h.fstestRun.CheckRemoteItems(t, remoteItem)
928	1123
929	1124	h.requireWriteLine("CHECKPRESENT SomeKey")
930	1125	h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
931	1126
932	1127	h.requireWriteLine("REMOVE SomeKey")
933	1128	h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
934		− require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
	1129	+
	1130	+ h.requireRemoteIsEmpty()
935	1131
936	1132	h.requireWriteLine("CHECKPRESENT SomeKey")
937	1133	h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
@@ -944,10 +1140,6 @@ var localBackendTestCases = []testCase{ · 6 unmodified lines
944	1140	testProtocolFunc: func(t testing.T, h testState) {
945	1141	h.preconfigureServer()
946	1142
947		− // Create temp file for transfer.
948		− fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
949		− require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
950		−
951	1143	h.requireReadLineExact("VERSION 1")
952	1144	h.requireWriteLine("INITREMOTE")
953	1145	h.requireReadLineExact("INITREMOTE-SUCCESS")
@@ -955,10 +1147,12 @@ var localBackendTestCases = []testCase{ · 1 unmodified line
955	1147	h.requireWriteLine("CHECKPRESENT SomeKey")
956	1148	h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
957	1149
958		− require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
	1150	+ h.requireRemoteIsEmpty()
	1151	+
959	1152	h.requireWriteLine("REMOVE SomeKey")
960	1153	h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
961		− require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
	1154	+
	1155	+ h.requireRemoteIsEmpty()
962	1156
963	1157	h.requireWriteLine("CHECKPRESENT SomeKey")
964	1158	h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
@@ -983,44 +1177,40 @@ var localBackendTestCases = []testCase{ · 18 unmodified lines
983	1177	},
984	1178	}
985	1179
986		−func TestGitAnnexLocalBackendCases(t *testing.T) {
987		− for _, testCase := range localBackendTestCases {
988		− // Clear global state left behind by tests that chdir to a temp directory.
989		− cache.Clear()
	1180	+// TestMain drives the tests
	1181	+func TestMain(m *testing.M) {
	1182	+ fstest.TestMain(m)
	1183	+}
	1184	+
	1185	+// Run fstest-compatible test cases with backend selected by `-remote`.
	1186	+func TestGitAnnexFstestBackendCases(t *testing.T) {
990	1187
	1188	+ for _, testCase := range fstestTestCases {
991	1189	// TODO: Remove this when rclone requires a Go version >= 1.22. Future
992	1190	// versions of Go fix the semantics of capturing a range variable.
993	1191	// https://go.dev/blog/loopvar-preview
994	1192	testCase := testCase
995	1193
996	1194	t.Run(testCase.label, func(t *testing.T) {
997		− tempDir := t.TempDir()
	1195	+ r := fstest.NewRun(t)
	1196	+ t.Cleanup(func() { r.Finalise() })
998	1197
999		− // Create temp dir for an rclone remote pointing at local filesystem.
1000		− localFsDir := filepath.Join(tempDir, "remoteTarget")
1001		− require.NoError(t, os.Mkdir(localFsDir, 0700))
	1198	+ // Parse the fstest-provided remote string. It might have a path!
	1199	+ remoteName, remotePath, err := fspath.SplitFs(r.FremoteName)
	1200	+ require.NoError(t, err)
1002	1201
1003		− // Create temp config
1004		− remoteName := getUniqueRemoteName(t)
1005		− configLines := []string{
1006		− fmt.Sprintf("[%s]", remoteName),
1007		− "type = local",
1008		− fmt.Sprintf("remote = %s", localFsDir),
	1202	+ // The gitannex command requires the `rcloneremotename` is the name
	1203	+ // of a remote or exactly ":local", so the empty string will not
	1204	+ // suffice.
	1205	+ if remoteName == "" {
	1206	+ require.True(t, r.Fremote.Features().IsLocal)
	1207	+ remoteName = ":local"
1009	1208	}
1010		− configContents := strings.Join(configLines, "\n")
1011		−
1012		− configPath := filepath.Join(tempDir, "rclone.conf")
1013		− require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
1014		− require.NoError(t, config.SetConfigPath(configPath))
1015		−
1016		− // The custom config file will be ignored unless we install the
1017		− // global config file handler.
1018		− configfile.Install()
1019	1209
1020	1210	handle := makeTestState(t)
1021		− handle.localFsDir = localFsDir
1022		− handle.configPath = configPath
	1211	+ handle.fstestRun = r
1023	1212	handle.remoteName = remoteName
	1213	+ handle.remotePrefix = remotePath
1024	1214
1025	1215	var wg sync.WaitGroup
1026	1216	wg.Add(1)
@@ -1042,54 +1232,3 @@ func TestGitAnnexLocalBackendCases(t *testing.T) { · 15 unmodified lines
1042	1232	})
1043	1233	}
1044	1234	}
1045		−
1046		−// Configure the git-annex client with a mockfs backend and send it the
1047		−// "INITREMOTE" command over mocked stdin. This should fail because mockfs does
1048		−// not support empty directories.
1049		−func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {
1050		− tempDir := t.TempDir()
1051		−
1052		− // Temporarily override the filesystem registry.
1053		− oldRegistry := fs.Registry
1054		− mockfs.Register()
1055		− defer func() { fs.Registry = oldRegistry }()
1056		−
1057		− // Create temp dir for an rclone remote pointing at local filesystem.
1058		− localFsDir := filepath.Join(tempDir, "remoteTarget")
1059		− require.NoError(t, os.Mkdir(localFsDir, 0700))
1060		−
1061		− // Create temp config
1062		− remoteName := getUniqueRemoteName(t)
1063		− configLines := []string{
1064		− fmt.Sprintf("[%s]", remoteName),
1065		− "type = mockfs",
1066		− fmt.Sprintf("remote = %s", localFsDir),
1067		− }
1068		− configContents := strings.Join(configLines, "\n")
1069		−
1070		− configPath := filepath.Join(tempDir, "rclone.conf")
1071		− require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
1072		−
1073		− // The custom config file will be ignored unless we install the global
1074		− // config file handler.
1075		− configfile.Install()
1076		− require.NoError(t, config.SetConfigPath(configPath))
1077		−
1078		− handle := makeTestState(t)
1079		− handle.server.configPrefix = localFsDir
1080		− handle.server.configRcloneRemoteName = remoteName
1081		− handle.server.configsDone = true
1082		−
1083		− var wg sync.WaitGroup
1084		− wg.Add(1)
1085		−
1086		− go func() {
1087		− require.NotNil(t, handle.server.run())
1088		− wg.Done()
1089		− }()
1090		− defer wg.Wait()
1091		−
1092		− handle.requireReadLineExact("VERSION 1")
1093		− handle.requireWriteLine("INITREMOTE")
1094		− handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")
1095		−}

cmd/gitannex/layout.go+10−5

@@ -3,6 +3,8 @@ package gitannex · 2 unmodified lines
3	3	import (
4	4	"fmt"
5	5	"strings"
	6	+
	7	+ "github.com/rclone/rclone/fs/fspath"
6	8	)
7	9
8	10	type layoutMode string
@@ -39,8 +41,11 @@ func parseLayoutMode(mode string) layoutMode { · 30 unmodified lines
39	41	type queryDirhashFunc func(msg string) (string, error)
40	42
41	43	func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
	44	+ remoteName = strings.TrimSuffix(remoteName, ":") + ":"
	45	+ remoteString := fspath.JoinRootPath(remoteName, prefix)
	46	+
42	47	if mode == layoutModeNodir {
43		− return fmt.Sprintf("%s:%s", remoteName, prefix), nil
	48	+ return remoteString, nil
44	49	}
45	50
46	51	var dirhash string
@@ -59,13 +64,13 @@ func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteNa · 12 unmodified lines
59	64
60	65	switch mode {
61	66	case layoutModeLower:
62		− return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
	67	+ return fmt.Sprintf("%s/%s", remoteString, dirhash), nil
63	68	case layoutModeDirectory:
64		− return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
	69	+ return fmt.Sprintf("%s/%s%s", remoteString, dirhash, key), nil
65	70	case layoutModeMixed:
66		− return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
	71	+ return fmt.Sprintf("%s/%s", remoteString, dirhash), nil
67	72	case layoutModeFrankencase:
68		− return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
	73	+ return fmt.Sprintf("%s/%s", remoteString, strings.ToLower(dirhash)), nil
69	74	default:
70	75	panic("unreachable")
71	76	}

cmd/gitannex: Port unit tests to fstest

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code