package cli import (	"bytes"	"compress/gzip"	"context"	"encoding/json"	"errors"	"fmt"	"io"	"os"	"path/filepath"	"strings"	"testing"	"time" 	"github.com/openclaw/gitcrawl/internal/config") func writeTestCompressedPortableSource(t *testing.T, dbPath string) string {	t.Helper()	manifestPath := writeTestPortableManifest(t, dbPath)	manifest, ok, err := readPortableDBManifest(manifestPath)	if err != nil || !ok {		t.Fatalf("read direct portable manifest: ok=%v err=%v", ok, err)	}	archivePath := dbPath + ".gz"	source, err := os.Open(dbPath)	if err != nil {		t.Fatalf("open portable db: %v", err)	}	archive, err := os.Create(archivePath)	if err != nil {		_ = source.Close()		t.Fatalf("create portable archive: %v", err)	}	writer := gzip.NewWriter(archive)	if _, err := io.Copy(writer, source); err != nil {		_ = writer.Close()		_ = archive.Close()		_ = source.Close()		t.Fatalf("compress portable db: %v", err)	}	if err := writer.Close(); err != nil {		_ = archive.Close()		_ = source.Close()		t.Fatalf("close portable gzip writer: %v", err)	}	if err := archive.Close(); err != nil {		_ = source.Close()		t.Fatalf("close portable archive: %v", err)	}	if err := source.Close(); err != nil {		t.Fatalf("close portable db: %v", err)	}	archiveInfo, err := os.Stat(archivePath)	if err != nil {		t.Fatalf("stat portable archive: %v", err)	}	archiveSum, err := fileSHA256(archivePath)	if err != nil {		t.Fatalf("hash portable archive: %v", err)	}	manifest.Compression = "gzip"	manifest.ArchivePath = filepath.Base(archivePath)	manifest.ArchiveBytes = archiveInfo.Size()	manifest.ArchiveSHA256 = fmt.Sprintf("%x", archiveSum)	data, err := json.Marshal(manifest)	if err != nil {		t.Fatalf("marshal compressed portable manifest: %v", err)	}	if err := os.WriteFile(manifestPath, data, 0o644); err != nil {		t.Fatalf("write compressed portable manifest: %v", err)	}	if err := os.Remove(dbPath); err != nil {		t.Fatalf("remove direct portable db: %v", err)	}	return archivePath} func TestLocalRuntimeDBTarget(t *testing.T) {	direct := localRuntime{Config: config.Config{DBPath: "/data/gitcrawl.db"}}	if got, want := direct.dbTarget(), (dbTargetInfo{DBTarget: "direct", DBTargetPath: "/data/gitcrawl.db"}); got != want {		t.Fatalf("direct db target = %+v, want %+v", got, want)	} 	redirected := localRuntime{		Config:       config.Config{DBPath: "/runtime/store/data/gitcrawl.db"},		SourceDBPath: "/checkout/data/gitcrawl.db",		RemoteSource: true,	}	want := dbTargetInfo{		DBTarget:         "runtime-mirror",		DBTargetPath:     "/runtime/store/data/gitcrawl.db",		PortableSourceDB: "/checkout/data/gitcrawl.db",	}	if got := redirected.dbTarget(); got != want {		t.Fatalf("redirected db target = %+v, want %+v", got, want)	}} func TestSweepOrphanPortableRuntimeTempFiles(t *testing.T) {	dir := t.TempDir()	mirrorPath := filepath.Join(dir, "x.db")	old := time.Now().Add(-2 * portableRuntimeTempMaxAge)	oldMatching := []string{".x.db.tmp-123", ".x.db.tmp-123-wal", ".x.db.tmp-123-shm"}	oldPreserved := []string{"old.db", ".other.db.tmp-9", ".notes.tmp-backup"}	for _, name := range append(append([]string(nil), oldMatching...), oldPreserved...) {		path := filepath.Join(dir, name)		if err := os.WriteFile(path, []byte(name), 0o600); err != nil {			t.Fatalf("write %s: %v", name, err)		}		if err := os.Chtimes(path, old, old); err != nil {			t.Fatalf("backdate %s: %v", name, err)		}	}	fresh := filepath.Join(dir, ".x.db.tmp-fresh")	if err := os.WriteFile(fresh, []byte("fresh"), 0o600); err != nil {		t.Fatalf("write fresh temp: %v", err)	} 	sweepOrphanPortableRuntimeTempFiles(mirrorPath, portableRuntimeTempMaxAge) 	for _, name := range oldMatching {		if _, err := os.Stat(filepath.Join(dir, name)); !os.IsNotExist(err) {			t.Fatalf("old matching temp %s still exists: %v", name, err)		}	}	preserved := []string{fresh}	for _, name := range oldPreserved {		preserved = append(preserved, filepath.Join(dir, name))	}	for _, path := range preserved {		if _, err := os.Stat(path); err != nil {			t.Fatalf("preserved file %s: %v", path, err)		}	}} func TestCopySQLiteFileAtomicVerifiedRemovesTempFiles(t *testing.T) {	dir := t.TempDir()	source := filepath.Join(dir, "source.db")	target := filepath.Join(dir, "target.db")	seedPortableThread(t, source, 1, "copy temp cleanup")	if err := copySQLiteFileAtomicVerified(context.Background(), source, target); err != nil {		t.Fatalf("copy verified sqlite file: %v", err)	}	matches, err := filepath.Glob(filepath.Join(dir, ".target.db.tmp-*"))	if err != nil {		t.Fatalf("glob temp files: %v", err)	}	if len(matches) != 0 {		t.Fatalf("verified copy left temp files: %v", matches)	}	invalidSource := filepath.Join(dir, "invalid.db")	failedTarget := filepath.Join(dir, "failed.db")	if err := os.WriteFile(invalidSource, []byte("not sqlite"), 0o600); err != nil {		t.Fatalf("write invalid source: %v", err)	}	if err := copySQLiteFileAtomicVerified(context.Background(), invalidSource, failedTarget); err == nil {		t.Fatal("invalid sqlite source should fail validation")	}	matches, err = filepath.Glob(filepath.Join(dir, ".failed.db.tmp-*"))	if err != nil {		t.Fatalf("glob failed-copy temp files: %v", err)	}	if len(matches) != 0 {		t.Fatalf("failed verified copy left temp files: %v", matches)	} 	tempPath := filepath.Join(dir, ".failed.db.tmp-123")	for _, suffix := range []string{"-wal", "-shm"} {		if err := os.WriteFile(tempPath+suffix, []byte("sidecar"), 0o600); err != nil {			t.Fatalf("write temp sidecar: %v", err)		}	}	removeSQLiteTempSidecars(tempPath)	for _, suffix := range []string{"-wal", "-shm"} {		if _, err := os.Stat(tempPath + suffix); !os.IsNotExist(err) {			t.Fatalf("temp sidecar %s still exists: %v", suffix, err)		}	}} func TestRefreshPortableRuntimeDBInflatesCompressedSource(t *testing.T) {	ctx := context.Background()	dir := t.TempDir()	checkoutDir := filepath.Join(dir, "checkout")	sourceDB := filepath.Join(checkoutDir, "data", "openclaw__openclaw.sync.db")	mirrorDB := filepath.Join(dir, "runtime", "openclaw__openclaw.sync.db")	seedPortableThread(t, sourceDB, 7, "compressed portable source")	writeTestCompressedPortableSource(t, sourceDB) 	changed, err := refreshPortableRuntimeDB(		ctx,		sourceDB,		mirrorDB,		false,		filepath.Join(dir, "config.toml"),	)	if err != nil || !changed {		t.Fatalf("refresh compressed portable source: changed=%v err=%v", changed, err)	}	if err := sqliteStoreHealth(ctx, mirrorDB); err != nil {		t.Fatalf("inflated mirror health: %v", err)	}	manifest, ok, err := readPortableDBManifest(portableDBManifestPath(sourceDB))	if err != nil || !ok {		t.Fatalf("read compressed manifest: ok=%v err=%v", ok, err)	}	sum, err := fileSHA256(mirrorDB)	if err != nil {		t.Fatalf("hash inflated mirror: %v", err)	}	if got := fmt.Sprintf("%x", sum); got != manifest.SHA256 {		t.Fatalf("inflated mirror sha256 = %s, want %s", got, manifest.SHA256)	}} func TestCompressedPortableSourceFailureKeepsLastGoodMirror(t *testing.T) {	ctx := context.Background()	dir := t.TempDir()	checkoutDir := filepath.Join(dir, "checkout")	sourceDB := filepath.Join(checkoutDir, "data", "openclaw__openclaw.sync.db")	mirrorDB := filepath.Join(dir, "runtime", "openclaw__openclaw.sync.db")	seedPortableThread(t, sourceDB, 8, "new compressed content")	archivePath := writeTestCompressedPortableSource(t, sourceDB)	seedPortableThread(t, mirrorDB, 9, "last good mirror")	past := time.Now().Add(-time.Hour)	if err := os.Chtimes(mirrorDB, past, past); err != nil {		t.Fatalf("age mirror: %v", err)	}	before, err := fileSHA256(mirrorDB)	if err != nil {		t.Fatalf("hash mirror before: %v", err)	}	if err := os.WriteFile(archivePath, []byte("truncated"), 0o644); err != nil {		t.Fatalf("corrupt portable archive: %v", err)	} 	changed, err := refreshPortableRuntimeDB(		ctx,		sourceDB,		mirrorDB,		false,		filepath.Join(dir, "config.toml"),	)	if err == nil || changed || !strings.Contains(err.Error(), "portable manifest mismatch") {		t.Fatalf("corrupt compressed source: changed=%v err=%v", changed, err)	}	after, err := fileSHA256(mirrorDB)	if err != nil {		t.Fatalf("hash mirror after: %v", err)	}	if after != before {		t.Fatal("failed compressed refresh replaced the last good mirror")	}} func TestCompressedPortableManifestRejectsEscapingArchivePath(t *testing.T) {	dir := t.TempDir()	dbPath := filepath.Join(dir, "data", "openclaw__openclaw.sync.db")	if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil {		t.Fatalf("mkdir data: %v", err)	}	manifest := portableDBManifest{		Schema:        "gitcrawl-portable-sync-v2",		OutputBytes:   1,		SHA256:        strings.Repeat("a", 64),		Compression:   "gzip",		ArchivePath:   "../escape.db.gz",		ArchiveBytes:  1,		ArchiveSHA256: strings.Repeat("b", 64),	}	data, err := json.Marshal(manifest)	if err != nil {		t.Fatalf("marshal manifest: %v", err)	}	if err := os.WriteFile(portableDBManifestPath(dbPath), data, 0o644); err != nil {		t.Fatalf("write manifest: %v", err)	}	if _, _, _, err := portableSourceArtifact(dbPath); err == nil ||		!strings.Contains(err.Error(), "archivePath escapes") {		t.Fatalf("escaping archivePath error = %v", err)	}} func TestCompressedPortableManifestRejectsInvalidMetadata(t *testing.T) {	dir := t.TempDir()	dbPath := filepath.Join(dir, "data", "openclaw__openclaw.sync.db")	if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil {		t.Fatalf("mkdir data: %v", err)	}	manifestPath := portableDBManifestPath(dbPath)	base := portableDBManifest{		Schema:        "gitcrawl-portable-sync-v2",		OutputBytes:   1,		SHA256:        strings.Repeat("a", 64),		Compression:   "gzip",		ArchivePath:   filepath.Base(dbPath) + ".gz",		ArchiveBytes:  1,		ArchiveSHA256: strings.Repeat("b", 64),	}	cases := []struct {		name    string		mutate  func(*portableDBManifest)		message string	}{		{			name:    "unsupported compression",			mutate:  func(manifest *portableDBManifest) { manifest.Compression = "zstd" },			message: "unsupported compression",		},		{			name:    "missing archive path",			mutate:  func(manifest *portableDBManifest) { manifest.ArchivePath = "" },			message: "archivePath must be relative",		},		{			name:    "absolute archive path",			mutate:  func(manifest *portableDBManifest) { manifest.ArchivePath = "/tmp/store.db.gz" },			message: "archivePath must be relative",		},		{			name:    "drive letter archive path",			mutate:  func(manifest *portableDBManifest) { manifest.ArchivePath = `C:\store.db.gz` },			message: "archivePath must be relative",		},		{			name:    "current directory archive path",			mutate:  func(manifest *portableDBManifest) { manifest.ArchivePath = "." },			message: "archivePath escapes the store",		},		{			name:    "missing archive bytes",			mutate:  func(manifest *portableDBManifest) { manifest.ArchiveBytes = 0 },			message: "archiveBytes missing",		},		{			name:    "missing archive sha",			mutate:  func(manifest *portableDBManifest) { manifest.ArchiveSHA256 = "" },			message: "archiveSha256 missing",		},	}	for _, tc := range cases {		t.Run(tc.name, func(t *testing.T) {			manifest := base			tc.mutate(&manifest)			data, err := json.Marshal(manifest)			if err != nil {				t.Fatalf("marshal manifest: %v", err)			}			if err := os.WriteFile(manifestPath, data, 0o644); err != nil {				t.Fatalf("write manifest: %v", err)			}			if _, _, _, err := portableSourceArtifact(dbPath); err == nil ||				!strings.Contains(err.Error(), tc.message) {				t.Fatalf("portable source error = %v, want %q", err, tc.message)			}		})	}} func TestCompressedPortableArchiveRejectsTampering(t *testing.T) {	dir := t.TempDir()	dbPath := filepath.Join(dir, "data", "openclaw__openclaw.sync.db")	seedPortableThread(t, dbPath, 10, "compressed portable source")	archivePath := writeTestCompressedPortableSource(t, dbPath)	manifestPath := portableDBManifestPath(dbPath)	manifest, ok, err := readPortableDBManifest(manifestPath)	if err != nil || !ok {		t.Fatalf("read compressed manifest: ok=%v err=%v", ok, err)	} 	manifest.ArchiveBytes++	if err := validatePortableArchive(archivePath, manifest); err == nil ||		!strings.Contains(err.Error(), "archive size") {		t.Fatalf("archive size error = %v", err)	} 	manifest.ArchiveBytes--	manifest.ArchiveSHA256 = strings.Repeat("0", 64)	if err := validatePortableArchive(archivePath, manifest); err == nil ||		!strings.Contains(err.Error(), "archive sha256") {		t.Fatalf("archive hash error = %v", err)	}} func TestCompressedPortableArchiveRejectsInvalidPayload(t *testing.T) {	dir := t.TempDir()	dbPath := filepath.Join(dir, "data", "openclaw__openclaw.sync.db")	seedPortableThread(t, dbPath, 11, "compressed portable source")	archivePath := writeTestCompressedPortableSource(t, dbPath)	manifestPath := portableDBManifestPath(dbPath)	targetPath := filepath.Join(dir, "runtime", filepath.Base(dbPath)) 	writeArchive := func(data []byte, outputBytes int64) {		t.Helper()		if err := os.WriteFile(archivePath, data, 0o644); err != nil {			t.Fatalf("write archive: %v", err)		}		manifest, ok, err := readPortableDBManifest(manifestPath)		if err != nil || !ok {			t.Fatalf("read compressed manifest: ok=%v err=%v", ok, err)		}		info, err := os.Stat(archivePath)		if err != nil {			t.Fatalf("stat archive: %v", err)		}		sum, err := fileSHA256(archivePath)		if err != nil {			t.Fatalf("hash archive: %v", err)		}		manifest.OutputBytes = outputBytes		manifest.ArchiveBytes = info.Size()		manifest.ArchiveSHA256 = fmt.Sprintf("%x", sum)		encoded, err := json.Marshal(manifest)		if err != nil {			t.Fatalf("marshal compressed manifest: %v", err)		}		if err := os.WriteFile(manifestPath, encoded, 0o644); err != nil {			t.Fatalf("write compressed manifest: %v", err)		}	} 	writeArchive([]byte("not gzip"), 1)	if _, err := stagePortableSQLiteSourceTemp(dbPath, targetPath, 0o600); err == nil ||		!strings.Contains(err.Error(), "open portable gzip archive") {		t.Fatalf("invalid gzip error = %v", err)	} 	var compressed bytes.Buffer	writer := gzip.NewWriter(&compressed)	if _, err := writer.Write([]byte("short")); err != nil {		t.Fatalf("write gzip payload: %v", err)	}	if err := writer.Close(); err != nil {		t.Fatalf("close gzip payload: %v", err)	}	writeArchive(compressed.Bytes(), 10)	if _, err := stagePortableSQLiteSourceTemp(dbPath, targetPath, 0o600); err == nil ||		!strings.Contains(err.Error(), "inflated size") {		t.Fatalf("inflated size error = %v", err)	}} func TestValidateCompressedPortableSQLiteSource(t *testing.T) {	ctx := context.Background()	dir := t.TempDir()	dbPath := filepath.Join(dir, "data", "openclaw__openclaw.sync.db")	seedPortableThread(t, dbPath, 12, "compressed validation source")	writeTestCompressedPortableSource(t, dbPath) 	if err := validatePortableSQLiteSourceFile(ctx, dbPath, dbPath); err != nil {		t.Fatalf("validate compressed portable source: %v", err)	}} func TestPreserveMalformedCompressedPortableSource(t *testing.T) {	dir := t.TempDir()	root := filepath.Join(dir, "checkout")	dbPath := filepath.Join(root, "data", "openclaw__openclaw.sync.db")	seedPortableThread(t, dbPath, 13, "compressed evidence source")	archivePath := writeTestCompressedPortableSource(t, dbPath) 	backupDir, err := preserveMalformedPortableDB(root, dbPath)	if err != nil {		t.Fatalf("preserve compressed portable source: %v", err)	}	for _, name := range []string{		filepath.Base(archivePath) + ".malformed",		filepath.Base(portableDBManifestPath(dbPath)),	} {		if _, err := os.Stat(filepath.Join(backupDir, name)); err != nil {			t.Fatalf("stat preserved evidence %s: %v", name, err)		}	}} func writeTestPortableManifest(t *testing.T, dbPath string) string {	t.Helper()	info, err := os.Stat(dbPath)	if err != nil {		t.Fatalf("stat db for manifest: %v", err)	}	sum, err := fileSHA256(dbPath)	if err != nil {		t.Fatalf("hash db for manifest: %v", err)	}	manifest := portableDBManifest{		Schema:      "gitcrawl-portable-sync-v2",		ExportedAt:  time.Now().UTC().Format(time.RFC3339Nano),		OutputPath:  filepath.Base(dbPath),		OutputBytes: info.Size(),		SHA256:      fmt.Sprintf("%x", sum),		QuickCheck:  "ok",	}	data, err := json.Marshal(manifest)	if err != nil {		t.Fatalf("marshal manifest: %v", err)

Lines 1–500 of 1073. Load more