package cli import (	"bytes"	"compress/gzip"	"context"	"database/sql"	"encoding/json"	"errors"	"fmt"	"io"	"net/http"	"net/http/httptest"	"os"	"os/exec"	"path/filepath"	"slices"	"sort"	"strconv"	"strings"	"sync/atomic"	"testing"	"time" 	"github.com/openclaw/crawlkit/control"	crawlremote "github.com/openclaw/crawlkit/remote"	clusterer "github.com/openclaw/gitcrawl/internal/cluster"	"github.com/openclaw/gitcrawl/internal/config"	gh "github.com/openclaw/gitcrawl/internal/github"	"github.com/openclaw/gitcrawl/internal/store"	"github.com/zalando/go-keyring") func TestInitWritesConfig(t *testing.T) {	dir := t.TempDir()	configPath := filepath.Join(dir, "config.toml")	dbPath := filepath.Join(dir, "custom", "gitcrawl.db")	wantVectorDir := filepath.Join(filepath.Dir(dbPath), "vectors")	app := New()	var stdout bytes.Buffer	app.Stdout = &stdout 	err := app.Run(context.Background(), []string{"--config", configPath, "--json", "init", "--db", dbPath})	if err != nil {		t.Fatalf("run init: %v", err)	}	if !strings.Contains(stdout.String(), `"config_path"`) {		t.Fatalf("expected json init output, got %q", stdout.String())	}	var result initResult	if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {		t.Fatalf("decode init output: %v\n%s", err, stdout.String())	}	if result.VectorDir != wantVectorDir {		t.Fatalf("vector dir = %q, want %q", result.VectorDir, wantVectorDir)	}	if _, err := os.Stat(wantVectorDir); err != nil {		t.Fatalf("stat vector dir: %v", err)	}} func TestInitRelativeDBPathIsAbsoluteAndWritable(t *testing.T) {	ctx := context.Background()	dir := t.TempDir()	workDir := filepath.Join(dir, "work")	if err := os.MkdirAll(workDir, 0o755); err != nil {		t.Fatalf("mkdir work dir: %v", err)	}	t.Chdir(workDir)	configPath := filepath.Join(dir, "config.toml") 	if err := New().Run(ctx, []string{"--config", configPath, "init", "--db", "archive.db"}); err != nil {		t.Fatalf("init relative db: %v", err)	}	cfg, err := config.Load(configPath)	if err != nil {		t.Fatalf("load config: %v", err)	}	wantDBPath := filepath.Join(workDir, "archive.db")	if cfg.DBPath != wantDBPath {		t.Fatalf("db path = %q, want %q", cfg.DBPath, wantDBPath)	}	if cfg.VectorDir != filepath.Join(workDir, "vectors") {		t.Fatalf("vector dir = %q, want sibling of resolved db", cfg.VectorDir)	}	st, err := store.Open(ctx, cfg.DBPath)	if err != nil {		t.Fatalf("open resolved db: %v", err)	}	if err := st.Close(); err != nil {		t.Fatalf("close resolved db: %v", err)	}	if info, err := os.Stat(wantDBPath); err != nil || info.Size() == 0 {		t.Fatalf("resolved db was not initialized: info=%v err=%v", info, err)	}} func TestInitRelativeRuntimeDirIsAbsolute(t *testing.T) {	dir := t.TempDir()	t.Chdir(dir)	configPath := filepath.Join(dir, "config.toml") 	if err := New().Run(context.Background(), []string{"--config", configPath, "init", "--runtime-dir", "runtime"}); err != nil {		t.Fatalf("init relative runtime: %v", err)	}	cfg, err := config.Load(configPath)	if err != nil {		t.Fatalf("load config: %v", err)	}	wantRoot := filepath.Join(dir, "runtime")	if cfg.DBPath != filepath.Join(wantRoot, "gitcrawl.db") || cfg.CacheDir != filepath.Join(wantRoot, "cache") || cfg.VectorDir != filepath.Join(wantRoot, "vectors") || cfg.LogDir != filepath.Join(wantRoot, "logs") {		t.Fatalf("relative runtime paths not anchored under %q: %+v", wantRoot, cfg)	}} func TestInitHomeRelativePathsExpandBeforeAnchoring(t *testing.T) {	homeDir := t.TempDir()	workDir := t.TempDir()	t.Setenv("HOME", homeDir)	t.Chdir(workDir) 	dbConfigPath := filepath.Join(workDir, "db.toml")	if err := New().Run(context.Background(), []string{"--config", dbConfigPath, "init", "--db", "~/archive.db"}); err != nil {		t.Fatalf("init home-relative db: %v", err)	}	dbConfig, err := config.Load(dbConfigPath)	if err != nil {		t.Fatalf("load db config: %v", err)	}	if want := filepath.Join(homeDir, "archive.db"); dbConfig.DBPath != want {		t.Fatalf("home-relative db path = %q, want %q", dbConfig.DBPath, want)	} 	runtimeConfigPath := filepath.Join(workDir, "runtime.toml")	if err := New().Run(context.Background(), []string{"--config", runtimeConfigPath, "init", "--runtime-dir", "~/runtime"}); err != nil {		t.Fatalf("init home-relative runtime: %v", err)	}	runtimeConfig, err := config.Load(runtimeConfigPath)	if err != nil {		t.Fatalf("load runtime config: %v", err)	}	wantRuntimeRoot := filepath.Join(homeDir, "runtime")	if runtimeConfig.DBPath != filepath.Join(wantRuntimeRoot, "gitcrawl.db") || runtimeConfig.CacheDir != filepath.Join(wantRuntimeRoot, "cache") || runtimeConfig.VectorDir != filepath.Join(wantRuntimeRoot, "vectors") || runtimeConfig.LogDir != filepath.Join(wantRuntimeRoot, "logs") {		t.Fatalf("home-relative runtime paths not anchored under %q: %+v", wantRuntimeRoot, runtimeConfig)	}} func TestInitResolvesWorkingDirectoryOnlyForRelativePaths(t *testing.T) {	app := New()	app.getWorkingDirectory = func() (string, error) {		return "", errors.New("working directory unavailable")	} 	relativeCases := []struct {		name string		args []string		flag string	}{		{name: "database", args: []string{"init", "--db", "archive.db"}, flag: "--db"},		{name: "runtime", args: []string{"init", "--runtime-dir", "runtime"}, flag: "--runtime-dir"},		{name: "portable store", args: []string{"init", "--portable-store", "https://example.com/store.git", "--store-dir", "store"}, flag: "--store-dir"},	}	for _, tc := range relativeCases {		err := app.Run(context.Background(), tc.args)		if err == nil || !strings.Contains(err.Error(), "resolve "+tc.flag+" path") {			t.Errorf("%s error = %v, want %s resolution failure", tc.name, err, tc.flag)		}	} 	dir := t.TempDir()	absoluteConfig := filepath.Join(dir, "absolute.toml")	if err := app.Run(context.Background(), []string{"--config", absoluteConfig, "init", "--db", filepath.Join(dir, "archive.db")}); err != nil {		t.Fatalf("absolute db init consulted working directory: %v", err)	}	remoteConfig := filepath.Join(dir, "remote.toml")	if err := app.Run(context.Background(), []string{"--config", remoteConfig, "init", "--remote", "https://archive.example.com", "--archive", "example"}); err != nil {		t.Fatalf("remote init consulted working directory: %v", err)	}} func TestInitRejectsNonFilesystemDBValues(t *testing.T) {	for _, dbPath := range []string{":memory:", "file:archive.db", "file:/tmp/archive.db?mode=rwc"} {		err := New().Run(context.Background(), []string{"init", "--db", dbPath})		if err == nil || !strings.Contains(err.Error(), "--db requires a filesystem path") {			t.Errorf("init --db %q error = %v", dbPath, err)		}	}} func TestInitRejectsHomeRelativePathsWithoutHomeDirectory(t *testing.T) {	t.Setenv("HOME", "")	tests := []struct {		name string		args []string	}{		{name: "database", args: []string{"init", "--db", "~/archive.db"}},		{name: "runtime", args: []string{"init", "--runtime-dir", "~/runtime"}},		{name: "portable store", args: []string{"init", "--portable-store", "https://example.com/store.git", "--store-dir", "~/store"}},	}	for _, tc := range tests {		err := New().Run(context.Background(), tc.args)		if err == nil || !strings.Contains(err.Error(), "home directory unavailable") {			t.Errorf("%s error = %v, want unavailable-home failure", tc.name, err)		}	}} func TestInitRuntimeDirIsolatesRuntimePaths(t *testing.T) {	dir := t.TempDir()	configPath := filepath.Join(dir, "config.toml")	runtimeDir := filepath.Join(dir, "runtime")	app := New()	var stdout bytes.Buffer	app.Stdout = &stdout 	if err := app.Run(context.Background(), []string{"--config", configPath, "--json", "init", "--runtime-dir", runtimeDir}); err != nil {		t.Fatalf("run isolated init: %v", err)	}	var result initResult	if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {		t.Fatalf("decode init output: %v\n%s", err, stdout.String())	}	if result.RuntimeDir != runtimeDir {		t.Fatalf("runtime dir = %q, want %q", result.RuntimeDir, runtimeDir)	}	wantPaths := map[string]string{		"db":      filepath.Join(runtimeDir, "gitcrawl.db"),		"cache":   filepath.Join(runtimeDir, "cache"),		"vectors": filepath.Join(runtimeDir, "vectors"),		"logs":    filepath.Join(runtimeDir, "logs"),	}	gotPaths := map[string]string{		"db":      result.DBPath,		"cache":   result.CacheDir,		"vectors": result.VectorDir,		"logs":    result.LogDir,	}	for name, want := range wantPaths {		if gotPaths[name] != want {			t.Errorf("%s path = %q, want %q", name, gotPaths[name], want)		}	}	for _, path := range []string{filepath.Dir(wantPaths["db"]), wantPaths["cache"], wantPaths["vectors"], wantPaths["logs"]} {		if info, err := os.Stat(path); err != nil || !info.IsDir() {			t.Errorf("runtime directory %q: info=%v err=%v", path, info, err)		}	}	loaded, err := config.Load(configPath)	if err != nil {		t.Fatalf("load isolated config: %v", err)	}	if loaded.DBPath != wantPaths["db"] || loaded.CacheDir != wantPaths["cache"] || loaded.VectorDir != wantPaths["vectors"] || loaded.LogDir != wantPaths["logs"] {		t.Fatalf("isolated config paths = %+v", loaded)	}} func TestInitRuntimeDirRejectsOtherArchiveModes(t *testing.T) {	dir := t.TempDir()	app := New()	err := app.Run(context.Background(), []string{"--config", filepath.Join(dir, "config.toml"), "init", "--runtime-dir", filepath.Join(dir, "runtime"), "--db", filepath.Join(dir, "other.db")})	if err == nil || !strings.Contains(err.Error(), "use only one of") {		t.Fatalf("runtime/db conflict error = %v", err)	}} func TestGitcrawlThreadBodyExpr(t *testing.T) {	ctx := context.Background()	cases := []struct {		name   string		schema string		want   string	}{		{name: "body and excerpt", schema: "create table threads(id text primary key, body text, body_excerpt text)", want: "coalesce(body, body_excerpt, '')"},		{name: "body only", schema: "create table threads(id text primary key, body text)", want: "coalesce(body, '')"},		{name: "excerpt only", schema: "create table threads(id text primary key, body_excerpt text)", want: "coalesce(body_excerpt, '')"},		{name: "neither", schema: "create table threads(id text primary key)", want: "''"},	}	for _, tc := range cases {		t.Run(tc.name, func(t *testing.T) {			db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "threads.db"))			if err != nil {				t.Fatalf("open sqlite: %v", err)			}			defer db.Close()			if _, err := db.ExecContext(ctx, tc.schema); err != nil {				t.Fatalf("create schema: %v", err)			}			got, err := gitcrawlThreadBodyExpr(ctx, db)			if err != nil {				t.Fatalf("body expr: %v", err)			}			if got != tc.want {				t.Fatalf("body expr = %q, want %q", got, tc.want)			}		})	}} func TestCloudSQLiteSnapshotFallbacks(t *testing.T) {	ctx := context.Background()	db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "closed.db"))	if err != nil {		t.Fatalf("open sqlite: %v", err)	}	if err := db.Close(); err != nil {		t.Fatalf("close sqlite: %v", err)	}	fallback := filepath.Join(t.TempDir(), "fallback.db")	got, cleanup, err := sqliteSnapshotPath(ctx, db, fallback)	if err != nil {		t.Fatalf("snapshot fallback: %v", err)	}	defer cleanup()	if got != fallback {		t.Fatalf("snapshot fallback path = %q, want %q", got, fallback)	}	if _, cleanup, err := sqliteSnapshotPath(ctx, db, ""); err == nil {		cleanup()		t.Fatal("snapshot without fallback unexpectedly succeeded")	}} func TestCloudSQLiteSnapshotVacuum(t *testing.T) {	ctx := context.Background()	db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "source.db"))	if err != nil {		t.Fatalf("open sqlite: %v", err)	}	defer db.Close()	if _, err := db.ExecContext(ctx, "create table repositories(id text primary key)"); err != nil {		t.Fatalf("create schema: %v", err)	}	if _, err := db.ExecContext(ctx, "insert into repositories(id) values('r1')"); err != nil {		t.Fatalf("seed schema: %v", err)	}	snapshot, cleanup, err := sqliteSnapshotPath(ctx, db, "")	if err != nil {		t.Fatalf("snapshot: %v", err)	}	defer cleanup()	snapshotDB, err := sql.Open("sqlite", snapshot)	if err != nil {		t.Fatalf("open snapshot: %v", err)	}	defer snapshotDB.Close()	var count int	if err := snapshotDB.QueryRowContext(ctx, "select count(*) from repositories").Scan(&count); err != nil {		t.Fatalf("query snapshot: %v", err)	}	if count != 1 {		t.Fatalf("snapshot count = %d, want 1", count)	}} func TestCloudSQLiteSnapshotDropsLocalCodeCorpus(t *testing.T) {	ctx := context.Background()	dbPath := filepath.Join(t.TempDir(), "source.db")	st, err := store.Open(ctx, dbPath)	if err != nil {		t.Fatalf("open store: %v", err)	}	repoID, err := st.UpsertRepository(ctx, store.Repository{Owner: "openclaw", Name: "gitcrawl", FullName: "openclaw/gitcrawl", RawJSON: "{}", UpdatedAt: "2026-06-06T00:00:00Z"})	if err != nil {		t.Fatalf("repo: %v", err)	}	if _, err := st.ReplaceCodeSnapshot(ctx, store.CodeSnapshot{		RepoID: repoID, SourceRoot: "/private/repo", GitSHA: "abc",		FileCount: 1, ByteCount: 6, IndexedAt: "2026-06-06T00:00:00Z",	}, []store.CodeDocument{{Path: "secret.txt", Language: "txt", ContentHash: "h", Text: "secret", ByteSize: 6, UpdatedAt: "2026-06-06T00:00:00Z"}}); err != nil {		t.Fatalf("replace code snapshot: %v", err)	}	if _, err := st.DB().ExecContext(ctx, `		insert into threads(			id, repo_id, github_id, number, kind, state, title, body, html_url,			labels_json, assignees_json, raw_json, content_hash, updated_at		) values(			1, ?, 'pr-1', 1, 'pull_request', 'open', 'PR', 'body',			'https://github.com/openclaw/gitcrawl/pull/1', '[]', '[]',			'{"secret":"thread"}', 'hash', '2026-06-06T00:00:00Z'		);		insert into pull_request_details(			thread_id, repo_id, number, raw_json, fetched_at, updated_at		) values(			1, ?, 1, '{"secret":"detail"}',			'2026-06-06T00:00:00Z', '2026-06-06T00:00:00Z'		);			insert into pull_request_files(				thread_id, position, path, patch, raw_json, fetched_at			) values(				1, 0, 'secret.go', '@@ private source', '{"secret":"file"}',				'2026-06-06T00:00:00Z'			);			insert into pull_request_review_threads(				thread_id, review_thread_id, path, comments_json, raw_json, fetched_at			) values(				1, 'review-thread-1', 'secret.go',				'[{"body":"review body","diffHunk":"@@ private review source"}]',				'{"secret":"review-thread"}', '2026-06-06T00:00:00Z'			);			insert into blobs(				sha256, media_type, compression, size_bytes, storage_kind, inline_text, created_at			) values(			'blob-hash', 'application/json', 'none', 6, 'inline', 'secret',			'2026-06-06T00:00:00Z'		);			insert into sync_runs(				repo_id, scope, status, started_at, finished_at, stats_json, error_text			) values(				?, 'all', 'failed', '2026-06-06T00:00:00Z',				'2026-06-06T00:01:00Z',				'{"message":"/private/host/token","threads":1}', '/private/host/token'			);			insert into summary_runs(				repo_id, scope, status, started_at, finished_at, stats_json, error_text			) values(				?, 'all', 'failed', '2026-06-06T00:00:00Z',				'2026-06-06T00:01:00Z',				'{"failures":[{"message":"provider secret"}]}', 'provider secret'			);			insert into embedding_runs(				repo_id, scope, status, started_at, finished_at, stats_json, error_text			) values(				?, 'all', 'failed', '2026-06-06T00:00:00Z',				'2026-06-06T00:01:00Z',				'{"failures":[{"message":"/private/embed"}]}', '/private/embed'			);			insert into cluster_runs(				repo_id, scope, status, started_at, finished_at, stats_json, error_text			) values(				?, 'all', 'failed', '2026-06-06T00:00:00Z',				'2026-06-06T00:01:00Z',				'{"message":"/private/cluster"}', '/private/cluster'			);			create table portable_metadata(key text primary key, value text not null);			insert into portable_metadata(key, value) values('source_path', '/private/archive.db')		`, repoID, repoID, repoID, repoID, repoID, repoID); err != nil {		t.Fatalf("seed private cloud payloads: %v", err)	}	snapshotPath, cleanup, err := cloudSQLiteSnapshotPath(ctx, st.DB(), dbPath)	if err != nil {		t.Fatalf("cloud snapshot: %v", err)	}	defer cleanup()	snapshotDB, err := sql.Open("sqlite", snapshotPath)	if err != nil {		t.Fatalf("open cloud snapshot: %v", err)	}	defer snapshotDB.Close()	for _, table := range []string{		"code_snapshots",		"code_documents",		"code_documents_fts",		"thread_code_snapshots",		"thread_changed_files",		"thread_hunk_signatures",	} {		var count int		if err := snapshotDB.QueryRowContext(ctx, `select count(*) from sqlite_schema where name = ?`, table).Scan(&count); err != nil {			t.Fatalf("inspect %s: %v", table, err)		}		if count != 0 {			t.Fatalf("cloud snapshot retained %s", table)		}	}	var repositoryRawJSON, threadRawJSON, detailRawJSON, filePatch, fileRawJSON string	if err := snapshotDB.QueryRowContext(ctx, `		select repository.raw_json, thread.raw_json, detail.raw_json, file.patch, file.raw_json		from repositories repository		join threads thread on thread.repo_id = repository.id		join pull_request_details detail on detail.thread_id = thread.id		join pull_request_files file on file.thread_id = thread.id	`).Scan(&repositoryRawJSON, &threadRawJSON, &detailRawJSON, &filePatch, &fileRawJSON); err != nil {		t.Fatalf("inspect scrubbed cloud payloads: %v", err)	}	if repositoryRawJSON != "" || threadRawJSON != "" || detailRawJSON != "" || fileRawJSON != "" {		t.Fatalf(			"cloud snapshot retained raw payloads: repository=%q thread=%q detail=%q file=%q",			repositoryRawJSON,			threadRawJSON,			detailRawJSON,			fileRawJSON,		)	}	if filePatch != "@@ private source" {		t.Fatalf("cloud snapshot patch = %q, want full structured patch text", filePatch)	}	var blobCount int	if err := snapshotDB.QueryRowContext(ctx, `select count(*) from blobs`).Scan(&blobCount); err != nil {		t.Fatalf("inspect cloud blobs: %v", err)	}	if blobCount != 0 {		t.Fatalf("cloud snapshot retained %d blobs", blobCount)	}	var reviewComments string	if err := snapshotDB.QueryRowContext(ctx, `			select comments_json from pull_request_review_threads limit 1		`).Scan(&reviewComments); err != nil {		t.Fatalf("inspect scrubbed review comments: %v", err)	}	if reviewComments != "" {

Lines 1–500 of 9226. Load more