| @@ -3,9 +3,11 @@ package cli · 2 unmodified lines |
| 3 | 3 | import ( |
| 4 | 4 | "bytes" |
| 5 | 5 | "context" |
| 6 | + "database/sql" |
| 6 | 7 | "encoding/json" |
| 7 | 8 | "errors" |
| 8 | 9 | "fmt" |
| 10 | + "io" |
| 9 | 11 | "net/http" |
| 10 | 12 | "net/http/httptest" |
| 11 | 13 | "os" |
| @@ -53,6 +55,231 @@ func TestInitWritesConfig(t *testing.T) { · 41 unmodified lines |
| 53 | 55 | } |
| 54 | 56 | } |
| 55 | 57 | |
| 58 | +func TestGitcrawlThreadBodyExpr(t *testing.T) { |
| 59 | + ctx := context.Background() |
| 60 | + cases := []struct { |
| 61 | + name string |
| 62 | + schema string |
| 63 | + want string |
| 64 | + }{ |
| 65 | + {name: "body and excerpt", schema: "create table threads(id text primary key, body text, body_excerpt text)", want: "coalesce(body, body_excerpt, '')"}, |
| 66 | + {name: "body only", schema: "create table threads(id text primary key, body text)", want: "coalesce(body, '')"}, |
| 67 | + {name: "excerpt only", schema: "create table threads(id text primary key, body_excerpt text)", want: "coalesce(body_excerpt, '')"}, |
| 68 | + {name: "neither", schema: "create table threads(id text primary key)", want: "''"}, |
| 69 | + } |
| 70 | + for _, tc := range cases { |
| 71 | + t.Run(tc.name, func(t *testing.T) { |
| 72 | + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "threads.db")) |
| 73 | + if err != nil { |
| 74 | + t.Fatalf("open sqlite: %v", err) |
| 75 | + } |
| 76 | + defer db.Close() |
| 77 | + if _, err := db.ExecContext(ctx, tc.schema); err != nil { |
| 78 | + t.Fatalf("create schema: %v", err) |
| 79 | + } |
| 80 | + got, err := gitcrawlThreadBodyExpr(ctx, db) |
| 81 | + if err != nil { |
| 82 | + t.Fatalf("body expr: %v", err) |
| 83 | + } |
| 84 | + if got != tc.want { |
| 85 | + t.Fatalf("body expr = %q, want %q", got, tc.want) |
| 86 | + } |
| 87 | + }) |
| 88 | + } |
| 89 | +} |
| 90 | + |
| 91 | +func TestCloudSQLiteSnapshotFallbacks(t *testing.T) { |
| 92 | + ctx := context.Background() |
| 93 | + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "closed.db")) |
| 94 | + if err != nil { |
| 95 | + t.Fatalf("open sqlite: %v", err) |
| 96 | + } |
| 97 | + if err := db.Close(); err != nil { |
| 98 | + t.Fatalf("close sqlite: %v", err) |
| 99 | + } |
| 100 | + fallback := filepath.Join(t.TempDir(), "fallback.db") |
| 101 | + got, cleanup, err := sqliteSnapshotPath(ctx, db, fallback) |
| 102 | + if err != nil { |
| 103 | + t.Fatalf("snapshot fallback: %v", err) |
| 104 | + } |
| 105 | + defer cleanup() |
| 106 | + if got != fallback { |
| 107 | + t.Fatalf("snapshot fallback path = %q, want %q", got, fallback) |
| 108 | + } |
| 109 | + if _, cleanup, err := sqliteSnapshotPath(ctx, db, ""); err == nil { |
| 110 | + cleanup() |
| 111 | + t.Fatal("snapshot without fallback unexpectedly succeeded") |
| 112 | + } |
| 113 | +} |
| 114 | + |
| 115 | +func TestCloudSQLiteSnapshotVacuum(t *testing.T) { |
| 116 | + ctx := context.Background() |
| 117 | + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "source.db")) |
| 118 | + if err != nil { |
| 119 | + t.Fatalf("open sqlite: %v", err) |
| 120 | + } |
| 121 | + defer db.Close() |
| 122 | + if _, err := db.ExecContext(ctx, "create table repositories(id text primary key)"); err != nil { |
| 123 | + t.Fatalf("create schema: %v", err) |
| 124 | + } |
| 125 | + if _, err := db.ExecContext(ctx, "insert into repositories(id) values('r1')"); err != nil { |
| 126 | + t.Fatalf("seed schema: %v", err) |
| 127 | + } |
| 128 | + snapshot, cleanup, err := sqliteSnapshotPath(ctx, db, "") |
| 129 | + if err != nil { |
| 130 | + t.Fatalf("snapshot: %v", err) |
| 131 | + } |
| 132 | + defer cleanup() |
| 133 | + snapshotDB, err := sql.Open("sqlite", snapshot) |
| 134 | + if err != nil { |
| 135 | + t.Fatalf("open snapshot: %v", err) |
| 136 | + } |
| 137 | + defer snapshotDB.Close() |
| 138 | + var count int |
| 139 | + if err := snapshotDB.QueryRowContext(ctx, "select count(*) from repositories").Scan(&count); err != nil { |
| 140 | + t.Fatalf("query snapshot: %v", err) |
| 141 | + } |
| 142 | + if count != 1 { |
| 143 | + t.Fatalf("snapshot count = %d, want 1", count) |
| 144 | + } |
| 145 | +} |
| 146 | + |
| 147 | +func TestCloudFileSHA256(t *testing.T) { |
| 148 | + path := filepath.Join(t.TempDir(), "payload.txt") |
| 149 | + if err := os.WriteFile(path, []byte("abc"), 0o600); err != nil { |
| 150 | + t.Fatalf("write payload: %v", err) |
| 151 | + } |
| 152 | + got, err := cloudFileSHA256(path) |
| 153 | + if err != nil { |
| 154 | + t.Fatalf("hash payload: %v", err) |
| 155 | + } |
| 156 | + const want = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" |
| 157 | + if got != want { |
| 158 | + t.Fatalf("hash = %q, want %q", got, want) |
| 159 | + } |
| 160 | + if _, err := cloudFileSHA256(filepath.Join(t.TempDir(), "missing")); err == nil { |
| 161 | + t.Fatal("missing payload unexpectedly hashed") |
| 162 | + } |
| 163 | +} |
| 164 | + |
| 165 | +func TestSendIngestRowsBatchesAndAcceptsEmpty(t *testing.T) { |
| 166 | + ctx := context.Background() |
| 167 | + var cursors []string |
| 168 | + var finals []bool |
| 169 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
| 170 | + if req.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw/ingest" { |
| 171 | + http.NotFound(w, req) |
| 172 | + return |
| 173 | + } |
| 174 | + var body crawlremote.IngestRequest |
| 175 | + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { |
| 176 | + http.Error(w, err.Error(), http.StatusBadRequest) |
| 177 | + return |
| 178 | + } |
| 179 | + cursors = append(cursors, body.Cursor) |
| 180 | + finals = append(finals, body.Final) |
| 181 | + w.Header().Set("Content-Type", "application/json") |
| 182 | + _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{Table: body.Table, RowsAccepted: int64(len(body.Rows)), Complete: body.Final}) |
| 183 | + })) |
| 184 | + defer server.Close() |
| 185 | + client, err := crawlremote.NewClient(crawlremote.Options{ |
| 186 | + Endpoint: server.URL, |
| 187 | + TokenProvider: crawlremote.StaticToken("token"), |
| 188 | + }) |
| 189 | + if err != nil { |
| 190 | + t.Fatalf("remote client: %v", err) |
| 191 | + } |
| 192 | + manifest := crawlremote.IngestManifest{App: "gitcrawl", Archive: "gitcrawl/openclaw"} |
| 193 | + empty, err := sendIngestRows(ctx, client, "gitcrawl", "gitcrawl/openclaw", manifest, "repositories", gitcrawlRepositoryColumns, nil, false) |
| 194 | + if err != nil { |
| 195 | + t.Fatalf("empty ingest: %v", err) |
| 196 | + } |
| 197 | + if empty != 0 { |
| 198 | + t.Fatalf("empty rows accepted = %d, want 0", empty) |
| 199 | + } |
| 200 | + rows := make([][]any, gitcrawlCloudBatchSize+1) |
| 201 | + for i := range rows { |
| 202 | + rows[i] = []any{fmt.Sprintf("r%d", i), "openclaw/repo", "openclaw", "repo", "", "", ""} |
| 203 | + } |
| 204 | + total, err := sendIngestRows(ctx, client, "gitcrawl", "gitcrawl/openclaw", manifest, "repositories", gitcrawlRepositoryColumns, rows, true) |
| 205 | + if err != nil { |
| 206 | + t.Fatalf("batched ingest: %v", err) |
| 207 | + } |
| 208 | + if total != int64(len(rows)) { |
| 209 | + t.Fatalf("rows accepted = %d, want %d", total, len(rows)) |
| 210 | + } |
| 211 | + if got, want := strings.Join(cursors, ","), ",,250"; got != want { |
| 212 | + t.Fatalf("cursors = %q, want %q", got, want) |
| 213 | + } |
| 214 | + if got, want := fmt.Sprint(finals), "[false false true]"; got != want { |
| 215 | + t.Fatalf("finals = %s, want %s", got, want) |
| 216 | + } |
| 217 | +} |
| 218 | + |
| 219 | +func TestRemoteValueCoercionHelpers(t *testing.T) { |
| 220 | + value := map[string]any{ |
| 221 | + "bool_string": "true", |
| 222 | + "bool_number": json.Number("1"), |
| 223 | + "int_string": "42", |
| 224 | + "int_number": json.Number("43"), |
| 225 | + "int64_string": "42000000000", |
| 226 | + "float_string": "4.25", |
| 227 | + "float_number": json.Number("5.5"), |
| 228 | + } |
| 229 | + if !boolValue(value, "bool_string") || !boolValue(value, "bool_number") || boolValue(value, "missing") { |
| 230 | + t.Fatalf("bool coercion failed") |
| 231 | + } |
| 232 | + if got := intValue(value, "int_string"); got != 42 { |
| 233 | + t.Fatalf("int string = %d, want 42", got) |
| 234 | + } |
| 235 | + if got := intValue(value, "int_number"); got != 43 { |
| 236 | + t.Fatalf("int number = %d, want 43", got) |
| 237 | + } |
| 238 | + if got := int64Value(value, "int64_string"); got != 42000000000 { |
| 239 | + t.Fatalf("int64 string = %d, want 42000000000", got) |
| 240 | + } |
| 241 | + if got := floatValue(value, "float_string"); got != 4.25 { |
| 242 | + t.Fatalf("float string = %v, want 4.25", got) |
| 243 | + } |
| 244 | + if got := floatValue(value, "float_number"); got != 5.5 { |
| 245 | + t.Fatalf("float number = %v, want 5.5", got) |
| 246 | + } |
| 247 | + ref := threadReference{Owner: "openclaw", Repo: "gitcrawl"} |
| 248 | + if got := ref.FullName(); got != "openclaw/gitcrawl" { |
| 249 | + t.Fatalf("full name = %q, want openclaw/gitcrawl", got) |
| 250 | + } |
| 251 | + if got := (threadReference{Owner: "openclaw"}).FullName(); got != "" { |
| 252 | + t.Fatalf("partial full name = %q, want empty", got) |
| 253 | + } |
| 254 | +} |
| 255 | + |
| 256 | +func TestUploadSQLiteArchiveReturnsRemoteError(t *testing.T) { |
| 257 | + ctx := context.Background() |
| 258 | + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "source.db")) |
| 259 | + if err != nil { |
| 260 | + t.Fatalf("open sqlite: %v", err) |
| 261 | + } |
| 262 | + defer db.Close() |
| 263 | + if _, err := db.ExecContext(ctx, "create table repositories(id text primary key)"); err != nil { |
| 264 | + t.Fatalf("create schema: %v", err) |
| 265 | + } |
| 266 | + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { |
| 267 | + http.Error(w, "nope", http.StatusBadGateway) |
| 268 | + })) |
| 269 | + defer server.Close() |
| 270 | + client, err := crawlremote.NewClient(crawlremote.Options{ |
| 271 | + Endpoint: server.URL, |
| 272 | + TokenProvider: crawlremote.StaticToken("token"), |
| 273 | + }) |
| 274 | + if err != nil { |
| 275 | + t.Fatalf("remote client: %v", err) |
| 276 | + } |
| 277 | + _, err = uploadSQLiteArchive(ctx, client, "gitcrawl", "gitcrawl/openclaw", db, "", crawlremote.IngestManifest{SchemaName: "gitcrawl-cloud-v1"}) |
| 278 | + if err == nil { |
| 279 | + t.Fatal("upload unexpectedly succeeded") |
| 280 | + } |
| 281 | +} |
| 282 | + |
| 56 | 283 | func TestInitDefaultOutputIsHumanReadable(t *testing.T) { |
| 57 | 284 | dir := t.TempDir() |
| 58 | 285 | configPath := filepath.Join(dir, "config.toml") |
| @@ -227,11 +454,36 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 168 unmodified lines |
| 227 | 454 | tokenEnv := "GITCRAWL_TEST_PUBLISH_TOKEN" |
| 228 | 455 | t.Setenv(tokenEnv, "publish-token") |
| 229 | 456 | seenTables := map[string]crawlremote.IngestRequest{} |
| 457 | + var sawSQLiteUpload bool |
| 230 | 458 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 231 | 459 | if got := r.Header.Get("authorization"); got != "Bearer publish-token" { |
| 232 | 460 | http.Error(w, "missing bearer", http.StatusUnauthorized) |
| 233 | 461 | return |
| 234 | 462 | } |
| 463 | + if r.Method == http.MethodPut && r.URL.EscapedPath() == "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/sqlite" { |
| 464 | + sawSQLiteUpload = true |
| 465 | + payload, err := io.ReadAll(r.Body) |
| 466 | + if err != nil { |
| 467 | + http.Error(w, err.Error(), http.StatusBadRequest) |
| 468 | + return |
| 469 | + } |
| 470 | + if !bytes.HasPrefix(payload, []byte("SQLite format 3")) { |
| 471 | + http.Error(w, "not sqlite", http.StatusBadRequest) |
| 472 | + return |
| 473 | + } |
| 474 | + if r.Header.Get("x-crawl-content-sha256") == "" { |
| 475 | + http.Error(w, "missing hash", http.StatusBadRequest) |
| 476 | + return |
| 477 | + } |
| 478 | + w.Header().Set("content-type", "application/json") |
| 479 | + _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ |
| 480 | + App: "gitcrawl", |
| 481 | + Archive: "gitcrawl/openclaw__openclaw", |
| 482 | + Complete: true, |
| 483 | + Object: &crawlremote.SQLiteObject{Key: "v1/gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/current.db", Size: int64(len(payload))}, |
| 484 | + }) |
| 485 | + return |
| 486 | + } |
| 235 | 487 | if r.Method != http.MethodPost || r.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/ingest" { |
| 236 | 488 | http.NotFound(w, r) |
| 237 | 489 | return |
| @@ -277,6 +529,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 39 unmodified lines |
| 277 | 529 | if !seenTables["threads"].Final { |
| 278 | 530 | t.Fatalf("threads batch should finish ingest") |
| 279 | 531 | } |
| 532 | + if !sawSQLiteUpload { |
| 533 | + t.Fatalf("cloud publish did not upload sqlite object") |
| 534 | + } |
| 280 | 535 | var payload map[string]any |
| 281 | 536 | if err := json.Unmarshal(out.Bytes(), &payload); err != nil { |
| 282 | 537 | t.Fatalf("decode output: %v\n%s", err, out.String()) |
| @@ -284,6 +539,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 1 unmodified line |
| 284 | 539 | if payload["repositories"] != float64(1) || payload["threads"] != float64(3) { |
| 285 | 540 | t.Fatalf("unexpected output: %#v", payload) |
| 286 | 541 | } |
| 542 | + if payload["sqlite_object"] == nil { |
| 543 | + t.Fatalf("missing sqlite object output: %#v", payload) |
| 544 | + } |
| 287 | 545 | } |
| 288 | 546 | |
| 289 | 547 | func TestRemoteLoginStoresKeyringToken(t *testing.T) { |