Showing the changes to internal/cli/app_test.go (1 of 5 files). Show all
Unified · Split
| @@ -3,9 +3,11 @@ package cli · 2 unmodified lines | |||
| 3 | import ( | 3 | import ( |
| 4 | "bytes" | 4 | "bytes" |
| 5 | "context" | 5 | "context" |
| 6 | "database/sql" | ||
| 6 | "encoding/json" | 7 | "encoding/json" |
| 7 | "errors" | 8 | "errors" |
| 8 | "fmt" | 9 | "fmt" |
| 10 | "io" | ||
| 9 | "net/http" | 11 | "net/http" |
| 10 | "net/http/httptest" | 12 | "net/http/httptest" |
| 11 | "os" | 13 | "os" |
| @@ -53,6 +55,231 @@ func TestInitWritesConfig(t *testing.T) { · 41 unmodified lines | |||
| 53 | } | 55 | } |
| 54 | } | 56 | } |
| 55 | 57 | ||
| 58 | func TestGitcrawlThreadBodyExpr(t *testing.T) { | ||
| 59 | ctx := context.Background() | ||
| 60 | cases := []struct { | ||
| 61 | name string | ||
| 62 | schema string | ||
| 63 | want string | ||
| 64 | }{ | ||
| 65 | {name: "body and excerpt", schema: "create table threads(id text primary key, body text, body_excerpt text)", want: "coalesce(body, body_excerpt, '')"}, | ||
| 66 | {name: "body only", schema: "create table threads(id text primary key, body text)", want: "coalesce(body, '')"}, | ||
| 67 | {name: "excerpt only", schema: "create table threads(id text primary key, body_excerpt text)", want: "coalesce(body_excerpt, '')"}, | ||
| 68 | {name: "neither", schema: "create table threads(id text primary key)", want: "''"}, | ||
| 69 | } | ||
| 70 | for _, tc := range cases { | ||
| 71 | t.Run(tc.name, func(t *testing.T) { | ||
| 72 | db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "threads.db")) | ||
| 73 | if err != nil { | ||
| 74 | t.Fatalf("open sqlite: %v", err) | ||
| 75 | } | ||
| 76 | defer db.Close() | ||
| 77 | if _, err := db.ExecContext(ctx, tc.schema); err != nil { | ||
| 78 | t.Fatalf("create schema: %v", err) | ||
| 79 | } | ||
| 80 | got, err := gitcrawlThreadBodyExpr(ctx, db) | ||
| 81 | if err != nil { | ||
| 82 | t.Fatalf("body expr: %v", err) | ||
| 83 | } | ||
| 84 | if got != tc.want { | ||
| 85 | t.Fatalf("body expr = %q, want %q", got, tc.want) | ||
| 86 | } | ||
| 87 | }) | ||
| 88 | } | ||
| 89 | } | ||
| 90 | |||
| 91 | func TestCloudSQLiteSnapshotFallbacks(t *testing.T) { | ||
| 92 | ctx := context.Background() | ||
| 93 | db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "closed.db")) | ||
| 94 | if err != nil { | ||
| 95 | t.Fatalf("open sqlite: %v", err) | ||
| 96 | } | ||
| 97 | if err := db.Close(); err != nil { | ||
| 98 | t.Fatalf("close sqlite: %v", err) | ||
| 99 | } | ||
| 100 | fallback := filepath.Join(t.TempDir(), "fallback.db") | ||
| 101 | got, cleanup, err := sqliteSnapshotPath(ctx, db, fallback) | ||
| 102 | if err != nil { | ||
| 103 | t.Fatalf("snapshot fallback: %v", err) | ||
| 104 | } | ||
| 105 | defer cleanup() | ||
| 106 | if got != fallback { | ||
| 107 | t.Fatalf("snapshot fallback path = %q, want %q", got, fallback) | ||
| 108 | } | ||
| 109 | if _, cleanup, err := sqliteSnapshotPath(ctx, db, ""); err == nil { | ||
| 110 | cleanup() | ||
| 111 | t.Fatal("snapshot without fallback unexpectedly succeeded") | ||
| 112 | } | ||
| 113 | } | ||
| 114 | |||
| 115 | func TestCloudSQLiteSnapshotVacuum(t *testing.T) { | ||
| 116 | ctx := context.Background() | ||
| 117 | db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "source.db")) | ||
| 118 | if err != nil { | ||
| 119 | t.Fatalf("open sqlite: %v", err) | ||
| 120 | } | ||
| 121 | defer db.Close() | ||
| 122 | if _, err := db.ExecContext(ctx, "create table repositories(id text primary key)"); err != nil { | ||
| 123 | t.Fatalf("create schema: %v", err) | ||
| 124 | } | ||
| 125 | if _, err := db.ExecContext(ctx, "insert into repositories(id) values('r1')"); err != nil { | ||
| 126 | t.Fatalf("seed schema: %v", err) | ||
| 127 | } | ||
| 128 | snapshot, cleanup, err := sqliteSnapshotPath(ctx, db, "") | ||
| 129 | if err != nil { | ||
| 130 | t.Fatalf("snapshot: %v", err) | ||
| 131 | } | ||
| 132 | defer cleanup() | ||
| 133 | snapshotDB, err := sql.Open("sqlite", snapshot) | ||
| 134 | if err != nil { | ||
| 135 | t.Fatalf("open snapshot: %v", err) | ||
| 136 | } | ||
| 137 | defer snapshotDB.Close() | ||
| 138 | var count int | ||
| 139 | if err := snapshotDB.QueryRowContext(ctx, "select count(*) from repositories").Scan(&count); err != nil { | ||
| 140 | t.Fatalf("query snapshot: %v", err) | ||
| 141 | } | ||
| 142 | if count != 1 { | ||
| 143 | t.Fatalf("snapshot count = %d, want 1", count) | ||
| 144 | } | ||
| 145 | } | ||
| 146 | |||
| 147 | func TestCloudFileSHA256(t *testing.T) { | ||
| 148 | path := filepath.Join(t.TempDir(), "payload.txt") | ||
| 149 | if err := os.WriteFile(path, []byte("abc"), 0o600); err != nil { | ||
| 150 | t.Fatalf("write payload: %v", err) | ||
| 151 | } | ||
| 152 | got, err := cloudFileSHA256(path) | ||
| 153 | if err != nil { | ||
| 154 | t.Fatalf("hash payload: %v", err) | ||
| 155 | } | ||
| 156 | const want = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" | ||
| 157 | if got != want { | ||
| 158 | t.Fatalf("hash = %q, want %q", got, want) | ||
| 159 | } | ||
| 160 | if _, err := cloudFileSHA256(filepath.Join(t.TempDir(), "missing")); err == nil { | ||
| 161 | t.Fatal("missing payload unexpectedly hashed") | ||
| 162 | } | ||
| 163 | } | ||
| 164 | |||
| 165 | func TestSendIngestRowsBatchesAndAcceptsEmpty(t *testing.T) { | ||
| 166 | ctx := context.Background() | ||
| 167 | var cursors []string | ||
| 168 | var finals []bool | ||
| 169 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { | ||
| 170 | if req.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw/ingest" { | ||
| 171 | http.NotFound(w, req) | ||
| 172 | return | ||
| 173 | } | ||
| 174 | var body crawlremote.IngestRequest | ||
| 175 | if err := json.NewDecoder(req.Body).Decode(&body); err != nil { | ||
| 176 | http.Error(w, err.Error(), http.StatusBadRequest) | ||
| 177 | return | ||
| 178 | } | ||
| 179 | cursors = append(cursors, body.Cursor) | ||
| 180 | finals = append(finals, body.Final) | ||
| 181 | w.Header().Set("Content-Type", "application/json") | ||
| 182 | _ = json.NewEncoder(w).Encode(crawlremote.IngestResult{Table: body.Table, RowsAccepted: int64(len(body.Rows)), Complete: body.Final}) | ||
| 183 | })) | ||
| 184 | defer server.Close() | ||
| 185 | client, err := crawlremote.NewClient(crawlremote.Options{ | ||
| 186 | Endpoint: server.URL, | ||
| 187 | TokenProvider: crawlremote.StaticToken("token"), | ||
| 188 | }) | ||
| 189 | if err != nil { | ||
| 190 | t.Fatalf("remote client: %v", err) | ||
| 191 | } | ||
| 192 | manifest := crawlremote.IngestManifest{App: "gitcrawl", Archive: "gitcrawl/openclaw"} | ||
| 193 | empty, err := sendIngestRows(ctx, client, "gitcrawl", "gitcrawl/openclaw", manifest, "repositories", gitcrawlRepositoryColumns, nil, false) | ||
| 194 | if err != nil { | ||
| 195 | t.Fatalf("empty ingest: %v", err) | ||
| 196 | } | ||
| 197 | if empty != 0 { | ||
| 198 | t.Fatalf("empty rows accepted = %d, want 0", empty) | ||
| 199 | } | ||
| 200 | rows := make([][]any, gitcrawlCloudBatchSize+1) | ||
| 201 | for i := range rows { | ||
| 202 | rows[i] = []any{fmt.Sprintf("r%d", i), "openclaw/repo", "openclaw", "repo", "", "", ""} | ||
| 203 | } | ||
| 204 | total, err := sendIngestRows(ctx, client, "gitcrawl", "gitcrawl/openclaw", manifest, "repositories", gitcrawlRepositoryColumns, rows, true) | ||
| 205 | if err != nil { | ||
| 206 | t.Fatalf("batched ingest: %v", err) | ||
| 207 | } | ||
| 208 | if total != int64(len(rows)) { | ||
| 209 | t.Fatalf("rows accepted = %d, want %d", total, len(rows)) | ||
| 210 | } | ||
| 211 | if got, want := strings.Join(cursors, ","), ",,250"; got != want { | ||
| 212 | t.Fatalf("cursors = %q, want %q", got, want) | ||
| 213 | } | ||
| 214 | if got, want := fmt.Sprint(finals), "[false false true]"; got != want { | ||
| 215 | t.Fatalf("finals = %s, want %s", got, want) | ||
| 216 | } | ||
| 217 | } | ||
| 218 | |||
| 219 | func TestRemoteValueCoercionHelpers(t *testing.T) { | ||
| 220 | value := map[string]any{ | ||
| 221 | "bool_string": "true", | ||
| 222 | "bool_number": json.Number("1"), | ||
| 223 | "int_string": "42", | ||
| 224 | "int_number": json.Number("43"), | ||
| 225 | "int64_string": "42000000000", | ||
| 226 | "float_string": "4.25", | ||
| 227 | "float_number": json.Number("5.5"), | ||
| 228 | } | ||
| 229 | if !boolValue(value, "bool_string") || !boolValue(value, "bool_number") || boolValue(value, "missing") { | ||
| 230 | t.Fatalf("bool coercion failed") | ||
| 231 | } | ||
| 232 | if got := intValue(value, "int_string"); got != 42 { | ||
| 233 | t.Fatalf("int string = %d, want 42", got) | ||
| 234 | } | ||
| 235 | if got := intValue(value, "int_number"); got != 43 { | ||
| 236 | t.Fatalf("int number = %d, want 43", got) | ||
| 237 | } | ||
| 238 | if got := int64Value(value, "int64_string"); got != 42000000000 { | ||
| 239 | t.Fatalf("int64 string = %d, want 42000000000", got) | ||
| 240 | } | ||
| 241 | if got := floatValue(value, "float_string"); got != 4.25 { | ||
| 242 | t.Fatalf("float string = %v, want 4.25", got) | ||
| 243 | } | ||
| 244 | if got := floatValue(value, "float_number"); got != 5.5 { | ||
| 245 | t.Fatalf("float number = %v, want 5.5", got) | ||
| 246 | } | ||
| 247 | ref := threadReference{Owner: "openclaw", Repo: "gitcrawl"} | ||
| 248 | if got := ref.FullName(); got != "openclaw/gitcrawl" { | ||
| 249 | t.Fatalf("full name = %q, want openclaw/gitcrawl", got) | ||
| 250 | } | ||
| 251 | if got := (threadReference{Owner: "openclaw"}).FullName(); got != "" { | ||
| 252 | t.Fatalf("partial full name = %q, want empty", got) | ||
| 253 | } | ||
| 254 | } | ||
| 255 | |||
| 256 | func TestUploadSQLiteArchiveReturnsRemoteError(t *testing.T) { | ||
| 257 | ctx := context.Background() | ||
| 258 | db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "source.db")) | ||
| 259 | if err != nil { | ||
| 260 | t.Fatalf("open sqlite: %v", err) | ||
| 261 | } | ||
| 262 | defer db.Close() | ||
| 263 | if _, err := db.ExecContext(ctx, "create table repositories(id text primary key)"); err != nil { | ||
| 264 | t.Fatalf("create schema: %v", err) | ||
| 265 | } | ||
| 266 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { | ||
| 267 | http.Error(w, "nope", http.StatusBadGateway) | ||
| 268 | })) | ||
| 269 | defer server.Close() | ||
| 270 | client, err := crawlremote.NewClient(crawlremote.Options{ | ||
| 271 | Endpoint: server.URL, | ||
| 272 | TokenProvider: crawlremote.StaticToken("token"), | ||
| 273 | }) | ||
| 274 | if err != nil { | ||
| 275 | t.Fatalf("remote client: %v", err) | ||
| 276 | } | ||
| 277 | _, err = uploadSQLiteArchive(ctx, client, "gitcrawl", "gitcrawl/openclaw", db, "", crawlremote.IngestManifest{SchemaName: "gitcrawl-cloud-v1"}) | ||
| 278 | if err == nil { | ||
| 279 | t.Fatal("upload unexpectedly succeeded") | ||
| 280 | } | ||
| 281 | } | ||
| 282 | |||
| 56 | func TestInitDefaultOutputIsHumanReadable(t *testing.T) { | 283 | func TestInitDefaultOutputIsHumanReadable(t *testing.T) { |
| 57 | dir := t.TempDir() | 284 | dir := t.TempDir() |
| 58 | configPath := filepath.Join(dir, "config.toml") | 285 | configPath := filepath.Join(dir, "config.toml") |
| @@ -227,11 +454,36 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 168 unmodified lines | |||
| 227 | tokenEnv := "GITCRAWL_TEST_PUBLISH_TOKEN" | 454 | tokenEnv := "GITCRAWL_TEST_PUBLISH_TOKEN" |
| 228 | t.Setenv(tokenEnv, "publish-token") | 455 | t.Setenv(tokenEnv, "publish-token") |
| 229 | seenTables := map[string]crawlremote.IngestRequest{} | 456 | seenTables := map[string]crawlremote.IngestRequest{} |
| 457 | var sawSQLiteUpload bool | ||
| 230 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | 458 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 231 | if got := r.Header.Get("authorization"); got != "Bearer publish-token" { | 459 | if got := r.Header.Get("authorization"); got != "Bearer publish-token" { |
| 232 | http.Error(w, "missing bearer", http.StatusUnauthorized) | 460 | http.Error(w, "missing bearer", http.StatusUnauthorized) |
| 233 | return | 461 | return |
| 234 | } | 462 | } |
| 463 | if r.Method == http.MethodPut && r.URL.EscapedPath() == "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/sqlite" { | ||
| 464 | sawSQLiteUpload = true | ||
| 465 | payload, err := io.ReadAll(r.Body) | ||
| 466 | if err != nil { | ||
| 467 | http.Error(w, err.Error(), http.StatusBadRequest) | ||
| 468 | return | ||
| 469 | } | ||
| 470 | if !bytes.HasPrefix(payload, []byte("SQLite format 3")) { | ||
| 471 | http.Error(w, "not sqlite", http.StatusBadRequest) | ||
| 472 | return | ||
| 473 | } | ||
| 474 | if r.Header.Get("x-crawl-content-sha256") == "" { | ||
| 475 | http.Error(w, "missing hash", http.StatusBadRequest) | ||
| 476 | return | ||
| 477 | } | ||
| 478 | w.Header().Set("content-type", "application/json") | ||
| 479 | _ = json.NewEncoder(w).Encode(crawlremote.SQLiteUploadResult{ | ||
| 480 | App: "gitcrawl", | ||
| 481 | Archive: "gitcrawl/openclaw__openclaw", | ||
| 482 | Complete: true, | ||
| 483 | Object: &crawlremote.SQLiteObject{Key: "v1/gitcrawl/gitcrawl%2Fopenclaw__openclaw/sqlite/current.db", Size: int64(len(payload))}, | ||
| 484 | }) | ||
| 485 | return | ||
| 486 | } | ||
| 235 | if r.Method != http.MethodPost || r.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/ingest" { | 487 | if r.Method != http.MethodPost || r.URL.EscapedPath() != "/v1/apps/gitcrawl/archives/gitcrawl%2Fopenclaw__openclaw/ingest" { |
| 236 | http.NotFound(w, r) | 488 | http.NotFound(w, r) |
| 237 | return | 489 | return |
| @@ -277,6 +529,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 39 unmodified lines | |||
| 277 | if !seenTables["threads"].Final { | 529 | if !seenTables["threads"].Final { |
| 278 | t.Fatalf("threads batch should finish ingest") | 530 | t.Fatalf("threads batch should finish ingest") |
| 279 | } | 531 | } |
| 532 | if !sawSQLiteUpload { | ||
| 533 | t.Fatalf("cloud publish did not upload sqlite object") | ||
| 534 | } | ||
| 280 | var payload map[string]any | 535 | var payload map[string]any |
| 281 | if err := json.Unmarshal(out.Bytes(), &payload); err != nil { | 536 | if err := json.Unmarshal(out.Bytes(), &payload); err != nil { |
| 282 | t.Fatalf("decode output: %v\n%s", err, out.String()) | 537 | t.Fatalf("decode output: %v\n%s", err, out.String()) |
| @@ -284,6 +539,9 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 1 unmodified line | |||
| 284 | if payload["repositories"] != float64(1) || payload["threads"] != float64(3) { | 539 | if payload["repositories"] != float64(1) || payload["threads"] != float64(3) { |
| 285 | t.Fatalf("unexpected output: %#v", payload) | 540 | t.Fatalf("unexpected output: %#v", payload) |
| 286 | } | 541 | } |
| 542 | if payload["sqlite_object"] == nil { | ||
| 543 | t.Fatalf("missing sqlite object output: %#v", payload) | ||
| 544 | } | ||
| 287 | } | 545 | } |
| 288 | 546 | ||
| 289 | func TestRemoteLoginStoresKeyringToken(t *testing.T) { | 547 | func TestRemoteLoginStoresKeyringToken(t *testing.T) { |