Unified · Split
| @@ -41,7 +41,7 @@ These work on every command. · 40 unmodified lines | ||
| 41 | 41 | |
| 42 | 42 | | Command | Purpose | Docs | |
| 43 | 43 | | --- | --- | --- | |
| 44 | −| `gitcrawl sync owner/repo [--state --since --numbers <refs> --limit --include-comments --include-pr-details --with pr-details --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) | | |
| 44 | +| `gitcrawl sync owner/repo [--state --since --numbers <refs> --limit --include-comments --include-pr-details --with pr-details --progress-file <absolute-path> --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) | | |
| 45 | 45 | | `gitcrawl sync-failures owner/repo [--include-resolved --limit N --json]` | List failed PR hydration attempts and optional resolved history | [Sync](/sync/#hydration-depth) | |
| 46 | 46 | | `gitcrawl coverage [owner/repo \| --repos owner/a,owner/b] [--min-missing-pr-details N --json]` | Report archive, PR-detail, and enrichment coverage/freshness | — | |
| 47 | 47 | | `gitcrawl fill-pr-details owner/repo [--limit --order --batch-size --reserve-rate-limit --include-comments --json-progress --json]` | Hydrate locally missing pull request detail rows in bounded batches | — | |
| @@ -82,6 +82,7 @@ issue or pull request URLs. · 81 unmodified lines | ||
| 82 | 82 | | `--include-comments` | Issue comments, PR review comments, reviews | |
| 83 | 83 | | `--include-pr-details` | PR files, commits, status checks, workflow runs | |
| 84 | 84 | | `--with pr-details` | Same as `--include-pr-details` (gh-style flag) | |
| 85 | +| `--progress-file <absolute-path>` | Atomically publish sanitized machine-readable activity | | |
| 85 | 86 | |
| 86 | 87 | PR details land in `pr_files`, `pr_commits`, `pr_checks`, and `pr_runs` tables for local review, search, clustering, and TUI workflows. |
| 87 | 88 | |
| @@ -91,6 +92,30 @@ Use `gitcrawl coverage [owner/repo] --json` to inspect archive completeness afte · 3 unmodified lines | ||
| 91 | 92 | |
| 92 | 93 | `--include-code` is accepted for compatibility but is currently a no-op. |
| 93 | 94 | |
| 95 | +## Machine-readable progress | |
| 96 | + | |
| 97 | +Automation may pass an absolute `--progress-file` path. Gitcrawl atomically | |
| 98 | +replaces that private file as synchronization advances, then leaves the | |
| 99 | +terminal snapshot in place: | |
| 100 | + | |
| 101 | +```json | |
| 102 | +{ | |
| 103 | + "schema": "gitcrawl.sync-progress.v1", | |
| 104 | + "repository": "owner/repo", | |
| 105 | + "state": "running", | |
| 106 | + "stage": "syncing_threads", | |
| 107 | + "issues_received": 56, | |
| 108 | + "pull_requests_received": 238, | |
| 109 | + "comments_received": 350, | |
| 110 | + "observed_at": "2026-07-31T04:17:36.000000000Z" | |
| 111 | +} | |
| 112 | +``` | |
| 113 | + | |
| 114 | +`state` is `running`, `succeeded`, or `failed`; `stage` is `connecting`, | |
| 115 | +`syncing_threads`, or `finalizing`. Counts describe upstream activity observed | |
| 116 | +by the current process, not rows committed to SQLite. The snapshot never | |
| 117 | +contains credentials, issue content, request URLs, or raw diagnostics. | |
| 118 | + | |
| 94 | 119 | ## Limit and pagination |
| 95 | 120 | |
| 96 | 121 | ```bash |
| @@ -2884,8 +2884,9 @@ func (a *App) runSync(ctx context.Context, args []string) error { · 2883 unmodified lines | ||
| 2884 | 2884 | includeComments := fs.Bool("include-comments", false, "hydrate issue comments, PR reviews, and PR review comments") |
| 2885 | 2885 | includePRDetails := fs.Bool("include-pr-details", false, "hydrate PR files, commits, checks, and workflow runs") |
| 2886 | 2886 | withRaw := fs.String("with", "", "extra hydration: pr-details") |
| 2887 | + progressFile := fs.String("progress-file", "", "write an atomic sanitized sync progress snapshot") | |
| 2887 | 2888 | fs.Bool("include-code", false, "accepted for compatibility; code hydration is not implemented yet") |
| 2888 | − if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"numbers": true, "since": true, "state": true, "limit": true, "with": true})); err != nil { | |
| 2889 | + if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"numbers": true, "since": true, "state": true, "limit": true, "with": true, "progress-file": true})); err != nil { | |
| 2889 | 2890 | return usageErr(err) |
| 2890 | 2891 | } |
| 2891 | 2892 | a.applyCommandJSON(*jsonOut) |
| @@ -2908,6 +2909,18 @@ func (a *App) runSync(ctx context.Context, args []string) error { · 16 unmodified lines | ||
| 2908 | 2909 | if err != nil { |
| 2909 | 2910 | return usageErr(err) |
| 2910 | 2911 | } |
| 2912 | + progress, err := newSyncProgressWriter( | |
| 2913 | + *progressFile, | |
| 2914 | + owner+"/"+repo, | |
| 2915 | + ) | |
| 2916 | + if err != nil { | |
| 2917 | + return usageErr(err) | |
| 2918 | + } | |
| 2919 | + if err := progress.report(syncer.SyncProgress{ | |
| 2920 | + Stage: syncer.SyncProgressConnecting, | |
| 2921 | + }); err != nil { | |
| 2922 | + return err | |
| 2923 | + } | |
| 2911 | 2924 | |
| 2912 | 2925 | stats, target, err := a.syncRepository(ctx, owner, repo, syncOptions{ |
| 2913 | 2926 | Since: strings.TrimSpace(*since), |
| @@ -2916,8 +2929,15 @@ func (a *App) runSync(ctx context.Context, args []string) error { · 2 unmodified lines | ||
| 2916 | 2929 | Numbers: numbers, |
| 2917 | 2930 | IncludeComments: *includeComments, |
| 2918 | 2931 | IncludePRDetails: *includePRDetails || with["pr-details"], |
| 2932 | + Progress: progress.report, | |
| 2919 | 2933 | }) |
| 2920 | 2934 | if err != nil { |
| 2935 | + if progressErr := progress.finish(syncProgressFailed); progressErr != nil { | |
| 2936 | + return progressErr | |
| 2937 | + } | |
| 2938 | + return err | |
| 2939 | + } | |
| 2940 | + if err := progress.finish(syncProgressSucceeded); err != nil { | |
| 2921 | 2941 | return err |
| 2922 | 2942 | } |
| 2923 | 2943 | result := struct { |
| @@ -2936,6 +2956,7 @@ type syncOptions struct { · 12 unmodified lines | ||
| 2936 | 2956 | IncludePRDetails bool |
| 2937 | 2957 | Quiet bool |
| 2938 | 2958 | RateLimitReserve int |
| 2959 | + Progress syncer.SyncProgressReporter | |
| 2939 | 2960 | } |
| 2940 | 2961 | |
| 2941 | 2962 | type fillPRDetailsResult struct { |
| @@ -3241,6 +3262,7 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy · 299 unmodified lines | ||
| 3241 | 3262 | IncludePRDetails: options.IncludePRDetails, |
| 3242 | 3263 | Reporter: reporter, |
| 3243 | 3264 | Logger: logger, |
| 3265 | + Progress: options.Progress, | |
| 3244 | 3266 | }) |
| 3245 | 3267 | if err != nil { |
| 3246 | 3268 | return syncer.Stats{}, target, err |
| @@ -6281,6 +6281,7 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 6280 unmodified lines | ||
| 6281 | 6281 | dir := t.TempDir() |
| 6282 | 6282 | configPath := filepath.Join(dir, "config.toml") |
| 6283 | 6283 | dbPath := filepath.Join(dir, "gitcrawl.db") |
| 6284 | + progressPath := filepath.Join(dir, "sync-progress.json") | |
| 6284 | 6285 | init := New() |
| 6285 | 6286 | if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil { |
| 6286 | 6287 | t.Fatalf("init: %v", err) |
| @@ -6318,7 +6319,14 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 31 unmodified lines | ||
| 6318 | 6319 | run := New() |
| 6319 | 6320 | var stdout bytes.Buffer |
| 6320 | 6321 | run.Stdout = &stdout |
| 6321 | − if err := run.Run(ctx, []string{"--config", configPath, "sync", "openclaw/openclaw", "--numbers", "101,102", "--include-comments", "--json"}); err != nil { | |
| 6322 | + if err := run.Run(ctx, []string{ | |
| 6323 | + "--config", configPath, | |
| 6324 | + "sync", "openclaw/openclaw", | |
| 6325 | + "--numbers", "101,102", | |
| 6326 | + "--include-comments", | |
| 6327 | + "--progress-file", progressPath, | |
| 6328 | + "--json", | |
| 6329 | + }); err != nil { | |
| 6322 | 6330 | t.Fatalf("sync: %v", err) |
| 6323 | 6331 | } |
| 6324 | 6332 | var stats struct { |
| @@ -6337,6 +6345,24 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 12 unmodified lines | ||
| 6337 | 6345 | if stats.DBTarget != "direct" || stats.DBTargetPath != dbPath { |
| 6338 | 6346 | t.Fatalf("sync db target = %+v", stats) |
| 6339 | 6347 | } |
| 6348 | + progressData, err := os.ReadFile(progressPath) | |
| 6349 | + if err != nil { | |
| 6350 | + t.Fatalf("read sync progress: %v", err) | |
| 6351 | + } | |
| 6352 | + var syncProgress syncProgressSnapshot | |
| 6353 | + if err := json.Unmarshal(progressData, &syncProgress); err != nil { | |
| 6354 | + t.Fatalf("decode sync progress: %v", err) | |
| 6355 | + } | |
| 6356 | + if syncProgress.Schema != syncProgressSchemaV1 || | |
| 6357 | + syncProgress.Repository != "openclaw/openclaw" || | |
| 6358 | + syncProgress.State != syncProgressSucceeded || | |
| 6359 | + syncProgress.Stage != syncProgressFinalizing || | |
| 6360 | + syncProgress.IssuesReceived != 1 || | |
| 6361 | + syncProgress.PullRequestsReceived != 1 || | |
| 6362 | + syncProgress.CommentsReceived != 4 || | |
| 6363 | + syncProgress.ObservedAt == "" { | |
| 6364 | + t.Fatalf("sync progress = %+v", syncProgress) | |
| 6365 | + } | |
| 6340 | 6366 | |
| 6341 | 6367 | st, err := store.Open(ctx, dbPath) |
| 6342 | 6368 | if err != nil { |
| @@ -6352,6 +6378,53 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 9 unmodified lines | ||
| 6352 | 6378 | } |
| 6353 | 6379 | } |
| 6354 | 6380 | |
| 6381 | +func TestSyncCommandPublishesSanitizedFailureProgress(t *testing.T) { | |
| 6382 | + ctx := context.Background() | |
| 6383 | + dir := t.TempDir() | |
| 6384 | + configPath := filepath.Join(dir, "config.toml") | |
| 6385 | + dbPath := filepath.Join(dir, "gitcrawl.db") | |
| 6386 | + progressPath := filepath.Join(dir, "sync-progress.json") | |
| 6387 | + if err := New().Run(ctx, []string{ | |
| 6388 | + "--config", configPath, "init", "--db", dbPath, | |
| 6389 | + }); err != nil { | |
| 6390 | + t.Fatalf("init: %v", err) | |
| 6391 | + } | |
| 6392 | + server := httptest.NewServer(http.HandlerFunc(func( | |
| 6393 | + w http.ResponseWriter, | |
| 6394 | + _ *http.Request, | |
| 6395 | + ) { | |
| 6396 | + http.Error(w, "private upstream diagnostic", http.StatusBadGateway) | |
| 6397 | + })) | |
| 6398 | + defer server.Close() | |
| 6399 | + t.Setenv("GITHUB_TOKEN", "test-gh-token") | |
| 6400 | + t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL) | |
| 6401 | + | |
| 6402 | + err := New().Run(ctx, []string{ | |
| 6403 | + "--config", configPath, | |
| 6404 | + "sync", "openclaw/gitcrawl", | |
| 6405 | + "--progress-file", progressPath, | |
| 6406 | + }) | |
| 6407 | + if err == nil { | |
| 6408 | + t.Fatal("sync unexpectedly succeeded") | |
| 6409 | + } | |
| 6410 | + data, readErr := os.ReadFile(progressPath) | |
| 6411 | + if readErr != nil { | |
| 6412 | + t.Fatalf("read progress: %v", readErr) | |
| 6413 | + } | |
| 6414 | + var progress syncProgressSnapshot | |
| 6415 | + if err := json.Unmarshal(data, &progress); err != nil { | |
| 6416 | + t.Fatalf("decode progress: %v", err) | |
| 6417 | + } | |
| 6418 | + if progress.State != syncProgressFailed || | |
| 6419 | + progress.Stage != syncProgressConnecting { | |
| 6420 | + t.Fatalf("failure progress = %+v", progress) | |
| 6421 | + } | |
| 6422 | + if strings.Contains(string(data), "diagnostic") || | |
| 6423 | + strings.Contains(string(data), "test-gh-token") { | |
| 6424 | + t.Fatalf("failure progress retained sensitive data: %s", data) | |
| 6425 | + } | |
| 6426 | +} | |
| 6427 | + | |
| 6355 | 6428 | func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) { |
| 6356 | 6429 | ctx := context.Background() |
| 6357 | 6430 | dir := t.TempDir() |
| @@ -0,0 +1,103 @@ | ||
| 1 | +package cli | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "encoding/json" | |
| 5 | + "fmt" | |
| 6 | + "path/filepath" | |
| 7 | + "strings" | |
| 8 | + "time" | |
| 9 | + | |
| 10 | + "github.com/openclaw/gitcrawl/internal/syncer" | |
| 11 | +) | |
| 12 | + | |
| 13 | +const syncProgressSchemaV1 = "gitcrawl.sync-progress.v1" | |
| 14 | + | |
| 15 | +type syncProgressState string | |
| 16 | + | |
| 17 | +const ( | |
| 18 | + syncProgressRunning syncProgressState = "running" | |
| 19 | + syncProgressSucceeded syncProgressState = "succeeded" | |
| 20 | + syncProgressFailed syncProgressState = "failed" | |
| 21 | +) | |
| 22 | + | |
| 23 | +const ( | |
| 24 | + syncProgressConnecting = syncer.SyncProgressConnecting | |
| 25 | + syncProgressThreads = syncer.SyncProgressThreads | |
| 26 | + syncProgressFinalizing = syncer.SyncProgressFinalizing | |
| 27 | +) | |
| 28 | + | |
| 29 | +type syncProgressSnapshot struct { | |
| 30 | + Schema string `json:"schema"` | |
| 31 | + Repository string `json:"repository"` | |
| 32 | + State syncProgressState `json:"state"` | |
| 33 | + Stage syncer.SyncProgressStage `json:"stage"` | |
| 34 | + IssuesReceived int `json:"issues_received"` | |
| 35 | + PullRequestsReceived int `json:"pull_requests_received"` | |
| 36 | + CommentsReceived int `json:"comments_received"` | |
| 37 | + ObservedAt string `json:"observed_at"` | |
| 38 | +} | |
| 39 | + | |
| 40 | +type syncProgressWriter struct { | |
| 41 | + path string | |
| 42 | + repository string | |
| 43 | + now func() time.Time | |
| 44 | + latest syncer.SyncProgress | |
| 45 | +} | |
| 46 | + | |
| 47 | +func newSyncProgressWriter( | |
| 48 | + path string, | |
| 49 | + repository string, | |
| 50 | +) (*syncProgressWriter, error) { | |
| 51 | + path = strings.TrimSpace(path) | |
| 52 | + if path == "" { | |
| 53 | + return nil, nil | |
| 54 | + } | |
| 55 | + if !filepath.IsAbs(path) { | |
| 56 | + return nil, fmt.Errorf("sync progress path must be absolute") | |
| 57 | + } | |
| 58 | + if strings.TrimSpace(repository) == "" { | |
| 59 | + return nil, fmt.Errorf("sync progress repository is required") | |
| 60 | + } | |
| 61 | + return &syncProgressWriter{ | |
| 62 | + path: path, | |
| 63 | + repository: repository, | |
| 64 | + now: func() time.Time { return time.Now().UTC() }, | |
| 65 | + }, nil | |
| 66 | +} | |
| 67 | + | |
| 68 | +func (w *syncProgressWriter) report(progress syncer.SyncProgress) error { | |
| 69 | + if w == nil { | |
| 70 | + return nil | |
| 71 | + } | |
| 72 | + w.latest = progress | |
| 73 | + return w.write(syncProgressRunning) | |
| 74 | +} | |
| 75 | + | |
| 76 | +func (w *syncProgressWriter) finish(state syncProgressState) error { | |
| 77 | + if w == nil { | |
| 78 | + return nil | |
| 79 | + } | |
| 80 | + return w.write(state) | |
| 81 | +} | |
| 82 | + | |
| 83 | +func (w *syncProgressWriter) write(state syncProgressState) error { | |
| 84 | + snapshot := syncProgressSnapshot{ | |
| 85 | + Schema: syncProgressSchemaV1, | |
| 86 | + Repository: w.repository, | |
| 87 | + State: state, | |
| 88 | + Stage: w.latest.Stage, | |
| 89 | + IssuesReceived: w.latest.IssuesReceived, | |
| 90 | + PullRequestsReceived: w.latest.PullRequestsReceived, | |
| 91 | + CommentsReceived: w.latest.CommentsReceived, | |
| 92 | + ObservedAt: w.now().Format(time.RFC3339Nano), | |
| 93 | + } | |
| 94 | + data, err := json.Marshal(snapshot) | |
| 95 | + if err != nil { | |
| 96 | + return fmt.Errorf("encode sync progress: %w", err) | |
| 97 | + } | |
| 98 | + data = append(data, '\n') | |
| 99 | + if err := writeCaptureAtomically(w.path, data); err != nil { | |
| 100 | + return fmt.Errorf("write sync progress: %w", err) | |
| 101 | + } | |
| 102 | + return nil | |
| 103 | +} | |
| @@ -0,0 +1,69 @@ | ||
| 1 | +package cli | |
| 2 | + | |
| 3 | +import ( | |
| 4 | + "encoding/json" | |
| 5 | + "os" | |
| 6 | + "path/filepath" | |
| 7 | + "testing" | |
| 8 | + "time" | |
| 9 | + | |
| 10 | + "github.com/openclaw/gitcrawl/internal/syncer" | |
| 11 | +) | |
| 12 | + | |
| 13 | +func TestSyncProgressWriterPublishesPrivateSanitizedSnapshot(t *testing.T) { | |
| 14 | + path := filepath.Join(t.TempDir(), "progress.json") | |
| 15 | + writer, err := newSyncProgressWriter(path, "openclaw/gitcrawl") | |
| 16 | + if err != nil { | |
| 17 | + t.Fatalf("new writer: %v", err) | |
| 18 | + } | |
| 19 | + writer.now = func() time.Time { | |
| 20 | + return time.Date(2026, 7, 31, 4, 0, 0, 0, time.UTC) | |
| 21 | + } | |
| 22 | + if err := writer.report(syncer.SyncProgress{ | |
| 23 | + Stage: syncer.SyncProgressThreads, | |
| 24 | + IssuesReceived: 56, | |
| 25 | + PullRequestsReceived: 238, | |
| 26 | + CommentsReceived: 350, | |
| 27 | + }); err != nil { | |
| 28 | + t.Fatalf("report progress: %v", err) | |
| 29 | + } | |
| 30 | + if err := writer.finish(syncProgressFailed); err != nil { | |
| 31 | + t.Fatalf("finish progress: %v", err) | |
| 32 | + } | |
| 33 | + | |
| 34 | + data, err := os.ReadFile(path) | |
| 35 | + if err != nil { | |
| 36 | + t.Fatalf("read progress: %v", err) | |
| 37 | + } | |
| 38 | + var snapshot map[string]any | |
| 39 | + if err := json.Unmarshal(data, &snapshot); err != nil { | |
| 40 | + t.Fatalf("decode progress: %v", err) | |
| 41 | + } | |
| 42 | + if len(snapshot) != 8 || | |
| 43 | + snapshot["schema"] != syncProgressSchemaV1 || | |
| 44 | + snapshot["repository"] != "openclaw/gitcrawl" || | |
| 45 | + snapshot["state"] != string(syncProgressFailed) || | |
| 46 | + snapshot["stage"] != string(syncer.SyncProgressThreads) || | |
| 47 | + snapshot["issues_received"] != float64(56) || | |
| 48 | + snapshot["pull_requests_received"] != float64(238) || | |
| 49 | + snapshot["comments_received"] != float64(350) || | |
| 50 | + snapshot["observed_at"] != "2026-07-31T04:00:00Z" { | |
| 51 | + t.Fatalf("progress snapshot = %#v", snapshot) | |
| 52 | + } | |
| 53 | + info, err := os.Stat(path) | |
| 54 | + if err != nil { | |
| 55 | + t.Fatalf("stat progress: %v", err) | |
| 56 | + } | |
| 57 | + if info.Mode().Perm() != 0o600 { | |
| 58 | + t.Fatalf("progress mode = %o", info.Mode().Perm()) | |
| 59 | + } | |
| 60 | +} | |
| 61 | + | |
| 62 | +func TestSyncProgressWriterRequiresAbsolutePath(t *testing.T) { | |
| 63 | + if _, err := newSyncProgressWriter( | |
| 64 | + "progress.json", | |
| 65 | + "openclaw/gitcrawl", | |
| 66 | + ); err == nil { | |
| 67 | + t.Fatal("relative progress path unexpectedly accepted") | |
| 68 | + } | |
| 69 | +} | |
| @@ -0,0 +1,21 @@ | ||
| 1 | +package syncer | |
| 2 | + | |
| 3 | +// SyncProgressStage is one sanitized phase of repository synchronization. | |
| 4 | +type SyncProgressStage string | |
| 5 | + | |
| 6 | +const ( | |
| 7 | + SyncProgressConnecting SyncProgressStage = "connecting" | |
| 8 | + SyncProgressThreads SyncProgressStage = "syncing_threads" | |
| 9 | + SyncProgressFinalizing SyncProgressStage = "finalizing" | |
| 10 | +) | |
| 11 | + | |
| 12 | +// SyncProgress reports content-free activity observed during one sync. | |
| 13 | +type SyncProgress struct { | |
| 14 | + Stage SyncProgressStage | |
| 15 | + IssuesReceived int | |
| 16 | + PullRequestsReceived int | |
| 17 | + CommentsReceived int | |
| 18 | +} | |
| 19 | + | |
| 20 | +// SyncProgressReporter accepts one sanitized repository activity snapshot. | |
| 21 | +type SyncProgressReporter func(SyncProgress) error | |
| @@ -51,6 +51,7 @@ type Options struct { · 50 unmodified lines | ||
| 51 | 51 | IncludePRDetails bool |
| 52 | 52 | Reporter gh.Reporter |
| 53 | 53 | Logger *slog.Logger |
| 54 | + Progress SyncProgressReporter | |
| 54 | 55 | } |
| 55 | 56 | |
| 56 | 57 | type Stats struct { |
| @@ -116,6 +117,11 @@ func New(client GitHubClient, st *store.Store) *Syncer { · 59 unmodified lines | ||
| 116 | 117 | |
| 117 | 118 | func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { |
| 118 | 119 | started := s.now().Format(time.RFC3339Nano) |
| 120 | + if err := reportSyncProgress(options.Progress, SyncProgress{ | |
| 121 | + Stage: SyncProgressConnecting, | |
| 122 | + }); err != nil { | |
| 123 | + return Stats{}, err | |
| 124 | + } | |
| 119 | 125 | since, err := normalizeSince(options.Since, s.now()) |
| 120 | 126 | if err != nil { |
| 121 | 127 | return Stats{}, err |
| @@ -160,6 +166,10 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 38 unmodified lines | ||
| 160 | 166 | } |
| 161 | 167 | rows = mergeIssueRows(rows, closedOverlapRows) |
| 162 | 168 | } |
| 169 | + received := receivedThreadCounts(rows) | |
| 170 | + if err := reportSyncProgress(options.Progress, received); err != nil { | |
| 171 | + return Stats{}, err | |
| 172 | + } | |
| 163 | 173 | closedOverlapNumbers := make(map[int]struct{}, len(closedOverlapRows)) |
| 164 | 174 | for _, row := range closedOverlapRows { |
| 165 | 175 | closedOverlapNumbers[intValue(row["number"])] = struct{}{} |
| @@ -177,6 +187,7 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 11 unmodified lines | ||
| 177 | 187 | return Stats{}, err |
| 178 | 188 | } |
| 179 | 189 | payload.commentRows = commentRows |
| 190 | + received.CommentsReceived += len(commentRows) | |
| 180 | 191 | } |
| 181 | 192 | if options.IncludePRDetails && kind == "pull_request" { |
| 182 | 193 | reviewThreads, reviewThreadsFetchedAt, err := s.fetchPullReviewThreadRows(ctx, options, number) |
| @@ -201,6 +212,9 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 18 unmodified lines | ||
| 201 | 212 | payload.hasPullDetails = true |
| 202 | 213 | } |
| 203 | 214 | payloads = append(payloads, payload) |
| 215 | + if err := reportSyncProgress(options.Progress, received); err != nil { | |
| 216 | + return Stats{}, err | |
| 217 | + } | |
| 204 | 218 | } |
| 205 | 219 | if err := s.consolidateWorkflowSnapshots(ctx, options, payloads); err != nil { |
| 206 | 220 | return Stats{}, err |
| @@ -219,6 +233,10 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 12 unmodified lines | ||
| 219 | 233 | if s.beforePersist != nil { |
| 220 | 234 | s.beforePersist() |
| 221 | 235 | } |
| 236 | + received.Stage = SyncProgressFinalizing | |
| 237 | + if err := reportSyncProgress(options.Progress, received); err != nil { | |
| 238 | + return Stats{}, err | |
| 239 | + } | |
| 222 | 240 | stats := Stats{ |
| 223 | 241 | Repository: options.Owner + "/" + options.Repo, |
| 224 | 242 | RequestedSince: since, |
| @@ -482,6 +500,28 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 257 unmodified lines | ||
| 482 | 500 | return stats, nil |
| 483 | 501 | } |
| 484 | 502 | |
| 503 | +func receivedThreadCounts(rows []map[string]any) SyncProgress { | |
| 504 | + result := SyncProgress{Stage: SyncProgressThreads} | |
| 505 | + for _, row := range rows { | |
| 506 | + if issueKind(row) == "pull_request" { | |
| 507 | + result.PullRequestsReceived++ | |
| 508 | + } else { | |
| 509 | + result.IssuesReceived++ | |
| 510 | + } | |
| 511 | + } | |
| 512 | + return result | |
| 513 | +} | |
| 514 | + | |
| 515 | +func reportSyncProgress( | |
| 516 | + reporter SyncProgressReporter, | |
| 517 | + snapshot SyncProgress, | |
| 518 | +) error { | |
| 519 | + if reporter == nil { | |
| 520 | + return nil | |
| 521 | + } | |
| 522 | + return reporter(snapshot) | |
| 523 | +} | |
| 524 | + | |
| 485 | 525 | func (s *Syncer) recordPullRequestSyncFailure(ctx context.Context, options Options, repoRaw, row map[string]any, operation string, syncErr error) error { |
| 486 | 526 | if syncErr == nil { |
| 487 | 527 | return nil |
| @@ -1363,12 +1363,17 @@ func TestSyncPersistsIssuesAndPullRequests(t *testing.T) { · 1362 unmodified lines | ||
| 1363 | 1363 | s := New(fakeGitHub{}, st) |
| 1364 | 1364 | s.now = func() time.Time { return time.Date(2026, 4, 26, 0, 0, 0, 0, time.UTC) } |
| 1365 | 1365 | var progressLogs bytes.Buffer |
| 1366 | + var snapshots []SyncProgress | |
| 1366 | 1367 | stats, err := s.Sync(ctx, Options{ |
| 1367 | 1368 | Owner: "openclaw", |
| 1368 | 1369 | Repo: "gitcrawl", |
| 1369 | 1370 | IncludeComments: true, |
| 1370 | 1371 | IncludePRDetails: true, |
| 1371 | 1372 | Logger: testProgressLogger(&progressLogs), |
| 1373 | + Progress: func(snapshot SyncProgress) error { | |
| 1374 | + snapshots = append(snapshots, snapshot) | |
| 1375 | + return nil | |
| 1376 | + }, | |
| 1372 | 1377 | }) |
| 1373 | 1378 | if err != nil { |
| 1374 | 1379 | t.Fatalf("sync: %v", err) |
| @@ -1385,6 +1390,18 @@ func TestSyncPersistsIssuesAndPullRequests(t *testing.T) { · 10 unmodified lines | ||
| 1385 | 1390 | if stats.MetadataOnly { |
| 1386 | 1391 | t.Fatal("metadata only: got true want false") |
| 1387 | 1392 | } |
| 1393 | + if len(snapshots) < 4 { | |
| 1394 | + t.Fatalf("progress snapshots = %+v", snapshots) | |
| 1395 | + } | |
| 1396 | + if snapshots[0].Stage != SyncProgressConnecting { | |
| 1397 | + t.Fatalf("initial progress = %+v", snapshots[0]) | |
| 1398 | + } | |
| 1399 | + if got := snapshots[len(snapshots)-1]; got.Stage != SyncProgressFinalizing || | |
| 1400 | + got.IssuesReceived != 1 || | |
| 1401 | + got.PullRequestsReceived != 1 || | |
| 1402 | + got.CommentsReceived != 1 { | |
| 1403 | + t.Fatalf("terminal progress = %+v", got) | |
| 1404 | + } | |
| 1388 | 1405 | |
| 1389 | 1406 | repo, err := st.RepositoryByFullName(ctx, "openclaw/gitcrawl") |
| 1390 | 1407 | if err != nil { |