History

feat: export sync progress snapshots (lore-b062.16)

Josh Lehman committed 3 months ago

Commit 8bb0a211 parent e3cf8bb9 files · +373 −3Browse files

Unified · Split

9 files
  1. docs/commands.md +1 −1
  2. docs/sync.md +25 −0
  3. internal/cli/app.go +23 −1
  4. internal/cli/app_test.go +74 −1
  5. internal/cli/sync_progress.go +103 −0
  6. internal/cli/sync_progress_test.go +69 −0
  7. internal/syncer/sync_progress.go +21 −0
  8. internal/syncer/syncer.go +40 −0
  9. internal/syncer/syncer_test.go +17 −0
docs/commands.md+1−1
@@ -41,7 +41,7 @@ These work on every command. · 40 unmodified lines
41 41
42| Command | Purpose | Docs |42| Command | Purpose | Docs |
43| --- | --- | --- |43| --- | --- | --- |
44| `gitcrawl sync owner/repo [--state --since --numbers <refs> --limit --include-comments --include-pr-details --with pr-details --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) |44| `gitcrawl sync owner/repo [--state --since --numbers <refs> --limit --include-comments --include-pr-details --with pr-details --progress-file <absolute-path> --json]` | Sync issues and PRs from GitHub into local SQLite | [Sync](/sync/) |
45| `gitcrawl sync-failures owner/repo [--include-resolved --limit N --json]` | List failed PR hydration attempts and optional resolved history | [Sync](/sync/#hydration-depth) |45| `gitcrawl sync-failures owner/repo [--include-resolved --limit N --json]` | List failed PR hydration attempts and optional resolved history | [Sync](/sync/#hydration-depth) |
46| `gitcrawl coverage [owner/repo \| --repos owner/a,owner/b] [--min-missing-pr-details N --json]` | Report archive, PR-detail, and enrichment coverage/freshness | — |46| `gitcrawl coverage [owner/repo \| --repos owner/a,owner/b] [--min-missing-pr-details N --json]` | Report archive, PR-detail, and enrichment coverage/freshness | — |
47| `gitcrawl fill-pr-details owner/repo [--limit --order --batch-size --reserve-rate-limit --include-comments --json-progress --json]` | Hydrate locally missing pull request detail rows in bounded batches | — |47| `gitcrawl fill-pr-details owner/repo [--limit --order --batch-size --reserve-rate-limit --include-comments --json-progress --json]` | Hydrate locally missing pull request detail rows in bounded batches | — |
docs/sync.md+25−0
@@ -82,6 +82,7 @@ issue or pull request URLs. · 81 unmodified lines
82| `--include-comments` | Issue comments, PR review comments, reviews |82| `--include-comments` | Issue comments, PR review comments, reviews |
83| `--include-pr-details` | PR files, commits, status checks, workflow runs |83| `--include-pr-details` | PR files, commits, status checks, workflow runs |
84| `--with pr-details` | Same as `--include-pr-details` (gh-style flag) |84| `--with pr-details` | Same as `--include-pr-details` (gh-style flag) |
85| `--progress-file <absolute-path>` | Atomically publish sanitized machine-readable activity |
85 86
86PR details land in `pr_files`, `pr_commits`, `pr_checks`, and `pr_runs` tables for local review, search, clustering, and TUI workflows.87PR details land in `pr_files`, `pr_commits`, `pr_checks`, and `pr_runs` tables for local review, search, clustering, and TUI workflows.
87 88
@@ -91,6 +92,30 @@ Use `gitcrawl coverage [owner/repo] --json` to inspect archive completeness afte · 3 unmodified lines
91 92
92`--include-code` is accepted for compatibility but is currently a no-op.93`--include-code` is accepted for compatibility but is currently a no-op.
93 94
95## Machine-readable progress
96
97Automation may pass an absolute `--progress-file` path. Gitcrawl atomically
98replaces that private file as synchronization advances, then leaves the
99terminal snapshot in place:
100
101```json
102{
103 "schema": "gitcrawl.sync-progress.v1",
104 "repository": "owner/repo",
105 "state": "running",
106 "stage": "syncing_threads",
107 "issues_received": 56,
108 "pull_requests_received": 238,
109 "comments_received": 350,
110 "observed_at": "2026-07-31T04:17:36.000000000Z"
111}
112```
113
114`state` is `running`, `succeeded`, or `failed`; `stage` is `connecting`,
115`syncing_threads`, or `finalizing`. Counts describe upstream activity observed
116by the current process, not rows committed to SQLite. The snapshot never
117contains credentials, issue content, request URLs, or raw diagnostics.
118
94## Limit and pagination119## Limit and pagination
95 120
96```bash121```bash
internal/cli/app.go+23−1
@@ -2884,8 +2884,9 @@ func (a *App) runSync(ctx context.Context, args []string) error { · 2883 unmodified lines
2884 includeComments := fs.Bool("include-comments", false, "hydrate issue comments, PR reviews, and PR review comments")2884 includeComments := fs.Bool("include-comments", false, "hydrate issue comments, PR reviews, and PR review comments")
2885 includePRDetails := fs.Bool("include-pr-details", false, "hydrate PR files, commits, checks, and workflow runs")2885 includePRDetails := fs.Bool("include-pr-details", false, "hydrate PR files, commits, checks, and workflow runs")
2886 withRaw := fs.String("with", "", "extra hydration: pr-details")2886 withRaw := fs.String("with", "", "extra hydration: pr-details")
2887 progressFile := fs.String("progress-file", "", "write an atomic sanitized sync progress snapshot")
2887 fs.Bool("include-code", false, "accepted for compatibility; code hydration is not implemented yet")2888 fs.Bool("include-code", false, "accepted for compatibility; code hydration is not implemented yet")
2888 if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"numbers": true, "since": true, "state": true, "limit": true, "with": true})); err != nil {2889 if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"numbers": true, "since": true, "state": true, "limit": true, "with": true, "progress-file": true})); err != nil {
2889 return usageErr(err)2890 return usageErr(err)
2890 }2891 }
2891 a.applyCommandJSON(*jsonOut)2892 a.applyCommandJSON(*jsonOut)
@@ -2908,6 +2909,18 @@ func (a *App) runSync(ctx context.Context, args []string) error { · 16 unmodified lines
2908 if err != nil {2909 if err != nil {
2909 return usageErr(err)2910 return usageErr(err)
2910 }2911 }
2912 progress, err := newSyncProgressWriter(
2913 *progressFile,
2914 owner+"/"+repo,
2915 )
2916 if err != nil {
2917 return usageErr(err)
2918 }
2919 if err := progress.report(syncer.SyncProgress{
2920 Stage: syncer.SyncProgressConnecting,
2921 }); err != nil {
2922 return err
2923 }
2911 2924
2912 stats, target, err := a.syncRepository(ctx, owner, repo, syncOptions{2925 stats, target, err := a.syncRepository(ctx, owner, repo, syncOptions{
2913 Since: strings.TrimSpace(*since),2926 Since: strings.TrimSpace(*since),
@@ -2916,8 +2929,15 @@ func (a *App) runSync(ctx context.Context, args []string) error { · 2 unmodified lines
2916 Numbers: numbers,2929 Numbers: numbers,
2917 IncludeComments: *includeComments,2930 IncludeComments: *includeComments,
2918 IncludePRDetails: *includePRDetails || with["pr-details"],2931 IncludePRDetails: *includePRDetails || with["pr-details"],
2932 Progress: progress.report,
2919 })2933 })
2920 if err != nil {2934 if err != nil {
2935 if progressErr := progress.finish(syncProgressFailed); progressErr != nil {
2936 return progressErr
2937 }
2938 return err
2939 }
2940 if err := progress.finish(syncProgressSucceeded); err != nil {
2921 return err2941 return err
2922 }2942 }
2923 result := struct {2943 result := struct {
@@ -2936,6 +2956,7 @@ type syncOptions struct { · 12 unmodified lines
2936 IncludePRDetails bool2956 IncludePRDetails bool
2937 Quiet bool2957 Quiet bool
2938 RateLimitReserve int2958 RateLimitReserve int
2959 Progress syncer.SyncProgressReporter
2939}2960}
2940 2961
2941type fillPRDetailsResult struct {2962type fillPRDetailsResult struct {
@@ -3241,6 +3262,7 @@ func (a *App) syncRepository(ctx context.Context, owner, repo string, options sy · 299 unmodified lines
3241 IncludePRDetails: options.IncludePRDetails,3262 IncludePRDetails: options.IncludePRDetails,
3242 Reporter: reporter,3263 Reporter: reporter,
3243 Logger: logger,3264 Logger: logger,
3265 Progress: options.Progress,
3244 })3266 })
3245 if err != nil {3267 if err != nil {
3246 return syncer.Stats{}, target, err3268 return syncer.Stats{}, target, err
internal/cli/app_test.go+74−1
@@ -6281,6 +6281,7 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 6280 unmodified lines
6281 dir := t.TempDir()6281 dir := t.TempDir()
6282 configPath := filepath.Join(dir, "config.toml")6282 configPath := filepath.Join(dir, "config.toml")
6283 dbPath := filepath.Join(dir, "gitcrawl.db")6283 dbPath := filepath.Join(dir, "gitcrawl.db")
6284 progressPath := filepath.Join(dir, "sync-progress.json")
6284 init := New()6285 init := New()
6285 if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil {6286 if err := init.Run(ctx, []string{"--config", configPath, "init", "--db", dbPath}); err != nil {
6286 t.Fatalf("init: %v", err)6287 t.Fatalf("init: %v", err)
@@ -6318,7 +6319,14 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 31 unmodified lines
6318 run := New()6319 run := New()
6319 var stdout bytes.Buffer6320 var stdout bytes.Buffer
6320 run.Stdout = &stdout6321 run.Stdout = &stdout
6321 if err := run.Run(ctx, []string{"--config", configPath, "sync", "openclaw/openclaw", "--numbers", "101,102", "--include-comments", "--json"}); err != nil {6322 if err := run.Run(ctx, []string{
6323 "--config", configPath,
6324 "sync", "openclaw/openclaw",
6325 "--numbers", "101,102",
6326 "--include-comments",
6327 "--progress-file", progressPath,
6328 "--json",
6329 }); err != nil {
6322 t.Fatalf("sync: %v", err)6330 t.Fatalf("sync: %v", err)
6323 }6331 }
6324 var stats struct {6332 var stats struct {
@@ -6337,6 +6345,24 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 12 unmodified lines
6337 if stats.DBTarget != "direct" || stats.DBTargetPath != dbPath {6345 if stats.DBTarget != "direct" || stats.DBTargetPath != dbPath {
6338 t.Fatalf("sync db target = %+v", stats)6346 t.Fatalf("sync db target = %+v", stats)
6339 }6347 }
6348 progressData, err := os.ReadFile(progressPath)
6349 if err != nil {
6350 t.Fatalf("read sync progress: %v", err)
6351 }
6352 var syncProgress syncProgressSnapshot
6353 if err := json.Unmarshal(progressData, &syncProgress); err != nil {
6354 t.Fatalf("decode sync progress: %v", err)
6355 }
6356 if syncProgress.Schema != syncProgressSchemaV1 ||
6357 syncProgress.Repository != "openclaw/openclaw" ||
6358 syncProgress.State != syncProgressSucceeded ||
6359 syncProgress.Stage != syncProgressFinalizing ||
6360 syncProgress.IssuesReceived != 1 ||
6361 syncProgress.PullRequestsReceived != 1 ||
6362 syncProgress.CommentsReceived != 4 ||
6363 syncProgress.ObservedAt == "" {
6364 t.Fatalf("sync progress = %+v", syncProgress)
6365 }
6340 6366
6341 st, err := store.Open(ctx, dbPath)6367 st, err := store.Open(ctx, dbPath)
6342 if err != nil {6368 if err != nil {
@@ -6352,6 +6378,53 @@ func TestSyncCommandUsesConfiguredGitHubBaseURLAndHydratesComments(t *testing.T) · 9 unmodified lines
6352 }6378 }
6353}6379}
6354 6380
6381func TestSyncCommandPublishesSanitizedFailureProgress(t *testing.T) {
6382 ctx := context.Background()
6383 dir := t.TempDir()
6384 configPath := filepath.Join(dir, "config.toml")
6385 dbPath := filepath.Join(dir, "gitcrawl.db")
6386 progressPath := filepath.Join(dir, "sync-progress.json")
6387 if err := New().Run(ctx, []string{
6388 "--config", configPath, "init", "--db", dbPath,
6389 }); err != nil {
6390 t.Fatalf("init: %v", err)
6391 }
6392 server := httptest.NewServer(http.HandlerFunc(func(
6393 w http.ResponseWriter,
6394 _ *http.Request,
6395 ) {
6396 http.Error(w, "private upstream diagnostic", http.StatusBadGateway)
6397 }))
6398 defer server.Close()
6399 t.Setenv("GITHUB_TOKEN", "test-gh-token")
6400 t.Setenv("GITCRAWL_GITHUB_BASE_URL", server.URL)
6401
6402 err := New().Run(ctx, []string{
6403 "--config", configPath,
6404 "sync", "openclaw/gitcrawl",
6405 "--progress-file", progressPath,
6406 })
6407 if err == nil {
6408 t.Fatal("sync unexpectedly succeeded")
6409 }
6410 data, readErr := os.ReadFile(progressPath)
6411 if readErr != nil {
6412 t.Fatalf("read progress: %v", readErr)
6413 }
6414 var progress syncProgressSnapshot
6415 if err := json.Unmarshal(data, &progress); err != nil {
6416 t.Fatalf("decode progress: %v", err)
6417 }
6418 if progress.State != syncProgressFailed ||
6419 progress.Stage != syncProgressConnecting {
6420 t.Fatalf("failure progress = %+v", progress)
6421 }
6422 if strings.Contains(string(data), "diagnostic") ||
6423 strings.Contains(string(data), "test-gh-token") {
6424 t.Fatalf("failure progress retained sensitive data: %s", data)
6425 }
6426}
6427
6355func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) {6428func TestFillPRDetailsHydratesMissingPullRequestDetails(t *testing.T) {
6356 ctx := context.Background()6429 ctx := context.Background()
6357 dir := t.TempDir()6430 dir := t.TempDir()
internal/cli/sync_progress.go+103−0
@@ -0,0 +1,103 @@
1package cli
2
3import (
4 "encoding/json"
5 "fmt"
6 "path/filepath"
7 "strings"
8 "time"
9
10 "github.com/openclaw/gitcrawl/internal/syncer"
11)
12
13const syncProgressSchemaV1 = "gitcrawl.sync-progress.v1"
14
15type syncProgressState string
16
17const (
18 syncProgressRunning syncProgressState = "running"
19 syncProgressSucceeded syncProgressState = "succeeded"
20 syncProgressFailed syncProgressState = "failed"
21)
22
23const (
24 syncProgressConnecting = syncer.SyncProgressConnecting
25 syncProgressThreads = syncer.SyncProgressThreads
26 syncProgressFinalizing = syncer.SyncProgressFinalizing
27)
28
29type syncProgressSnapshot struct {
30 Schema string `json:"schema"`
31 Repository string `json:"repository"`
32 State syncProgressState `json:"state"`
33 Stage syncer.SyncProgressStage `json:"stage"`
34 IssuesReceived int `json:"issues_received"`
35 PullRequestsReceived int `json:"pull_requests_received"`
36 CommentsReceived int `json:"comments_received"`
37 ObservedAt string `json:"observed_at"`
38}
39
40type syncProgressWriter struct {
41 path string
42 repository string
43 now func() time.Time
44 latest syncer.SyncProgress
45}
46
47func newSyncProgressWriter(
48 path string,
49 repository string,
50) (*syncProgressWriter, error) {
51 path = strings.TrimSpace(path)
52 if path == "" {
53 return nil, nil
54 }
55 if !filepath.IsAbs(path) {
56 return nil, fmt.Errorf("sync progress path must be absolute")
57 }
58 if strings.TrimSpace(repository) == "" {
59 return nil, fmt.Errorf("sync progress repository is required")
60 }
61 return &syncProgressWriter{
62 path: path,
63 repository: repository,
64 now: func() time.Time { return time.Now().UTC() },
65 }, nil
66}
67
68func (w *syncProgressWriter) report(progress syncer.SyncProgress) error {
69 if w == nil {
70 return nil
71 }
72 w.latest = progress
73 return w.write(syncProgressRunning)
74}
75
76func (w *syncProgressWriter) finish(state syncProgressState) error {
77 if w == nil {
78 return nil
79 }
80 return w.write(state)
81}
82
83func (w *syncProgressWriter) write(state syncProgressState) error {
84 snapshot := syncProgressSnapshot{
85 Schema: syncProgressSchemaV1,
86 Repository: w.repository,
87 State: state,
88 Stage: w.latest.Stage,
89 IssuesReceived: w.latest.IssuesReceived,
90 PullRequestsReceived: w.latest.PullRequestsReceived,
91 CommentsReceived: w.latest.CommentsReceived,
92 ObservedAt: w.now().Format(time.RFC3339Nano),
93 }
94 data, err := json.Marshal(snapshot)
95 if err != nil {
96 return fmt.Errorf("encode sync progress: %w", err)
97 }
98 data = append(data, '\n')
99 if err := writeCaptureAtomically(w.path, data); err != nil {
100 return fmt.Errorf("write sync progress: %w", err)
101 }
102 return nil
103}
internal/cli/sync_progress_test.go+69−0
@@ -0,0 +1,69 @@
1package cli
2
3import (
4 "encoding/json"
5 "os"
6 "path/filepath"
7 "testing"
8 "time"
9
10 "github.com/openclaw/gitcrawl/internal/syncer"
11)
12
13func TestSyncProgressWriterPublishesPrivateSanitizedSnapshot(t *testing.T) {
14 path := filepath.Join(t.TempDir(), "progress.json")
15 writer, err := newSyncProgressWriter(path, "openclaw/gitcrawl")
16 if err != nil {
17 t.Fatalf("new writer: %v", err)
18 }
19 writer.now = func() time.Time {
20 return time.Date(2026, 7, 31, 4, 0, 0, 0, time.UTC)
21 }
22 if err := writer.report(syncer.SyncProgress{
23 Stage: syncer.SyncProgressThreads,
24 IssuesReceived: 56,
25 PullRequestsReceived: 238,
26 CommentsReceived: 350,
27 }); err != nil {
28 t.Fatalf("report progress: %v", err)
29 }
30 if err := writer.finish(syncProgressFailed); err != nil {
31 t.Fatalf("finish progress: %v", err)
32 }
33
34 data, err := os.ReadFile(path)
35 if err != nil {
36 t.Fatalf("read progress: %v", err)
37 }
38 var snapshot map[string]any
39 if err := json.Unmarshal(data, &snapshot); err != nil {
40 t.Fatalf("decode progress: %v", err)
41 }
42 if len(snapshot) != 8 ||
43 snapshot["schema"] != syncProgressSchemaV1 ||
44 snapshot["repository"] != "openclaw/gitcrawl" ||
45 snapshot["state"] != string(syncProgressFailed) ||
46 snapshot["stage"] != string(syncer.SyncProgressThreads) ||
47 snapshot["issues_received"] != float64(56) ||
48 snapshot["pull_requests_received"] != float64(238) ||
49 snapshot["comments_received"] != float64(350) ||
50 snapshot["observed_at"] != "2026-07-31T04:00:00Z" {
51 t.Fatalf("progress snapshot = %#v", snapshot)
52 }
53 info, err := os.Stat(path)
54 if err != nil {
55 t.Fatalf("stat progress: %v", err)
56 }
57 if info.Mode().Perm() != 0o600 {
58 t.Fatalf("progress mode = %o", info.Mode().Perm())
59 }
60}
61
62func TestSyncProgressWriterRequiresAbsolutePath(t *testing.T) {
63 if _, err := newSyncProgressWriter(
64 "progress.json",
65 "openclaw/gitcrawl",
66 ); err == nil {
67 t.Fatal("relative progress path unexpectedly accepted")
68 }
69}
internal/syncer/sync_progress.go+21−0
@@ -0,0 +1,21 @@
1package syncer
2
3// SyncProgressStage is one sanitized phase of repository synchronization.
4type SyncProgressStage string
5
6const (
7 SyncProgressConnecting SyncProgressStage = "connecting"
8 SyncProgressThreads SyncProgressStage = "syncing_threads"
9 SyncProgressFinalizing SyncProgressStage = "finalizing"
10)
11
12// SyncProgress reports content-free activity observed during one sync.
13type SyncProgress struct {
14 Stage SyncProgressStage
15 IssuesReceived int
16 PullRequestsReceived int
17 CommentsReceived int
18}
19
20// SyncProgressReporter accepts one sanitized repository activity snapshot.
21type SyncProgressReporter func(SyncProgress) error
internal/syncer/syncer.go+40−0
@@ -51,6 +51,7 @@ type Options struct { · 50 unmodified lines
51 IncludePRDetails bool51 IncludePRDetails bool
52 Reporter gh.Reporter52 Reporter gh.Reporter
53 Logger *slog.Logger53 Logger *slog.Logger
54 Progress SyncProgressReporter
54}55}
55 56
56type Stats struct {57type Stats struct {
@@ -116,6 +117,11 @@ func New(client GitHubClient, st *store.Store) *Syncer { · 59 unmodified lines
116 117
117func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) {118func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) {
118 started := s.now().Format(time.RFC3339Nano)119 started := s.now().Format(time.RFC3339Nano)
120 if err := reportSyncProgress(options.Progress, SyncProgress{
121 Stage: SyncProgressConnecting,
122 }); err != nil {
123 return Stats{}, err
124 }
119 since, err := normalizeSince(options.Since, s.now())125 since, err := normalizeSince(options.Since, s.now())
120 if err != nil {126 if err != nil {
121 return Stats{}, err127 return Stats{}, err
@@ -160,6 +166,10 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 38 unmodified lines
160 }166 }
161 rows = mergeIssueRows(rows, closedOverlapRows)167 rows = mergeIssueRows(rows, closedOverlapRows)
162 }168 }
169 received := receivedThreadCounts(rows)
170 if err := reportSyncProgress(options.Progress, received); err != nil {
171 return Stats{}, err
172 }
163 closedOverlapNumbers := make(map[int]struct{}, len(closedOverlapRows))173 closedOverlapNumbers := make(map[int]struct{}, len(closedOverlapRows))
164 for _, row := range closedOverlapRows {174 for _, row := range closedOverlapRows {
165 closedOverlapNumbers[intValue(row["number"])] = struct{}{}175 closedOverlapNumbers[intValue(row["number"])] = struct{}{}
@@ -177,6 +187,7 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 11 unmodified lines
177 return Stats{}, err187 return Stats{}, err
178 }188 }
179 payload.commentRows = commentRows189 payload.commentRows = commentRows
190 received.CommentsReceived += len(commentRows)
180 }191 }
181 if options.IncludePRDetails && kind == "pull_request" {192 if options.IncludePRDetails && kind == "pull_request" {
182 reviewThreads, reviewThreadsFetchedAt, err := s.fetchPullReviewThreadRows(ctx, options, number)193 reviewThreads, reviewThreadsFetchedAt, err := s.fetchPullReviewThreadRows(ctx, options, number)
@@ -201,6 +212,9 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 18 unmodified lines
201 payload.hasPullDetails = true212 payload.hasPullDetails = true
202 }213 }
203 payloads = append(payloads, payload)214 payloads = append(payloads, payload)
215 if err := reportSyncProgress(options.Progress, received); err != nil {
216 return Stats{}, err
217 }
204 }218 }
205 if err := s.consolidateWorkflowSnapshots(ctx, options, payloads); err != nil {219 if err := s.consolidateWorkflowSnapshots(ctx, options, payloads); err != nil {
206 return Stats{}, err220 return Stats{}, err
@@ -219,6 +233,10 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 12 unmodified lines
219 if s.beforePersist != nil {233 if s.beforePersist != nil {
220 s.beforePersist()234 s.beforePersist()
221 }235 }
236 received.Stage = SyncProgressFinalizing
237 if err := reportSyncProgress(options.Progress, received); err != nil {
238 return Stats{}, err
239 }
222 stats := Stats{240 stats := Stats{
223 Repository: options.Owner + "/" + options.Repo,241 Repository: options.Owner + "/" + options.Repo,
224 RequestedSince: since,242 RequestedSince: since,
@@ -482,6 +500,28 @@ func (s *Syncer) Sync(ctx context.Context, options Options) (Stats, error) { · 257 unmodified lines
482 return stats, nil500 return stats, nil
483}501}
484 502
503func receivedThreadCounts(rows []map[string]any) SyncProgress {
504 result := SyncProgress{Stage: SyncProgressThreads}
505 for _, row := range rows {
506 if issueKind(row) == "pull_request" {
507 result.PullRequestsReceived++
508 } else {
509 result.IssuesReceived++
510 }
511 }
512 return result
513}
514
515func reportSyncProgress(
516 reporter SyncProgressReporter,
517 snapshot SyncProgress,
518) error {
519 if reporter == nil {
520 return nil
521 }
522 return reporter(snapshot)
523}
524
485func (s *Syncer) recordPullRequestSyncFailure(ctx context.Context, options Options, repoRaw, row map[string]any, operation string, syncErr error) error {525func (s *Syncer) recordPullRequestSyncFailure(ctx context.Context, options Options, repoRaw, row map[string]any, operation string, syncErr error) error {
486 if syncErr == nil {526 if syncErr == nil {
487 return nil527 return nil
internal/syncer/syncer_test.go+17−0
@@ -1363,12 +1363,17 @@ func TestSyncPersistsIssuesAndPullRequests(t *testing.T) { · 1362 unmodified lines
1363 s := New(fakeGitHub{}, st)1363 s := New(fakeGitHub{}, st)
1364 s.now = func() time.Time { return time.Date(2026, 4, 26, 0, 0, 0, 0, time.UTC) }1364 s.now = func() time.Time { return time.Date(2026, 4, 26, 0, 0, 0, 0, time.UTC) }
1365 var progressLogs bytes.Buffer1365 var progressLogs bytes.Buffer
1366 var snapshots []SyncProgress
1366 stats, err := s.Sync(ctx, Options{1367 stats, err := s.Sync(ctx, Options{
1367 Owner: "openclaw",1368 Owner: "openclaw",
1368 Repo: "gitcrawl",1369 Repo: "gitcrawl",
1369 IncludeComments: true,1370 IncludeComments: true,
1370 IncludePRDetails: true,1371 IncludePRDetails: true,
1371 Logger: testProgressLogger(&progressLogs),1372 Logger: testProgressLogger(&progressLogs),
1373 Progress: func(snapshot SyncProgress) error {
1374 snapshots = append(snapshots, snapshot)
1375 return nil
1376 },
1372 })1377 })
1373 if err != nil {1378 if err != nil {
1374 t.Fatalf("sync: %v", err)1379 t.Fatalf("sync: %v", err)
@@ -1385,6 +1390,18 @@ func TestSyncPersistsIssuesAndPullRequests(t *testing.T) { · 10 unmodified lines
1385 if stats.MetadataOnly {1390 if stats.MetadataOnly {
1386 t.Fatal("metadata only: got true want false")1391 t.Fatal("metadata only: got true want false")
1387 }1392 }
1393 if len(snapshots) < 4 {
1394 t.Fatalf("progress snapshots = %+v", snapshots)
1395 }
1396 if snapshots[0].Stage != SyncProgressConnecting {
1397 t.Fatalf("initial progress = %+v", snapshots[0])
1398 }
1399 if got := snapshots[len(snapshots)-1]; got.Stage != SyncProgressFinalizing ||
1400 got.IssuesReceived != 1 ||
1401 got.PullRequestsReceived != 1 ||
1402 got.CommentsReceived != 1 {
1403 t.Fatalf("terminal progress = %+v", got)
1404 }
1388 1405
1389 repo, err := st.RepositoryByFullName(ctx, "openclaw/gitcrawl")1406 repo, err := st.RepositoryByFullName(ctx, "openclaw/gitcrawl")
1390 if err != nil {1407 if err != nil {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t