History

fix(cloud): scope publisher status to snapshots

Vincent Koc committed 4 months ago

Commit e26ec001 parent 72a412c6 files · +68 −9Browse files

Unified · Split

6 files
  1. CHANGELOG.md +1 −0
  2. go.mod +1 −1
  3. go.sum +2 −2
  4. internal/cli/app_test.go +44 −4
  5. internal/cli/cloud_commands.go +12 −2
  6. internal/cli/cloud_commands_test.go +8 −0
CHANGELOG.md+1−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
3## 0.7.2 - Unreleased3## 0.7.2 - Unreleased
4 4
5- Require every Gitcrawl cloud publisher manifest to opt into snapshot staging so `--stage-only` and normal publish-then-cutover runs cannot activate serving state through the legacy compatibility path.5- Require every Gitcrawl cloud publisher manifest to opt into snapshot staging so `--stage-only` and normal publish-then-cutover runs cannot activate serving state through the legacy compatibility path.
6- Update CrawlKit to v0.14.2 and bind publisher resume plus post-cutover verification to the exact snapshot ID, so a newer concurrently staged candidate cannot be mistaken for the snapshot this publisher owns.
6 7
7## 0.7.1 - 2026-07-098## 0.7.1 - 2026-07-09
8 9
go.mod+1−1
@@ -9,7 +9,7 @@ require ( · 8 unmodified lines
9 github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c6608349 github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834
10 github.com/charmbracelet/x/ansi v0.11.710 github.com/charmbracelet/x/ansi v0.11.7
11 github.com/mattn/go-isatty v0.0.2211 github.com/mattn/go-isatty v0.0.22
12 github.com/openclaw/crawlkit v0.14.112 github.com/openclaw/crawlkit v0.14.2
13 github.com/zalando/go-keyring v0.2.813 github.com/zalando/go-keyring v0.2.8
14 golang.org/x/sys v0.46.014 golang.org/x/sys v0.46.0
15 modernc.org/sqlite v1.53.015 modernc.org/sqlite v1.53.0
go.sum+2−2
@@ -64,8 +64,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc · 63 unmodified lines
64github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=64github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
65github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=65github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
66github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=66github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
67github.com/openclaw/crawlkit v0.14.1 h1:B1exhgP/PZ//JCvr5asdhBZt6jZ5JwW8LHxGx0jfprc=67github.com/openclaw/crawlkit v0.14.2 h1:NA0+fDlyYnQjrI6krkhajHDKMTOt9q8JTlMFsGH37pg=
68github.com/openclaw/crawlkit v0.14.1/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs=68github.com/openclaw/crawlkit v0.14.2/go.mod h1:u5oK8v0gtLzIXtj6gehF0JV2cGX1D/xiogZGCJwONGs=
69github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=69github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
70github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=70github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
71github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=71github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
internal/cli/app_test.go+44−4
@@ -1099,6 +1099,7 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 1098 unmodified lines
1099 var snapshotID string1099 var snapshotID string
1100 var sqliteImage []byte1100 var sqliteImage []byte
1101 var publishedSnapshot *crawlremote.ArchiveSnapshot1101 var publishedSnapshot *crawlremote.ArchiveSnapshot
1102 var publisherStatusSnapshotIDs []string
1102 mutationCounter := 01103 mutationCounter := 0
1103 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {1104 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1104 if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" {1105 if r.Method == http.MethodGet && r.URL.EscapedPath() == "/v1/contract" {
@@ -1216,6 +1217,10 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 111 unmodified lines
1216 return1217 return
1217 }1218 }
1218 if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") {1219 if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") {
1220 publisherStatusSnapshotIDs = append(
1221 publisherStatusSnapshotIDs,
1222 r.URL.Query().Get("snapshot_id"),
1223 )
1219 if publishedSnapshot == nil {1224 if publishedSnapshot == nil {
1220 http.NotFound(w, r)1225 http.NotFound(w, r)
1221 return1226 return
@@ -1382,6 +1387,15 @@ func TestCloudPublishSendsLocalRows(t *testing.T) { · 160 unmodified lines
1382 if payload["sqlite_bundle"] == nil {1387 if payload["sqlite_bundle"] == nil {
1383 t.Fatalf("missing sqlite bundle output: %#v", payload)1388 t.Fatalf("missing sqlite bundle output: %#v", payload)
1384 }1389 }
1390 for _, requestedSnapshotID := range publisherStatusSnapshotIDs {
1391 if requestedSnapshotID != snapshotID {
1392 t.Fatalf(
1393 "publisher status requested snapshot %q, want %q",
1394 requestedSnapshotID,
1395 snapshotID,
1396 )
1397 }
1398 }
1385 privacy, ok := payload["sqlite_bundle_privacy"].(map[string]any)1399 privacy, ok := payload["sqlite_bundle_privacy"].(map[string]any)
1386 if !ok || privacy["includes_private_messages"] != true {1400 if !ok || privacy["includes_private_messages"] != true {
1387 t.Fatalf("missing sqlite bundle privacy output: %#v", payload)1401 t.Fatalf("missing sqlite bundle privacy output: %#v", payload)
@@ -1749,6 +1763,7 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { · 361 unmodified lines
1749 uploadRequests := 01763 uploadRequests := 0
1750 ingestRequests := 01764 ingestRequests := 0
1751 publisherStatusRequests := 01765 publisherStatusRequests := 0
1766 var publisherStatusSnapshotIDs []string
1752 readerStatusRequests := 01767 readerStatusRequests := 0
1753 sqliteReadRequests := 01768 sqliteReadRequests := 0
1754 cutovers := 01769 cutovers := 0
@@ -1829,16 +1844,32 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { · 74 unmodified lines
1829 }1844 }
1830 if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") {1845 if r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status") {
1831 publisherStatusRequests++1846 publisherStatusRequests++
1847 requestedSnapshotID := r.URL.Query().Get("snapshot_id")
1848 publisherStatusSnapshotIDs = append(
1849 publisherStatusSnapshotIDs,
1850 requestedSnapshotID,
1851 )
1852 if stagedSnapshot == nil {
1853 http.NotFound(w, r)
1854 return
1855 }
1856 statusSnapshot := stagedSnapshot
1857 if requestedSnapshotID == "" {
1858 concurrent := *stagedSnapshot
1859 concurrent.ID = strings.Repeat("c", 64)
1860 concurrent.SourceSHA256 = concurrent.ID
1861 statusSnapshot = &concurrent
1862 }
1832 activeSnapshotID := ""1863 activeSnapshotID := ""
1833 if stagedSnapshot != nil {1864 if statusSnapshot != nil {
1834 activeSnapshotID = stagedSnapshot.ID1865 activeSnapshotID = statusSnapshot.ID
1835 }1866 }
1836 _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{1867 _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{
1837 App: "gitcrawl",1868 App: "gitcrawl",
1838 Archive: "gitcrawl/openclaw__openclaw",1869 Archive: "gitcrawl/openclaw__openclaw",
1839 ActiveSnapshotID: activeSnapshotID,1870 ActiveSnapshotID: activeSnapshotID,
1840 CoverageComplete: stagedSnapshot != nil,1871 CoverageComplete: statusSnapshot != nil,
1841 Snapshot: stagedSnapshot,1872 Snapshot: statusSnapshot,
1842 })1873 })
1843 return1874 return
1844 }1875 }
@@ -2095,6 +2126,15 @@ func TestCloudPublishStageOnlyThenResumesDefaultCutover(t *testing.T) { · 250 unmodified lines
2095 if publisherStatusRequests != 7 {2126 if publisherStatusRequests != 7 {
2096 t.Fatalf("publisher status requests = %d, want 7", publisherStatusRequests)2127 t.Fatalf("publisher status requests = %d, want 7", publisherStatusRequests)
2097 }2128 }
2129 for _, requestedSnapshotID := range publisherStatusSnapshotIDs {
2130 if requestedSnapshotID != snapshotID {
2131 t.Fatalf(
2132 "publisher status requested snapshot %q, want %q",
2133 requestedSnapshotID,
2134 snapshotID,
2135 )
2136 }
2137 }
2098 if readerStatusRequests != 3 {2138 if readerStatusRequests != 3 {
2099 t.Fatalf("reader status requests = %d, want 3 serving-state checks", readerStatusRequests)2139 t.Fatalf("reader status requests = %d, want 3 serving-state checks", readerStatusRequests)
2100 }2140 }
internal/cli/cloud_commands.go+12−2
@@ -168,7 +168,12 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 167 unmodified lines
168 }168 }
169 169
170 alreadyStaged := false170 alreadyStaged := false
171 status, statusErr := client.PublishStatus(ctx, "gitcrawl", archiveID)171 status, statusErr := client.PublishStatusForSnapshot(
172 ctx,
173 "gitcrawl",
174 archiveID,
175 snapshot.ID,
176 )
172 if statusErr == nil {177 if statusErr == nil {
173 alreadyStaged = gitcrawlPublisherStatusMatches(178 alreadyStaged = gitcrawlPublisherStatusMatches(
174 status,179 status,
@@ -1080,7 +1085,12 @@ func verifyGitcrawlSnapshotPublication( · 905 unmodified lines
1080 publicationCapabilities []string,1085 publicationCapabilities []string,
1081 sourceSize int64,1086 sourceSize int64,
1082) error {1087) error {
1083 status, err := client.PublishStatus(ctx, "gitcrawl", archive)1088 status, err := client.PublishStatusForSnapshot(
1089 ctx,
1090 "gitcrawl",
1091 archive,
1092 snapshot.ID,
1093 )
1084 if err != nil {1094 if err != nil {
1085 return fmt.Errorf("read post-cutover publisher status: %w", err)1095 return fmt.Errorf("read post-cutover publisher status: %w", err)
1086 }1096 }
internal/cli/cloud_commands_test.go+8−0
@@ -476,6 +476,14 @@ func TestVerifyGitcrawlSnapshotPublicationRejectsUnreadableOrMismatchedSQLite(t · 475 unmodified lines
476 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {476 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
477 switch {477 switch {
478 case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"):478 case r.Method == http.MethodGet && strings.HasSuffix(r.URL.EscapedPath(), "/publish-status"):
479 if got := r.URL.Query().Get("snapshot_id"); got != snapshotID {
480 http.Error(
481 w,
482 fmt.Sprintf("snapshot_id = %q, want %q", got, snapshotID),
483 http.StatusBadRequest,
484 )
485 return
486 }
479 w.Header().Set("content-type", "application/json")487 w.Header().Set("content-type", "application/json")
480 _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{488 _ = json.NewEncoder(w).Encode(crawlremote.PublisherStatus{
481 App: "gitcrawl",489 App: "gitcrawl",