Showing the changes to internal/cli/cloud_commands.go (1 of 5 files). Show all
Unified · Split
| @@ -2,11 +2,16 @@ package cli · 1 unmodified line | ||
| 2 | 2 | |
| 3 | 3 | import ( |
| 4 | 4 | "context" |
| 5 | + "crypto/sha256" | |
| 5 | 6 | "database/sql" |
| 6 | 7 | "flag" |
| 7 | 8 | "fmt" |
| 8 | 9 | "io" |
| 10 | + "net/http" | |
| 11 | + "os" | |
| 12 | + "path/filepath" | |
| 9 | 13 | "strings" |
| 14 | + "time" | |
| 10 | 15 | |
| 11 | 16 | crawlremote "github.com/openclaw/crawlkit/remote" |
| 12 | 17 | "github.com/openclaw/gitcrawl/internal/config" |
| @@ -67,7 +72,8 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 54 unmodified lines | ||
| 67 | 72 | TokenEnv: firstNonEmpty(*tokenEnv, cfg.Remote.TokenEnv, crawlremote.DefaultTokenEnv), |
| 68 | 73 | } |
| 69 | 74 | client, err := crawlremote.NewClientFromConfig(remoteCfg, crawlremote.Options{ |
| 70 | − UserAgent: "gitcrawl/" + version, | |
| 75 | + UserAgent: "gitcrawl/" + version, | |
| 76 | + HTTPClient: &http.Client{Timeout: 10 * time.Minute}, | |
| 71 | 77 | }) |
| 72 | 78 | if err != nil { |
| 73 | 79 | return err |
| @@ -85,7 +91,11 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 11 unmodified lines | ||
| 85 | 91 | if err != nil { |
| 86 | 92 | return err |
| 87 | 93 | } |
| 88 | − threadRows, err := publishRows(ctx, rt.Store.DB(), gitcrawlThreadExportSQL, func(values []any) []any { return values }) | |
| 94 | + threadSQL, err := gitcrawlThreadExportSQL(ctx, rt.Store.DB()) | |
| 95 | + if err != nil { | |
| 96 | + return err | |
| 97 | + } | |
| 98 | + threadRows, err := publishRows(ctx, rt.Store.DB(), threadSQL, func(values []any) []any { return values }) | |
| 89 | 99 | if err != nil { |
| 90 | 100 | return err |
| 91 | 101 | } |
| @@ -97,11 +107,16 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 5 unmodified lines | ||
| 97 | 107 | if err != nil { |
| 98 | 108 | return err |
| 99 | 109 | } |
| 110 | + sqliteObject, err := uploadSQLiteArchive(ctx, client, "gitcrawl", archiveID, rt.Store.DB(), rt.SourceDBPath, manifest) | |
| 111 | + if err != nil { | |
| 112 | + return err | |
| 113 | + } | |
| 100 | 114 | return a.writeOutput("cloud publish", map[string]any{ |
| 101 | − "remote": strings.TrimRight(endpoint, "/"), | |
| 102 | − "archive": archiveID, | |
| 103 | − "repositories": repoAccepted, | |
| 104 | − "threads": threadAccepted, | |
| 115 | + "remote": strings.TrimRight(endpoint, "/"), | |
| 116 | + "archive": archiveID, | |
| 117 | + "repositories": repoAccepted, | |
| 118 | + "threads": threadAccepted, | |
| 119 | + "sqlite_object": sqliteObject, | |
| 105 | 120 | }, true) |
| 106 | 121 | } |
| 107 | 122 | |
| @@ -175,6 +190,74 @@ func cursorFor(start int) string { · 67 unmodified lines | ||
| 175 | 190 | return fmt.Sprintf("%d", start) |
| 176 | 191 | } |
| 177 | 192 | |
| 193 | +func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, archive string, db *sql.DB, dbPath string, manifest crawlremote.IngestManifest) (*crawlremote.SQLiteObject, error) { | |
| 194 | + snapshotPath, cleanup, err := sqliteSnapshotPath(ctx, db, dbPath) | |
| 195 | + if err != nil { | |
| 196 | + return nil, err | |
| 197 | + } | |
| 198 | + defer cleanup() | |
| 199 | + file, err := os.Open(snapshotPath) | |
| 200 | + if err != nil { | |
| 201 | + return nil, fmt.Errorf("open sqlite snapshot: %w", err) | |
| 202 | + } | |
| 203 | + defer file.Close() | |
| 204 | + info, err := file.Stat() | |
| 205 | + if err != nil { | |
| 206 | + return nil, fmt.Errorf("stat sqlite snapshot: %w", err) | |
| 207 | + } | |
| 208 | + sum, err := cloudFileSHA256(snapshotPath) | |
| 209 | + if err != nil { | |
| 210 | + return nil, err | |
| 211 | + } | |
| 212 | + if _, err := file.Seek(0, io.SeekStart); err != nil { | |
| 213 | + return nil, fmt.Errorf("rewind sqlite snapshot: %w", err) | |
| 214 | + } | |
| 215 | + result, err := client.UploadSQLite(ctx, app, archive, crawlremote.SQLiteUploadRequest{ | |
| 216 | + Body: file, | |
| 217 | + Size: info.Size(), | |
| 218 | + ContentSHA256: sum, | |
| 219 | + SchemaName: manifest.SchemaName, | |
| 220 | + SchemaVersion: manifest.SchemaVersion, | |
| 221 | + SchemaHash: manifest.SchemaHash, | |
| 222 | + SourceSyncAt: manifest.SourceSyncAt, | |
| 223 | + }) | |
| 224 | + if err != nil { | |
| 225 | + return nil, err | |
| 226 | + } | |
| 227 | + return result.Object, nil | |
| 228 | +} | |
| 229 | + | |
| 230 | +func sqliteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (string, func(), error) { | |
| 231 | + tmpDir, err := os.MkdirTemp("", "gitcrawl-cloud-sqlite-*") | |
| 232 | + if err != nil { | |
| 233 | + return "", func() {}, fmt.Errorf("create sqlite snapshot dir: %w", err) | |
| 234 | + } | |
| 235 | + cleanup := func() { _ = os.RemoveAll(tmpDir) } | |
| 236 | + snapshotPath := filepath.Join(tmpDir, "archive.db") | |
| 237 | + if _, err := db.ExecContext(ctx, "vacuum main into ?", snapshotPath); err == nil { | |
| 238 | + return snapshotPath, cleanup, nil | |
| 239 | + } | |
| 240 | + source := strings.TrimSpace(dbPath) | |
| 241 | + if source == "" { | |
| 242 | + cleanup() | |
| 243 | + return "", func() {}, fmt.Errorf("sqlite snapshot failed and no source db path is available") | |
| 244 | + } | |
| 245 | + return source, cleanup, nil | |
| 246 | +} | |
| 247 | + | |
| 248 | +func cloudFileSHA256(path string) (string, error) { | |
| 249 | + file, err := os.Open(path) | |
| 250 | + if err != nil { | |
| 251 | + return "", fmt.Errorf("open sqlite snapshot for hash: %w", err) | |
| 252 | + } | |
| 253 | + defer file.Close() | |
| 254 | + hash := sha256.New() | |
| 255 | + if _, err := io.Copy(hash, file); err != nil { | |
| 256 | + return "", fmt.Errorf("hash sqlite snapshot: %w", err) | |
| 257 | + } | |
| 258 | + return fmt.Sprintf("%x", hash.Sum(nil)), nil | |
| 259 | +} | |
| 260 | + | |
| 178 | 261 | var gitcrawlRepositoryColumns = []string{"id", "full_name", "owner", "name", "html_url", "default_branch", "updated_at"} |
| 179 | 262 | |
| 180 | 263 | const gitcrawlRepositoryExportSQL = ` |
| @@ -184,11 +267,60 @@ order by id` · 3 unmodified lines | ||
| 184 | 267 | |
| 185 | 268 | var gitcrawlThreadColumns = []string{"id", "repo_id", "github_id", "number", "kind", "state", "title", "body", "author_login", "author_type", "html_url", "labels_json", "assignees_json", "is_draft", "created_at_gh", "updated_at_gh", "closed_at_gh", "merged_at_gh", "updated_at"} |
| 186 | 269 | |
| 187 | −const gitcrawlThreadExportSQL = ` | |
| 188 | −select id, repo_id, github_id, number, kind, state, title, coalesce(body, ''), | |
| 270 | +func gitcrawlThreadExportSQL(ctx context.Context, db *sql.DB) (string, error) { | |
| 271 | + bodyExpr, err := gitcrawlThreadBodyExpr(ctx, db) | |
| 272 | + if err != nil { | |
| 273 | + return "", err | |
| 274 | + } | |
| 275 | + return fmt.Sprintf(` | |
| 276 | +select id, repo_id, github_id, number, kind, state, title, %s, | |
| 189 | 277 | coalesce(author_login, ''), coalesce(author_type, ''), html_url, |
| 190 | 278 | labels_json, assignees_json, is_draft, coalesce(created_at_gh, ''), |
| 191 | 279 | coalesce(updated_at_gh, ''), coalesce(closed_at_gh, ''), |
| 192 | 280 | coalesce(merged_at_gh, ''), updated_at |
| 193 | 281 | from threads |
| 194 | −order by repo_id, number` | |
| 282 | +order by repo_id, number`, bodyExpr), nil | |
| 283 | +} | |
| 284 | + | |
| 285 | +func gitcrawlThreadBodyExpr(ctx context.Context, db *sql.DB) (string, error) { | |
| 286 | + hasBody, err := sqliteColumnExists(ctx, db, "threads", "body") | |
| 287 | + if err != nil { | |
| 288 | + return "", err | |
| 289 | + } | |
| 290 | + hasExcerpt, err := sqliteColumnExists(ctx, db, "threads", "body_excerpt") | |
| 291 | + if err != nil { | |
| 292 | + return "", err | |
| 293 | + } | |
| 294 | + switch { | |
| 295 | + case hasBody && hasExcerpt: | |
| 296 | + return "coalesce(body, body_excerpt, '')", nil | |
| 297 | + case hasBody: | |
| 298 | + return "coalesce(body, '')", nil | |
| 299 | + case hasExcerpt: | |
| 300 | + return "coalesce(body_excerpt, '')", nil | |
| 301 | + default: | |
| 302 | + return "''", nil | |
| 303 | + } | |
| 304 | +} | |
| 305 | + | |
| 306 | +func sqliteColumnExists(ctx context.Context, db *sql.DB, table, column string) (bool, error) { | |
| 307 | + rows, err := db.QueryContext(ctx, `pragma table_info(`+table+`)`) | |
| 308 | + if err != nil { | |
| 309 | + return false, err | |
| 310 | + } | |
| 311 | + defer rows.Close() | |
| 312 | + for rows.Next() { | |
| 313 | + var cid int | |
| 314 | + var name, typ string | |
| 315 | + var notNull int | |
| 316 | + var defaultValue any | |
| 317 | + var pk int | |
| 318 | + if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { | |
| 319 | + return false, err | |
| 320 | + } | |
| 321 | + if name == column { | |
| 322 | + return true, nil | |
| 323 | + } | |
| 324 | + } | |
| 325 | + return false, rows.Err() | |
| 326 | +} | |