Showing the changes to internal/cli/cloud_commands.go (1 of 5 files). Show all
Unified · Split
| @@ -2,11 +2,16 @@ package cli · 1 unmodified line | |||
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | "context" | 4 | "context" |
| 5 | "crypto/sha256" | ||
| 5 | "database/sql" | 6 | "database/sql" |
| 6 | "flag" | 7 | "flag" |
| 7 | "fmt" | 8 | "fmt" |
| 8 | "io" | 9 | "io" |
| 10 | "net/http" | ||
| 11 | "os" | ||
| 12 | "path/filepath" | ||
| 9 | "strings" | 13 | "strings" |
| 14 | "time" | ||
| 10 | 15 | ||
| 11 | crawlremote "github.com/openclaw/crawlkit/remote" | 16 | crawlremote "github.com/openclaw/crawlkit/remote" |
| 12 | "github.com/openclaw/gitcrawl/internal/config" | 17 | "github.com/openclaw/gitcrawl/internal/config" |
| @@ -67,7 +72,8 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 54 unmodified lines | |||
| 67 | TokenEnv: firstNonEmpty(*tokenEnv, cfg.Remote.TokenEnv, crawlremote.DefaultTokenEnv), | 72 | TokenEnv: firstNonEmpty(*tokenEnv, cfg.Remote.TokenEnv, crawlremote.DefaultTokenEnv), |
| 68 | } | 73 | } |
| 69 | client, err := crawlremote.NewClientFromConfig(remoteCfg, crawlremote.Options{ | 74 | client, err := crawlremote.NewClientFromConfig(remoteCfg, crawlremote.Options{ |
| 70 | UserAgent: "gitcrawl/" + version, | 75 | UserAgent: "gitcrawl/" + version, |
| 76 | HTTPClient: &http.Client{Timeout: 10 * time.Minute}, | ||
| 71 | }) | 77 | }) |
| 72 | if err != nil { | 78 | if err != nil { |
| 73 | return err | 79 | return err |
| @@ -85,7 +91,11 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 11 unmodified lines | |||
| 85 | if err != nil { | 91 | if err != nil { |
| 86 | return err | 92 | return err |
| 87 | } | 93 | } |
| 88 | threadRows, err := publishRows(ctx, rt.Store.DB(), gitcrawlThreadExportSQL, func(values []any) []any { return values }) | 94 | threadSQL, err := gitcrawlThreadExportSQL(ctx, rt.Store.DB()) |
| 95 | if err != nil { | ||
| 96 | return err | ||
| 97 | } | ||
| 98 | threadRows, err := publishRows(ctx, rt.Store.DB(), threadSQL, func(values []any) []any { return values }) | ||
| 89 | if err != nil { | 99 | if err != nil { |
| 90 | return err | 100 | return err |
| 91 | } | 101 | } |
| @@ -97,11 +107,16 @@ func (a *App) runCloudPublish(ctx context.Context, args []string) error { · 5 unmodified lines | |||
| 97 | if err != nil { | 107 | if err != nil { |
| 98 | return err | 108 | return err |
| 99 | } | 109 | } |
| 110 | sqliteObject, err := uploadSQLiteArchive(ctx, client, "gitcrawl", archiveID, rt.Store.DB(), rt.SourceDBPath, manifest) | ||
| 111 | if err != nil { | ||
| 112 | return err | ||
| 113 | } | ||
| 100 | return a.writeOutput("cloud publish", map[string]any{ | 114 | return a.writeOutput("cloud publish", map[string]any{ |
| 101 | "remote": strings.TrimRight(endpoint, "/"), | 115 | "remote": strings.TrimRight(endpoint, "/"), |
| 102 | "archive": archiveID, | 116 | "archive": archiveID, |
| 103 | "repositories": repoAccepted, | 117 | "repositories": repoAccepted, |
| 104 | "threads": threadAccepted, | 118 | "threads": threadAccepted, |
| 119 | "sqlite_object": sqliteObject, | ||
| 105 | }, true) | 120 | }, true) |
| 106 | } | 121 | } |
| 107 | 122 | ||
| @@ -175,6 +190,74 @@ func cursorFor(start int) string { · 67 unmodified lines | |||
| 175 | return fmt.Sprintf("%d", start) | 190 | return fmt.Sprintf("%d", start) |
| 176 | } | 191 | } |
| 177 | 192 | ||
| 193 | func uploadSQLiteArchive(ctx context.Context, client *crawlremote.Client, app, archive string, db *sql.DB, dbPath string, manifest crawlremote.IngestManifest) (*crawlremote.SQLiteObject, error) { | ||
| 194 | snapshotPath, cleanup, err := sqliteSnapshotPath(ctx, db, dbPath) | ||
| 195 | if err != nil { | ||
| 196 | return nil, err | ||
| 197 | } | ||
| 198 | defer cleanup() | ||
| 199 | file, err := os.Open(snapshotPath) | ||
| 200 | if err != nil { | ||
| 201 | return nil, fmt.Errorf("open sqlite snapshot: %w", err) | ||
| 202 | } | ||
| 203 | defer file.Close() | ||
| 204 | info, err := file.Stat() | ||
| 205 | if err != nil { | ||
| 206 | return nil, fmt.Errorf("stat sqlite snapshot: %w", err) | ||
| 207 | } | ||
| 208 | sum, err := cloudFileSHA256(snapshotPath) | ||
| 209 | if err != nil { | ||
| 210 | return nil, err | ||
| 211 | } | ||
| 212 | if _, err := file.Seek(0, io.SeekStart); err != nil { | ||
| 213 | return nil, fmt.Errorf("rewind sqlite snapshot: %w", err) | ||
| 214 | } | ||
| 215 | result, err := client.UploadSQLite(ctx, app, archive, crawlremote.SQLiteUploadRequest{ | ||
| 216 | Body: file, | ||
| 217 | Size: info.Size(), | ||
| 218 | ContentSHA256: sum, | ||
| 219 | SchemaName: manifest.SchemaName, | ||
| 220 | SchemaVersion: manifest.SchemaVersion, | ||
| 221 | SchemaHash: manifest.SchemaHash, | ||
| 222 | SourceSyncAt: manifest.SourceSyncAt, | ||
| 223 | }) | ||
| 224 | if err != nil { | ||
| 225 | return nil, err | ||
| 226 | } | ||
| 227 | return result.Object, nil | ||
| 228 | } | ||
| 229 | |||
| 230 | func sqliteSnapshotPath(ctx context.Context, db *sql.DB, dbPath string) (string, func(), error) { | ||
| 231 | tmpDir, err := os.MkdirTemp("", "gitcrawl-cloud-sqlite-*") | ||
| 232 | if err != nil { | ||
| 233 | return "", func() {}, fmt.Errorf("create sqlite snapshot dir: %w", err) | ||
| 234 | } | ||
| 235 | cleanup := func() { _ = os.RemoveAll(tmpDir) } | ||
| 236 | snapshotPath := filepath.Join(tmpDir, "archive.db") | ||
| 237 | if _, err := db.ExecContext(ctx, "vacuum main into ?", snapshotPath); err == nil { | ||
| 238 | return snapshotPath, cleanup, nil | ||
| 239 | } | ||
| 240 | source := strings.TrimSpace(dbPath) | ||
| 241 | if source == "" { | ||
| 242 | cleanup() | ||
| 243 | return "", func() {}, fmt.Errorf("sqlite snapshot failed and no source db path is available") | ||
| 244 | } | ||
| 245 | return source, cleanup, nil | ||
| 246 | } | ||
| 247 | |||
| 248 | func cloudFileSHA256(path string) (string, error) { | ||
| 249 | file, err := os.Open(path) | ||
| 250 | if err != nil { | ||
| 251 | return "", fmt.Errorf("open sqlite snapshot for hash: %w", err) | ||
| 252 | } | ||
| 253 | defer file.Close() | ||
| 254 | hash := sha256.New() | ||
| 255 | if _, err := io.Copy(hash, file); err != nil { | ||
| 256 | return "", fmt.Errorf("hash sqlite snapshot: %w", err) | ||
| 257 | } | ||
| 258 | return fmt.Sprintf("%x", hash.Sum(nil)), nil | ||
| 259 | } | ||
| 260 | |||
| 178 | var gitcrawlRepositoryColumns = []string{"id", "full_name", "owner", "name", "html_url", "default_branch", "updated_at"} | 261 | var gitcrawlRepositoryColumns = []string{"id", "full_name", "owner", "name", "html_url", "default_branch", "updated_at"} |
| 179 | 262 | ||
| 180 | const gitcrawlRepositoryExportSQL = ` | 263 | const gitcrawlRepositoryExportSQL = ` |
| @@ -184,11 +267,60 @@ order by id` · 3 unmodified lines | |||
| 184 | 267 | ||
| 185 | var gitcrawlThreadColumns = []string{"id", "repo_id", "github_id", "number", "kind", "state", "title", "body", "author_login", "author_type", "html_url", "labels_json", "assignees_json", "is_draft", "created_at_gh", "updated_at_gh", "closed_at_gh", "merged_at_gh", "updated_at"} | 268 | var gitcrawlThreadColumns = []string{"id", "repo_id", "github_id", "number", "kind", "state", "title", "body", "author_login", "author_type", "html_url", "labels_json", "assignees_json", "is_draft", "created_at_gh", "updated_at_gh", "closed_at_gh", "merged_at_gh", "updated_at"} |
| 186 | 269 | ||
| 187 | const gitcrawlThreadExportSQL = ` | 270 | func gitcrawlThreadExportSQL(ctx context.Context, db *sql.DB) (string, error) { |
| 188 | select id, repo_id, github_id, number, kind, state, title, coalesce(body, ''), | 271 | bodyExpr, err := gitcrawlThreadBodyExpr(ctx, db) |
| 272 | if err != nil { | ||
| 273 | return "", err | ||
| 274 | } | ||
| 275 | return fmt.Sprintf(` | ||
| 276 | select id, repo_id, github_id, number, kind, state, title, %s, | ||
| 189 | coalesce(author_login, ''), coalesce(author_type, ''), html_url, | 277 | coalesce(author_login, ''), coalesce(author_type, ''), html_url, |
| 190 | labels_json, assignees_json, is_draft, coalesce(created_at_gh, ''), | 278 | labels_json, assignees_json, is_draft, coalesce(created_at_gh, ''), |
| 191 | coalesce(updated_at_gh, ''), coalesce(closed_at_gh, ''), | 279 | coalesce(updated_at_gh, ''), coalesce(closed_at_gh, ''), |
| 192 | coalesce(merged_at_gh, ''), updated_at | 280 | coalesce(merged_at_gh, ''), updated_at |
| 193 | from threads | 281 | from threads |
| 194 | order by repo_id, number` | 282 | order by repo_id, number`, bodyExpr), nil |
| 283 | } | ||
| 284 | |||
| 285 | func gitcrawlThreadBodyExpr(ctx context.Context, db *sql.DB) (string, error) { | ||
| 286 | hasBody, err := sqliteColumnExists(ctx, db, "threads", "body") | ||
| 287 | if err != nil { | ||
| 288 | return "", err | ||
| 289 | } | ||
| 290 | hasExcerpt, err := sqliteColumnExists(ctx, db, "threads", "body_excerpt") | ||
| 291 | if err != nil { | ||
| 292 | return "", err | ||
| 293 | } | ||
| 294 | switch { | ||
| 295 | case hasBody && hasExcerpt: | ||
| 296 | return "coalesce(body, body_excerpt, '')", nil | ||
| 297 | case hasBody: | ||
| 298 | return "coalesce(body, '')", nil | ||
| 299 | case hasExcerpt: | ||
| 300 | return "coalesce(body_excerpt, '')", nil | ||
| 301 | default: | ||
| 302 | return "''", nil | ||
| 303 | } | ||
| 304 | } | ||
| 305 | |||
| 306 | func sqliteColumnExists(ctx context.Context, db *sql.DB, table, column string) (bool, error) { | ||
| 307 | rows, err := db.QueryContext(ctx, `pragma table_info(`+table+`)`) | ||
| 308 | if err != nil { | ||
| 309 | return false, err | ||
| 310 | } | ||
| 311 | defer rows.Close() | ||
| 312 | for rows.Next() { | ||
| 313 | var cid int | ||
| 314 | var name, typ string | ||
| 315 | var notNull int | ||
| 316 | var defaultValue any | ||
| 317 | var pk int | ||
| 318 | if err := rows.Scan(&cid, &name, &typ, ¬Null, &defaultValue, &pk); err != nil { | ||
| 319 | return false, err | ||
| 320 | } | ||
| 321 | if name == column { | ||
| 322 | return true, nil | ||
| 323 | } | ||
| 324 | } | ||
| 325 | return false, rows.Err() | ||
| 326 | } | ||