package cli import (	"compress/gzip"	"context"	"encoding/json"	"errors"	"fmt"	"io"	"os"	"os/exec"	"path/filepath"	"strings"	"sync"	"time" 	crawlremote "github.com/openclaw/crawlkit/remote"	"github.com/openclaw/gitcrawl/internal/config"	"github.com/openclaw/gitcrawl/internal/store") type localRuntime struct {	Config       config.Config	Store        *store.Store	SourceDBPath string	RemoteSource bool} type dbTargetInfo struct {	DBTarget         string `json:"db_target,omitempty"`	DBTargetPath     string `json:"db_target_path,omitempty"`	PortableSourceDB string `json:"portable_source_db,omitempty"`} func (rt localRuntime) dbTarget() dbTargetInfo {	if rt.RemoteSource {		return dbTargetInfo{			DBTarget:         "runtime-mirror",			DBTargetPath:     rt.Config.DBPath,			PortableSourceDB: rt.SourceDBPath,		}	}	return dbTargetInfo{DBTarget: "direct", DBTargetPath: rt.Config.DBPath}} const portableStoreRefreshTimeout = 15 * time.Secondconst portableStoreRepairTimeout = 90 * time.Secondconst portableStoreRefreshTTL = 2 * time.Minuteconst portableStoreRefreshFailureBackoff = time.Minuteconst portableRuntimeTempMaxAge = time.Hourconst portableSourceRecoveryBackoff = 15 * time.Minuteconst portableStoreMarkerFile = "gitcrawl-portable-store"const staleGitIndexLockAge = 2 * time.Second var errPortableStoreDirty = errors.New("portable store checkout has local changes") func (a *App) openLocalRuntime(ctx context.Context) (localRuntime, error) {	cfg, err := config.LoadRuntime(a.configPath)	if err != nil {		return localRuntime{}, err	}	if cfg.Remote.Enabled() && cfg.Remote.Mode == crawlremote.ModeCloud {		return localRuntime{}, fmt.Errorf("command requires a local gitcrawl database; config is remote cloud mode")	}	sourceDBPath := cfg.DBPath	remoteSource := false	if _, ok, err := portableStoreRoot(ctx, cfg.DBPath); err != nil {		return localRuntime{}, err	} else if ok {		mirrorPath, _, err := a.ensurePortableRuntimeDB(ctx, cfg.DBPath, false)		if err != nil {			return localRuntime{}, err		}		cfg.DBPath = mirrorPath		remoteSource = true		a.dbTargetNoticeOnce.Do(func() {			fmt.Fprintf(a.Stderr, "gitcrawl: portable store checkout detected; writes go to the runtime mirror at %s, not the checkout database %s. Run 'gitcrawl portable prune' to publish.\n", mirrorPath, sourceDBPath)		})	}	st, err := store.Open(ctx, cfg.DBPath)	if err != nil {		return localRuntime{}, err	}	return localRuntime{Config: cfg, Store: st, SourceDBPath: sourceDBPath, RemoteSource: remoteSource}, nil} func (a *App) openLocalRuntimeReadOnly(ctx context.Context) (localRuntime, error) {	cfg, err := config.LoadRuntime(a.configPath)	if err != nil {		return localRuntime{}, err	}	return a.openLocalRuntimeReadOnlyWithConfig(ctx, cfg)} func (a *App) openLocalRuntimeReadOnlyWithConfig(ctx context.Context, cfg config.Config) (localRuntime, error) {	if cfg.Remote.Enabled() && cfg.Remote.Mode == crawlremote.ModeCloud {		return localRuntime{}, fmt.Errorf("command requires a local gitcrawl database; config is remote cloud mode")	}	sourceDBPath := cfg.DBPath	remoteSource := false	if _, ok, err := portableStoreRoot(ctx, cfg.DBPath); err != nil {		return localRuntime{}, err	} else if ok {		mirrorPath, _, err := a.ensurePortableRuntimeDB(ctx, cfg.DBPath, true)		if err != nil {			return localRuntime{}, err		}		cfg.DBPath = mirrorPath		remoteSource = true	}	st, err := store.OpenReadOnly(ctx, cfg.DBPath)	if err != nil {		return localRuntime{}, err	}	return localRuntime{Config: cfg, Store: st, SourceDBPath: sourceDBPath, RemoteSource: remoteSource}, nil} func (rt localRuntime) repository(ctx context.Context, owner, repo string) (store.Repository, error) {	return rt.Store.RepositoryByFullName(ctx, owner+"/"+repo)} func (rt localRuntime) defaultRepository(ctx context.Context) (store.Repository, error) {	repos, err := rt.Store.ListRepositories(ctx)	if err != nil {		return store.Repository{}, err	}	if len(repos) == 0 {		return store.Repository{}, fmt.Errorf("no local repositories found")	}	return repos[0], nil} func refreshPortableStoreForDB(ctx context.Context, dbPath string) error {	root, ok, err := portableStoreRoot(ctx, dbPath)	if err != nil {		return err	}	if !ok {		return nil	}	clean := gitWorktreeClean(ctx, root)	if !clean {		removed, _ := removeStaleGitIndexLock(ctx, root, staleGitIndexLockAge)		if removed {			clean = gitWorktreeClean(ctx, root)		}	}	if !clean {		return errPortableStoreDirty	}	pullCtx, cancel := context.WithTimeout(ctx, portableStoreRefreshTimeout)	defer cancel()	if _, err := fastForwardGitCheckoutWithStaleIndexLockRetry(pullCtx, root, true); err != nil {		return err	}	return removePortableSQLiteSidecars(root)} type portableRepairResult struct {	Action           string	DBBackupPath     string	StoreBackupPath  string	RemovedIndexLock bool} func repairMalformedPortableStoreForDB(ctx context.Context, dbPath, configPath string) (portableRepairResult, error) {	result := portableRepairResult{Action: "reset-pulled"}	root, ok, err := portableStoreRoot(ctx, dbPath)	if err != nil {		return result, err	}	if !ok {		return result, nil	}	if !portableStoreRepairAllowed(root, configPath) {		return result, fmt.Errorf("refuse destructive repair for unmarked portable store checkout %s", root)	}	backupPath, err := preserveMalformedPortableDB(root, dbPath)	if err != nil {		return result, err	}	result.DBBackupPath = backupPath	pullCtx, cancel := context.WithTimeout(ctx, portableStoreRepairTimeout)	defer cancel()	if !gitWorktreeClean(pullCtx, root) {		removed, err := runGitWithStaleIndexLockRetry(pullCtx, root, "-C", root, "reset", "--hard", "HEAD")		result.RemovedIndexLock = result.RemovedIndexLock || removed		if err != nil {			return result, err		}	}	removed, err := fastForwardGitCheckoutWithStaleIndexLockRetry(pullCtx, root, true)	result.RemovedIndexLock = result.RemovedIndexLock || removed	if err != nil {		return result, err	}	return result, removePortableSQLiteSidecars(root)} func recloneMalformedPortableStoreForDB(ctx context.Context, dbPath, configPath string) (portableRepairResult, error) {	result := portableRepairResult{Action: "recloned"}	root, ok, err := portableStoreRoot(ctx, dbPath)	if err != nil {		return result, err	}	if !ok {		return result, nil	}	if !portableStoreRepairAllowed(root, configPath) {		return result, fmt.Errorf("refuse reclone for unmarked portable store checkout %s", root)	}	remote := portableStoreRemoteURL(ctx, root)	if strings.TrimSpace(remote) == "" {		return result, fmt.Errorf("portable store remote not found for %s", root)	}	branch := currentGitBranch(ctx, root)	timestamp := time.Now().UTC().Format("20060102T150405Z")	backupPath := filepath.Join(filepath.Dir(root), "backups", "checkout-malformed-"+timestamp)	if err := os.MkdirAll(filepath.Dir(backupPath), 0o755); err != nil {		return result, fmt.Errorf("create portable checkout backup parent: %w", err)	}	if err := os.Rename(root, backupPath); err != nil {		return result, fmt.Errorf("preserve malformed portable checkout: %w", err)	}	result.StoreBackupPath = backupPath	cloneCtx, cancel := context.WithTimeout(ctx, portableStoreRepairTimeout)	defer cancel()	cloneArgs := []string{"clone", "--depth", "1"}	if strings.TrimSpace(branch) != "" {		cloneArgs = append(cloneArgs, "--branch", branch)	}	cloneArgs = append(cloneArgs, remote, root)	if err := runGit(cloneCtx, "", cloneArgs...); err != nil {		_ = os.RemoveAll(root)		_ = os.Rename(backupPath, root)		return result, err	}	if err := markPortableStoreCheckout(root); err != nil {		return result, err	}	return result, removePortableSQLiteSidecars(root)} var portableRuntimeMu sync.Mutex func (a *App) ensurePortableRuntimeDB(ctx context.Context, sourceDBPath string, refresh bool) (string, bool, error) {	mirrorPath, err := a.portableRuntimeDBPath(ctx, sourceDBPath)	if err != nil {		return "", false, err	}	changed, err := refreshPortableRuntimeDB(ctx, sourceDBPath, mirrorPath, refresh, a.configPath)	return mirrorPath, changed, err} func (a *App) portableRuntimeDBPath(ctx context.Context, sourceDBPath string) (string, error) {	root, ok, err := portableStoreRoot(ctx, sourceDBPath)	if err != nil {		return "", err	}	if !ok {		return "", fmt.Errorf("portable store root not found for %s", sourceDBPath)	}	rel, err := filepath.Rel(root, sourceDBPath)	if err != nil || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) || rel == ".." || filepath.IsAbs(rel) {		return "", fmt.Errorf("portable database %s is outside store root %s", sourceDBPath, root)	}	name := safePathName(filepath.Base(root))	if name == "" {		name = "portable-store"	}	return filepath.Join(filepath.Dir(config.ResolvePath(a.configPath)), "runtime", name, rel), nil} func refreshPortableRuntimeDB(ctx context.Context, sourceDBPath, mirrorPath string, refresh bool, configPath string) (bool, error) {	portableRuntimeMu.Lock()	defer portableRuntimeMu.Unlock()	sweepOrphanPortableRuntimeTempFiles(mirrorPath, portableRuntimeTempMaxAge)	_, isPortableSource, err := portableStoreRoot(ctx, sourceDBPath)	if err != nil {		return false, err	}	isRepairablePortableSource := isPortableSource	if refresh {		_ = refreshPortableStoreForDBIfDue(ctx, sourceDBPath, mirrorPath)	}	needsCopy, err := portableRuntimeNeedsCopy(sourceDBPath, mirrorPath)	statePath := portableStoreRefreshStatePath(mirrorPath)	if err != nil {		if !isRepairablePortableSource || !errors.Is(err, os.ErrNotExist) {			return false, err		}		// A recovered source is preferred, but any failure along the		// repair/reclone chain degrades to serving a healthy mirror so a		// missing source cannot take reads down. Recovery runs reset/pull and		// potentially a full reclone, so attempts are backed off via the		// recorded repair timestamp instead of repeating on every read. The		// backoff only gates this stat-failure branch: an externally restored		// source makes the stat succeed and skips the gate entirely.		recoverErr := err		state := readPortableStoreRefreshState(statePath)		if !recentPortableRefresh(state.LastRepairAt, time.Now().UTC(), portableSourceRecoveryBackoff) {			recoverErr = recoverMissingPortableSource(ctx, sourceDBPath, configPath, statePath)			if recoverErr == nil {				needsCopy, recoverErr = portableRuntimeNeedsCopy(sourceDBPath, mirrorPath)			}		}		if recoverErr != nil {			if mirrorHealthErr := sqliteStoreHealth(ctx, mirrorPath); mirrorHealthErr == nil {				return false, nil			}			return false, recoverErr		}	}	mirrorCorrupt := false	if isRepairablePortableSource && !needsCopy {		mirrorHealthErr := portableMirrorCachedHealth(ctx, mirrorPath, sourceDBPath, statePath)		if mirrorHealthErr != nil {			if isSQLiteCorruption(mirrorHealthErr) {				mirrorCorrupt = true				needsCopy = true			} else if isPortableManifestMismatch(mirrorHealthErr) {				needsCopy = true			} else {				return false, fmt.Errorf("check portable runtime db: %w", mirrorHealthErr)			}		}	}	if needsCopy && isRepairablePortableSource {		sourceHealthErr := validatePortableSQLiteSourceFile(ctx, sourceDBPath, sourceDBPath)		if sourceHealthErr != nil && isPortableSourceRepairableHealthError(sourceHealthErr) {			repair, err := repairMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)			recordPortableRepairState(statePath, repair, err)			if err != nil {				state := readPortableStoreRefreshState(statePath)				if !recentPortableRefresh(state.LastRecloneAttempt, time.Now().UTC(), portableSourceRecoveryBackoff) {					reclone, recloneErr := recloneMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)					recordPortableRepairState(statePath, reclone, recloneErr)					if recloneErr == nil {						err = nil					}				}			}			if err != nil {				if !mirrorCorrupt {					if mirrorHealthErr := sqliteStoreHealth(ctx, mirrorPath); mirrorHealthErr == nil {						return false, nil					}				}				return false, fmt.Errorf("repair malformed portable store db: %w", err)			}			sourceHealthErr = validatePortableSQLiteSourceFile(ctx, sourceDBPath, sourceDBPath)			if sourceHealthErr != nil && isPortableSourceRepairableHealthError(sourceHealthErr) {				reclone, err := recloneMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)				recordPortableRepairState(statePath, reclone, err)				if err != nil {					return false, fmt.Errorf("reclone malformed portable store db: %w", err)				}				sourceHealthErr = validatePortableSQLiteSourceFile(ctx, sourceDBPath, sourceDBPath)			}		}		if sourceHealthErr != nil {			return false, fmt.Errorf("check portable source db: %w", sourceHealthErr)		}	}	if !needsCopy {		return false, nil	}	if err := copySQLiteFileAtomicVerified(ctx, sourceDBPath, mirrorPath); err != nil {		return false, err	}	if isRepairablePortableSource {		_ = markPortableMirrorHealthVerified(mirrorPath, statePath, sourceDBPath)	}	return true, nil} type portableStoreRefreshState struct {	LastAttempt                 string `json:"last_attempt,omitempty"`	LastSuccess                 string `json:"last_success,omitempty"`	LastFailure                 string `json:"last_failure,omitempty"`	Error                       string `json:"error,omitempty"`	MirrorHealthModTime         string `json:"mirror_health_mod_time,omitempty"`	MirrorHealthSize            int64  `json:"mirror_health_size,omitempty"`	MirrorHealthManifestModTime string `json:"mirror_health_manifest_mod_time,omitempty"`	MirrorHealthManifestSize    int64  `json:"mirror_health_manifest_size,omitempty"`	MirrorHealthSourceSHA256    string `json:"mirror_health_source_sha256,omitempty"`	LastRepair                  string `json:"last_repair,omitempty"`	LastRepairBackup            string `json:"last_repair_backup,omitempty"`	LastRepairAt                string `json:"last_repair_at,omitempty"`	LastRepairError             string `json:"last_repair_error,omitempty"`	LastRecloneAttempt          string `json:"last_reclone_attempt,omitempty"`} func recoverMissingPortableSource(ctx context.Context, sourceDBPath, configPath, statePath string) error {	repair, err := repairMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)	recordPortableRepairState(statePath, repair, err)	if err != nil {		state := readPortableStoreRefreshState(statePath)		if !recentPortableRefresh(state.LastRecloneAttempt, time.Now().UTC(), portableSourceRecoveryBackoff) {			reclone, recloneErr := recloneMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)			recordPortableRepairState(statePath, reclone, recloneErr)			if recloneErr == nil {				// The caller re-stats the source and falls back to a healthy				// mirror if the recloned store still lacks the database;				// falling through here would reclone a second time.				return nil			}			return fmt.Errorf("repair malformed portable store db: %w; reclone fallback: %v", err, recloneErr)		}		return fmt.Errorf("repair malformed portable store db: %w", err)	}	if _, statErr := os.Stat(sourceDBPath); errors.Is(statErr, os.ErrNotExist) {		reclone, recloneErr := recloneMalformedPortableStoreForDB(ctx, sourceDBPath, configPath)		recordPortableRepairState(statePath, reclone, recloneErr)		if recloneErr != nil {			return fmt.Errorf("reclone malformed portable store db: %w", recloneErr)		}	}	return nil} func refreshPortableStoreForDBIfDue(ctx context.Context, sourceDBPath, mirrorPath string) error {	ttl := portableStoreRefreshInterval()	statePath := portableStoreRefreshStatePath(mirrorPath)	state := readPortableStoreRefreshState(statePath)	now := time.Now().UTC()	if ttl > 0 && recentPortableRefresh(state.LastSuccess, now, ttl) {		return nil	}	if ttl > 0 && recentPortableRefresh(state.LastFailure, now, portableStoreRefreshFailureBackoff) {		return nil	}	lockPath := statePath + ".lock"	if err := os.MkdirAll(filepath.Dir(statePath), 0o755); err != nil {		return err	}	removeStalePortableRefreshLock(lockPath, now)	lock, locked := tryGHCommandCacheLock(lockPath)	if !locked {		return nil	}	defer func() {		_ = lock.Close()		_ = os.Remove(lockPath)	}()	state = readPortableStoreRefreshState(statePath)	now = time.Now().UTC()	if ttl > 0 && recentPortableRefresh(state.LastSuccess, now, ttl) {		return nil	}	state.LastAttempt = now.Format(time.RFC3339Nano)	err := refreshPortableStoreForDB(ctx, sourceDBPath)	if err != nil {		state.LastFailure = time.Now().UTC().Format(time.RFC3339Nano)		state.Error = err.Error()		_ = writePortableStoreRefreshState(statePath, state)		return err	}	state.LastSuccess = time.Now().UTC().Format(time.RFC3339Nano)	state.LastFailure = ""	state.Error = ""	return writePortableStoreRefreshState(statePath, state)} func removeStalePortableRefreshLock(path string, now time.Time) {	info, err := os.Stat(path)	if err != nil {		return	}	if now.Sub(info.ModTime()) <= 2*portableStoreRefreshTimeout {		return	}	_ = os.Remove(path)} func portableStoreRefreshInterval() time.Duration {	if raw := strings.TrimSpace(os.Getenv("GITCRAWL_PORTABLE_REFRESH_TTL")); raw != "" {		if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 {			return duration		}	}	return portableStoreRefreshTTL} func portableStoreRefreshStatePath(mirrorPath string) string {	return filepath.Join(filepath.Dir(mirrorPath), ".portable-refresh.json")} func readPortableStoreRefreshState(path string) portableStoreRefreshState {	data, err := os.ReadFile(path)	if err != nil {		return portableStoreRefreshState{}	}	var state portableStoreRefreshState	if err := json.Unmarshal(data, &state); err != nil {		return portableStoreRefreshState{}	}	return state} 

Lines 1–500 of 1458. Load more