package cli import (	"bytes"	"context"	"encoding/json"	"errors"	"flag"	"fmt"	"io"	"log/slog"	"math"	"net/url"	"os"	"os/exec"	"path/filepath"	"regexp"	"sort"	"strconv"	"strings"	"sync"	"text/tabwriter"	"time" 	"github.com/alecthomas/kong"	crawlconfig "github.com/openclaw/crawlkit/config"	"github.com/openclaw/crawlkit/control"	"github.com/openclaw/crawlkit/mirror"	crawlremote "github.com/openclaw/crawlkit/remote"	clusterer "github.com/openclaw/gitcrawl/internal/cluster"	"github.com/openclaw/gitcrawl/internal/config"	gh "github.com/openclaw/gitcrawl/internal/github"	"github.com/openclaw/gitcrawl/internal/openai"	"github.com/openclaw/gitcrawl/internal/store"	"github.com/openclaw/gitcrawl/internal/syncer"	"github.com/openclaw/gitcrawl/internal/vector") const (	defaultTUIMinSize          = 5	defaultTUIWorkingSetLimit  = 500	defaultClusterMaxSize      = 40	defaultClusterFanout       = 16	defaultClusterThreshold    = 0.80	defaultCrossKindMinScore   = 0.93	highConfidenceEdgeScore    = 0.90	weakEdgeMinTitleOverlap    = 0.18	deterministicRefScore      = 0.94	bodyRefEvidencePrefixChars = 240) var errNoSemanticVectors = errors.New("no semantic vectors") var threadReferencePattern = regexp.MustCompile(`(?i)(?:\b([\w.-]+/[\w.-]+)#(\d+)|(?:\b([\w.-]+/[\w.-]+)/)?(?:issues|pull)/(\d+)|#(\d{2,}))`)var githubThreadURLPattern = regexp.MustCompile(`(?i)^https?://github\.com/([\w.-]+)/([\w.-]+)/(?:issues|pull)/(\d+)(?:[/?#].*)?$`)var ownerRepoThreadPattern = regexp.MustCompile(`(?i)^([\w.-]+)/([\w.-]+)#(\d+)$`)var pathThreadPattern = regexp.MustCompile(`(?i)(?:^|/)(?:issues|pull)/(\d+)(?:[/?#].*)?$`)var titleTokenPattern = regexp.MustCompile(`[A-Za-z0-9]{4,}`) type referenceEvidence struct {	Title     bool	EarlyBody bool} type App struct {	Stdout io.Writer	Stderr io.Writer 	configPath            string	format                OutputFormat	getWorkingDirectory   func() (string, error)	githubAuthTokenLookup func(context.Context) (string, error)	dbTargetNoticeOnce    sync.Once} type initResult struct {	ConfigPath       string `json:"config_path"`	RuntimeDir       string `json:"runtime_dir,omitempty"`	DBPath           string `json:"db_path"`	CacheDir         string `json:"cache_dir"`	VectorDir        string `json:"vector_dir"`	LogDir           string `json:"log_dir"`	PortableStoreURL string `json:"portable_store_url,omitempty"`	PortableStoreDir string `json:"portable_store_dir,omitempty"`	PortableStore    string `json:"portable_store,omitempty"`	RemoteMode       string `json:"remote_mode,omitempty"`	RemoteEndpoint   string `json:"remote_endpoint,omitempty"`	RemoteArchive    string `json:"remote_archive,omitempty"`} type OutputFormat string const (	FormatText OutputFormat = "text"	FormatJSON OutputFormat = "json"	FormatLog  OutputFormat = "log") var version = "dev" func New() *App {	return &App{		Stdout:              os.Stdout,		Stderr:              os.Stderr,		format:              FormatText,		getWorkingDirectory: os.Getwd,	}} func (a *App) Run(ctx context.Context, args []string) error {	if len(args) == 0 || rootHelpRequested(args, "config", "format") {		a.printUsage()		return nil	}	var global gitcrawlRootArgs	if err := parseKongArgs(&global, args, "gitcrawl", a.Stdout, a.Stderr); err != nil {		return usageErr(err)	} 	resolvedFormat, err := resolveOutputFormat(global.Format, global.JSON)	if err != nil {		return usageErr(err)	}	a.configPath = strings.TrimSpace(global.Config)	a.format = resolvedFormat 	rest := global.Args	if global.Version {		return a.writeOutput("version", map[string]string{"version": version}, false)	}	if len(rest) == 0 || rest[0] == "--help" || rest[0] == "-h" {		a.printUsage()		return nil	}	if rest[0] == "help" {		if len(rest) > 1 {			return a.printCommandUsage(rest[1])		}		a.printUsage()		return nil	}	if releaseNotificationAllowed(rest) {		a.maybeNotifyRelease(ctx, rest)	} 	switch rest[0] {	case "version":		return a.writeOutput("version", map[string]string{"version": version}, false)	case "check-update":		return a.runCheckUpdate(ctx, rest[1:])	case "metadata":		return a.runMetadata(rest[1:])	case "whoami":		return a.runRemoteWhoami(ctx, rest[1:])	case "remote":		return a.runRemote(ctx, rest[1:])	case "cloud":		return a.runCloud(ctx, rest[1:])	case "serve":		return usageErr(fmt.Errorf("serve is not supported in gitcrawl"))	case "init":		return a.runInit(ctx, rest[1:])	case "doctor":		return a.runDoctor(ctx, rest[1:])	case "status":		return a.runStatus(ctx, rest[1:])	case "sync":		return a.runSync(ctx, rest[1:])	case "fill-pr-details":		return a.runFillPRDetails(ctx, rest[1:])	case "threads":		return a.runThreads(ctx, rest[1:])	case "capture":		return a.runCapture(ctx, rest[1:])	case "close-thread":		return a.runCloseThread(ctx, rest[1:])	case "reopen-thread":		return a.runReopenThread(ctx, rest[1:])	case "close-cluster":		return a.runCloseCluster(ctx, rest[1:])	case "reopen-cluster":		return a.runReopenCluster(ctx, rest[1:])	case "exclude-cluster-member":		return a.runExcludeClusterMember(ctx, rest[1:])	case "include-cluster-member":		return a.runIncludeClusterMember(ctx, rest[1:])	case "set-cluster-canonical":		return a.runSetClusterCanonical(ctx, rest[1:])	case "runs":		return a.runRuns(ctx, rest[1:])	case "sync-failures":		return a.runSyncFailures(ctx, rest[1:])	case "coverage":		return a.runCoverage(ctx, rest[1:])	case "search":		return a.runSearch(ctx, rest[1:])	case "code":		return a.runCode(ctx, rest[1:])	case "gh":		return a.runGHShim(ctx, rest[1:])	case "configure":		return a.runConfigure(rest[1:])	case "refresh":		return a.runRefresh(ctx, rest[1:])	case "summarize":		return a.runSummarize(ctx, rest[1:])	case "embed":		return a.runEmbed(ctx, rest[1:])	case "clusters":		return a.runClusters(ctx, rest[1:])	case "clusters-report":		return a.runClustersReport(ctx, rest[1:])	case "durable-clusters":		return a.runDurableClusters(ctx, rest[1:])	case "cluster-detail":		return a.runClusterDetail(ctx, rest[1:])	case "cluster-explain":		return a.runClusterDetail(ctx, rest[1:])	case "neighbors":		return a.runNeighbors(ctx, rest[1:])	case "cluster":		return a.runCluster(ctx, rest[1:])	case "portable":		return a.runPortable(ctx, rest[1:])	case "tui":		return a.runTUI(ctx, rest[1:])	case "key-summaries", "cluster-experiment", "merge-clusters", "split-cluster", "export-sync", "import-sync", "validate-sync", "portable-size", "sync-status", "optimize", "completion":		_ = ctx		return notImplemented(rest[0])	default:		return usageErr(fmt.Errorf("unknown command %q", rest[0]))	}} type gitcrawlRootArgs struct {	Config  string   `help:"Config path."`	Format  string   `default:"text" help:"Output format: text, json, or log."`	JSON    bool     `name:"json" help:"Write JSON output."`	Version bool     `help:"Print version."`	NoColor bool     `name:"no-color" help:"Disable color output."`	Args    []string `arg:"" optional:"" passthrough:"partial" name:"command" help:"Command and arguments."`} func rootHelpRequested(args []string, valueFlags ...string) bool {	valueFlagSet := make(map[string]struct{}, len(valueFlags))	for _, flag := range valueFlags {		valueFlagSet[flag] = struct{}{}	}	for i := 0; i < len(args); i++ {		arg := args[i]		if arg == "--help" || arg == "-h" || (arg == "help" && i == len(args)-1) {			return true		}		if !strings.HasPrefix(arg, "-") {			return false		}		if name, ok := strings.CutPrefix(arg, "--"); ok {			if strings.Contains(name, "=") {				continue			}			if _, ok := valueFlagSet[name]; ok {				i++			}		}	}	return false} func parseKongArgs(target any, args []string, name string, stdout, stderr io.Writer, options ...kong.Option) error {	_, err := parseKongContext(target, args, name, stdout, stderr, options...)	return err} func parseKongContext(target any, args []string, name string, stdout, stderr io.Writer, options ...kong.Option) (*kong.Context, error) {	opts := []kong.Option{		kong.Name(name),		kong.NoDefaultHelp(),		kong.Writers(stdout, stderr),		kong.Exit(func(int) {}),	}	opts = append(opts, options...)	parser, err := kong.New(target, opts...)	if err != nil {		return nil, err	}	return parser.Parse(args)} func (a *App) runConfigure(args []string) error {	fs := flag.NewFlagSet("configure", flag.ContinueOnError)	fs.SetOutput(io.Discard)	summaryModel := fs.String("summary-model", "", "summary model")	embedModel := fs.String("embed-model", "", "embedding model")	embedBaseURLFlag := fs.String("embed-base-url", "", "custom endpoint for the embedding model")	embeddingBasis := fs.String("embedding-basis", "", "embedding basis")	jsonOut := fs.Bool("json", false, "write JSON output")	if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"summary-model": true, "embed-model": true, "embed-base-url": true, "embedding-basis": true})); err != nil {		return usageErr(err)	}	a.applyCommandJSON(*jsonOut) 	cfg, err := config.Load(a.configPath)	configExists := true	if err != nil {		if !errors.Is(err, os.ErrNotExist) {			return err		}		configExists = false		cfg = config.Default()	}	updated := false	if strings.TrimSpace(*summaryModel) != "" {		cfg.OpenAI.SummaryModel = strings.TrimSpace(*summaryModel)		updated = true	}	if strings.TrimSpace(*embedModel) != "" {		cfg.OpenAI.EmbedModel = strings.TrimSpace(*embedModel)		updated = true	}	if flagWasSet(fs, "embed-base-url") {		cfg.OpenAI.EmbedBaseURL = strings.TrimSpace(*embedBaseURLFlag)		updated = true	}	if strings.TrimSpace(*embeddingBasis) != "" {		cfg.EmbeddingBasis = strings.TrimSpace(*embeddingBasis)		updated = true	}	if updated || !configExists {		if err := config.Save(a.configPath, cfg); err != nil {			return err		}	}	return a.writeOutput("configure", map[string]any{		"config_path":     config.ResolvePath(a.configPath),		"updated":         updated || !configExists,		"summary_model":   cfg.OpenAI.SummaryModel,		"embed_model":     cfg.OpenAI.EmbedModel,		"embed_base_url":  embedBaseURL(cfg),		"embedding_basis": cfg.EmbeddingBasis,	}, true)} func flagWasSet(fs *flag.FlagSet, name string) bool {	found := false	fs.Visit(func(value *flag.Flag) {		found = found || value.Name == name	})	return found} type refreshResult struct {	Repository string          `json:"repository"`	Selected   map[string]bool `json:"selected"`	Sync       *syncer.Stats   `json:"sync,omitempty"`	Embed      *embedResult    `json:"embed,omitempty"`	Cluster    map[string]any  `json:"cluster,omitempty"`	dbTargetInfo} func (a *App) runRefresh(ctx context.Context, args []string) error {	fs := flag.NewFlagSet("refresh", flag.ContinueOnError)	fs.SetOutput(io.Discard)	noSync := fs.Bool("no-sync", false, "skip GitHub sync stage")	noEmbed := fs.Bool("no-embed", false, "skip embedding stage")	noCluster := fs.Bool("no-cluster", false, "skip clustering stage")	includeComments := fs.Bool("include-comments", false, "hydrate comments during sync")	includePRDetails := fs.Bool("include-pr-details", false, "hydrate PR files, commits, checks, workflow runs, and review threads")	withRaw := fs.String("with", "", "additional sync hydration: pr-details")	fs.Bool("include-code", false, "accepted for compatibility; code hydration is not implemented yet")	since := fs.String("since", "", "GitHub since timestamp")	state := fs.String("state", "", "GitHub issue state: open|closed|all; default open")	limitRaw := fs.String("limit", "", "maximum sync or embedding rows")	thresholdRaw := fs.String("threshold", fmt.Sprintf("%.2f", defaultClusterThreshold), "minimum cluster cosine score")	minSizeRaw := fs.String("min-size", "1", "minimum cluster member count")	maxClusterSizeRaw := fs.String("max-cluster-size", strconv.Itoa(defaultClusterMaxSize), "maximum members per generated cluster")	fanoutRaw := fs.String("k", strconv.Itoa(defaultClusterFanout), "nearest-neighbor fanout per thread")	crossKindThresholdRaw := fs.String("cross-kind-threshold", fmt.Sprintf("%.2f", defaultCrossKindMinScore), "minimum score for issue/pull request edges")	strictVectors := fs.Bool("strict-vectors", false, "require complete fresh configured vectors before clustering")	jsonOut := fs.Bool("json", false, "write JSON output")	if err := fs.Parse(normalizeCommandArgs(args, map[string]bool{"since": true, "state": true, "limit": true, "threshold": true, "min-size": true, "max-cluster-size": true, "k": true, "cross-kind-threshold": true, "with": true})); err != nil {		return usageErr(err)	}	a.applyCommandJSON(*jsonOut)	if fs.NArg() != 1 {		return usageErr(fmt.Errorf("refresh requires owner/repo"))	}	if *noSync && *noEmbed && *noCluster {		return usageErr(fmt.Errorf("refresh requires at least one selected stage"))	}	with, err := parseSyncWith(*withRaw)	if err != nil {		return usageErr(err)	}	owner, repoName, err := parseOwnerRepo(fs.Arg(0))	if err != nil {		return usageErr(err)	}	limit, err := parseOptionalPositiveInt(*limitRaw)	if err != nil {		return usageErr(err)	}	threshold, err := parseOptionalFloat(*thresholdRaw)	if err != nil {		return usageErr(err)	}	if threshold <= 0 || threshold > 1 {		return usageErr(fmt.Errorf("refresh requires --threshold between 0 and 1"))	}	minSize, err := parseOptionalPositiveInt(*minSizeRaw)	if err != nil {		return usageErr(err)	}	if minSize <= 0 {		minSize = 2	}	maxClusterSize, fanout, crossKindThreshold, err := parseClusterShapeOptions("refresh", *maxClusterSizeRaw, *fanoutRaw, *crossKindThresholdRaw)	if err != nil {		return err	} 	result := refreshResult{		Repository: owner + "/" + repoName,		Selected: map[string]bool{			"sync":    !*noSync,			"embed":   !*noEmbed,			"cluster": !*noCluster,		},	}	if !*noSync {		fmt.Fprintln(a.Stderr, "[refresh] sync")		stats, target, err := a.syncRepository(ctx, owner, repoName, syncOptions{			Since:            strings.TrimSpace(*since),			State:            strings.TrimSpace(*state),			Limit:            limit,			IncludeComments:  *includeComments,			IncludePRDetails: *includePRDetails || with["pr-details"],		})		if err != nil {			return err		}		result.Repository = stats.Repository		result.Sync = &stats		result.dbTargetInfo = target	}	if !*noEmbed {		fmt.Fprintln(a.Stderr, "[refresh] embed")		embed, err := a.embedRepository(ctx, owner, repoName, embedOptions{Limit: limit, IncludeClosed: stateIncludesClosed(*state)})		if err != nil {			return err		}		result.Repository = embed.Repository		result.Embed = &embed		if result.DBTarget == "" {			result.dbTargetInfo = embed.dbTarget		}	}	if !*noCluster {		fmt.Fprintln(a.Stderr, "[refresh] cluster")		rt, err := a.openLocalRuntime(ctx)		if err != nil {			return err		}		if result.DBTarget == "" {			result.dbTargetInfo = rt.dbTarget()		}		repo, err := rt.repository(ctx, owner, repoName)		if err != nil {			_ = rt.Store.Close()			return err		}		query := store.ThreadVectorQuery{RepoID: repo.ID, Model: rt.Config.OpenAI.EmbedModel, Basis: rt.Config.EmbeddingBasis}		query.IncludeClosed = stateIncludesClosed(*state)		vectors, err := rt.Store.ListThreadVectorsFiltered(ctx, query)		if err != nil {			_ = rt.Store.Close()			return err		}		coverage, freshVectors, err := evaluateClusterVectorCoverage(ctx, rt.Store, query, vectors)		if err != nil {			_ = rt.Store.Close()			return err		}		if *strictVectors {			if err := requireCompleteClusterVectorCoverage(owner, repoName, query, coverage, len(freshVectors), false); err != nil {				_ = rt.Store.Close()				return err			}		}		vectors = freshVectors		if len(vectors) == 0 && !*strictVectors {			fallbackQuery := store.ThreadVectorQuery{RepoID: repo.ID, IncludeClosed: stateIncludesClosed(*state)}			fallbackVectors, err := rt.Store.ListThreadVectorsFiltered(ctx, fallbackQuery)			if err != nil {				_ = rt.Store.Close()				return err			}			if len(fallbackVectors) > 0 {				query = fallbackQuery				vectors, err = freshFallbackClusterVectors(ctx, rt.Store, fallbackQuery, fallbackVectors)				if err != nil {

Lines 1–500 of 5407. Load more