package documents import ( "encoding/json" "regexp" "sort" "strings" "github.com/openclaw/gitcrawl/internal/store" ) var whitespacePattern = regexp.MustCompile(`\s+`) const ( maxChangedFilePaths = 200 maxCommitSubjects = 100 ) func Build(thread store.Thread) store.Document { return BuildWithComments(thread, nil) } func BuildWithComments(thread store.Thread, comments []store.Comment) store.Document { return BuildWithContext(thread, comments, nil, nil) } func BuildWithContext(thread store.Thread, comments []store.Comment, files []store.PullRequestFile, commits []store.PullRequestCommit) store.Document { labels := labelNames(thread.LabelsJSON) sections := []string{ "# " + thread.Title, } if strings.TrimSpace(thread.Body) != "" { sections = append(sections, strings.TrimSpace(thread.Body)) } if len(labels) > 0 { sections = append(sections, "Labels: "+strings.Join(labels, ", ")) } for _, comment := range comments { if comment.IsBot || strings.TrimSpace(comment.Body) == "" { continue } sections = append(sections, comment.AuthorLogin+": "+strings.TrimSpace(comment.Body)) } changedPaths := changedFilePaths(files) if len(changedPaths) > 0 { sections = append(sections, "Changed files:\n- "+strings.Join(changedPaths, "\n- ")) } commitSubjects := pullCommitSubjects(commits) if len(commitSubjects) > 0 { sections = append(sections, "Commits:\n- "+strings.Join(commitSubjects, "\n- ")) } rawText := strings.Join(sections, "\n\n") dedupeParts := []string{thread.Title, thread.Body, strings.Join(labels, " ")} for _, comment := range comments { if comment.IsBot { continue } dedupeParts = append(dedupeParts, comment.Body) } dedupeParts = append(dedupeParts, changedPaths...) dedupeParts = append(dedupeParts, commitSubjects...) return store.Document{ ThreadID: thread.ID, Title: thread.Title, Body: thread.Body, RawText: rawText, DedupeText: normalizeDedupe(strings.Join(dedupeParts, " ")), UpdatedAt: thread.UpdatedAt, } } func changedFilePaths(files []store.PullRequestFile) []string { paths := make([]string, 0, min(len(files), maxChangedFilePaths)) seen := make(map[string]bool, len(files)) for _, file := range files { for _, path := range []string{file.Path, file.PreviousPath} { path = strings.TrimSpace(path) if path == "" || seen[path] { continue } seen[path] = true paths = append(paths, path) } } sort.Strings(paths) if len(paths) > maxChangedFilePaths { paths = paths[:maxChangedFilePaths] } return paths } func pullCommitSubjects(commits []store.PullRequestCommit) []string { subjects := make([]string, 0, min(len(commits), maxCommitSubjects)) for _, commit := range commits { message := strings.TrimSpace(commit.Message) if message == "" { continue } if newline := strings.IndexByte(message, '\n'); newline >= 0 { message = strings.TrimSpace(message[:newline]) } if message != "" { subjects = append(subjects, message) } if len(subjects) == maxCommitSubjects { break } } return subjects } func normalizeDedupe(value string) string { value = strings.ToLower(strings.TrimSpace(value)) value = strings.ReplaceAll(value, "\x00", " ") value = whitespacePattern.ReplaceAllString(value, " ") return strings.TrimSpace(value) } func labelNames(raw string) []string { var labels []struct { Name string `json:"name"` } if err := json.Unmarshal([]byte(raw), &labels); err == nil { out := make([]string, 0, len(labels)) for _, label := range labels { name := strings.TrimSpace(label.Name) if name != "" { out = append(out, name) } } if len(out) > 0 { return out } } var names []string if err := json.Unmarshal([]byte(raw), &names); err != nil { return nil } out := make([]string, 0, len(names)) for _, name := range names { name = strings.TrimSpace(name) if name != "" { out = append(out, name) } } return out }