package codeindex import ( "bytes" "context" "crypto/sha256" "encoding/hex" "fmt" "io" "math" "os" "os/exec" "path/filepath" "strings" "unicode/utf8") const ( DefaultMaxFileBytes int64 = 512 * 1024 DefaultMaxTotalBytes int64 = 256 * 1024 * 1024 DefaultMaxFiles = 100000) type Options struct { Path string MaxFileBytes int64 MaxTotalBytes int64 MaxFiles int} type Document struct { Path string Language string ContentHash string Text string ByteSize int64} type Result struct { SourceRoot string GitSHA string WorktreeDirty bool FilesSeen int FilesIndexed int BytesIndexed int64 SkippedBinary int SkippedLarge int SkippedMissing int SkippedNonRegular int SkippedOutsideRoot int Documents []Document} func Scan(ctx context.Context, options Options) (Result, error) { path := strings.TrimSpace(options.Path) if path == "" { path = "." } if options.MaxFileBytes <= 0 { options.MaxFileBytes = DefaultMaxFileBytes } if options.MaxTotalBytes <= 0 { options.MaxTotalBytes = DefaultMaxTotalBytes } if options.MaxFiles <= 0 { options.MaxFiles = DefaultMaxFiles } root, err := gitText(ctx, path, "rev-parse", "--show-toplevel") if err != nil { return Result{}, err } root, err = filepath.Abs(strings.TrimSpace(root)) if err != nil { return Result{}, fmt.Errorf("resolve source root: %w", err) } root, err = filepath.EvalSymlinks(root) if err != nil { return Result{}, fmt.Errorf("resolve source root symlinks: %w", err) } sha, err := gitText(ctx, root, "rev-parse", "HEAD") if err != nil { return Result{}, err } status, err := gitBytes(ctx, root, "status", "--porcelain", "--untracked-files=no") if err != nil { return Result{}, err } rawEntries, err := gitBytes(ctx, root, "ls-files", "-z", "--cached", "--stage") if err != nil { return Result{}, err } rootFS, err := os.OpenRoot(root) if err != nil { return Result{}, fmt.Errorf("open source root: %w", err) } defer rootFS.Close() result := Result{ SourceRoot: root, GitSHA: strings.TrimSpace(string(sha)), WorktreeDirty: len(bytes.TrimSpace(status)) > 0, } for _, rawEntry := range bytes.Split(rawEntries, []byte{0}) { if len(rawEntry) == 0 { continue } result.FilesSeen++ metadata, rawPath, ok := bytes.Cut(rawEntry, []byte{'\t'}) fields := bytes.Fields(metadata) if !ok || len(fields) != 3 { return Result{}, fmt.Errorf("parse tracked entry %q", rawEntry) } mode := string(fields[0]) stage := string(fields[2]) if stage != "0" || (mode != "100644" && mode != "100755") { result.SkippedNonRegular++ continue } path := filepath.ToSlash(string(rawPath)) if !safeRelativePath(path) { return Result{}, fmt.Errorf("git returned unsafe tracked path %q", path) } rootPath := filepath.FromSlash(path) info, err := rootFS.Lstat(rootPath) if err != nil { if os.IsNotExist(err) { result.SkippedMissing++ continue } if pathEscapesRoot(err) { result.SkippedOutsideRoot++ continue } return Result{}, fmt.Errorf("stat tracked file %s: %w", path, err) } if !info.Mode().IsRegular() { result.SkippedNonRegular++ continue } if info.Size() > options.MaxFileBytes { result.SkippedLarge++ continue } data, tooLarge, err := readFileAtMost(rootFS, rootPath, options.MaxFileBytes) if err != nil { if os.IsNotExist(err) { result.SkippedMissing++ continue } if pathEscapesRoot(err) { result.SkippedOutsideRoot++ continue } return Result{}, fmt.Errorf("read tracked file %s: %w", path, err) } if tooLarge { result.SkippedLarge++ continue } if bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data) { result.SkippedBinary++ continue } if result.FilesIndexed >= options.MaxFiles { return Result{}, fmt.Errorf("tracked text file count exceeds --max-files %d", options.MaxFiles) } if result.BytesIndexed+int64(len(data)) > options.MaxTotalBytes { return Result{}, fmt.Errorf("tracked text corpus exceeds --max-total-bytes %d", options.MaxTotalBytes) } sum := sha256.Sum256(data) result.Documents = append(result.Documents, Document{ Path: path, Language: languageForPath(path), ContentHash: hex.EncodeToString(sum[:]), Text: string(data), ByteSize: int64(len(data)), }) result.FilesIndexed++ result.BytesIndexed += int64(len(data)) } finalSHA, err := gitText(ctx, root, "rev-parse", "HEAD") if err != nil { return Result{}, err } if strings.TrimSpace(finalSHA) != result.GitSHA { return Result{}, fmt.Errorf("repository HEAD changed during code scan; retry indexing") } finalStatus, err := gitBytes(ctx, root, "status", "--porcelain", "--untracked-files=no") if err != nil { return Result{}, err } result.WorktreeDirty = result.WorktreeDirty || len(bytes.TrimSpace(finalStatus)) > 0 return result, nil} func readFileAtMost(root *os.Root, path string, maxBytes int64) ([]byte, bool, error) { file, err := root.Open(path) if err != nil { return nil, false, err } limit := maxBytes if limit < math.MaxInt64 { limit++ } data, readErr := io.ReadAll(io.LimitReader(file, limit)) closeErr := file.Close() if readErr != nil { return nil, false, readErr } if closeErr != nil { return nil, false, closeErr } if int64(len(data)) > maxBytes { return nil, true, nil } return data, false, nil} func pathEscapesRoot(err error) bool { return err != nil && strings.Contains(err.Error(), "path escapes from parent")} func safeRelativePath(path string) bool { if path == "" || filepath.IsAbs(filepath.FromSlash(path)) { return false } clean := filepath.ToSlash(filepath.Clean(filepath.FromSlash(path))) return clean != ".." && !strings.HasPrefix(clean, "../")} func languageForPath(path string) string { base := strings.ToLower(filepath.Base(path)) switch base { case "dockerfile": return "dockerfile" case "makefile": return "makefile" } ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".") if ext == "" { return "text" } return ext} func gitText(ctx context.Context, dir string, args ...string) (string, error) { out, err := gitBytes(ctx, dir, args...) if err != nil { return "", err } return strings.TrimSpace(string(out)), nil} func gitBytes(ctx context.Context, dir string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { return nil, fmt.Errorf("git %s failed: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) } return stdout.Bytes(), nil}
package codeindex
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"math"
"os"
"os/exec"
"path/filepath"
"strings"
"unicode/utf8"
)
const (
DefaultMaxFileBytes int64 = 512 * 1024
DefaultMaxTotalBytes int64 = 256 * 1024 * 1024
DefaultMaxFiles = 100000
)
type Options struct {
Path string
MaxFileBytes int64
MaxTotalBytes int64
MaxFiles int
}
type Document struct {
Path string
Language string
ContentHash string
Text string
ByteSize int64
}
type Result struct {
SourceRoot string
GitSHA string
WorktreeDirty bool
FilesSeen int
FilesIndexed int
BytesIndexed int64
SkippedBinary int
SkippedLarge int
SkippedMissing int
SkippedNonRegular int
SkippedOutsideRoot int
Documents []Document
}
func Scan(ctx context.Context, options Options) (Result, error) {
path := strings.TrimSpace(options.Path)
if path == "" {
path = "."
}
if options.MaxFileBytes <= 0 {
options.MaxFileBytes = DefaultMaxFileBytes
}
if options.MaxTotalBytes <= 0 {
options.MaxTotalBytes = DefaultMaxTotalBytes
}
if options.MaxFiles <= 0 {
options.MaxFiles = DefaultMaxFiles
}
root, err := gitText(ctx, path, "rev-parse", "--show-toplevel")
if err != nil {
return Result{}, err
}
root, err = filepath.Abs(strings.TrimSpace(root))
if err != nil {
return Result{}, fmt.Errorf("resolve source root: %w", err)
}
root, err = filepath.EvalSymlinks(root)
if err != nil {
return Result{}, fmt.Errorf("resolve source root symlinks: %w", err)
}
sha, err := gitText(ctx, root, "rev-parse", "HEAD")
if err != nil {
return Result{}, err
}
status, err := gitBytes(ctx, root, "status", "--porcelain", "--untracked-files=no")
if err != nil {
return Result{}, err
}
rawEntries, err := gitBytes(ctx, root, "ls-files", "-z", "--cached", "--stage")
if err != nil {
return Result{}, err
}
rootFS, err := os.OpenRoot(root)
if err != nil {
return Result{}, fmt.Errorf("open source root: %w", err)
}
defer rootFS.Close()
result := Result{
SourceRoot: root,
GitSHA: strings.TrimSpace(string(sha)),
WorktreeDirty: len(bytes.TrimSpace(status)) > 0,
}
for _, rawEntry := range bytes.Split(rawEntries, []byte{0}) {
if len(rawEntry) == 0 {
continue
}
result.FilesSeen++
metadata, rawPath, ok := bytes.Cut(rawEntry, []byte{'\t'})
fields := bytes.Fields(metadata)
if !ok || len(fields) != 3 {
return Result{}, fmt.Errorf("parse tracked entry %q", rawEntry)
}
mode := string(fields[0])
stage := string(fields[2])
if stage != "0" || (mode != "100644" && mode != "100755") {
result.SkippedNonRegular++
continue
}
path := filepath.ToSlash(string(rawPath))
if !safeRelativePath(path) {
return Result{}, fmt.Errorf("git returned unsafe tracked path %q", path)
}
rootPath := filepath.FromSlash(path)
info, err := rootFS.Lstat(rootPath)
if err != nil {
if os.IsNotExist(err) {
result.SkippedMissing++
continue
}
if pathEscapesRoot(err) {
result.SkippedOutsideRoot++
continue
}
return Result{}, fmt.Errorf("stat tracked file %s: %w", path, err)
}
if !info.Mode().IsRegular() {
result.SkippedNonRegular++
continue
}
if info.Size() > options.MaxFileBytes {
result.SkippedLarge++
continue
}
data, tooLarge, err := readFileAtMost(rootFS, rootPath, options.MaxFileBytes)
if err != nil {
if os.IsNotExist(err) {
result.SkippedMissing++
continue
}
if pathEscapesRoot(err) {
result.SkippedOutsideRoot++
continue
}
return Result{}, fmt.Errorf("read tracked file %s: %w", path, err)
}
if tooLarge {
result.SkippedLarge++
continue
}
if bytes.IndexByte(data, 0) >= 0 || !utf8.Valid(data) {
result.SkippedBinary++
continue
}
if result.FilesIndexed >= options.MaxFiles {
return Result{}, fmt.Errorf("tracked text file count exceeds --max-files %d", options.MaxFiles)
}
if result.BytesIndexed+int64(len(data)) > options.MaxTotalBytes {
return Result{}, fmt.Errorf("tracked text corpus exceeds --max-total-bytes %d", options.MaxTotalBytes)
}
sum := sha256.Sum256(data)
result.Documents = append(result.Documents, Document{
Path: path,
Language: languageForPath(path),
ContentHash: hex.EncodeToString(sum[:]),
Text: string(data),
ByteSize: int64(len(data)),
})
result.FilesIndexed++
result.BytesIndexed += int64(len(data))
}
finalSHA, err := gitText(ctx, root, "rev-parse", "HEAD")
if err != nil {
return Result{}, err
}
if strings.TrimSpace(finalSHA) != result.GitSHA {
return Result{}, fmt.Errorf("repository HEAD changed during code scan; retry indexing")
}
finalStatus, err := gitBytes(ctx, root, "status", "--porcelain", "--untracked-files=no")
if err != nil {
return Result{}, err
}
result.WorktreeDirty = result.WorktreeDirty || len(bytes.TrimSpace(finalStatus)) > 0
return result, nil
}
func readFileAtMost(root *os.Root, path string, maxBytes int64) ([]byte, bool, error) {
file, err := root.Open(path)
if err != nil {
return nil, false, err
}
limit := maxBytes
if limit < math.MaxInt64 {
limit++
}
data, readErr := io.ReadAll(io.LimitReader(file, limit))
closeErr := file.Close()
if readErr != nil {
return nil, false, readErr
}
if closeErr != nil {
return nil, false, closeErr
}
if int64(len(data)) > maxBytes {
return nil, true, nil
}
return data, false, nil
}
func pathEscapesRoot(err error) bool {
return err != nil && strings.Contains(err.Error(), "path escapes from parent")
}
func safeRelativePath(path string) bool {
if path == "" || filepath.IsAbs(filepath.FromSlash(path)) {
return false
}
clean := filepath.ToSlash(filepath.Clean(filepath.FromSlash(path)))
return clean != ".." && !strings.HasPrefix(clean, "../")
}
func languageForPath(path string) string {
base := strings.ToLower(filepath.Base(path))
switch base {
case "dockerfile":
return "dockerfile"
case "makefile":
return "makefile"
}
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
if ext == "" {
return "text"
}
return ext
}
func gitText(ctx context.Context, dir string, args ...string) (string, error) {
out, err := gitBytes(ctx, dir, args...)
if err != nil {
return "", err
}
return strings.TrimSpace(string(out)), nil
}
func gitBytes(ctx context.Context, dir string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...)
cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("git %s failed: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
}
return stdout.Bytes(), nil
}