Search - query parser: phrases, prefix*, -exclude, OR, from:, subject:, group:, after:/before: date bounds - meta_fts index over subject + from_addr (npm run index:meta) - drop the FTS->messages join from counts (11s -> ~25ms for "unix"); group scope seeks by rowid range instead of joining - highlighted snippets, relevance/newest/oldest sort, timing, clear errors UI - group links with @, spaces, commas no longer 404 (decode route params) - home: hierarchy overview, largest/all views, name filter, sortable columns - group: oldest/newest toggle, scoped search, windowed pager + jump-to-page - message: quote-depth tinting, dimmed signatures, clickable URLs, author/thread links, index link returns to the right page - decode RFC 2047 encoded subjects/authors - fixed table layout (no column overflow), 14px text, mobile layout - keyboard: j/k next/prev, u up, / search; help, 404, error, loading pages
Showing the changes to scripts (1 of 19 files). Show all
Unified · Split
| @@ -0,0 +1,44 @@ | ||
| 1 | +/** | |
| 2 | + * Build (or rebuild) the header search index. | |
| 3 | + * | |
| 4 | + * The importers only index message bodies (messages_fts). This adds | |
| 5 | + * meta_fts, an external-content FTS5 index over subject + from_addr, | |
| 6 | + * which powers subject:/from: search and makes header matches rank. | |
| 7 | + * | |
| 8 | + * Safe to re-run: the index is derived entirely from `messages`. | |
| 9 | + * | |
| 10 | + * npm run index:meta | |
| 11 | + */ | |
| 12 | +import Database from "better-sqlite3"; | |
| 13 | +import path from "path"; | |
| 14 | + | |
| 15 | +const DB_PATH = process.argv[2] ?? path.join(process.cwd(), "data", "usenet.db"); | |
| 16 | +const db = new Database(DB_PATH); | |
| 17 | +db.pragma("journal_mode = WAL"); | |
| 18 | +db.pragma("cache_size = -262144"); // 256 MiB | |
| 19 | + | |
| 20 | +const t0 = Date.now(); | |
| 21 | +const log = (s: string) => | |
| 22 | + console.log(`[${((Date.now() - t0) / 1000).toFixed(1)}s] ${s}`); | |
| 23 | + | |
| 24 | +log(`db: ${DB_PATH}`); | |
| 25 | +db.exec(` | |
| 26 | + CREATE VIRTUAL TABLE IF NOT EXISTS meta_fts USING fts5( | |
| 27 | + subject, | |
| 28 | + from_addr, | |
| 29 | + content='messages', | |
| 30 | + content_rowid='id' | |
| 31 | + ); | |
| 32 | +`); | |
| 33 | + | |
| 34 | +log("rebuilding meta_fts (subject, from_addr)..."); | |
| 35 | +db.exec(`INSERT INTO meta_fts(meta_fts) VALUES('rebuild')`); | |
| 36 | +log("optimizing..."); | |
| 37 | +db.exec(`INSERT INTO meta_fts(meta_fts) VALUES('optimize')`); | |
| 38 | + | |
| 39 | +log("refreshing planner statistics..."); | |
| 40 | +db.exec(`ANALYZE newsgroups; ANALYZE messages;`); | |
| 41 | + | |
| 42 | +const n = (db.prepare(`SELECT COUNT(*) AS c FROM messages`).get() as { c: number }).c; | |
| 43 | +log(`done — ${n.toLocaleString()} messages indexed`); | |
| 44 | +db.close(); | |