fast search, readable UI, fix broken group links

Search - query parser: phrases, prefix*, -exclude, OR, from:, subject:, group:, after:/before: date bounds - meta_fts index over subject + from_addr (npm run index:meta) - drop the FTS->messages join from counts (11s -> ~25ms for "unix"); group scope seeks by rowid range instead of joining - highlighted snippets, relevance/newest/oldest sort, timing, clear errors UI - group links with @, spaces, commas no longer 404 (decode route params) - home: hierarchy overview, largest/all views, name filter, sortable columns - group: oldest/newest toggle, scoped search, windowed pager + jump-to-page - message: quote-depth tinting, dimmed signatures, clickable URLs, author/thread links, index link returns to the right page - decode RFC 2047 encoded subjects/authors - fixed table layout (no column overflow), 14px text, mobile layout - keyboard: j/k next/prev, u up, / search; help, 404, error, loading pages

Kris Yotam committed 1 month agomain

Commit c20846c1 parent 4ca247219 files · +2058 −474Browse files

Showing the changes to lib (3 of 19 files). Show all

Unified · Split

3 files

lib/db.ts +310 −116
lib/format.tsx +189 −0
lib/query.ts +201 −0

lib/db.ts+310−116

@@ -1,18 +1,34 @@
1	1	import Database from "better-sqlite3";
2	2	import path from "path";
	3	+import type { ParsedQuery } from "./query";
	4	+import { HL_OPEN } from "./format";
3	5
4		−const DB_PATH = path.join(process.cwd(), "data", "usenet.db");
	6	+const DB_PATH = process.env.USENET_DB ?? path.join(process.cwd(), "data", "usenet.db");
5	7
6	8	let _db: Database.Database \| null = null;
7	9
8		−function getDb(): Database.Database {
	10	+function db(): Database.Database {
9	11	if (!_db) {
10		− _db = new Database(DB_PATH, { readonly: true });
11		− _db.pragma("journal_mode = WAL");
	12	+ _db = new Database(DB_PATH, { readonly: true, fileMustExist: true });
	13	+ _db.pragma("query_only = ON");
	14	+ _db.pragma("cache_size = -131072"); // 128 MiB page cache
	15	+ _db.pragma("mmap_size = 2147483648"); // 2 GiB memory map
	16	+ _db.pragma("temp_store = MEMORY");
12	17	}
13	18	return _db;
14	19	}
15	20
	21	+// Prepared-statement cache: SQL text is the key.
	22	+const stmts = new Map<string, Database.Statement>();
	23	+function q(sql: string): Database.Statement {
	24	+ let s = stmts.get(sql);
	25	+ if (!s) {
	26	+ s = db().prepare(sql);
	27	+ stmts.set(sql, s);
	28	+ }
	29	+ return s;
	30	+}
	31	+
16	32	export interface Newsgroup {
17	33	name: string;
18	34	message_count: number;
@@ -20,7 +36,7 @@ export interface Newsgroup { · 1 unmodified line
20	36	last_date: string \| null;
21	37	}
22	38
23		−export interface Message {
	39	+export interface MessageRow {
24	40	id: number;
25	41	newsgroup: string;
26	42	original_id: number;
@@ -28,145 +44,323 @@ export interface Message { · 1 unmodified line
28	44	date: string;
29	45	subject: string;
30	46	message_id: string;
	47	+}
	48	+
	49	+export interface Message extends MessageRow {
31	50	body: string;
32	51	}
33	52
	53	+/* ------------------------------------------------------------------ groups */
	54	+
	55	+let _groups: Newsgroup[] \| null = null;
	56	+let _groupMap: Map<string, Newsgroup> \| null = null;
	57	+
	58	+/** All newsgroups (the archive is static, so this is cached for the process). */
34	59	export function getNewsgroups(): Newsgroup[] {
35		− const db = getDb();
36		− return db
37		− .prepare(
38		− `SELECT name, message_count, first_date, last_date
	60	+ if (!_groups) {
	61	+ _groups = q(
	62	+ `SELECT name, message_count,
	63	+ NULLIF(first_date, '') AS first_date,
	64	+ NULLIF(last_date, '') AS last_date
39	65	FROM newsgroups ORDER BY name`
40		− )
41		− .all() as Newsgroup[];
	66	+ ).all() as Newsgroup[];
	67	+ _groupMap = new Map(_groups.map((g) => [g.name, g]));
	68	+ }
	69	+ return _groups;
42	70	}
43	71
44	72	export function getNewsgroup(name: string): Newsgroup \| undefined {
45		− const db = getDb();
46		− return db
47		− .prepare(`SELECT * FROM newsgroups WHERE name = ?`)
48		− .get(name) as Newsgroup \| undefined;
	73	+ getNewsgroups();
	74	+ return _groupMap!.get(name);
	75	+}
	76	+
	77	+export function getTotals(): { groups: number; messages: number } {
	78	+ const g = getNewsgroups();
	79	+ return { groups: g.length, messages: g.reduce((s, x) => s + x.message_count, 0) };
	80	+}
	81	+
	82	+interface IdRange {
	83	+ lo: number;
	84	+ hi: number;
	85	+ /** every id in [lo, hi] belongs to this group */
	86	+ contiguous: boolean;
	87	+}
	88	+const ranges = new Map<string, IdRange>();
	89	+
	90	+/** Row-id span of a group — lets FTS5 seek straight to it instead of joining. */
	91	+function groupRange(name: string): IdRange \| null {
	92	+ const hit = ranges.get(name);
	93	+ if (hit) return hit;
	94	+ const g = getNewsgroup(name);
	95	+ if (!g) return null;
	96	+ const r = q(
	97	+ `SELECT (SELECT id FROM messages WHERE newsgroup = ? ORDER BY id ASC LIMIT 1) AS lo,
	98	+ (SELECT id FROM messages WHERE newsgroup = ? ORDER BY id DESC LIMIT 1) AS hi`
	99	+ ).get(name, name) as { lo: number \| null; hi: number \| null };
	100	+ if (r.lo == null \|\| r.hi == null) return null;
	101	+ const range = { lo: r.lo, hi: r.hi, contiguous: r.hi - r.lo + 1 === g.message_count };
	102	+ ranges.set(name, range);
	103	+ return range;
49	104	}
50	105
	106	+/* ---------------------------------------------------------------- messages */
	107	+
	108	+const LIST_COLS = `id, newsgroup, original_id, from_addr, date, subject, message_id`;
	109	+
51	110	export function getMessages(
52	111	newsgroup: string,
53	112	page: number,
54		− perPage: number = 50
55		−): { messages: Message[]; total: number } {
56		− const db = getDb();
57		− const total = (
58		− db
59		− .prepare(`SELECT COUNT(*) as c FROM messages WHERE newsgroup = ?`)
60		− .get(newsgroup) as { c: number }
61		− ).c;
62		− const messages = db
63		− .prepare(
64		− `SELECT id, newsgroup, original_id, from_addr, date, subject, message_id
65		− FROM messages WHERE newsgroup = ?
66		− ORDER BY original_id ASC
67		− LIMIT ? OFFSET ?`
68		− )
69		− .all(newsgroup, perPage, (page - 1) * perPage) as Message[];
	113	+ perPage = 50,
	114	+ order: "asc" \| "desc" = "asc"
	115	+): { messages: MessageRow[]; total: number } {
	116	+ const total = getNewsgroup(newsgroup)?.message_count ?? 0;
	117	+ const dir = order === "desc" ? "DESC" : "ASC";
	118	+ const messages = q(
	119	+ `SELECT ${LIST_COLS} FROM messages
	120	+ WHERE newsgroup = ? ORDER BY original_id ${dir} LIMIT ? OFFSET ?`
	121	+ ).all(newsgroup, perPage, (page - 1) * perPage) as MessageRow[];
70	122	return { messages, total };
71	123	}
72	124
73		−export function getMessage(
74		− newsgroup: string,
75		− originalId: number
76		−): Message \| undefined {
77		− const db = getDb();
78		− return db
79		− .prepare(
80		− `SELECT * FROM messages WHERE newsgroup = ? AND original_id = ?`
81		− )
82		− .get(newsgroup, originalId) as Message \| undefined;
	125	+export function getMessage(newsgroup: string, originalId: number): Message \| undefined {
	126	+ return q(`SELECT * FROM messages WHERE newsgroup = ? AND original_id = ?`).get(
	127	+ newsgroup,
	128	+ originalId
	129	+ ) as Message \| undefined;
83	130	}
84	131
85	132	export function getAdjacentIds(
86	133	newsgroup: string,
87	134	originalId: number
88		−): { prev: number \| null; next: number \| null } {
89		− const db = getDb();
90		− const prev = db
91		− .prepare(
92		− `SELECT original_id FROM messages
93		− WHERE newsgroup = ? AND original_id < ?
94		− ORDER BY original_id DESC LIMIT 1`
95		− )
96		− .get(newsgroup, originalId) as { original_id: number } \| undefined;
97		− const next = db
98		− .prepare(
99		− `SELECT original_id FROM messages
100		− WHERE newsgroup = ? AND original_id > ?
101		− ORDER BY original_id ASC LIMIT 1`
102		− )
103		− .get(newsgroup, originalId) as { original_id: number } \| undefined;
	135	+): { prev: number \| null; next: number \| null; position: number } {
	136	+ const prev = q(
	137	+ `SELECT original_id FROM messages WHERE newsgroup = ? AND original_id < ?
	138	+ ORDER BY original_id DESC LIMIT 1`
	139	+ ).get(newsgroup, originalId) as { original_id: number } \| undefined;
	140	+ const next = q(
	141	+ `SELECT original_id FROM messages WHERE newsgroup = ? AND original_id > ?
	142	+ ORDER BY original_id ASC LIMIT 1`
	143	+ ).get(newsgroup, originalId) as { original_id: number } \| undefined;
	144	+ const before = q(
	145	+ `SELECT COUNT(*) AS c FROM messages WHERE newsgroup = ? AND original_id < ?`
	146	+ ).get(newsgroup, originalId) as { c: number };
104	147	return {
105	148	prev: prev?.original_id ?? null,
106	149	next: next?.original_id ?? null,
	150	+ position: before.c,
107	151	};
108	152	}
109	153
	154	+/* ------------------------------------------------------------------ search */
	155	+
	156	+let _hasMeta: boolean \| null = null;
	157	+/** meta_fts is built by `npm run index:meta`; search degrades gracefully without it. */
	158	+export function hasMetaIndex(): boolean {
	159	+ if (_hasMeta === null) {
	160	+ _hasMeta = !!q(
	161	+ `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta_fts'`
	162	+ ).get();
	163	+ }
	164	+ return _hasMeta;
	165	+}
	166	+
	167	+export type SearchSort = "relevance" \| "newest" \| "oldest";
	168	+
	169	+export interface SearchHit extends MessageRow {
	170	+ /** body excerpt; \u0002…\u0003 delimit highlighted spans */
	171	+ excerpt: string;
	172	+ /** subject/from with highlight markers when they matched */
	173	+ subjectHl?: string;
	174	+ fromHl?: string;
	175	+}
	176	+
	177	+export interface SearchResult {
	178	+ hits: SearchHit[];
	179	+ total: number;
	180	+ /** total is a lower bound (counting was capped) */
	181	+ capped: boolean;
	182	+ /** requested sort could not be honoured (too many matches) */
	183	+ sortFallback: boolean;
	184	+ sort: SearchSort;
	185	+ ms: number;
	186	+ error?: string;
	187	+}
	188	+
	189	+/** Result sets larger than this are not re-sorted by date (it would need a full join). */
	190	+const DATE_SORT_LIMIT = 100_000;
	191	+/** When counting needs a join against `messages`, stop counting here. */
	192	+const JOIN_COUNT_CAP = 10_000;
	193	+
	194	+
	195	+function cleanExcerpt(body: string): string {
	196	+ // First meaningful lines: skip blank, quoted, attribution and header-ish lines.
	197	+ const out: string[] = [];
	198	+ for (const raw of body.split("\n")) {
	199	+ const line = raw.trim();
	200	+ if (!line \|\| line.startsWith(">") \|\| line.startsWith("\|")) continue;
	201	+ if (/^(in article\|on .{0,80} wrote:?$\|.{0,80} writes:$)/i.test(line)) continue;
	202	+ out.push(line);
	203	+ if (out.join(" ").length > 260) break;
	204	+ }
	205	+ const s = out.join(" ");
	206	+ return s.length > 280 ? s.slice(0, 280).replace(/\s+\S*$/, "") + " …" : s;
	207	+}
	208	+
110	209	export function searchMessages(
111		− query: string,
112		− newsgroup?: string,
113		− page: number = 1,
114		− perPage: number = 50
115		−): { messages: Message[]; total: number } {
116		− const db = getDb();
117		− const ftsQuery = query
118		− .split(/\s+/)
119		− .filter(Boolean)
120		− .map((t) => `"${t.replace(/"/g, '""')}"`)
121		− .join(" ");
122		−
123		− if (newsgroup) {
124		− const total = (
125		− db
126		− .prepare(
127		− `SELECT COUNT(*) as c FROM messages_fts
128		− JOIN messages ON messages.rowid = messages_fts.rowid
129		− WHERE messages_fts MATCH ? AND messages.newsgroup = ?`
130		− )
131		− .get(ftsQuery, newsgroup) as { c: number }
132		− ).c;
133		− const messages = db
134		− .prepare(
135		− `SELECT messages.id, messages.newsgroup, messages.original_id,
136		− messages.from_addr, messages.date, messages.subject,
137		− messages.message_id,
138		− snippet(messages_fts, 0, '<mark class="highlight">', '</mark>', '...', 40) as body
139		− FROM messages_fts
140		− JOIN messages ON messages.rowid = messages_fts.rowid
141		− WHERE messages_fts MATCH ? AND messages.newsgroup = ?
142		− ORDER BY rank
143		− LIMIT ? OFFSET ?`
144		− )
145		− .all(ftsQuery, newsgroup, perPage, (page - 1) * perPage) as Message[];
146		− return { messages, total };
	210	+ pq: ParsedQuery,
	211	+ opts: { page: number; perPage: number; sort: SearchSort }
	212	+): SearchResult {
	213	+ const t0 = performance.now();
	214	+ const done = (r: Omit<SearchResult, "ms">): SearchResult => ({
	215	+ ...r,
	216	+ ms: performance.now() - t0,
	217	+ });
	218	+ const empty = { hits: [], total: 0, capped: false, sortFallback: false, sort: opts.sort };
	219	+
	220	+ if (pq.error) return done({ ...empty, error: pq.error });
	221	+
	222	+ const meta = hasMetaIndex();
	223	+ if (!meta && (pq.meta.pos \|\| pq.meta.negOnly)) {
	224	+ return done({
	225	+ ...empty,
	226	+ error: "from: and subject: need the header index — run `npm run index:meta` once.",
	227	+ });
147	228	}
148	229
149		− const total = (
150		− db
151		− .prepare(
152		− `SELECT COUNT(*) as c FROM messages_fts
153		− JOIN messages ON messages.rowid = messages_fts.rowid
154		− WHERE messages_fts MATCH ?`
155		− )
156		− .get(ftsQuery) as { c: number }
157		− ).c;
158		− const messages = db
159		− .prepare(
160		− `SELECT messages.id, messages.newsgroup, messages.original_id,
161		− messages.from_addr, messages.date, messages.subject,
162		− messages.message_id,
163		− snippet(messages_fts, 0, '<mark class="highlight">', '</mark>', '...', 40) as body
164		− FROM messages_fts
165		− JOIN messages ON messages.rowid = messages_fts.rowid
166		− WHERE messages_fts MATCH ?
167		− ORDER BY rank
168		− LIMIT ? OFFSET ?`
169		− )
170		− .all(ftsQuery, perPage, (page - 1) * perPage) as Message[];
171		− return { messages, total };
	230	+ let groupFilter: string \| null = null;
	231	+ let range: IdRange \| null = null;
	232	+ if (pq.group) {
	233	+ if (!getNewsgroup(pq.group)) return done({ ...empty, error: `No newsgroup named “${pq.group}”.` });
	234	+ range = groupRange(pq.group);
	235	+ if (!range) return done(empty);
	236	+ if (!range.contiguous) groupFilter = pq.group;
	237	+ }
	238	+
	239	+ // Pick the driving index: body if it has positive terms, else headers.
	240	+ const bodyDrives = !!pq.body.pos;
	241	+ const T = bodyDrives ? "messages_fts" : "meta_fts";
	242	+ const where: string[] = [`${T} MATCH ?`];
	243	+ const params: unknown[] = [bodyDrives ? pq.body.pos : pq.meta.pos];
	244	+
	245	+ if (bodyDrives && pq.meta.pos) {
	246	+ where.push(`${T}.rowid IN (SELECT rowid FROM meta_fts WHERE meta_fts MATCH ?)`);
	247	+ params.push(pq.meta.pos);
	248	+ }
	249	+ if (!bodyDrives && pq.body.negOnly) {
	250	+ where.push(`${T}.rowid NOT IN (SELECT rowid FROM messages_fts WHERE messages_fts MATCH ?)`);
	251	+ params.push(pq.body.negOnly);
	252	+ }
	253	+ if (pq.meta.negOnly) {
	254	+ where.push(`${T}.rowid NOT IN (SELECT rowid FROM meta_fts WHERE meta_fts MATCH ?)`);
	255	+ params.push(pq.meta.negOnly);
	256	+ }
	257	+ if (range) {
	258	+ where.push(`${T}.rowid BETWEEN ? AND ?`);
	259	+ params.push(range.lo, range.hi);
	260	+ }
	261	+
	262	+ const joinConds: string[] = [];
	263	+ const joinParams: unknown[] = [];
	264	+ if (groupFilter) {
	265	+ joinConds.push(`m.newsgroup = ?`);
	266	+ joinParams.push(groupFilter);
	267	+ }
	268	+ if (pq.after) {
	269	+ joinConds.push(`m.date >= ?`);
	270	+ joinParams.push(pq.after);
	271	+ }
	272	+ if (pq.before) {
	273	+ joinConds.push(`m.date < ? AND m.date <> ''`);
	274	+ joinParams.push(pq.before);
	275	+ }
	276	+ const filterJoin = joinConds.length > 0;
	277	+
	278	+ const from = (join: boolean) =>
	279	+ `FROM ${T}${join ? ` JOIN messages m ON m.id = ${T}.rowid` : ""}
	280	+ WHERE ${where.join(" AND ")}${join && joinConds.length ? ` AND ${joinConds.join(" AND ")}` : ""}`;
	281	+
	282	+ try {
	283	+ // ---- count
	284	+ let total: number;
	285	+ let capped = false;
	286	+ if (filterJoin) {
	287	+ total = (
	288	+ q(`SELECT COUNT(*) AS c FROM (SELECT 1 ${from(true)} LIMIT ${JOIN_COUNT_CAP + 1})`).get(
	289	+ ...params,
	290	+ ...joinParams
	291	+ ) as { c: number }
	292	+ ).c;
	293	+ if (total > JOIN_COUNT_CAP) {
	294	+ total = JOIN_COUNT_CAP;
	295	+ capped = true;
	296	+ }
	297	+ } else {
	298	+ total = (q(`SELECT COUNT(*) AS c ${from(false)}`).get(...params) as { c: number }).c;
	299	+ }
	300	+ if (total === 0) return done({ ...empty, total: 0 });
	301	+
	302	+ // ---- sort
	303	+ let sort = opts.sort;
	304	+ let sortFallback = false;
	305	+ if (sort !== "relevance" && (capped \|\| total > DATE_SORT_LIMIT)) {
	306	+ sort = "relevance";
	307	+ sortFallback = true;
	308	+ }
	309	+ const needJoin = filterJoin \|\| sort !== "relevance";
	310	+ const order =
	311	+ sort === "newest"
	312	+ ? `ORDER BY (m.date = '') ASC, m.date DESC, m.id DESC`
	313	+ : sort === "oldest"
	314	+ ? `ORDER BY (m.date = '') ASC, m.date ASC, m.id ASC`
	315	+ : `ORDER BY ${T}.rank`;
	316	+
	317	+ const ids = (
	318	+ q(`SELECT ${T}.rowid AS id ${from(needJoin)} ${order} LIMIT ? OFFSET ?`).all(
	319	+ ...params,
	320	+ ...(needJoin ? joinParams : []),
	321	+ opts.perPage,
	322	+ (opts.page - 1) * opts.perPage
	323	+ ) as { id: number }[]
	324	+ ).map((r) => r.id);
	325	+ if (ids.length === 0) return done({ ...empty, total, capped, sortFallback, sort });
	326	+
	327	+ // ---- hydrate (only the rows on this page)
	328	+ const rowStmt = q(`SELECT ${LIST_COLS}, substr(body, 1, 1500) AS head FROM messages WHERE id = ?`);
	329	+ const snipStmt = pq.body.pos
	330	+ ? q(
	331	+ `SELECT snippet(messages_fts, 0, char(2), char(3), ' … ', 32) AS s
	332	+ FROM messages_fts WHERE messages_fts MATCH ? AND rowid = ?`
	333	+ )
	334	+ : null;
	335	+ const hlStmt =
	336	+ meta && pq.meta.pos
	337	+ ? q(
	338	+ `SELECT highlight(meta_fts, 0, char(2), char(3)) AS s,
	339	+ highlight(meta_fts, 1, char(2), char(3)) AS f
	340	+ FROM meta_fts WHERE meta_fts MATCH ? AND rowid = ?`
	341	+ )
	342	+ : null;
	343	+
	344	+ const hits: SearchHit[] = ids.map((id) => {
	345	+ const row = rowStmt.get(id) as MessageRow & { head: string };
	346	+ const { head, ...rest } = row;
	347	+ const hit: SearchHit = { ...rest, excerpt: "" };
	348	+ if (snipStmt) {
	349	+ const s = snipStmt.get(pq.body.pos, id) as { s: string } \| undefined;
	350	+ hit.excerpt = (s?.s ?? "").replace(/\s+/g, " ").trim();
	351	+ }
	352	+ if (!hit.excerpt) hit.excerpt = cleanExcerpt(head);
	353	+ if (hlStmt) {
	354	+ const h = hlStmt.get(pq.meta.pos, id) as { s: string; f: string } \| undefined;
	355	+ if (h?.s?.includes(HL_OPEN)) hit.subjectHl = h.s;
	356	+ if (h?.f?.includes(HL_OPEN)) hit.fromHl = h.f;
	357	+ }
	358	+ return hit;
	359	+ });
	360	+
	361	+ return done({ hits, total, capped, sortFallback, sort });
	362	+ } catch (e) {
	363	+ const msg = e instanceof Error ? e.message : String(e);
	364	+ return done({ ...empty, error: `Search failed: ${msg}` });
	365	+ }
172	366	}

lib/format.tsx+189−0

@@ -0,0 +1,189 @@
	1	+import type { ReactNode } from "react";
	2	+
	3	+export const HL_OPEN = "\u0002";
	4	+export const HL_CLOSE = "\u0003";
	5	+
	6	+/** Route params arrive percent-encoded (group names contain @, spaces, commas…). */
	7	+export function decodeParam(s: string): string {
	8	+ try {
	9	+ return decodeURIComponent(s);
	10	+ } catch {
	11	+ return s;
	12	+ }
	13	+}
	14	+
	15	+export const groupHref = (name: string, qs = "") =>
	16	+ `/group/${encodeURIComponent(name)}${qs}`;
	17	+
	18	+export const messageHref = (name: string, originalId: number) =>
	19	+ `/group/${encodeURIComponent(name)}/${originalId}`;
	20	+
	21	+export function searchHref(params: Record<string, string \| number \| undefined \| null>) {
	22	+ const sp = new URLSearchParams();
	23	+ for (const [k, v] of Object.entries(params)) {
	24	+ if (v === undefined \|\| v === null \|\| v === "") continue;
	25	+ sp.set(k, String(v));
	26	+ }
	27	+ const s = sp.toString();
	28	+ return s ? `/search?${s}` : "/search";
	29	+}
	30	+
	31	+/** Quote a value for use inside a query (from:"Rob Pike"). */
	32	+export function queryValue(v: string): string {
	33	+ const clean = v.replace(/"/g, " ").replace(/\s+/g, " ").trim();
	34	+ return /\s/.test(clean) ? `"${clean}"` : clean;
	35	+}
	36	+
	37	+export const fmt = (n: number) => n.toLocaleString("en-US");
	38	+
	39	+export function fmtDate(d: string \| null \| undefined): string {
	40	+ return d ? d : "—";
	41	+}
	42	+
	43	+export function intParam(v: string \| undefined, fallback = 1): number {
	44	+ const n = parseInt(v ?? "", 10);
	45	+ return Number.isFinite(n) && n > 0 ? n : fallback;
	46	+}
	47	+
	48	+/** Render text containing \u0002…\u0003 highlight markers as <mark>. */
	49	+export function Highlight({ text }: { text: string }): ReactNode {
	50	+ if (!text.includes(HL_OPEN)) return text;
	51	+ const out: ReactNode[] = [];
	52	+ let i = 0;
	53	+ let k = 0;
	54	+ while (i < text.length) {
	55	+ const a = text.indexOf(HL_OPEN, i);
	56	+ if (a === -1) {
	57	+ out.push(text.slice(i));
	58	+ break;
	59	+ }
	60	+ if (a > i) out.push(text.slice(i, a));
	61	+ const b = text.indexOf(HL_CLOSE, a + 1);
	62	+ const end = b === -1 ? text.length : b;
	63	+ out.push(<mark key={k++}>{text.slice(a + 1, end)}</mark>);
	64	+ i = end + 1;
	65	+ }
	66	+ return out;
	67	+}
	68	+
	69	+/** Pull the human name out of "addr (Name)" / "Name <addr>" / bare address. */
	70	+export function authorName(from: string): string {
	71	+ const paren = from.match(/$([^)]+)$\s*$/);
	72	+ if (paren) return paren[1].trim();
	73	+ const angle = from.match(/^\s"?([^"<]+?)"?\s<[^>]+>\s*$/);
	74	+ if (angle) return angle[1].trim();
	75	+ return from.trim();
	76	+}
	77	+
	78	+export function authorAddress(from: string): string {
	79	+ const angle = from.match(/<([^>]+)>/);
	80	+ if (angle) return angle[1];
	81	+ const bare = from.match(/[^\s()<>"]+@[^\s()<>"]+/);
	82	+ if (bare) return bare[0];
	83	+ const bang = from.match(/^[^\s()]+![^\s()]+/); // uucp paths
	84	+ if (bang) return bang[0];
	85	+ return "";
	86	+}
	87	+
	88	+/* ------------------------------------------------------------ message body */
	89	+
	90	+const URL_RE = /\b(?:https?\|ftp):\/\/[^\s<>"'`]+[^\s<>"'`.,;:!?)\]}]/g;
	91	+
	92	+function linkify(text: string, keyBase: string): ReactNode[] {
	93	+ const out: ReactNode[] = [];
	94	+ let last = 0;
	95	+ let k = 0;
	96	+ for (const m of text.matchAll(URL_RE)) {
	97	+ const i = m.index ?? 0;
	98	+ if (i > last) out.push(text.slice(last, i));
	99	+ out.push(
	100	+ <a key={`${keyBase}-${k++}`} href={m[0]} rel="noopener noreferrer nofollow" target="_blank">
	101	+ {m[0]}
	102	+ </a>
	103	+ );
	104	+ last = i + m[0].length;
	105	+ }
	106	+ if (last < text.length) out.push(text.slice(last));
	107	+ return out;
	108	+}
	109	+
	110	+function lineClass(line: string, inSig: boolean): string {
	111	+ if (inSig) return "sig";
	112	+ const m = line.match(/^((?:\s?[>\|])+)/);
	113	+ if (m) {
	114	+ const depth = (m[1].match(/[>\|]/g) ?? []).length;
	115	+ return `q${((depth - 1) % 3) + 1}`;
	116	+ }
	117	+ if (/^in article <[^>]>.writes:\s$/i.test(line) \|\| /^.{1,100} (writes\|wrote):\s$/.test(line))
	118	+ return "attr";
	119	+ return "";
	120	+}
	121	+
	122	+/**
	123	+ * Message body → spans grouped by line class (quote depth, attribution,
	124	+ * signature). Consecutive lines with the same class share one span so huge
	125	+ * posts don't turn into tens of thousands of nodes.
	126	+ */
	127	+export function formatBody(body: string): ReactNode[] {
	128	+ const lines = body.replace(/\r\n?/g, "\n").replace(/\s+$/, "").split("\n");
	129	+ const out: ReactNode[] = [];
	130	+ let inSig = false;
	131	+ let cls: string \| null = null;
	132	+ let buf: string[] = [];
	133	+ let k = 0;
	134	+ const flush = () => {
	135	+ if (!buf.length) return;
	136	+ const text = buf.join("\n") + "\n";
	137	+ const key = `b${k++}`;
	138	+ const kids = linkify(text, key);
	139	+ out.push(cls ? <span key={key} className={cls}>{kids}</span> : <span key={key}>{kids}</span>);
	140	+ buf = [];
	141	+ };
	142	+ for (const line of lines) {
	143	+ if (!inSig && (line === "-- " \|\| line === "--")) inSig = true;
	144	+ const c = lineClass(line, inSig);
	145	+ if (c !== cls) {
	146	+ flush();
	147	+ cls = c;
	148	+ }
	149	+ buf.push(line);
	150	+ }
	151	+ flush();
	152	+ return out;
	153	+}
	154	+
	155	+/* -------------------------------------------------------------- hierarchy */
	156	+
	157	+/** Top-level hierarchy of a group name; crossposted/garbled names go to "misc". */
	158	+export function hierarchyOf(name: string): string {
	159	+ if (/[\s,@^]/.test(name)) return "(crossposts & strays)";
	160	+ const dot = name.indexOf(".");
	161	+ return dot === -1 ? "(top-level)" : name.slice(0, dot);
	162	+}
	163	+
	164	+/* ------------------------------------------------- RFC 2047 encoded words */
	165	+
	166	+const ENCODED_WORD = /=\?([^?\s]+)\?([BbQq])\?([^?\s]*)\?=/g;
	167	+
	168	+/** Decode MIME encoded-words (=?utf-8?b?…?=) in headers; leaves anything malformed alone. */
	169	+export function decodeMime(s: string): string {
	170	+ if (!s \|\| !s.includes("=?")) return s;
	171	+ return s
	172	+ .replace(/(\?=)\s+(=\?)/g, "$1$2") // whitespace between adjacent encoded words is dropped
	173	+ .replace(ENCODED_WORD, (whole, charset: string, enc: string, text: string) => {
	174	+ try {
	175	+ const bytes =
	176	+ enc.toUpperCase() === "B"
	177	+ ? Buffer.from(text, "base64")
	178	+ : Buffer.from(
	179	+ text
	180	+ .replace(/_/g, " ")
	181	+ .replace(/=([0-9A-Fa-f]{2})/g, (_m, h: string) => String.fromCharCode(parseInt(h, 16))),
	182	+ "latin1"
	183	+ );
	184	+ return new TextDecoder(charset.split("*")[0].toLowerCase(), { fatal: false }).decode(bytes);
	185	+ } catch {
	186	+ return whole;
	187	+ }
	188	+ });
	189	+}

lib/query.ts+201−0

@@ -0,0 +1,201 @@
	1	+/**
	2	+ * Search query parser.
	3	+ *
	4	+ * Turns a user query into FTS5 expressions for the two indexes:
	5	+ * messages_fts(body) — free text
	6	+ * meta_fts(subject, from_addr) — subject: / from: terms
	7	+ *
	8	+ * Syntax
	9	+ * word must contain word
	10	+ * "a phrase" exact phrase
	11	+ * pref* prefix match
	12	+ * -word exclude (also -"phrase", -from:x)
	13	+ * a OR b either term
	14	+ * from:pike author (name or address) aliases: f:, by:, author:
	15	+ * subject:plan9 subject line aliases: s:, subj:, title:
	16	+ * group:comp.os.plan9 one newsgroup aliases: g:, in:
	17	+ * after:1990 before:1995-06 date bounds (YYYY, YYYY-MM, YYYY-MM-DD)
	18	+ */
	19	+
	20	+export type Field = "body" \| "subject" \| "from_addr";
	21	+
	22	+export interface IndexExpr {
	23	+ /** FTS5 expression for positive terms (NOT clauses folded in), or "" */
	24	+ pos: string;
	25	+ /** FTS5 expression of rows to exclude when this index has no positives, or "" */
	26	+ negOnly: string;
	27	+}
	28	+
	29	+export interface ParsedQuery {
	30	+ body: IndexExpr;
	31	+ meta: IndexExpr;
	32	+ group?: string;
	33	+ after?: string; // inclusive lower bound, YYYY-MM-DD
	34	+ before?: string; // exclusive upper bound, YYYY-MM-DD
	35	+ /** Positive plain terms, for echoing back to the user. */
	36	+ terms: string[];
	37	+ error?: string;
	38	+}
	39	+
	40	+interface Term {
	41	+ neg: boolean;
	42	+ text: string; // valid FTS5 phrase, possibly column-qualified
	43	+}
	44	+
	45	+type Special = "group" \| "after" \| "before";
	46	+
	47	+const FIELD_ALIASES: Record<string, Field \| Special> = {
	48	+ from: "from_addr",
	49	+ f: "from_addr",
	50	+ author: "from_addr",
	51	+ by: "from_addr",
	52	+ subject: "subject",
	53	+ subj: "subject",
	54	+ s: "subject",
	55	+ title: "subject",
	56	+ group: "group",
	57	+ g: "group",
	58	+ in: "group",
	59	+ newsgroup: "group",
	60	+ after: "after",
	61	+ since: "after",
	62	+ before: "before",
	63	+ until: "before",
	64	+};
	65	+
	66	+const HAS_WORD = /[\p{L}\p{N}]/u;
	67	+
	68	+function phrase(raw: string): string \| null {
	69	+ let prefix = false;
	70	+ let s = raw;
	71	+ if (s.endsWith("*")) {
	72	+ prefix = true;
	73	+ s = s.replace(/\*+$/, "");
	74	+ }
	75	+ if (!HAS_WORD.test(s)) return null;
	76	+ return `"${s.replace(/"/g, '""')}"${prefix ? "*" : ""}`;
	77	+}
	78	+
	79	+/** Normalise YYYY / YYYY-MM / YYYY-MM-DD. `end` rounds up to the start of the next period. */
	80	+export function parseDate(v: string, end: boolean): string \| null {
	81	+ const m = v.match(/^(\d{4})(?:-(\d{1,2}))?(?:-(\d{1,2}))?$/);
	82	+ if (!m) return null;
	83	+ const y = +m[1];
	84	+ const mo = m[2] ? +m[2] : 1;
	85	+ const d = m[3] ? +m[3] : 1;
	86	+ if (mo < 1 \|\| mo > 12 \|\| d < 1 \|\| d > 31) return null;
	87	+ let dt = new Date(Date.UTC(y, mo - 1, d));
	88	+ if (end) {
	89	+ if (m[3]) dt = new Date(Date.UTC(y, mo - 1, d + 1));
	90	+ else if (m[2]) dt = new Date(Date.UTC(y, mo, 1));
	91	+ else dt = new Date(Date.UTC(y + 1, 0, 1));
	92	+ }
	93	+ return dt.toISOString().slice(0, 10);
	94	+}
	95	+
	96	+function buildExpr(terms: Term[], orAfter: Set<number>): IndexExpr {
	97	+ const pos: string[] = [];
	98	+ const neg: string[] = [];
	99	+ let cur: string[] = [];
	100	+ const flush = () => {
	101	+ if (cur.length) pos.push(cur.length > 1 ? `(${cur.join(" OR ")})` : cur[0]);
	102	+ cur = [];
	103	+ };
	104	+ terms.forEach((t, i) => {
	105	+ if (t.neg) {
	106	+ neg.push(t.text);
	107	+ return;
	108	+ }
	109	+ cur.push(t.text);
	110	+ if (!orAfter.has(i)) flush();
	111	+ });
	112	+ flush();
	113	+
	114	+ const negExpr = neg.length > 1 ? `(${neg.join(" OR ")})` : (neg[0] ?? "");
	115	+ if (pos.length) {
	116	+ const p = pos.length > 1 ? pos.join(" AND ") : pos[0];
	117	+ return { pos: negExpr ? `(${p}) NOT ${negExpr}` : p, negOnly: "" };
	118	+ }
	119	+ return { pos: "", negOnly: negExpr };
	120	+}
	121	+
	122	+export function parseQuery(input: string): ParsedQuery {
	123	+ const re = /(-)?(?:([A-Za-z_]+):)?(?:"([^"]*)"?\|(\S+))/g;
	124	+ const out: ParsedQuery = {
	125	+ body: { pos: "", negOnly: "" },
	126	+ meta: { pos: "", negOnly: "" },
	127	+ terms: [],
	128	+ };
	129	+ const body: Term[] = [];
	130	+ const meta: Term[] = [];
	131	+ const bodyOr = new Set<number>();
	132	+ const metaOr = new Set<number>();
	133	+ let pendingOr = false;
	134	+ let lastList: Term[] \| null = null;
	135	+
	136	+ let m: RegExpExecArray \| null;
	137	+ while ((m = re.exec(input))) {
	138	+ if (m[0] === "") {
	139	+ re.lastIndex++;
	140	+ continue;
	141	+ }
	142	+ const [, minus, rawField, quoted, bare] = m;
	143	+ const neg = !!minus;
	144	+ const value = quoted ?? bare ?? "";
	145	+
	146	+ if (!rawField && quoted === undefined && !neg && (value === "OR" \|\| value === "\|")) {
	147	+ pendingOr = true;
	148	+ continue;
	149	+ }
	150	+
	151	+ const alias = rawField ? FIELD_ALIASES[rawField.toLowerCase()] : undefined;
	152	+ // Unknown "foo:bar" (a URL, "Re:") is just text.
	153	+ const text = rawField && !alias ? `${rawField}:${value}` : value;
	154	+
	155	+ if (alias === "group") {
	156	+ if (value) out.group = value;
	157	+ pendingOr = false;
	158	+ continue;
	159	+ }
	160	+ if (alias === "after" \|\| alias === "before") {
	161	+ const d = parseDate(value, alias === "before");
	162	+ if (!d) out.error = `Bad date “${value}” — use YYYY, YYYY-MM or YYYY-MM-DD.`;
	163	+ else if (alias === "after") out.after = d;
	164	+ else out.before = d;
	165	+ pendingOr = false;
	166	+ continue;
	167	+ }
	168	+
	169	+ const ph = phrase(text);
	170	+ if (!ph) {
	171	+ pendingOr = false;
	172	+ continue;
	173	+ }
	174	+ const field: Field = (alias as Field \| undefined) ?? "body";
	175	+ const isMeta = field !== "body";
	176	+ const list = isMeta ? meta : body;
	177	+ const orSet = isMeta ? metaOr : bodyOr;
	178	+
	179	+ // OR links two positive terms that land in the same index.
	180	+ if (pendingOr && !neg && lastList === list && list.length && !list[list.length - 1].neg) {
	181	+ orSet.add(list.length - 1);
	182	+ }
	183	+ list.push({ neg, text: isMeta ? `${field} : ${ph}` : ph });
	184	+ lastList = list;
	185	+ pendingOr = false;
	186	+ if (!neg) out.terms.push(text.replace(/\*+$/, ""));
	187	+ }
	188	+
	189	+ out.body = buildExpr(body, bodyOr);
	190	+ out.meta = buildExpr(meta, metaOr);
	191	+
	192	+ if (!out.error && !out.body.pos && !out.meta.pos) {
	193	+ if (out.body.negOnly \|\| out.meta.negOnly)
	194	+ out.error = "Exclusions need something to exclude from — add a word, phrase, from: or subject: term.";
	195	+ else if (out.group \|\| out.after \|\| out.before)
	196	+ out.error = "Add a word, phrase, from: or subject: term to search within that filter.";
	197	+ }
	198	+ if (!out.error && out.after && out.before && out.after >= out.before)
	199	+ out.error = "after: must be earlier than before:.";
	200	+ return out;
	201	+}