299 lines · 10.0 KB · TypeScript
RawDownload
import type {  ServerProcessResourceHistoryBucket,  ServerProcessResourceHistoryInput,  ServerProcessResourceHistoryResult,  ServerProcessResourceHistorySummary,} from "@t3tools/contracts";import * as Context from "effect/Context";import * as DateTime from "effect/DateTime";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as Option from "effect/Option";import * as Ref from "effect/Ref";import { ChildProcessSpawner } from "effect/unstable/process"; import {  buildDescendantEntries,  isDiagnosticsQueryProcess,  type ProcessRow,  readProcessRows,} from "./ProcessDiagnostics.ts"; const SAMPLE_INTERVAL_MS = 5_000;const RETENTION_MS = 60 * 60_000;const MAX_RETAINED_SAMPLES = 20_000; export interface ProcessResourceSample {  readonly sampledAt: DateTime.Utc;  readonly sampledAtMs: number;  readonly processKey: string;  readonly pid: number;  readonly ppid: number;  readonly command: string;  readonly cpuPercent: number;  readonly rssBytes: number;  readonly depth: number;  readonly isServerRoot: boolean;} interface MonitorState {  readonly samples: ReadonlyArray<ProcessResourceSample>;  readonly lastError: string | null;} export interface ProcessResourceMonitorShape {  readonly readHistory: (    input: ServerProcessResourceHistoryInput,  ) => Effect.Effect<ServerProcessResourceHistoryResult>;} export class ProcessResourceMonitor extends Context.Service<  ProcessResourceMonitor,  ProcessResourceMonitorShape>()("t3/diagnostics/ProcessResourceMonitor") {} function dateTimeFromMillis(ms: number): DateTime.Utc {  return DateTime.makeUnsafe(ms);} function sampleKey(row: Pick<ProcessRow, "pid" | "command">): string {  return `${row.pid}:${row.command}`;} function findServerRootRow(rows: ReadonlyArray<ProcessRow>, serverPid: number): ProcessRow | null {  return rows.find((row) => row.pid === serverPid) ?? null;} export function collectMonitoredSamples(input: {  readonly rows: ReadonlyArray<ProcessRow>;  readonly serverPid: number;  readonly sampledAt: DateTime.Utc;  readonly sampledAtMs: number;}): ReadonlyArray<ProcessResourceSample> {  const rows = input.rows.filter((row) => !isDiagnosticsQueryProcess(row, input.serverPid));  const root = findServerRootRow(rows, input.serverPid);  const descendants = buildDescendantEntries(rows, input.serverPid);  const samples: ProcessResourceSample[] = [];   if (root) {    samples.push({      sampledAt: input.sampledAt,      sampledAtMs: input.sampledAtMs,      processKey: sampleKey(root),      pid: root.pid,      ppid: root.ppid,      command: root.command,      cpuPercent: root.cpuPercent,      rssBytes: root.rssBytes,      depth: 0,      isServerRoot: true,    });  }   for (const process of descendants) {    samples.push({      sampledAt: input.sampledAt,      sampledAtMs: input.sampledAtMs,      processKey: sampleKey(process),      pid: process.pid,      ppid: process.ppid,      command: process.command,      cpuPercent: process.cpuPercent,      rssBytes: process.rssBytes,      depth: process.depth + 1,      isServerRoot: false,    });  }   return samples;} function trimSamples(  samples: ReadonlyArray<ProcessResourceSample>,  nowMs: number,): ReadonlyArray<ProcessResourceSample> {  const minSampledAtMs = nowMs - RETENTION_MS;  const retained = samples.filter((sample) => sample.sampledAtMs >= minSampledAtMs);  return retained.length <= MAX_RETAINED_SAMPLES    ? retained    : retained.slice(retained.length - MAX_RETAINED_SAMPLES);} function summarizeProcesses(  samples: ReadonlyArray<ProcessResourceSample>,): ReadonlyArray<ServerProcessResourceHistorySummary> {  const groups = new Map<string, ProcessResourceSample[]>();  for (const sample of samples) {    const processSamples = groups.get(sample.processKey) ?? [];    processSamples.push(sample);    groups.set(sample.processKey, processSamples);  }   return [...groups.entries()]    .map(([processKey, processSamples]) => {      const sorted = processSamples.toSorted((left, right) => left.sampledAtMs - right.sampledAtMs);      const first = sorted[0]!;      const latest = sorted[sorted.length - 1]!;      const cpuPercentTotal = sorted.reduce((total, sample) => total + sample.cpuPercent, 0);      const maxCpuPercent = Math.max(...sorted.map((sample) => sample.cpuPercent));      const maxRssBytes = Math.max(...sorted.map((sample) => sample.rssBytes));      const cpuSecondsApprox = sorted.reduce(        (total, sample) => total + (sample.cpuPercent / 100) * (SAMPLE_INTERVAL_MS / 1_000),        0,      );       return {        processKey,        pid: latest.pid,        ppid: latest.ppid,        command: latest.command,        depth: latest.depth,        isServerRoot: latest.isServerRoot,        firstSeenAt: first.sampledAt,        lastSeenAt: latest.sampledAt,        currentCpuPercent: latest.cpuPercent,        avgCpuPercent: cpuPercentTotal / sorted.length,        maxCpuPercent,        cpuSecondsApprox,        currentRssBytes: latest.rssBytes,        maxRssBytes,        sampleCount: sorted.length,      } satisfies ServerProcessResourceHistorySummary;    })    .toSorted((left, right) => right.cpuSecondsApprox - left.cpuSecondsApprox);} function buildBuckets(input: {  readonly samples: ReadonlyArray<ProcessResourceSample>;  readonly nowMs: number;  readonly windowMs: number;  readonly bucketMs: number;}): ReadonlyArray<ServerProcessResourceHistoryBucket> {  const bucketMs = Math.max(1_000, input.bucketMs);  const windowStartMs = input.nowMs - input.windowMs;  const buckets: ServerProcessResourceHistoryBucket[] = [];   for (let startedAtMs = windowStartMs; startedAtMs < input.nowMs; startedAtMs += bucketMs) {    const endedAtMs = Math.min(input.nowMs, startedAtMs + bucketMs);    const bucketSamples = input.samples.filter(      (sample) =>        sample.sampledAtMs >= startedAtMs &&        (endedAtMs === input.nowMs          ? sample.sampledAtMs <= endedAtMs          : sample.sampledAtMs < endedAtMs),    );    const samplesByRead = new Map<number, ProcessResourceSample[]>();    for (const sample of bucketSamples) {      const samplesAtTime = samplesByRead.get(sample.sampledAtMs) ?? [];      samplesAtTime.push(sample);      samplesByRead.set(sample.sampledAtMs, samplesAtTime);    }     const readTotals = [...samplesByRead.values()].map((samplesAtTime) => ({      cpuPercent: samplesAtTime.reduce((total, sample) => total + sample.cpuPercent, 0),      rssBytes: samplesAtTime.reduce((total, sample) => total + sample.rssBytes, 0),      processCount: samplesAtTime.length,    }));    const avgCpuPercent =      readTotals.length === 0        ? 0        : readTotals.reduce((total, read) => total + read.cpuPercent, 0) / readTotals.length;     buckets.push({      startedAt: dateTimeFromMillis(startedAtMs),      endedAt: dateTimeFromMillis(endedAtMs),      avgCpuPercent,      maxCpuPercent: readTotals.length ? Math.max(...readTotals.map((read) => read.cpuPercent)) : 0,      maxRssBytes: readTotals.length ? Math.max(...readTotals.map((read) => read.rssBytes)) : 0,      maxProcessCount: readTotals.length        ? Math.max(...readTotals.map((read) => read.processCount))        : 0,    });  }   return buckets;} export function aggregateProcessResourceHistory(input: {  readonly samples: ReadonlyArray<ProcessResourceSample>;  readonly readAt: DateTime.Utc;  readonly readAtMs: number;  readonly windowMs: number;  readonly bucketMs: number;  readonly lastError: string | null;}): ServerProcessResourceHistoryResult {  const windowMs = Math.max(1_000, input.windowMs);  const bucketMs = Math.max(1_000, input.bucketMs);  const minSampledAtMs = input.readAtMs - windowMs;  const samples = input.samples.filter((sample) => sample.sampledAtMs >= minSampledAtMs);  const topProcesses = summarizeProcesses(samples);  const totalCpuSecondsApprox = samples.reduce(    (total, sample) => total + (sample.cpuPercent / 100) * (SAMPLE_INTERVAL_MS / 1_000),    0,  );   return {    readAt: input.readAt,    windowMs,    bucketMs,    sampleIntervalMs: SAMPLE_INTERVAL_MS,    retainedSampleCount: input.samples.length,    totalCpuSecondsApprox,    buckets: buildBuckets({ samples, nowMs: input.readAtMs, windowMs, bucketMs }),    topProcesses,    error: input.lastError ? Option.some({ message: input.lastError }) : Option.none(),  };} export const make = Effect.fn("makeProcessResourceMonitor")(function* () {  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;  const state = yield* Ref.make<MonitorState>({ samples: [], lastError: null });   const sampleOnce = Effect.gen(function* () {    const sampledAt = yield* DateTime.now;    const sampledAtMs = DateTime.toEpochMillis(sampledAt);    const rows = yield* readProcessRows().pipe(      Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),    );    const samples = collectMonitoredSamples({      rows,      serverPid: process.pid,      sampledAt,      sampledAtMs,    });    yield* Ref.update(state, (current) => ({      samples: trimSamples([...current.samples, ...samples], sampledAtMs),      lastError: null,    }));  }).pipe(    Effect.catch((error: unknown) =>      Ref.update(state, (current) => ({        ...current,        lastError: error instanceof Error ? error.message : "Failed to sample process resources.",      })),    ),  );   yield* Effect.forever(sampleOnce.pipe(Effect.andThen(Effect.sleep(SAMPLE_INTERVAL_MS)))).pipe(    Effect.forkScoped,  );   const readHistory: ProcessResourceMonitorShape["readHistory"] = (input) =>    Effect.gen(function* () {      const readAt = yield* DateTime.now;      const readAtMs = DateTime.toEpochMillis(readAt);      const current = yield* Ref.get(state);      return aggregateProcessResourceHistory({        samples: current.samples,        readAt,        readAtMs,        windowMs: input.windowMs,        bucketMs: input.bucketMs,        lastError: current.lastError,      });    });   return ProcessResourceMonitor.of({ readHistory });}); export const layer = Layer.effect(ProcessResourceMonitor, make());