755 lines · 26.7 KB · TypeScript
RawDownload
import {  type EnvironmentId,  type ServerConfig,  type ServerConfigStreamEvent,  type ServerLifecycleWelcomePayload,  type ServerLifecycleStreamReadyEvent,  type ServerSelfUpdateProgressEvent,  type ServerSelfUpdateResult,  WS_METHODS,} from "@t3tools/contracts";import * as Cause from "effect/Cause";import * as Duration from "effect/Duration";import * as Effect from "effect/Effect";import * as Exit from "effect/Exit";import * as Option from "effect/Option";import * as Queue from "effect/Queue";import * as Ref from "effect/Ref";import * as Result from "effect/Result";import * as Schema from "effect/Schema";import * as Stream from "effect/Stream";import * as SubscriptionRef from "effect/SubscriptionRef";import { AsyncResult, Atom } from "effect/unstable/reactivity"; import {  createAtomCommandScheduler,  createEnvironmentRpcCommand,  createEnvironmentRpcQueryAtomFamily,  createEnvironmentRpcSubscriptionAtomFamily,  createRuntimeCommand,  scheduleAtomCommandEffect,} from "./runtime.ts";import { EnvironmentRegistry } from "../connection/registry.ts";import { EnvironmentSupervisor } from "../connection/supervisor.ts";import { safeErrorLogAttributes } from "../errors/safeLog.ts";import { EnvironmentCacheStore } from "../platform/persistence.ts";import {  isRpcClientError,  request,  runStream,  subscribe,  type EnvironmentRpcInput,} from "../rpc/client.ts";import { followStreamInEnvironment } from "./runtime.ts";import {  applyServerConfigProjection,  type ServerConfigProjection,  withoutEnvironmentThemes,} from "./serverConfigProjection.ts"; // Exported server state includes this type in its inferred public return type.export type { ServerConfigProjection } from "./serverConfigProjection.ts"; export type ServerUpdateStage = "downloading" | "installing" | "resuming"; export type ServerUpdateState =  | { readonly status: "idle" }  | {      readonly status: "running";      readonly stage: ServerUpdateStage;      readonly fromVersion: string;      readonly targetVersion: string;    }  | {      readonly status: "failed";      readonly stage: ServerUpdateStage;      readonly fromVersion: string;      readonly targetVersion: string;      readonly message: string;    }; export interface ServerUpdateTarget {  readonly environmentId: EnvironmentId;  readonly input: EnvironmentRpcInput<typeof WS_METHODS.serverUpdateServer>;} const IDLE_SERVER_UPDATE_STATE: ServerUpdateState = { status: "idle" };const EMPTY_SERVER_UPDATE_STATE_ATOM = Atom.make<ServerUpdateState>(IDLE_SERVER_UPDATE_STATE).pipe(  Atom.withLabel("environment-data:server:update-state:empty"),);const serverUpdateStateAtom = Atom.family((environmentId: EnvironmentId) =>  Atom.make<ServerUpdateState>(IDLE_SERVER_UPDATE_STATE).pipe(    Atom.withLabel(`environment-data:server:update-state:${environmentId}`),  ),); export class ServerUpdateResumeTimeoutError extends Schema.TaggedErrorClass<ServerUpdateResumeTimeoutError>()(  "ServerUpdateResumeTimeoutError",  {    environmentId: Schema.String,    targetVersion: Schema.String,  },) {  override get message(): string {    return `The server did not resume on t3@${this.targetVersion}.`;  }} export class ServerUpdateProgressIncompleteError extends Schema.TaggedErrorClass<ServerUpdateProgressIncompleteError>()(  "ServerUpdateProgressIncompleteError",  {    targetVersion: Schema.String,  },) {  override get message(): string {    return `The t3@${this.targetVersion} update ended before the server accepted the restart.`;  }} export class ServerUpdateTerminalError extends Schema.TaggedErrorClass<ServerUpdateTerminalError>()(  "ServerUpdateTerminalError",  {    targetVersion: Schema.String,    status: Schema.Literals(["committed", "rolled-back", "failed"]),    reason: Schema.optional(Schema.String),  },) {  override get message(): string {    return this.reason ?? `The t3@${this.targetVersion} update ${this.status}.`;  }} // Covers the 120-second trial deadline and a final restart of the previous// version when the trial rolls back.const SERVER_UPDATE_RESUME_TIMEOUT = Duration.minutes(4); export function matchesServerUpdateReadyEvent(  result: ServerSelfUpdateResult,  event: ServerLifecycleStreamReadyEvent,): boolean {  return result.updateId === undefined    ? event.payload.environment.serverVersion === result.targetVersion    : event.payload.updateOutcome?.id === result.updateId;} export function validateServerUpdateReadyEvent(  result: ServerSelfUpdateResult,  event: ServerLifecycleStreamReadyEvent,): Effect.Effect<void, ServerUpdateTerminalError> {  if (result.updateId === undefined) return Effect.void;  const outcome = event.payload.updateOutcome;  if (    outcome?.id === result.updateId &&    outcome.status === "committed" &&    outcome.targetVersion === result.targetVersion &&    event.payload.environment.serverVersion === result.targetVersion  ) {    return Effect.void;  }  return Effect.fail(    new ServerUpdateTerminalError({      targetVersion: result.targetVersion,      status: outcome?.status ?? "failed",      reason:        outcome?.reason ??        "The service launcher resumed without committing the requested server version.",    }),  );} /** * Keeps reconnect attempts ~1s apart for the whole update restart. * * A restart takes the server down for ~15 seconds, but the supervisor's normal * backoff ladder (1/2/4/8/16s) assumes an unexpected failure and lands attempts * at ~3, 5, 9, 17 and 33 seconds — so a 15-second restart is observed as a * 33-second "Resuming". Nudging on every backoff entry (not just the first) * holds the retry cadence flat until the server answers again. The sleep before * each nudge is the pacer: a connection that fails instantly re-enters backoff * immediately and would otherwise spin a tight retry loop. * * A newly restarted server can also reject the first environment credential. * Authentication blocks need the same paced retry during this known restart; * permission and configuration failures remain blocked. * * Callers fork this as a child of the update command so it is interrupted as * soon as the update settles, whether it succeeds, fails, or times out. */export function nudgeReconnectDuringUpdateRestart(input: {  readonly stateChanges: Stream.Stream<    {      readonly phase: string;      readonly lastFailure?: { readonly reason: string } | null;    },    unknown  >;  readonly retryNow: Effect.Effect<void>;  readonly interval?: Duration.Duration;}): Effect.Effect<void> {  return input.stateChanges.pipe(    Stream.filter(      (state) =>        state.phase === "backoff" ||        (state.phase === "blocked" && state.lastFailure?.reason === "authentication"),    ),    Stream.runForEach(() =>      Effect.sleep(input.interval ?? Duration.seconds(1)).pipe(Effect.andThen(input.retryNow)),    ),    Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT),    Effect.ignore,  );} export function serverUpdateStateForProgressEvent(  fromVersion: string,  targetVersion: string,  event: ServerSelfUpdateProgressEvent,): Extract<ServerUpdateState, { status: "running" }> {  return {    status: "running",    stage: event.type === "complete" ? "resuming" : event.stage,    fromVersion,    targetVersion,  };} export function serverUpdateStateForServerVersion(  state: ServerUpdateState,  serverVersion: string | null,): ServerUpdateState {  return state.status === "idle" ||    state.status === "running" ||    serverVersion === null ||    state.fromVersion === serverVersion    ? state    : IDLE_SERVER_UPDATE_STATE;} function serverUpdateFailureMessage(error: unknown): string {  return error instanceof Error ? error.message : "Server update failed.";} function isRpcSocketError(error: unknown): boolean {  if (!isRpcClientError(error)) {    return false;  }  switch (error.reason._tag) {    case "SocketReadError":    case "SocketWriteError":    case "SocketCloseError":      return true;    default:      return false;  }} export function isLegacyUpdateHandoffLoss(cause: Cause.Cause<unknown>): boolean {  if (Cause.hasInterruptsOnly(cause)) {    return true;  }  return (    cause.reasons.length > 0 &&    cause.reasons.every((reason) => Cause.isFailReason(reason) && isRpcSocketError(reason.error))  );} export function resolveServerUpdateProgressResult<E>(  targetVersion: string,  terminal: Option.Option<ServerSelfUpdateResult>,  streamExit: Exit.Exit<void, E>,): Effect.Effect<ServerSelfUpdateResult, E | ServerUpdateProgressIncompleteError> {  if (    Option.isSome(terminal) &&    (Exit.isSuccess(streamExit) || isLegacyUpdateHandoffLoss(streamExit.cause))  ) {    return Effect.succeed(terminal.value);  }  if (Exit.isFailure(streamExit)) {    return Effect.failCause(streamExit.cause);  }  return Effect.fail(new ServerUpdateProgressIncompleteError({ targetVersion }));} export function projectServerConfig(  current: Option.Option<ServerConfigProjection>,  event: ServerConfigStreamEvent,): readonly [Option.Option<ServerConfigProjection>, ReadonlyArray<ServerConfigProjection>] {  const next = applyServerConfigProjection(current, event);  return [next, Option.toArray(next)];} const cachedConfigSnapshotEvent = (config: ServerConfig): ServerConfigStreamEvent => ({  version: 1,  type: "snapshot",  config,}); export const makeEnvironmentServerConfigState = Effect.fn("EnvironmentServerConfigState.make")(  function* (environmentThemes?: boolean) {    const supervisor = yield* EnvironmentSupervisor;    const cache = yield* EnvironmentCacheStore;    const environmentId = supervisor.target.environmentId;    const cachedConfig = yield* cache.loadServerConfig(environmentId).pipe(      Effect.catch((error) =>        Effect.logWarning("Could not load cached server configuration.").pipe(          Effect.annotateLogs({            environmentId,            ...safeErrorLogAttributes(error),          }),          Effect.as(Option.none<ServerConfig>()),        ),      ),    );    const state = yield* SubscriptionRef.make<Option.Option<ServerConfigProjection>>(      // Stripped on load as well as on save: a cache written by an earlier      // build can still carry published themes.      Option.map(cachedConfig, (cached) => ({        config: withoutEnvironmentThemes(cached),        latestEvent: cachedConfigSnapshotEvent(withoutEnvironmentThemes(cached)),        source: "cache" as const,      })),    );    const persistence = yield* Queue.sliding<ServerConfig>(1);    const pendingPersistence = yield* Ref.make<Option.Option<ServerConfig>>(Option.none());     const persist = Effect.fn("EnvironmentServerConfigState.persist")(function* (      config: ServerConfig,    ) {      return yield* cache.saveServerConfig(environmentId, withoutEnvironmentThemes(config)).pipe(        Effect.as(true),        Effect.catch((error) =>          Effect.logWarning("Could not persist cached server configuration.").pipe(            Effect.annotateLogs({              environmentId,              ...safeErrorLogAttributes(error),            }),            Effect.as(false),          ),        ),      );    });     const persistPending = Effect.fn("EnvironmentServerConfigState.persistPending")(function* (      config: ServerConfig,    ) {      if (!(yield* persist(config))) {        return;      }      yield* Ref.update(pendingPersistence, (pending) =>        Option.isSome(pending) && pending.value === config ? Option.none() : pending,      );    });     yield* Stream.fromQueue(persistence).pipe(      Stream.debounce("500 millis"),      Stream.runForEach(persistPending),      Effect.forkScoped,    );     yield* subscribe(      WS_METHODS.subscribeServerConfig,      environmentThemes === true ? { environmentThemes: true } : {},    ).pipe(      Stream.runForEach((event) =>        Effect.gen(function* () {          const next = applyServerConfigProjection(yield* SubscriptionRef.get(state), event);          if (Option.isNone(next)) {            return;          }          yield* Ref.set(pendingPersistence, Option.some(next.value.config));          yield* SubscriptionRef.set(state, next);          yield* Queue.offer(persistence, next.value.config);        }),      ),      Effect.forkScoped,    );     yield* Effect.addFinalizer(() =>      Ref.get(pendingPersistence).pipe(        Effect.flatMap(          Option.match({            onNone: () => Effect.void,            onSome: (config) => persist(config).pipe(Effect.asVoid),          }),        ),      ),    );     return state;  },); export function serverConfigStateChanges(  environmentId: EnvironmentId,  environmentThemes?: boolean,) {  return followStreamInEnvironment(    environmentId,    Stream.unwrap(      makeEnvironmentServerConfigState(environmentThemes).pipe(        Effect.map((state) =>          SubscriptionRef.changes(state).pipe(            Stream.filterMap((projection) =>              Option.match(projection, {                onNone: () => Result.failVoid,                onSome: (value) => Result.succeed(value),              }),            ),          ),        ),      ),    ),  );} export function projectServerWelcome(  current: Option.Option<ServerLifecycleWelcomePayload>,  event: {    readonly type: "welcome" | "ready";    readonly payload: unknown;  },): readonly [  Option.Option<ServerLifecycleWelcomePayload>,  ReadonlyArray<ServerLifecycleWelcomePayload>,] {  if (event.type !== "welcome") {    return [current, []];  }  const welcome = event.payload as ServerLifecycleWelcomePayload;  return [Option.some(welcome), [welcome]];} export function resolveServerConfigValue(  projection: ServerConfigProjection | null,  initialConfig: ServerConfig | null,): ServerConfig | null {  if (    projection?.source === "live" &&    (initialConfig === null ||      projection.config.environment.serverVersion === initialConfig.environment.serverVersion)  ) {    return projection.config;  }  return initialConfig ?? projection?.config ?? null;} export function createServerEnvironmentAtoms<R, E>(  runtime: Atom.AtomRuntime<EnvironmentRegistry | EnvironmentCacheStore | R, E>,  options: {    readonly initialConfigValueAtom: (      environmentId: EnvironmentId,    ) => Atom.Atom<ServerConfig | null>;    /**     * Whether this surface renders themes the environment publishes. Mobile     * keeps its own appearance settings, so it neither asks for the stream nor     * receives the payload.     */    readonly environmentThemes?: boolean;  },) {  const configScheduler = createAtomCommandScheduler();  // Updates stay serial end-to-end, but only their handoff phase occupies the config lane.  const updateScheduler = createAtomCommandScheduler();  const configConcurrency = {    mode: "serial" as const,    key: ({ environmentId }: { readonly environmentId: string }) => environmentId,  };  const configProjectionFamily = Atom.family((environmentId: EnvironmentId) =>    runtime      .atom(serverConfigStateChanges(environmentId, options.environmentThemes))      .pipe(        Atom.setIdleTTL(5 * 60_000),        Atom.withLabel(`environment-data:server:config-projection:${environmentId}`),      ),  );  const configProjection = (target: {    readonly environmentId: EnvironmentId;    readonly input: EnvironmentRpcInput<typeof WS_METHODS.subscribeServerConfig>;  }) => configProjectionFamily(target.environmentId);  const emptyConfigAtom = Atom.make<ServerConfig | null>(null).pipe(    Atom.withLabel("environment-data:server:config:empty"),  );  const configValueAtom = Atom.family((environmentId: EnvironmentId | null) => {    if (environmentId === null) {      return emptyConfigAtom;    }    return Atom.make((get): ServerConfig | null => {      const projection = Option.getOrNull(        AsyncResult.value(get(configProjection({ environmentId, input: {} }))),      );      return resolveServerConfigValue(        projection,        get(options.initialConfigValueAtom(environmentId)),      );    }).pipe(Atom.withLabel(`environment-data:server:config:${environmentId}`));  });  const updateStateValueAtom = Atom.family((environmentId: EnvironmentId) =>    Atom.make((get) =>      serverUpdateStateForServerVersion(        get(serverUpdateStateAtom(environmentId)),        get(configValueAtom(environmentId))?.environment.serverVersion ?? null,      ),    ).pipe(Atom.withLabel(`environment-data:server:update-state-value:${environmentId}`)),  );  const updateStateAtom = (environmentId: EnvironmentId | null) =>    environmentId === null ? EMPTY_SERVER_UPDATE_STATE_ATOM : updateStateValueAtom(environmentId);  const updateServer = createRuntimeCommand<    EnvironmentRegistry | EnvironmentCacheStore | R,    E,    ServerUpdateTarget,    ServerSelfUpdateResult,

Lines 1–500 of 755.