799 lines · 25.3 KB · TypeScript
RawDownload
import { withRelayClientTracing } from "@t3tools/shared/relayTracing";import * as Cause from "effect/Cause";import * as Clock from "effect/Clock";import * as Context from "effect/Context";import * as Effect from "effect/Effect";import * as Exit from "effect/Exit";import * as Fiber from "effect/Fiber";import * as Option from "effect/Option";import * as Queue from "effect/Queue";import * as Ref from "effect/Ref";import * as Scope from "effect/Scope";import * as Stream from "effect/Stream";import * as SubscriptionRef from "effect/SubscriptionRef";import * as Tracer from "effect/Tracer"; import type { ConnectionCatalogEntry } from "./catalog.ts";import * as Connectivity from "./connectivity.ts";import * as ConnectionDriver from "./driver.ts";import {  type ConnectionAttemptError,  type ConnectionTarget,  ConnectionTransientError,  type NetworkStatus,  type PreparedConnection,  type SupervisorConnectionState,} from "./model.ts";import * as RpcSession from "../rpc/session.ts";import { safeErrorLogAttributes } from "../errors/safeLog.ts";import { NETWORK_BLOCKING_HINT } from "../errors/network.ts";import * as ConnectionWakeups from "./wakeups.ts"; const RETRY_DELAYS_MS = [3_000, 4_000, 8_000, 16_000] as const;const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";const CONNECTION_PROBE_TIMEOUT = "15 seconds";const MOBILE_CONNECTION_PROBE_TIMEOUT = "3 seconds";const BACKOFF_RESET_AFTER_MS = 30_000; interface SupervisorIntent {  readonly desired: boolean;  readonly network: NetworkStatus;} type SupervisorSignal =  | { readonly _tag: "ConnectRequested" }  | { readonly _tag: "DisconnectRequested" }  | { readonly _tag: "RetryRequested" }  | { readonly _tag: "NetworkChanged"; readonly network: NetworkStatus }  | { readonly _tag: "Wakeup"; readonly reason: ConnectionWakeups.ConnectionWakeup }; interface PendingRetryTrace {  readonly previousAttempt: Tracer.Span;  readonly failureCount: number;  readonly delayMs: number;  readonly reason: ConnectionAttemptError["reason"];} interface TracedAttemptFailure {  readonly error: ConnectionAttemptError;  readonly attemptSpan: Option.Option<Tracer.Span>;} type AttemptOutcome =  | {      readonly _tag: "Interrupted";      readonly established: boolean;      readonly stable: boolean;      readonly resetRetry: boolean;    }  | {      readonly _tag: "Failure";      readonly established: boolean;      readonly stable: boolean;      readonly failure: TracedAttemptFailure;    }; type EstablishmentEvent =  | {      readonly _tag: "Completed";      readonly exit: Exit.Exit<        {          readonly attemptSpan: Option.Option<Tracer.Span>;          readonly lease: ConnectionDriver.EnvironmentConnectionLease;        },        TracedAttemptFailure      >;    }  | { readonly _tag: "Interrupted"; readonly resetRetry: boolean }  | { readonly _tag: "TimedOut" }; function exitUnlessInterrupted<A, E, R>(  effect: Effect.Effect<A, E, R>,): Effect.Effect<Exit.Exit<A, E>, never, R> {  return Effect.matchCauseEffect(effect, {    onFailure: (cause) =>      Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(Exit.failCause(cause)),    onSuccess: (value) => Effect.succeed(Exit.succeed(value)),  });} export interface EnvironmentSupervisorOptions {  readonly initiallyDesired?: boolean;} function retryDelayMs(failureCount: number): number {  return RETRY_DELAYS_MS[Math.min(failureCount, RETRY_DELAYS_MS.length - 1)] ?? 16_000;} function annotateTarget(target: ConnectionTarget) {  return Effect.annotateCurrentSpan({    "environment.id": target.environmentId,    "environment.label": target.label,    "environment.target.kind": target._tag,  });} function availableState(intent: SupervisorIntent, generation: number): SupervisorConnectionState {  return {    desired: false,    network: intent.network,    phase: "available",    stage: null,    attempt: 0,    generation,    lastFailure: null,    retryAt: null,  };} function offlineState(  intent: SupervisorIntent,  generation: number,  attempt: number,  lastFailure: ConnectionAttemptError | null,): SupervisorConnectionState {  return {    desired: true,    network: intent.network,    phase: "offline",    stage: null,    attempt,    generation,    lastFailure,    retryAt: null,  };} function connectingState(  intent: SupervisorIntent,  generation: number,  attempt: number,  lastFailure: ConnectionAttemptError | null,  stage: SupervisorConnectionState["stage"] = "preparing",): SupervisorConnectionState {  return {    desired: true,    network: intent.network,    phase: "connecting",    stage,    attempt,    generation,    lastFailure,    retryAt: null,  };} function failureFromExit<A>(  target: ConnectionTarget,  exit: Exit.Exit<A, TracedAttemptFailure>,  established: boolean,  stable: boolean,): AttemptOutcome {  if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) {    return { _tag: "Interrupted", established, stable, resetRetry: false };  }  const typedFailure = exit.cause.reasons.find(Cause.isFailReason);  if (typedFailure) {    return {      _tag: "Failure",      established,      stable,      failure: typedFailure.error,    };  }  return {    _tag: "Failure",    established,    stable,    failure: {      error: new ConnectionTransientError({        reason: "transport",        detail: `${target.label} connection failed unexpectedly.`,      }),      attemptSpan: Option.none(),    },  };} export class EnvironmentSupervisor extends Context.Service<  EnvironmentSupervisor,  {    readonly target: ConnectionTarget;    readonly state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>;    readonly session: SubscriptionRef.SubscriptionRef<Option.Option<RpcSession.RpcSession>>;    readonly prepared: SubscriptionRef.SubscriptionRef<Option.Option<PreparedConnection>>;    readonly connect: Effect.Effect<void>;    readonly disconnect: Effect.Effect<void>;    readonly retryNow: Effect.Effect<void>;  }>()("@t3tools/client-runtime/connection/supervisor/EnvironmentSupervisor") {} export const make = Effect.fn("EnvironmentSupervisor.make")(function* (  entry: ConnectionCatalogEntry,  options?: EnvironmentSupervisorOptions,): Effect.fn.Return<  EnvironmentSupervisor["Service"],  never,  | Connectivity.Connectivity  | ConnectionDriver.ConnectionDriver  | Scope.Scope  | ConnectionWakeups.ConnectionWakeups> {  const target = entry.target;  const setupTimeoutDetail = `${target.label} did not respond during connection setup.${    target._tag === "RelayConnectionTarget" ? ` ${NETWORK_BLOCKING_HINT}` : ""  }`;  yield* annotateTarget(target);   const connectivity = yield* Connectivity.Connectivity;  const driver = yield* ConnectionDriver.ConnectionDriver;  const wakeups = yield* ConnectionWakeups.ConnectionWakeups;  const initialIntent: SupervisorIntent = {    desired: options?.initiallyDesired ?? false,    network: yield* connectivity.status,  };  const intent = yield* Ref.make(initialIntent);  const signals = yield* Queue.unbounded<SupervisorSignal>();  const resetRetryState = yield* Ref.make(false);  // Set when a foreground wake probe fails or times out: the user is actively  // returning to the app on a dead transport, so the follow-up reconnect skips  // the first backoff rung instead of sleeping.  const wakeProbeFailed = yield* Ref.make(false);  const state = yield* SubscriptionRef.make<SupervisorConnectionState>(    !initialIntent.desired      ? availableState(initialIntent, 0)      : initialIntent.network === "offline"        ? offlineState(initialIntent, 0, 0, null)        : connectingState(initialIntent, 0, 1, null),  );  const session = yield* SubscriptionRef.make<Option.Option<RpcSession.RpcSession>>(Option.none());  const prepared = yield* SubscriptionRef.make<Option.Option<PreparedConnection>>(Option.none());   const clearLease = Effect.all(    [SubscriptionRef.set(session, Option.none()), SubscriptionRef.set(prepared, Option.none())],    { discard: true },  );   const setState = Effect.fn("EnvironmentSupervisor.setState")(function* (    next: SupervisorConnectionState,  ) {    yield* SubscriptionRef.set(state, next);  });   const signal = Effect.fn("EnvironmentSupervisor.signal")(function* (next: SupervisorSignal) {    yield* Queue.offer(signals, next);  });   const logManagedRelayAccountChange = Effect.logInfo(    "Managed relay account changed; restarting the environment connection.",  ).pipe(    Effect.annotateLogs({      "environment.id": target.environmentId,      "environment.label": target.label,    }),  );   const reportProgress = Effect.fn("EnvironmentSupervisor.reportProgress")(function* (    attempt: number,    generation: number,    lastFailure: ConnectionAttemptError | null,    progress: ConnectionDriver.ConnectionDriverProgress,  ) {    if ("prepared" in progress) {      yield* SubscriptionRef.set(prepared, Option.some(progress.prepared));    }    yield* setState(      connectingState(yield* Ref.get(intent), generation, attempt, lastFailure, progress.stage),    );  });   const establishConnection = Effect.fnUntraced(function* (    attempt: number,    generation: number,    lastFailure: ConnectionAttemptError | null,  ) {    return yield* driver.connect(entry, (progress) =>      reportProgress(attempt, generation, lastFailure, progress),    );  });   const traceRelayEstablishment = (    effect: Effect.Effect<      ConnectionDriver.EnvironmentConnectionLease,      ConnectionAttemptError,      Scope.Scope    >,    attempt: number,    generation: number,    pendingRetry: Option.Option<PendingRetryTrace>,  ) => {    const traced = Effect.gen(function* () {      const attemptSpan = yield* Effect.currentSpan.pipe(Effect.orDie);      yield* annotateTarget(target);      yield* Effect.annotateCurrentSpan({        "connection.attempt": attempt,        "connection.generation": generation,        "connection.retry.failure_count": Option.match(pendingRetry, {          onNone: () => 0,          onSome: (retry) => retry.failureCount,        }),      });      const lease = yield* effect.pipe(        Effect.mapError((error): TracedAttemptFailure => ({          error,          attemptSpan: Option.some(attemptSpan),        })),      );      return { attemptSpan: Option.some(attemptSpan), lease };    }).pipe(Effect.withSpan("relay.connection.attempt", { root: true }));     return Option.match(pendingRetry, {      onNone: () => traced,      onSome: (retry) =>        traced.pipe(          Effect.linkSpans(retry.previousAttempt, {            "connection.retry.delay_ms": retry.delayMs,            "connection.retry.reason": retry.reason,          }),        ),    }).pipe(withRelayClientTracing);  };   const establishTracedConnection = Effect.fnUntraced(function* (    attempt: number,    generation: number,    lastFailure: ConnectionAttemptError | null,    pendingRetry: Option.Option<PendingRetryTrace>,  ) {    if (target._tag === "RelayConnectionTarget") {      return yield* traceRelayEstablishment(        establishConnection(attempt, generation, lastFailure),        attempt,        generation,        pendingRetry,      );    }    return yield* establishConnection(attempt, generation, lastFailure).pipe(      Effect.map((lease) => ({        attemptSpan: Option.none<Tracer.Span>(),        lease,      })),      Effect.mapError((error): TracedAttemptFailure => ({        error,        attemptSpan: Option.none(),      })),    );  });   const waitForEstablishmentInterrupt = Effect.fnUntraced(function* () {    for (;;) {      const next = yield* Queue.take(signals);      switch (next._tag) {        case "DisconnectRequested":        case "RetryRequested":          return false;        case "NetworkChanged":          if (next.network === "offline") {            return false;          }          break;        case "ConnectRequested":          break;        case "Wakeup":          if (next.reason === "application-active-reconnect") {            return true;          }          if (next.reason === "credentials-changed" && target._tag === "RelayConnectionTarget") {            yield* logManagedRelayAccountChange;            return false;          }          break;      }    }  });   const monitorConnectedLease = Effect.fnUntraced(function* (    lease: ConnectionDriver.EnvironmentConnectionLease,  ) {    for (;;) {      const next = yield* Queue.take(signals);      switch (next._tag) {        case "DisconnectRequested":        case "RetryRequested":          return false;        case "NetworkChanged":          if (next.network === "offline") {            return false;          }          break;        case "Wakeup":          if (next.reason === "credentials-changed" && target._tag === "RelayConnectionTarget") {            yield* logManagedRelayAccountChange;            return false;          }          if (next.reason === "application-active-reconnect") {            // Mobile operating systems commonly suspend sockets without            // delivering a close event. A long background resume deliberately            // replaces that lease and starts a fresh attempt without backoff.            return true;          }          if (next.reason === "application-active" || next.reason === "application-active-probe") {            const probe = yield* lease.session.probe.pipe(              Effect.timeoutOrElse({                duration:                  next.reason === "application-active-probe"                    ? MOBILE_CONNECTION_PROBE_TIMEOUT                    : CONNECTION_PROBE_TIMEOUT,                orElse: () =>                  Effect.fail(                    new ConnectionTransientError({                      reason: "timeout",                      detail: `${target.label} did not respond to a connection health check.`,                    }),                  ),              }),              Effect.forkChild,            );            for (;;) {              const probeEvent = yield* Effect.raceFirst(                Fiber.await(probe).pipe(                  Effect.map((exit) => ({ _tag: "ProbeCompleted" as const, exit })),                ),                Queue.take(signals).pipe(                  Effect.map((signal) => ({ _tag: "Signal" as const, signal })),                ),              );              if (probeEvent._tag === "ProbeCompleted") {                if (Exit.isFailure(probeEvent.exit)) {                  yield* Ref.set(wakeProbeFailed, true);                }                yield* probeEvent.exit;                break;              }              switch (probeEvent.signal._tag) {                case "DisconnectRequested":                case "RetryRequested":                  yield* Fiber.interrupt(probe);                  return false;                case "NetworkChanged":                  if (probeEvent.signal.network === "offline") {                    yield* Fiber.interrupt(probe);                    return false;                  }                  break;                case "Wakeup":                  if (probeEvent.signal.reason === "application-active-reconnect") {                    yield* Fiber.interrupt(probe);                    return true;                  }                  if (                    probeEvent.signal.reason === "credentials-changed" &&                    target._tag === "RelayConnectionTarget"                  ) {                    yield* Fiber.interrupt(probe);                    return false;                  }                  break;                case "ConnectRequested":                  break;              }            }          }          break;        case "ConnectRequested":          break;      }    }  });   const runAttempt = Effect.fnUntraced(function* (    attempt: number,    generation: number,    lastFailure: ConnectionAttemptError | null,    pendingRetry: Option.Option<PendingRetryTrace>,  ) {    yield* SubscriptionRef.set(prepared, Option.none());    const establishment = yield* Effect.raceAllFirst([      exitUnlessInterrupted(        establishTracedConnection(attempt, generation, lastFailure, pendingRetry),      ).pipe(        Effect.map((exit): EstablishmentEvent => ({

Lines 1–500 of 799.