// CRITICAL /** * Canary comparison harness for controller endpoints. */ import { existsSync, unlinkSync, writeFileSync } from "node:fs"; type HttpMethod = "GET" | "POST" | "PUT" | "DELETE"; /** * Response snapshot for comparison. */ interface Snapshot { status: number; contentType: string | null; body: unknown; } /** * Build a shape signature for a JSON payload. * @param value - Response body. * @returns Shape signature string. */ const shapeSignature = (value: unknown): string => { if (Array.isArray(value)) { const first = value[0]; const elementShape = first ? shapeSignature(first) : "empty"; return `array:${value.length}:${elementShape}`; } if (value && typeof value === "object") { const keys = Object.keys(value as Record).sort(); return `object:${keys.join(",")}`; } return typeof value; }; /** * Fetch and parse a response as JSON or text. * @param url - URL to fetch. * @param method - HTTP method. * @param body - Optional request body. * @param timeoutMs - Timeout in milliseconds. * @returns Snapshot of response. */ const fetchSnapshot = async ( url: string, method: HttpMethod, body?: unknown, timeoutMs = 10_000, ): Promise => { const init: RequestInit = { method }; if (body !== undefined) { init.headers = { "Content-Type": "application/json" }; init.body = JSON.stringify(body); } const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { ...init, signal: controller.signal }); const contentType = response.headers.get("content-type"); let payload: unknown = null; if (contentType && contentType.includes("application/json")) { payload = await response.json(); } else { payload = await response.text(); } return { status: response.status, contentType, body: payload }; } catch (error) { return { status: 0, contentType: null, body: String(error) }; } finally { clearTimeout(timeout); } }; /** * Fetch a streaming response without consuming the body. * @param url - URL to fetch. * @param method - HTTP method. * @param body - Optional request body. * @param timeoutMs - Timeout in milliseconds. * @returns Snapshot of response metadata. */ const fetchStreamSnapshot = async ( url: string, method: HttpMethod, body?: unknown, timeoutMs = 10_000, ): Promise => { const init: RequestInit = { method }; if (body !== undefined) { init.headers = { "Content-Type": "application/json" }; init.body = JSON.stringify(body); } const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { ...init, signal: controller.signal }); const contentType = response.headers.get("content-type"); if (response.body) { await response.body.cancel(); } return { status: response.status, contentType, body: null }; } catch (error) { return { status: 0, contentType: null, body: String(error) }; } finally { clearTimeout(timeout); } }; /** * Compare two snapshots. * @param left - Left snapshot. * @param right - Right snapshot. * @returns True if comparable. */ const compareSnapshots = (left: Snapshot, right: Snapshot): boolean => { if (left.status !== right.status) { return false; } const leftShape = shapeSignature(left.body); const rightShape = shapeSignature(right.body); return leftShape === rightShape; }; /** * Compare two streaming snapshots. * @param left - Left snapshot. * @param right - Right snapshot. * @returns True if comparable. */ const compareStreamSnapshots = (left: Snapshot, right: Snapshot): boolean => { if (left.status !== right.status) { return false; } const normalizeContentType = (contentType: string | null): string => contentType?.split(";")[0]?.trim() ?? ""; return normalizeContentType(left.contentType) === normalizeContentType(right.contentType); }; /** * Execute and compare a request against both controllers. * @param name - Check name. * @param method - HTTP method. * @param path - Endpoint path. * @param body - Optional body. * @param baseOld - Old controller URL. * @param baseNew - New controller URL. * @param beforeNew - Optional hook before new request. * @returns Comparison result. */ const compareEndpoint = async ( name: string, method: HttpMethod, path: string, body: unknown | undefined, baseOld: string, baseNew: string, beforeNew?: () => Promise, ): Promise<{ name: string; ok: boolean; old: Snapshot; next: Snapshot }> => { const oldSnapshot = await fetchSnapshot(`${baseOld}${path}`, method, body); if (beforeNew) { await beforeNew(); } const newSnapshot = await fetchSnapshot(`${baseNew}${path}`, method, body); return { name, ok: compareSnapshots(oldSnapshot, newSnapshot), old: oldSnapshot, next: newSnapshot, }; }; /** * Execute and compare a streaming request against both controllers. * @param name - Check name. * @param method - HTTP method. * @param path - Endpoint path. * @param body - Optional body. * @param baseOld - Old controller URL. * @param baseNew - New controller URL. * @param beforeNew - Optional hook before new request. * @returns Comparison result. */ const compareStreamEndpoint = async ( name: string, method: HttpMethod, path: string, body: unknown | undefined, baseOld: string, baseNew: string, beforeNew?: () => Promise, ): Promise<{ name: string; ok: boolean; old: Snapshot; next: Snapshot }> => { const oldSnapshot = await fetchStreamSnapshot(`${baseOld}${path}`, method, body); if (beforeNew) { await beforeNew(); } const newSnapshot = await fetchStreamSnapshot(`${baseNew}${path}`, method, body); return { name, ok: compareStreamSnapshots(oldSnapshot, newSnapshot), old: oldSnapshot, next: newSnapshot, }; }; /** * Run the comparison harness. * @returns Promise that resolves after reporting. */ const run = async (): Promise => { const baseOld = process.env["CONTROLLER_OLD_URL"] ?? "http://localhost:8080"; const baseNew = process.env["CONTROLLER_NEW_URL"] ?? "http://localhost:8002"; const results: Array<{ name: string; ok: boolean; old: Snapshot; next: Snapshot }> = []; results.push(await compareEndpoint("GET /health", "GET", "/health", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /status", "GET", "/status", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /gpus", "GET", "/gpus", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /config", "GET", "/config", undefined, baseOld, baseNew)); const recipeId = `canary-${Date.now()}`; const recipePayload = { id: recipeId, name: "Canary Recipe", model_path: "/models/canary", backend: "vllm", }; results.push(await compareEndpoint("POST /recipes", "POST", "/recipes", recipePayload, baseOld, baseNew)); results.push(await compareEndpoint("GET /recipes/:id", "GET", `/recipes/${recipeId}`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /recipes", "GET", "/recipes", undefined, baseOld, baseNew)); const updatedRecipe = { ...recipePayload, name: "Canary Recipe Updated" }; results.push(await compareEndpoint("PUT /recipes/:id", "PUT", `/recipes/${recipeId}`, updatedRecipe, baseOld, baseNew)); results.push(await compareEndpoint("POST /launch/:id", "POST", `/launch/${recipeId}`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("POST /launch/:id/cancel", "POST", `/launch/${recipeId}/cancel`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("POST /evict", "POST", "/evict", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /wait-ready", "GET", "/wait-ready", undefined, baseOld, baseNew)); const logSessionId = `canary-log-${Date.now()}`; const isSnap = Boolean(process.env["SNAP"]); const oldLogSessionId = process.env["CANARY_LOG_SESSION_OLD"] ?? (isSnap ? logSessionId : `${logSessionId}-old`); const newLogSessionId = process.env["CANARY_LOG_SESSION_NEW"] ?? (isSnap ? logSessionId : `${logSessionId}-new`); const oldLogPath = `/tmp/vllm_${oldLogSessionId}.log`; const logPath = `/tmp/vllm_${newLogSessionId}.log`; if (!isSnap) { writeFileSync(oldLogPath, "canary log line\n", "utf-8"); } writeFileSync(logPath, "canary log line\n", "utf-8"); results.push(await compareEndpoint("GET /logs", "GET", "/logs", undefined, baseOld, baseNew)); const oldLogSnapshot = await fetchSnapshot(`${baseOld}/logs/${oldLogSessionId}`, "GET"); const newLogSnapshot = await fetchSnapshot(`${baseNew}/logs/${newLogSessionId}`, "GET"); results.push({ name: "GET /logs/:id", ok: compareSnapshots(oldLogSnapshot, newLogSnapshot), old: oldLogSnapshot, next: newLogSnapshot, }); const oldLogStream = await fetchStreamSnapshot(`${baseOld}/logs/${oldLogSessionId}/stream`, "GET"); const newLogStream = await fetchStreamSnapshot(`${baseNew}/logs/${newLogSessionId}/stream`, "GET"); results.push({ name: "GET /logs/:id/stream", ok: compareStreamSnapshots(oldLogStream, newLogStream), old: oldLogStream, next: newLogStream, }); const deleteOldLog = await fetchSnapshot(`${baseOld}/logs/${oldLogSessionId}`, "DELETE"); const deleteNewLog = await fetchSnapshot(`${baseNew}/logs/${newLogSessionId}`, "DELETE"); results.push({ name: "DELETE /logs/:id", ok: compareSnapshots(deleteOldLog, deleteNewLog), old: deleteOldLog, next: deleteNewLog, }); results.push(await compareStreamEndpoint("GET /events", "GET", "/events", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /events/stats", "GET", "/events/stats", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /metrics", "GET", "/metrics", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /peak-metrics", "GET", "/peak-metrics", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /lifetime-metrics", "GET", "/lifetime-metrics", undefined, baseOld, baseNew)); results.push(await compareEndpoint("POST /benchmark", "POST", "/benchmark", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /usage", "GET", "/usage", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /v1/models", "GET", "/v1/models", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /v1/models/:id", "GET", `/v1/models/${recipeId}`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /v1/studio/models", "GET", "/v1/studio/models", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /v1/huggingface/models", "GET", "/v1/huggingface/models", undefined, baseOld, baseNew)); const chatCreate = await fetchSnapshot(`${baseOld}/chats`, "POST", { title: "Canary Chat", model: "default" }); const chatSession = (chatCreate.body as Record)?.["session"] as Record | undefined; const sessionId = chatSession && typeof chatSession["id"] === "string" ? String(chatSession["id"]) : `canary-chat-${Date.now()}`; results.push(await compareEndpoint("POST /chats", "POST", "/chats", { title: "Canary Chat", model: "default" }, baseOld, baseNew)); results.push(await compareEndpoint("GET /chats", "GET", "/chats", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /chats/:id", "GET", `/chats/${sessionId}`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("PUT /chats/:id", "PUT", `/chats/${sessionId}`, { title: "Canary Chat Updated", model: "default", }, baseOld, baseNew)); results.push(await compareEndpoint("POST /chats/:id/messages", "POST", `/chats/${sessionId}/messages`, { id: `msg-${Date.now()}`, role: "user", content: "Hello", }, baseOld, baseNew)); results.push(await compareEndpoint("GET /chats/:id/usage", "GET", `/chats/${sessionId}/usage`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("POST /chats/:id/fork", "POST", `/chats/${sessionId}/fork`, {}, baseOld, baseNew)); const deleteOldSession = await fetchSnapshot(`${baseOld}/chats`, "POST", { title: "Canary Delete Chat", model: "default" }); const deleteNewSession = await fetchSnapshot(`${baseNew}/chats`, "POST", { title: "Canary Delete Chat", model: "default" }); const deleteOldPayload = deleteOldSession.body as Record; const deleteNewPayload = deleteNewSession.body as Record; const deleteOldId = typeof (deleteOldPayload["session"] as Record | undefined)?.["id"] === "string" ? String((deleteOldPayload["session"] as Record)["id"]) : `canary-delete-old-${Date.now()}`; const deleteNewId = typeof (deleteNewPayload["session"] as Record | undefined)?.["id"] === "string" ? String((deleteNewPayload["session"] as Record)["id"]) : `canary-delete-new-${Date.now()}`; const deleteOldSnapshot = await fetchSnapshot(`${baseOld}/chats/${deleteOldId}`, "DELETE"); const deleteNewSnapshot = await fetchSnapshot(`${baseNew}/chats/${deleteNewId}`, "DELETE"); results.push({ name: "DELETE /chats/:id", ok: compareSnapshots(deleteOldSnapshot, deleteNewSnapshot), old: deleteOldSnapshot, next: deleteNewSnapshot, }); const mcpServerId = `canary-mcp-${Date.now()}`; const mcpServerPayload = { id: mcpServerId, name: "Canary MCP", enabled: true, command: "npx", args: ["-y", "exa-mcp-server"], env: {}, description: "Canary MCP server", url: "https://example.com", }; results.push(await compareEndpoint("POST /mcp/servers", "POST", "/mcp/servers", mcpServerPayload, baseOld, baseNew)); results.push(await compareEndpoint("GET /mcp/servers/:id", "GET", `/mcp/servers/${mcpServerId}`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("PUT /mcp/servers/:id", "PUT", `/mcp/servers/${mcpServerId}`, { name: "Canary MCP Updated", }, baseOld, baseNew)); results.push(await compareEndpoint("POST /mcp/servers/:id/disable", "POST", `/mcp/servers/${mcpServerId}/disable`, undefined, baseOld, baseNew)); results.push(await compareEndpoint("POST /mcp/servers/:id/enable", "POST", `/mcp/servers/${mcpServerId}/enable`, undefined, baseOld, baseNew)); results.push(await compareEndpoint( "DELETE /mcp/servers/:id", "DELETE", `/mcp/servers/${mcpServerId}`, undefined, baseOld, baseNew, async () => { await fetchSnapshot(`${baseOld}/mcp/servers`, "POST", mcpServerPayload); }, )); results.push(await compareEndpoint("GET /mcp/servers", "GET", "/mcp/servers", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /mcp/tools", "GET", "/mcp/tools", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /mcp/servers/exa", "GET", "/mcp/servers/exa", undefined, baseOld, baseNew)); results.push(await compareEndpoint("GET /mcp/servers/exa/tools", "GET", "/mcp/servers/exa/tools", undefined, baseOld, baseNew)); results.push(await compareEndpoint("POST /v1/tokenize", "POST", "/v1/tokenize", { model: "default", prompt: "hi" }, baseOld, baseNew)); results.push(await compareEndpoint("POST /v1/detokenize", "POST", "/v1/detokenize", { model: "default", tokens: [1, 2] }, baseOld, baseNew)); results.push(await compareEndpoint("POST /v1/count-tokens", "POST", "/v1/count-tokens", { model: "default", text: "hello" }, baseOld, baseNew)); results.push(await compareEndpoint("POST /v1/tokenize-chat-completions", "POST", "/v1/tokenize-chat-completions", { model: "default", messages: [{ role: "user", content: "hello" }] }, baseOld, baseNew)); results.push(await compareEndpoint("POST /v1/chat/completions", "POST", "/v1/chat/completions", { model: "default", messages: [{ role: "user", content: "ping" }], max_tokens: 1, stream: false, }, baseOld, baseNew)); results.push(await compareEndpoint("POST /api/title", "POST", "/api/title", { model: "default", user: "Hello", assistant: "World" }, baseOld, baseNew)); results.push(await compareEndpoint( "DELETE /recipes/:id", "DELETE", `/recipes/${recipeId}`, undefined, baseOld, baseNew, async () => { await fetchSnapshot(`${baseOld}/recipes`, "POST", recipePayload); }, )); if (existsSync(logPath)) { unlinkSync(logPath); } if (!isSnap && oldLogPath !== logPath && existsSync(oldLogPath)) { unlinkSync(oldLogPath); } const lines = results.map((result) => { const marker = result.ok ? "✅" : "⚠️"; const oldShape = shapeSignature(result.old.body); const newShape = shapeSignature(result.next.body); return `${marker} ${result.name} | old:${result.old.status}/${oldShape} new:${result.next.status}/${newShape}`; }); console.log(lines.join("\n")); process.exit(0); }; void run();