// CRITICAL import type { Hono } from "hono"; import type { AppContext } from "../types/context"; /** * Register tokenization and title routes. * @param app - Hono app. * @param context - App context. */ export const registerTokenizationRoutes = (app: Hono, context: AppContext): void => { app.post("/v1/tokenize", async (ctx) => { const current = await context.processManager.findInferenceProcess(context.config.inference_port); if (!current) { return ctx.json({ error: "No model running", num_tokens: 0 }); } let body: unknown = {}; try { body = await ctx.req.json(); } catch (error) { return ctx.json({ error: String(error), num_tokens: 0 }); } try { const response = await fetch(`http://localhost:${context.config.inference_port}/tokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), }); if (response.status === 200) { return ctx.json(await response.json()); } return ctx.json({ error: `Tokenization failed: ${response.status}`, num_tokens: 0 }); } catch (error) { return ctx.json({ error: String(error), num_tokens: 0 }); } }); app.post("/v1/detokenize", async (ctx) => { const current = await context.processManager.findInferenceProcess(context.config.inference_port); if (!current) { return ctx.json({ error: "No model running", text: "" }); } let body: unknown = {}; try { body = await ctx.req.json(); } catch (error) { return ctx.json({ error: String(error), text: "" }); } try { const response = await fetch(`http://localhost:${context.config.inference_port}/detokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), }); if (response.status === 200) { return ctx.json(await response.json()); } return ctx.json({ error: `Detokenization failed: ${response.status}`, text: "" }); } catch (error) { return ctx.json({ error: String(error), text: "" }); } }); app.post("/v1/count-tokens", async (ctx) => { const current = await context.processManager.findInferenceProcess(context.config.inference_port); if (!current) { return ctx.json({ error: "No model running", num_tokens: 0 }); } let body: Record = {}; try { body = (await ctx.req.json()) as Record; } catch (error) { return ctx.json({ error: String(error), num_tokens: 0 }); } const text = typeof body["text"] === "string" ? body["text"] : ""; const model = typeof body["model"] === "string" ? body["model"] : (current.served_model_name ?? "default"); try { const response = await fetch(`http://localhost:${context.config.inference_port}/tokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model, prompt: text }), }); if (response.status === 200) { const data = (await response.json()) as { tokens?: unknown[] }; const tokens = Array.isArray(data.tokens) ? data.tokens : []; return ctx.json({ num_tokens: tokens.length, model }); } return ctx.json({ error: `Token count failed: ${response.status}`, num_tokens: 0 }); } catch (error) { return ctx.json({ error: String(error), num_tokens: 0 }); } }); app.post("/v1/tokenize-chat-completions", async (ctx) => { const current = await context.processManager.findInferenceProcess(context.config.inference_port); if (!current) { return ctx.json({ error: "No model running", input_tokens: 0 }); } let body: Record = {}; try { body = (await ctx.req.json()) as Record; } catch (error) { return ctx.json({ error: String(error), input_tokens: 0 }); } const messages = Array.isArray(body["messages"]) ? body["messages"] : []; const tools = Array.isArray(body["tools"]) ? body["tools"] : []; const model = typeof body["model"] === "string" ? body["model"] : (current.served_model_name ?? "default"); try { const testRequest: Record = { model, messages, max_tokens: 1, stream: false, }; if (tools.length > 0) { testRequest["tools"] = tools; } const response = await fetch(`http://localhost:${context.config.inference_port}/v1/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(testRequest), }); if (response.status === 200) { const data = (await response.json()) as { usage?: Record }; const promptTokens = data.usage?.["prompt_tokens"] ?? 0; return ctx.json({ input_tokens: promptTokens, breakdown: { messages: promptTokens, tools: 0 }, model, }); } } catch { await Promise.resolve(); } let messagesTokens = 0; let toolsTokens = 0; try { let allText = ""; for (const message of messages) { const record = message as Record; const content = record["content"]; if (typeof content === "string") { allText += `${content}\n`; } else if (Array.isArray(content)) { for (const part of content) { const partRecord = part as Record; if (partRecord["type"] === "text") { allText += `${String(partRecord["text"] ?? "")}\n`; } } } } const response = await fetch(`http://localhost:${context.config.inference_port}/tokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model, prompt: allText }), }); if (response.status === 200) { const data = (await response.json()) as { tokens?: unknown[] }; messagesTokens = Array.isArray(data.tokens) ? data.tokens.length : 0; } if (tools.length > 0) { const toolsText = JSON.stringify(tools); const toolsResponse = await fetch(`http://localhost:${context.config.inference_port}/tokenize`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model, prompt: toolsText }), }); if (toolsResponse.status === 200) { const data = (await toolsResponse.json()) as { tokens?: unknown[] }; toolsTokens = Array.isArray(data.tokens) ? data.tokens.length : 0; } } } catch { await Promise.resolve(); } const overhead = messages.length * 4; return ctx.json({ input_tokens: messagesTokens + toolsTokens + overhead, breakdown: { messages: messagesTokens + overhead, tools: toolsTokens, }, model, }); }); app.post("/api/title", async (ctx) => { try { let body: Record = {}; try { body = (await ctx.req.json()) as Record; } catch { return ctx.json({ title: "New Chat" }); } const model = typeof body["model"] === "string" ? body["model"] : undefined; const userMessage = typeof body["user"] === "string" ? body["user"] : ""; const assistantMessage = typeof body["assistant"] === "string" ? body["assistant"] : ""; if (!model || !userMessage) { return ctx.json({ title: "New Chat" }); } const prompt = `Generate a short, descriptive title (3-6 words) for this conversation. Only output the title, nothing else. User: ${userMessage.slice(0, 500)} Assistant: ${assistantMessage ? assistantMessage.slice(0, 500) : "(response pending)"} Title:`; const litellmKey = process.env["LITELLM_MASTER_KEY"] ?? "sk-master"; const response = await fetch("http://localhost:4100/v1/chat/completions", { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${litellmKey}`, }, body: JSON.stringify({ model, messages: [{ role: "user", content: prompt }], max_tokens: 20, temperature: 0.7, }), }); if (response.status === 200) { const data = (await response.json()) as Record; const choices = data["choices"] as Array> | undefined; const firstChoice = choices?.[0]; const titleRaw = firstChoice && (firstChoice["message"] as Record)?.["content"]; let title = typeof titleRaw === "string" ? titleRaw.trim() : ""; title = title.replace(/^["']|["']$/g, "").trim(); if (title.length > 60) { title = `${title.slice(0, 57)}...`; } return ctx.json({ title: title || "New Chat" }); } return ctx.json({ title: "New Chat" }); } catch (error) { context.logger.error("Title generation error", { error: String(error) }); return ctx.json({ title: "New Chat" }); } }); };