243 lines · 7.1 KB · TypeScript
RawDownload
import { spawn } from "node:child_process";import { chmod, readFile, unlink, writeFile } from "node:fs/promises";import { join } from "node:path";import { randomUUID } from "node:crypto";import { Effect } from "effect";import { resolveBinary } from "../../core/command";import { secureSpeechDirectory } from "./storage"; export const MAX_VOICE_UPLOAD_BYTES = 20 * 1024 * 1024;const MAX_NORMALIZED_BYTES = 1_100_000;const TRANSCODE_TIMEOUT_MS = 60_000; export class VoiceReferenceError extends Error {  public constructor(    public readonly status: number,    public readonly code: string,    message: string,  ) {    super(message);  }} export interface NormalizedVoiceReference {  audio: Uint8Array;  durationMs: number;} interface VoiceReferenceDependencies {  ffmpegPath: () => string | null;  transcode: (    command: string,    input: Uint8Array,    format: VoiceInputFormat,    output: string,  ) => Promise<void>;} type VoiceInputFormat = "aiff" | "caf" | "flac" | "matroska" | "mov" | "mp3" | "ogg" | "wav"; const FFMPEG_ARGS = [  "-hide_banner",  "-nostdin",  "-y",  "-v",  "error",  "-max_alloc",  "67108864",  "-protocol_whitelist",  "pipe",  "-probesize",  "1048576",  "-analyzeduration",  "5000000",] as const; const transcode = (  command: string,  input: Uint8Array,  format: VoiceInputFormat,  output: string,): Promise<void> => {  const conversion = Effect.callback<void, VoiceReferenceError>((resume) => {    const child = spawn(      command,      [        ...FFMPEG_ARGS,        "-f",        format,        "-i",        "pipe:0",        "-map",        "0:a:0",        "-vn",        "-sn",        "-dn",        "-threads",        "1",        "-filter_threads",        "1",        "-ac",        "1",        "-ar",        "24000",        "-c:a",        "pcm_s16le",        "-t",        "20.1",        "-f",        "wav",        output,      ],      { stdio: ["pipe", "ignore", "ignore"] },    );    let settled = false;    const settle = (effect: Effect.Effect<void, VoiceReferenceError>): void => {      if (settled) return;      settled = true;      resume(effect);    };    child.stdin.on("error", () => {});    child.stdin.end(input);    child.once("error", () =>      settle(        Effect.fail(          new VoiceReferenceError(503, "ffmpeg_unavailable", "FFmpeg could not be started"),        ),      ),    );    child.once("close", (code) =>      settle(        code === 0          ? Effect.void          : Effect.fail(              new VoiceReferenceError(                400,                "voice_audio_invalid",                "Voice reference could not be decoded",              ),            ),      ),    );    return Effect.sync(() => {      if (settled) return;      settled = true;      child.stdin.destroy();      child.kill("SIGKILL");    });  }).pipe(    Effect.timeoutOrElse({      duration: TRANSCODE_TIMEOUT_MS,      orElse: () =>        Effect.fail(          new VoiceReferenceError(504, "voice_decode_timeout", "Voice reference decode timed out"),        ),    }),  );  return Effect.runPromise(conversion);}; const defaultDependencies: VoiceReferenceDependencies = {  ffmpegPath: () => resolveBinary(process.env["LOCAL_STUDIO_FFMPEG_CLI"] ?? "ffmpeg"),  transcode,}; const ascii = (bytes: Buffer, offset: number): string =>  bytes.subarray(offset, offset + 4).toString("ascii"); const detectedFormat = (input: Uint8Array): VoiceInputFormat => {  const bytes = Buffer.from(input.buffer, input.byteOffset, input.byteLength);  if (ascii(bytes, 0) === "RIFF" && ascii(bytes, 8) === "WAVE") return "wav";  if (ascii(bytes, 0) === "OggS") return "ogg";  if (ascii(bytes, 0) === "fLaC") return "flac";  if (ascii(bytes, 0) === "FORM" && ["AIFF", "AIFC"].includes(ascii(bytes, 8))) return "aiff";  if (ascii(bytes, 0) === "caff") return "caf";  if (ascii(bytes, 4) === "ftyp") return "mov";  if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) return "matroska";  if (bytes.subarray(0, 3).toString("ascii") === "ID3") return "mp3";  if (bytes.length >= 2 && bytes[0] === 0xff && ((bytes[1] ?? 0) & 0xe0) === 0xe0) return "mp3";  throw new VoiceReferenceError(    400,    "voice_audio_invalid",    "Voice reference must be WAV, WebM, Ogg, FLAC, MP3, AIFF, CAF, or MP4 audio",  );}; const wavDuration = (audio: Uint8Array): number => {  const bytes = Buffer.from(audio);  if (bytes.length < 44 || ascii(bytes, 0) !== "RIFF" || ascii(bytes, 8) !== "WAVE") {    throw new VoiceReferenceError(400, "voice_audio_invalid", "Voice reference is not valid audio");  }  let byteRate = 0;  let dataBytes = 0;  for (let offset = 12; offset + 8 <= bytes.length; ) {    const id = ascii(bytes, offset);    const size = bytes.readUInt32LE(offset + 4);    const start = offset + 8;    const end = start + size;    if (end > bytes.length) break;    if (id === "fmt " && size >= 16) {      const pcm = bytes.readUInt16LE(start) === 1;      const mono = bytes.readUInt16LE(start + 2) === 1;      const sampleRate = bytes.readUInt32LE(start + 4);      byteRate = bytes.readUInt32LE(start + 8);      const bits = bytes.readUInt16LE(start + 14);      if (!pcm || !mono || sampleRate !== 24_000 || bits !== 16) byteRate = 0;    }    if (id === "data") dataBytes = size;    offset = end + (size % 2);  }  if (!byteRate || !dataBytes) {    throw new VoiceReferenceError(400, "voice_audio_invalid", "Voice reference is not valid audio");  }  return Math.round((dataBytes / byteRate) * 1000);}; export const normalizeVoiceReference = async (  input: Uint8Array,  dataDirectory: string,  dependencies: VoiceReferenceDependencies = defaultDependencies,): Promise<NormalizedVoiceReference> => {  if (!input.length) {    throw new VoiceReferenceError(400, "voice_audio_invalid", "Voice reference is empty");  }  if (input.length > MAX_VOICE_UPLOAD_BYTES) {    throw new VoiceReferenceError(      413,      "voice_audio_too_large",      `Voice reference must be smaller than ${MAX_VOICE_UPLOAD_BYTES / 1024 / 1024} MB`,    );  }  const format = detectedFormat(input);  const ffmpeg = dependencies.ffmpegPath();  if (!ffmpeg) {    throw new VoiceReferenceError(      503,      "ffmpeg_missing",      "FFmpeg is required to create a voice profile",    );  }  const directory = join(dataDirectory, "runtime", "speech", "uploads");  const output = join(directory, `${randomUUID()}.wav`);  try {    secureSpeechDirectory(directory);    await writeFile(output, new Uint8Array(), { mode: 0o600, flag: "wx" });    await dependencies.transcode(ffmpeg, input, format, output);    await chmod(output, 0o600);    const audio = await readFile(output);    if (audio.length > MAX_NORMALIZED_BYTES) {      throw new VoiceReferenceError(400, "voice_audio_invalid", "Voice reference is too long");    }    const durationMs = wavDuration(audio);    if (durationMs < 6_000 || durationMs > 20_000) {      throw new VoiceReferenceError(        400,        "voice_duration_invalid",        "Voice reference must be 6 to 20 seconds",      );    }    return { audio, durationMs };  } finally {    await unlink(output).catch(() => undefined);  }};