// Openrouter provider module implements model/runtime integration. import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation"; import type { MusicGenerationProvider, MusicGenerationRequest, MusicGenerationSourceImage, } from "openclaw/plugin-sdk/music-generation"; import { resolvePositiveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, createProviderOperationDeadline, postJsonRequest, resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, type ProviderOperationDeadline, } from "openclaw/plugin-sdk/provider-http"; import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { OPENROUTER_BASE_URL } from "./provider-catalog.js"; const DEFAULT_OPENROUTER_MUSIC_MODEL = "google/lyria-3-pro-preview"; const OPENROUTER_CLIP_MUSIC_MODEL = "google/lyria-3-clip-preview"; const DEFAULT_TIMEOUT_MS = 180_000; const OPENROUTER_MUSIC_MODELS = [ DEFAULT_OPENROUTER_MUSIC_MODEL, OPENROUTER_CLIP_MUSIC_MODEL, ] as const; type OpenRouterAudioStreamResult = { audioBuffer: Buffer; transcript: string; }; function resolveOpenRouterMusicModel(model: string | undefined): string { return normalizeOptionalString(model) ?? DEFAULT_OPENROUTER_MUSIC_MODEL; } function outputFormatToMimeType(format: "mp3" | "wav" | undefined): string { return format === "mp3" ? "audio/mpeg" : "audio/wav"; } function imageToContentPart(image: MusicGenerationSourceImage): { type: "image_url"; image_url: { url: string }; } { const url = normalizeOptionalString(image.url) ?? (image.buffer ? toImageDataUrl({ ...image, buffer: image.buffer, defaultMimeType: "image/png" }) : undefined); if (!url) { throw new Error("OpenRouter music generation reference image is missing data."); } return { type: "image_url", image_url: { url }, }; } function buildOpenRouterMusicPrompt(req: MusicGenerationRequest): string { const parts = [req.prompt.trim()]; const lyrics = normalizeOptionalString(req.lyrics); if (req.instrumental === true) { parts.push("Instrumental only. No vocals, no sung lyrics, no spoken word."); } if (lyrics) { parts.push(`Lyrics:\n${lyrics}`); } if (typeof req.durationSeconds === "number") { parts.push(`Target duration: about ${Math.round(req.durationSeconds)} seconds.`); } return parts.join("\n\n"); } function buildOpenRouterMessageContent( req: MusicGenerationRequest, ): | string | Array<{ type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }> { const prompt = buildOpenRouterMusicPrompt(req); const images = req.inputImages ?? []; if (images.length === 0) { return prompt; } return [{ type: "text", text: prompt }, ...images.map((image) => imageToContentPart(image))]; } function readDeltaAudio(part: unknown): { data?: string; transcript?: string } | undefined { if (!isRecord(part)) { return undefined; } const choices = part.choices; if (!Array.isArray(choices)) { return undefined; } const first = choices[0]; if (!isRecord(first)) { return undefined; } const delta = first.delta; if (!isRecord(delta)) { return undefined; } const audio = delta.audio; if (!isRecord(audio)) { return undefined; } return { data: normalizeOptionalString(audio.data), transcript: typeof audio.transcript === "string" ? audio.transcript : undefined, }; } function processOpenRouterSseLine( line: string, result: { audioBuffers: Buffer[]; transcriptChunks: string[] }, ): boolean { if (!line.startsWith("data:")) { return false; } const data = line.slice("data:".length).trim(); if (!data) { return false; } if (data === "[DONE]") { return true; } const audio = readDeltaAudio(JSON.parse(data)); if (audio?.data) { result.audioBuffers.push(Buffer.from(audio.data, "base64")); } if (audio?.transcript) { result.transcriptChunks.push(audio.transcript); } return false; } function resolveOpenRouterStreamRemainingMs(deadline: ProviderOperationDeadline): number { return resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: deadline.timeoutMs ?? DEFAULT_TIMEOUT_MS, }); } async function readOpenRouterStreamChunk( reader: ReadableStreamDefaultReader, deadline: ProviderOperationDeadline, ): Promise> { const timeoutMs = resolveOpenRouterStreamRemainingMs(deadline); let timeoutId: ReturnType | undefined; try { return await Promise.race([ reader.read(), new Promise((_, reject) => { timeoutId = setTimeout(() => { reject(new Error(`${deadline.label} timed out after ${deadline.timeoutMs}ms`)); }, timeoutMs); }), ]); } catch (error) { await reader.cancel().catch(() => {}); throw error; } finally { if (timeoutId) { clearTimeout(timeoutId); } } } async function readOpenRouterAudioStream( response: Response, deadline: ProviderOperationDeadline, ): Promise { if (!response.body) { throw new Error("OpenRouter music generation response missing stream body"); } const reader = response.body.getReader(); const decoder = new TextDecoder(); const result = { audioBuffers: [] as Buffer[], transcriptChunks: [] as string[] }; let buffer = ""; let doneSeen = false; try { for (;;) { const { value, done } = await readOpenRouterStreamChunk(reader, deadline); if (done) { break; } buffer += decoder.decode(value, { stream: true }); const lines = buffer.split(/\r?\n/u); buffer = lines.pop() ?? ""; for (const line of lines) { if (processOpenRouterSseLine(line.trim(), result)) { await reader.cancel(); return { audioBuffer: Buffer.concat(result.audioBuffers), transcript: result.transcriptChunks.join(""), }; } } } resolveOpenRouterStreamRemainingMs(deadline); buffer += decoder.decode(); if (buffer.trim()) { for (const line of buffer.split(/\r?\n/u)) { if (processOpenRouterSseLine(line.trim(), result)) { doneSeen = true; } } } if (!doneSeen) { throw new Error("OpenRouter music generation stream ended before completion"); } return { audioBuffer: Buffer.concat(result.audioBuffers), transcript: result.transcriptChunks.join(""), }; } finally { try { reader.releaseLock(); } catch {} } } export function buildOpenRouterMusicGenerationProvider(): MusicGenerationProvider { return { id: "openrouter", label: "OpenRouter", defaultModel: DEFAULT_OPENROUTER_MUSIC_MODEL, models: [...OPENROUTER_MUSIC_MODELS], isConfigured: ({ agentDir }) => isProviderApiKeyConfigured({ provider: "openrouter", agentDir, }), capabilities: { generate: { maxTracks: 1, maxDurationSeconds: 180, supportsLyrics: true, supportsInstrumental: true, supportsDuration: true, supportsFormat: true, supportedFormats: ["mp3", "wav"], }, edit: { enabled: true, maxTracks: 1, maxInputImages: 1, maxDurationSeconds: 180, supportsLyrics: true, supportsInstrumental: true, supportsDuration: true, supportsFormat: true, supportedFormats: ["mp3", "wav"], }, }, async generateMusic(req) { if ((req.inputImages?.length ?? 0) > 1) { throw new Error("OpenRouter music generation supports at most one reference image."); } const auth = await resolveApiKeyForProvider({ provider: "openrouter", cfg: req.cfg, agentDir: req.agentDir, store: req.authStore, }); if (!auth.apiKey) { throw new Error("OpenRouter API key missing"); } const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl, defaultBaseUrl: OPENROUTER_BASE_URL, allowPrivateNetwork: false, defaultHeaders: { Authorization: `Bearer ${auth.apiKey}`, "Content-Type": "application/json", "HTTP-Referer": "https://openclaw.ai", "X-OpenRouter-Title": "OpenClaw", }, provider: "openrouter", capability: "audio", transport: "http", }); const model = resolveOpenRouterMusicModel(req.model); const format = req.format ?? "wav"; const requestedTimeoutMs = resolvePositiveTimerTimeoutMs(req.timeoutMs, DEFAULT_TIMEOUT_MS); const streamDeadline = createProviderOperationDeadline({ timeoutMs: requestedTimeoutMs, label: "OpenRouter music generation", }); const timeoutMs = resolveOpenRouterStreamRemainingMs(streamDeadline); const { response, release } = await postJsonRequest({ url: `${baseUrl}/chat/completions`, headers, body: { model, messages: [{ role: "user", content: buildOpenRouterMessageContent(req) }], modalities: ["text", "audio"], audio: { format }, stream: true, }, timeoutMs, fetchFn: fetch, allowPrivateNetwork, dispatcherPolicy, }); try { await assertOkOrThrowHttpError(response, "OpenRouter music generation failed"); const streamResult = await readOpenRouterAudioStream(response, streamDeadline); if (streamResult.audioBuffer.byteLength === 0) { throw new Error("OpenRouter music generation response missing audio data"); } return { tracks: [ { buffer: streamResult.audioBuffer, mimeType: outputFormatToMimeType(format), fileName: `track-1.${format}`, }, ], model, ...(streamResult.transcript ? { lyrics: [streamResult.transcript] } : {}), metadata: { inputImageCount: req.inputImages?.length ?? 0, instrumental: req.instrumental === true, requestedFormat: format, }, }; } finally { await release(); } }, }; }