diff --git a/src/index.ts b/src/index.ts index 71c3ef6c..abf2f702 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2043,6 +2043,65 @@ const plugin: OpenClawPluginDefinition = { }, }); + // /cr-speech [--model=speech-2.8-hd] [--format=mp3|wav|flac|pcm] + api.registerCommand({ + name: "cr-speech", + description: "Generate speech audio (paid via wallet)", + acceptsArgs: true, + requireAuth: false, + handler: async (ctx: PluginCommandContext) => { + const tokens = + (ctx.args ?? "") + .match(/(?:[^\s"]+|"[^"]*")+/g) + ?.map((token) => + token.startsWith('"') && token.endsWith('"') ? token.slice(1, -1) : token, + ) ?? []; + let model = "speech-2.8-hd"; + let format = "mp3"; + const textParts: string[] = []; + for (const token of tokens) { + const option = token.match(/^--(model|format)=(.+)$/); + if (option?.[1] === "model") model = option[2]!; + else if (option?.[1] === "format") format = option[2]!; + else textParts.push(token); + } + const text = textParts.join(" ").trim(); + if (!text) { + return { + text: "Usage: `/cr-speech [--model=speech-2.8-hd] [--format=mp3|wav|flac|pcm]`", + }; + } + + const port = getProxyPort(); + try { + const resp = await fetch(`http://127.0.0.1:${port}/v1/speech/generations`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, text, audio_setting: { format } }), + signal: AbortSignal.timeout(180_000), + }); + const responseText = await resp.text(); + if (!resp.ok) { + if (resp.status === 402) { + return { + text: `Insufficient wallet balance for speech generation. Top up with \`/wallet\`.\n\n${responseText}`, + }; + } + return { text: `Speech generation failed (${resp.status}): ${responseText}` }; + } + const result = JSON.parse(responseText) as { data?: Array<{ url?: string }> }; + const url = result.data?.[0]?.url; + return url + ? { text: `Speech audio: ${url}` } + : { text: "Speech generation returned no audio." }; + } catch (err) { + return { + text: `Speech generation error: ${err instanceof Error ? err.message : String(err)}`, + }; + } + }, + }); + // /cr-call <+E.164> "" [--voice nat] [--max-duration 5] [--from +1...] [--language en-US] // Places a REAL outbound AI voice call via BlockRun → Bland.ai. Returns // immediately with call_id + poll_url; the call itself runs in the cloud @@ -2141,7 +2200,7 @@ const plugin: OpenClawPluginDefinition = { api.registerCommand(createExcludeCommand()); if (shouldLogRegistration) { api.logger.info( - "Commands registered: /wallet, /blockrun, /stats, /exclude, /partners, /cr-imagegen, /videogen, /cr-call", + "Commands registered: /wallet, /blockrun, /stats, /exclude, /partners, /cr-imagegen, /videogen, /cr-speech, /cr-call", ); } diff --git a/src/proxy.speech.test.ts b/src/proxy.speech.test.ts new file mode 100644 index 00000000..1ff8c2fe --- /dev/null +++ b/src/proxy.speech.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; + +import { + DEFAULT_SPEECH_MODEL, + SPEECH_MODELS, + normalizeSpeechRequest, + parseSpeechResponse, +} from "./proxy.js"; + +describe("speech request normalization", () => { + it("defaults the model and audio fields", () => { + expect(normalizeSpeechRequest({ text: "Hello" })).toEqual({ + model: DEFAULT_SPEECH_MODEL, + text: "Hello", + stream: false, + output_format: "hex", + audio_setting: { format: "mp3" }, + }); + }); + + it("accepts every supported speech model and request field", () => { + for (const model of SPEECH_MODELS) { + expect( + normalizeSpeechRequest({ + model, + text: "Hello", + language_boost: "English", + voice_setting: { voice_id: "female-shaonv" }, + pronunciation_dict: { tone: ["hello/(he lou)"] }, + audio_setting: { format: "wav", sample_rate: 32000 }, + voice_modify: { pitch: 1 }, + subtitle_enable: true, + }).model, + ).toBe(model); + } + }); + + it("rejects missing text, unknown models, and unsupported audio formats", () => { + expect(() => normalizeSpeechRequest({ model: DEFAULT_SPEECH_MODEL })).toThrow("requires text"); + expect(() => normalizeSpeechRequest({ model: "unknown", text: "Hello" })).toThrow( + "Unsupported speech model", + ); + expect(() => + normalizeSpeechRequest({ text: "Hello", audio_setting: { format: "ogg" } }), + ).toThrow("Unsupported speech audio format"); + }); +}); + +describe("speech response parsing", () => { + it("decodes hex audio and returns response metadata", () => { + expect( + parseSpeechResponse({ + data: { audio: "48656c6c6f", status: 2 }, + base_resp: { status_code: 0 }, + extra_info: { audio_format: "wav" }, + }), + ).toEqual({ audio: Buffer.from("Hello"), format: "wav", status: 2 }); + }); + + it("decodes base64 audio and rejects failed or empty responses", () => { + expect(parseSpeechResponse({ data: { audio: "SGVsbG8=" } }).audio).toEqual( + Buffer.from("Hello"), + ); + expect(() => + parseSpeechResponse({ base_resp: { status_code: 1001, status_msg: "Invalid request" } }), + ).toThrow("Invalid request"); + expect(() => parseSpeechResponse({ data: {} })).toThrow("did not contain audio"); + }); +}); diff --git a/src/proxy.ts b/src/proxy.ts index f99b6ef8..a58fac38 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -110,6 +110,94 @@ const BLOCKRUN_SOLANA_API = "https://sol.blockrun.ai/api"; const IMAGE_DIR = join(homedir(), ".openclaw", "blockrun", "images"); const AUDIO_DIR = join(homedir(), ".openclaw", "blockrun", "audio"); const VIDEO_DIR = join(homedir(), ".openclaw", "blockrun", "videos"); + +export const SPEECH_MODELS = [ + "speech-2.8-hd", + "speech-2.8-turbo", + "speech-2.6-hd", + "speech-2.6-turbo", + "speech-02-hd", + "speech-02-turbo", + "speech-01-hd", + "speech-01-turbo", +] as const; + +export const DEFAULT_SPEECH_MODEL = SPEECH_MODELS[0]; + +const SPEECH_AUDIO_FORMATS = new Set(["mp3", "wav", "flac", "pcm"]); + +type SpeechRequestBody = { + model?: string; + text?: string; + stream?: boolean; + language_boost?: string; + output_format?: string; + voice_setting?: Record; + pronunciation_dict?: Record; + audio_setting?: { format?: string; [key: string]: unknown }; + voice_modify?: Record; + subtitle_enable?: boolean; +}; + +export function normalizeSpeechRequest( + input: unknown, +): Required> & SpeechRequestBody { + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new Error("Speech request body must be a JSON object"); + } + const request = input as SpeechRequestBody; + const model = request.model ?? DEFAULT_SPEECH_MODEL; + const text = request.text?.trim(); + if (!SPEECH_MODELS.includes(model as (typeof SPEECH_MODELS)[number])) { + throw new Error(`Unsupported speech model: ${model}`); + } + if (!text) throw new Error("Speech request requires text"); + + const format = request.audio_setting?.format ?? "mp3"; + if (!SPEECH_AUDIO_FORMATS.has(format)) { + throw new Error(`Unsupported speech audio format: ${format}`); + } + + return { + ...request, + model, + text, + stream: request.stream ?? false, + output_format: request.output_format ?? "hex", + audio_setting: { ...request.audio_setting, format }, + }; +} + +export function parseSpeechResponse(input: unknown): { + audio: Buffer; + format: string; + status?: number; +} { + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new Error("Speech response must be a JSON object"); + } + const response = input as { + data?: { audio?: string; status?: number }; + base_resp?: { status_code?: number; status_msg?: string }; + extra_info?: { audio_format?: string }; + }; + if (response.base_resp?.status_code && response.base_resp.status_code !== 0) { + throw new Error(response.base_resp.status_msg || "Speech generation failed"); + } + if (!response.data?.audio) throw new Error("Speech response did not contain audio"); + + const audio = response.data.audio.trim(); + const hex = audio.replace(/^0x/, ""); + const isHex = hex.length > 0 && hex.length % 2 === 0 && /^[0-9a-f]+$/i.test(hex); + const decoded = Buffer.from(isHex ? hex : audio, isHex ? "hex" : "base64"); + if (decoded.length === 0) throw new Error("Speech response contained empty audio"); + + return { + audio: decoded, + format: response.extra_info?.audio_format ?? "mp3", + status: response.data.status, + }; +} // Routing profile models - virtual models that trigger intelligent routing const AUTO_MODEL = "blockrun/auto"; @@ -2905,6 +2993,80 @@ export async function startProxy(options: ProxyOptions): Promise { return; } + // --- Handle /v1/speech/generations through the wallet-backed speech gateway --- + if (req.url === "/v1/speech/generations" && req.method === "POST") { + const speechStartTime = Date.now(); + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + + let request: ReturnType; + try { + request = normalizeSpeechRequest(JSON.parse(Buffer.concat(chunks).toString())); + } catch (err) { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: err instanceof Error ? err.message : "Invalid speech request", + }), + ); + return; + } + + try { + const upstream = await payFetch(`${apiBase}/v1/t2a_v2`, { + method: "POST", + headers: { "content-type": "application/json", "user-agent": USER_AGENT }, + body: JSON.stringify(request), + }); + const text = await upstream.text(); + if (!upstream.ok) { + res.writeHead(upstream.status, { "Content-Type": "application/json" }); + res.end(text); + return; + } + + const speech = parseSpeechResponse(JSON.parse(text)); + await mkdir(AUDIO_DIR, { recursive: true }); + const format = SPEECH_AUDIO_FORMATS.has(speech.format) ? speech.format : "mp3"; + const filename = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}.${format}`; + await writeFile(join(AUDIO_DIR, filename), speech.audio); + const port = (server.address() as AddressInfo | null)?.port ?? 8402; + const actualCost = paymentStore.getStore()?.amountUsd ?? 0; + logUsage({ + timestamp: new Date().toISOString(), + model: request.model, + tier: "SPEECH", + cost: actualCost, + baselineCost: actualCost, + savings: 0, + latencyMs: Date.now() - speechStartTime, + }).catch(() => {}); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + model: request.model, + data: [ + { + url: `http://localhost:${port}/audio/${filename}`, + format, + status: speech.status, + }, + ], + }), + ); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`[ClawRouter] Speech generation error: ${msg}`); + if (!res.headersSent) { + res.writeHead(502, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Speech generation failed", details: msg })); + } + } + return; + } + // --- Handle /v1/videos/generations: async submit + poll --- // Server protocol (BlockRun 2026-04-23+): // POST /v1/videos/generations → 202 { id, poll_url } (payment verified, not settled)