Skip to content

feat: add wallet-backed speech generation #243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
octo-patch wants to merge 1 commit into from
+291 −1
Closed

feat: add wallet-backed speech generation #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 60 additions & 1 deletion src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2043,6 +2043,65 @@ const plugin: OpenClawPluginDefinition = {
},
});

// /cr-speech <text> [--model=speech-2.8-hd] [--format=mp3|wav|flac|pcm]
api.registerCommand({
name: "cr-speech",
description: "Generate speech audio (paid via wallet)",
acceptsArgs: true,
requireAuth: false,
handler: async (ctx: PluginCommandContext) => {
const tokens =
(ctx.args ?? "")
.match(/(?:[^\s"]+|"[^"]*")+/g)
?.map((token) =>
token.startsWith('"') && token.endsWith('"') ? token.slice(1, -1) : token,
) ?? [];
let model = "speech-2.8-hd";
let format = "mp3";
const textParts: string[] = [];
for (const token of tokens) {
const option = token.match(/^--(model|format)=(.+)$/);
if (option?.[1] === "model") model = option[2]!;
else if (option?.[1] === "format") format = option[2]!;
else textParts.push(token);
}
const text = textParts.join(" ").trim();
if (!text) {
return {
text: "Usage: `/cr-speech <text> [--model=speech-2.8-hd] [--format=mp3|wav|flac|pcm]`",
};
}

const port = getProxyPort();
try {
const resp = await fetch(`http://127.0.0.1:${port}/v1/speech/generations`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ model, text, audio_setting: { format } }),
signal: AbortSignal.timeout(180_000),
});
const responseText = await resp.text();
if (!resp.ok) {
if (resp.status === 402) {
return {
text: `Insufficient wallet balance for speech generation. Top up with \`/wallet\`.\n\n${responseText}`,
};
}
return { text: `Speech generation failed (${resp.status}): ${responseText}` };
}
const result = JSON.parse(responseText) as { data?: Array<{ url?: string }> };
const url = result.data?.[0]?.url;
return url
? { text: `Speech audio: ${url}` }
: { text: "Speech generation returned no audio." };
} catch (err) {
return {
text: `Speech generation error: ${err instanceof Error ? err.message : String(err)}`,
};
}
},
});

// /cr-call <+E.164> "<task>" [--voice nat] [--max-duration 5] [--from +1...] [--language en-US]
// Places a REAL outbound AI voice call via BlockRun → Bland.ai. Returns
// immediately with call_id + poll_url; the call itself runs in the cloud
Expand Down Expand Up @@ -2141,7 +2200,7 @@ const plugin: OpenClawPluginDefinition = {
api.registerCommand(createExcludeCommand());
if (shouldLogRegistration) {
api.logger.info(
"Commands registered: /wallet, /blockrun, /stats, /exclude, /partners, /cr-imagegen, /videogen, /cr-call",
"Commands registered: /wallet, /blockrun, /stats, /exclude, /partners, /cr-imagegen, /videogen, /cr-speech, /cr-call",
);
}

Expand Down
69 changes: 69 additions & 0 deletions src/proxy.speech.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import { describe, expect, it } from "vitest";

import {
DEFAULT_SPEECH_MODEL,
SPEECH_MODELS,
normalizeSpeechRequest,
parseSpeechResponse,
} from "./proxy.js";

describe("speech request normalization", () => {
it("defaults the model and audio fields", () => {
expect(normalizeSpeechRequest({ text: "Hello" })).toEqual({
model: DEFAULT_SPEECH_MODEL,
text: "Hello",
stream: false,
output_format: "hex",
audio_setting: { format: "mp3" },
});
});

it("accepts every supported speech model and request field", () => {
for (const model of SPEECH_MODELS) {
expect(
normalizeSpeechRequest({
model,
text: "Hello",
language_boost: "English",
voice_setting: { voice_id: "female-shaonv" },
pronunciation_dict: { tone: ["hello/(he lou)"] },
audio_setting: { format: "wav", sample_rate: 32000 },
voice_modify: { pitch: 1 },
subtitle_enable: true,
}).model,
).toBe(model);
}
});

it("rejects missing text, unknown models, and unsupported audio formats", () => {
expect(() => normalizeSpeechRequest({ model: DEFAULT_SPEECH_MODEL })).toThrow("requires text");
expect(() => normalizeSpeechRequest({ model: "unknown", text: "Hello" })).toThrow(
"Unsupported speech model",
);
expect(() =>
normalizeSpeechRequest({ text: "Hello", audio_setting: { format: "ogg" } }),
).toThrow("Unsupported speech audio format");
});
});

describe("speech response parsing", () => {
it("decodes hex audio and returns response metadata", () => {
expect(
parseSpeechResponse({
data: { audio: "48656c6c6f", status: 2 },
base_resp: { status_code: 0 },
extra_info: { audio_format: "wav" },
}),
).toEqual({ audio: Buffer.from("Hello"), format: "wav", status: 2 });
});

it("decodes base64 audio and rejects failed or empty responses", () => {
expect(parseSpeechResponse({ data: { audio: "SGVsbG8=" } }).audio).toEqual(
Buffer.from("Hello"),
);
expect(() =>
parseSpeechResponse({ base_resp: { status_code: 1001, status_msg: "Invalid request" } }),
).toThrow("Invalid request");
expect(() => parseSpeechResponse({ data: {} })).toThrow("did not contain audio");
});
});
Comment on lines +10 to +69

@coderabbitai coderabbitai Bot Aug 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🟠 Major | 🏗️ Heavy lift

Add speech endpoint lifecycle and integration tests.

These tests cover only pure request and response helpers. Add Vitest coverage that sends a request through startProxy, mocks the paid upstream response, verifies the generated /audio/ URL and media response, and verifies that a client disconnect aborts the upstream request.

As per coding guidelines: Use Vitest tests to cover error and lifecycle resilience, end-to-end tool ID sanitization, and Docker installation, edge-case, and integration behavior where applicable. Based on learnings: Use Vitest tests to cover error and lifecycle resilience, end-to-end tool ID sanitization, and Docker installation, edge-case, and integration behavior where applicable.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/proxy.speech.test.ts` around lines 10 - 69, Add Vitest integration
coverage around startProxy that mocks the paid speech upstream, sends a request
through the proxy, verifies the generated /audio/ URL and returned media
response, and confirms client disconnects abort the upstream request. Keep the
existing normalizeSpeechRequest and parseSpeechResponse unit tests, and use the
speech endpoint’s existing lifecycle and mocking conventions.

Sources: Coding guidelines, Learnings

162 changes: 162 additions & 0 deletions src/proxy.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,94 @@ const BLOCKRUN_SOLANA_API = "https://sol.blockrun.ai/api";
const IMAGE_DIR = join(homedir(), ".openclaw", "blockrun", "images");
const AUDIO_DIR = join(homedir(), ".openclaw", "blockrun", "audio");
const VIDEO_DIR = join(homedir(), ".openclaw", "blockrun", "videos");

export const SPEECH_MODELS = [
"speech-2.8-hd",
"speech-2.8-turbo",
"speech-2.6-hd",
"speech-2.6-turbo",
"speech-02-hd",
"speech-02-turbo",
"speech-01-hd",
"speech-01-turbo",
] as const;

export const DEFAULT_SPEECH_MODEL = SPEECH_MODELS[0];

const SPEECH_AUDIO_FORMATS = new Set(["mp3", "wav", "flac", "pcm"]);

type SpeechRequestBody = {
model?: string;
text?: string;
stream?: boolean;
language_boost?: string;
output_format?: string;
voice_setting?: Record<string, unknown>;
pronunciation_dict?: Record<string, unknown>;
audio_setting?: { format?: string; [key: string]: unknown };
voice_modify?: Record<string, unknown>;
subtitle_enable?: boolean;
};

export function normalizeSpeechRequest(
input: unknown,
): Required<Pick<SpeechRequestBody, "model" | "text">> & SpeechRequestBody {
if (!input || typeof input !== "object" || Array.isArray(input)) {
throw new Error("Speech request body must be a JSON object");
}
const request = input as SpeechRequestBody;
const model = request.model ?? DEFAULT_SPEECH_MODEL;
const text = request.text?.trim();
if (!SPEECH_MODELS.includes(model as (typeof SPEECH_MODELS)[number])) {
throw new Error(`Unsupported speech model: ${model}`);
}
if (!text) throw new Error("Speech request requires text");

const format = request.audio_setting?.format ?? "mp3";
if (!SPEECH_AUDIO_FORMATS.has(format)) {
throw new Error(`Unsupported speech audio format: ${format}`);
}

return {
...request,
model,
text,
stream: request.stream ?? false,
output_format: request.output_format ?? "hex",
audio_setting: { ...request.audio_setting, format },
};
}

export function parseSpeechResponse(input: unknown): {
audio: Buffer;
format: string;
status?: number;
} {
if (!input || typeof input !== "object" || Array.isArray(input)) {
throw new Error("Speech response must be a JSON object");
}
const response = input as {
data?: { audio?: string; status?: number };
base_resp?: { status_code?: number; status_msg?: string };
extra_info?: { audio_format?: string };
};
if (response.base_resp?.status_code && response.base_resp.status_code !== 0) {
throw new Error(response.base_resp.status_msg || "Speech generation failed");
}
if (!response.data?.audio) throw new Error("Speech response did not contain audio");

const audio = response.data.audio.trim();
const hex = audio.replace(/^0x/, "");
const isHex = hex.length > 0 && hex.length % 2 === 0 && /^[0-9a-f]+$/i.test(hex);
const decoded = Buffer.from(isHex ? hex : audio, isHex ? "hex" : "base64");
if (decoded.length === 0) throw new Error("Speech response contained empty audio");

return {
audio: decoded,
format: response.extra_info?.audio_format ?? "mp3",
status: response.data.status,
};
}
// Routing profile models - virtual models that trigger intelligent routing
const AUTO_MODEL = "blockrun/auto";

Expand Down Expand Up @@ -2905,6 +2993,80 @@ export async function startProxy(options: ProxyOptions): Promise<ProxyHandle> {
return;
}

// --- Handle /v1/speech/generations through the wallet-backed speech gateway ---
if (req.url === "/v1/speech/generations" && req.method === "POST") {
const speechStartTime = Date.now();
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}

let request: ReturnType<typeof normalizeSpeechRequest>;
try {
request = normalizeSpeechRequest(JSON.parse(Buffer.concat(chunks).toString()));
} catch (err) {
res.writeHead(400, { "Content-Type": "application/json" });
res.end(
JSON.stringify({
error: err instanceof Error ? err.message : "Invalid speech request",
}),
);
return;
}

try {
const upstream = await payFetch(`${apiBase}/v1/t2a_v2`, {
method: "POST",
headers: { "content-type": "application/json", "user-agent": USER_AGENT },
body: JSON.stringify(request),
});
const text = await upstream.text();
Comment on lines +2997 to +3023

@coderabbitai coderabbitai Bot Aug 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

Abort the upstream request when the client disconnects.

This route continues payFetch() after the client closes the response. The upstream speech request can generate audio and settle a wallet payment after the caller cannot receive the result.

Create an AbortController, abort it from res.on("close") when !res.writableEnded, and pass its signal to payFetch. Return without writing an error when that signal is aborted.

Proposed fix 
+        const clientAbort = new AbortController();
+        res.on("close", () => {
+          if (!res.writableEnded) clientAbort.abort();
+        });
+
         try {
           const upstream = await payFetch(`${apiBase}/v1/t2a_v2`, {
             method: "POST",
             headers: { "content-type": "application/json", "user-agent": USER_AGENT },
             body: JSON.stringify(request),
+            signal: clientAbort.signal,
           });
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/proxy.ts` around lines 2997 - 3023, Update the speech-generation handler
around payFetch to create an AbortController and abort it from res.on("close")
when !res.writableEnded. Pass the controller’s signal to payFetch, and in the
upstream error path return without writing an error response when the signal is
aborted; preserve existing error handling for other failures.

if (!upstream.ok) {
res.writeHead(upstream.status, { "Content-Type": "application/json" });
res.end(text);
return;
}

const speech = parseSpeechResponse(JSON.parse(text));
await mkdir(AUDIO_DIR, { recursive: true });
const format = SPEECH_AUDIO_FORMATS.has(speech.format) ? speech.format : "mp3";
const filename = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}.${format}`;
await writeFile(join(AUDIO_DIR, filename), speech.audio);
Comment on lines +3030 to +3034

@coderabbitai coderabbitai Bot Aug 12, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

Preserve and serve the selected audio format correctly.

If the upstream omits extra_info.audio_format, Lines 197-198 set the format to mp3 even when the request selected wav, flac, or pcm. Lines 3030-3034 then save bytes with an MP3 extension. The /audio/ handler also has no FLAC or PCM MIME mapping and falls back to audio/mpeg.

  • src/proxy.ts#L3030-L3034: use the normalized request format when upstream metadata is absent. Reject an incompatible upstream format instead of relabeling its bytes. Add matching MIME handling for every retained format.
  • src/proxy.ts#L127-L127: retain only formats that the cache and local audio endpoint can serve correctly.
  • src/proxy.ts#L197-L198: preserve an absent upstream format so the endpoint can apply the requested-format fallback.
  • src/index.ts#L2046-L2046: keep the command help aligned with the end-to-end supported format set.
📍 Affects 2 files
  • src/proxy.ts#L3030-L3034 (this comment)
  • src/proxy.ts#L127-L127
  • src/proxy.ts#L197-L198
  • src/index.ts#L2046-L2046
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/proxy.ts` around lines 3030 - 3034, Normalize and validate the selected
format across the audio flow: in src/proxy.ts lines 3030-3034, use the requested
format when upstream metadata is absent, reject incompatible upstream bytes, and
add MIME mappings for every retained format in the /audio/ handler; update
src/proxy.ts lines 127-127 to retain only end-to-end servable formats, preserve
an absent upstream format at lines 197-198 so the endpoint can apply the
requested fallback, and align the supported-format help in src/index.ts lines
2046-2046.

const port = (server.address() as AddressInfo | null)?.port ?? 8402;
const actualCost = paymentStore.getStore()?.amountUsd ?? 0;
logUsage({
timestamp: new Date().toISOString(),
model: request.model,
tier: "SPEECH",
cost: actualCost,
baselineCost: actualCost,
savings: 0,
latencyMs: Date.now() - speechStartTime,
}).catch(() => {});
res.writeHead(200, { "Content-Type": "application/json" });
res.end(
JSON.stringify({
model: request.model,
data: [
{
url: `http://localhost:${port}/audio/${filename}`,
format,
status: speech.status,
},
],
}),
);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error(`[ClawRouter] Speech generation error: ${msg}`);
if (!res.headersSent) {
res.writeHead(502, { "Content-Type": "application/json" });
res.end(JSON.stringify({ error: "Speech generation failed", details: msg }));
}
}
return;
}

// --- Handle /v1/videos/generations: async submit + poll ---
// Server protocol (BlockRun 2026-04-23+):
// POST /v1/videos/generations → 202 { id, poll_url } (payment verified, not settled)
Expand Down
Loading