feat: add ModelsLab as TTS and Video generation provider

apps/sim/app/api/tools/tts/unified/route.ts

@@ -76,6 +76,9 @@ interface TtsUnifiedRequestBody {
   voiceGuidance?: number
   textGuidance?: number
 
+  // ModelsLab specific
+  voice_id?: string
+
   // Execution context
   workspaceId?: string
   workflowId?: string
@@ -235,6 +238,17 @@ export async function POST(request: NextRequest) {
         audioBuffer = result.audioBuffer
         format = result.format
         mimeType = result.mimeType
+      } else if (provider === 'modelslab') {
+        const result = await synthesizeWithModelsLab({
+          text,
+          apiKey,
+          voice_id: body.voice_id,
+          language: body.language,
+          speed: body.speed,
+        })
+        audioBuffer = result.audioBuffer
+        format = result.format
+        mimeType = result.mimeType
       } else {
         return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
       }
@@ -749,6 +763,106 @@ async function synthesizeWithAzure(
   }
 }
 
+async function synthesizeWithModelsLab(params: {
+  text: string
+  apiKey: string
+  voice_id?: string
+  language?: string
+  speed?: number
+}): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {
+  const { text, apiKey, voice_id = 'madison', language = 'en', speed = 1.0 } = params
+
+  // Initial TTS request
+  const response = await fetch('https://modelslab.com/api/v6/voice/text_to_speech', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      key: apiKey,
+      prompt: text,
+      voice_id,
+      language,
+      speed,
+    }),
+  })
+
+  if (!response.ok) {
+    const error = await response.json().catch(() => ({}))
+    const errorMessage = error.message || error.error || response.statusText
+    throw new Error(`ModelsLab TTS API error: ${errorMessage}`)
+  }
+
+  const data = await response.json()
+
+  // Handle async processing
+  if (data.status === 'processing' && data.id) {
+    const requestId = data.id
+    const maxAttempts = 30
+    let attempts = 0
+
+    while (attempts < maxAttempts) {
+      await new Promise((resolve) => setTimeout(resolve, 3000))
+
+      const fetchResponse = await fetch('https://modelslab.com/api/v6/voice/fetch', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          key: apiKey,
+          request_id: requestId,
+        }),
+      })
+
+      if (!fetchResponse.ok) {
+        throw new Error(`ModelsLab fetch error: ${fetchResponse.status}`)
+      }
+
+      const fetchData = await fetchResponse.json()
+
+      if (fetchData.status === 'success' && fetchData.output) {
+        const audioUrl = Array.isArray(fetchData.output) ? fetchData.output[0] : fetchData.output
+        const audioResponse = await fetch(audioUrl)
+        if (!audioResponse.ok) {
+          throw new Error(`Failed to download ModelsLab audio: ${audioResponse.status}`)
+        }
+        const arrayBuffer = await audioResponse.arrayBuffer()
+        return {
+          audioBuffer: Buffer.from(arrayBuffer),
+          format: 'mp3',
+          mimeType: 'audio/mpeg',
+        }
+      }
+
+      if (fetchData.status === 'error' || fetchData.status === 'failed') {
+        throw new Error(`ModelsLab TTS generation failed: ${fetchData.message || 'Unknown error'}`)
+      }
+
+      attempts++
+    }
+
+    throw new Error('ModelsLab TTS generation timed out')
+  }
+
+  // Handle immediate success
+  if (data.status === 'success' && data.output) {
+    const audioUrl = Array.isArray(data.output) ? data.output[0] : data.output
+    const audioResponse = await fetch(audioUrl)
+    if (!audioResponse.ok) {
+      throw new Error(`Failed to download ModelsLab audio: ${audioResponse.status}`)
+    }
+    const arrayBuffer = await audioResponse.arrayBuffer()
+    return {
+      audioBuffer: Buffer.from(arrayBuffer),
+      format: 'mp3',
+      mimeType: 'audio/mpeg',
+    }
+  }
+
+  throw new Error(`ModelsLab TTS error: ${data.message || 'Unexpected response format'}`)
+}
+
 async function synthesizeWithPlayHT(
   params: Partial<PlayHtTtsParams>
 ): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> {

apps/sim/app/api/tools/video/route.ts

@@ -31,7 +31,7 @@ export async function POST(request: NextRequest) {
       )
     }
 
-    const validProviders = ['runway', 'veo', 'luma', 'minimax', 'falai']
+    const validProviders = ['runway', 'veo', 'luma', 'minimax', 'falai', 'modelslab']
     if (!validProviders.includes(provider)) {
       return NextResponse.json(
         { error: `Invalid provider. Must be one of: ${validProviders.join(', ')}` },
@@ -176,6 +176,23 @@ export async function POST(request: NextRequest) {
         height = result.height
         jobId = result.jobId
         actualDuration = result.duration
+      } else if (provider === 'modelslab') {
+        const result = await generateWithModelsLab(
+          apiKey,
+          prompt,
+          model || 'text2video',
+          body.imageUrl,
+          body.width || 512,
+          body.height || 512,
+          body.num_frames || 16,
+          requestId,
+          logger
+        )
+        videoBuffer = result.buffer
+        width = result.width
+        height = result.height
+        jobId = result.jobId
+        actualDuration = result.duration
       } else {
         return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 })
       }
@@ -945,6 +962,137 @@ async function generateWithFalAI(
   throw new Error('Fal.ai generation timed out')
 }
 
+async function generateWithModelsLab(
+  apiKey: string,
+  prompt: string,
+  mode: string,
+  imageUrl: string | undefined,
+  width: number,
+  height: number,
+  num_frames: number,
+  requestId: string,
+  logger: ReturnType<typeof createLogger>
+): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> {
+  logger.info(`[${requestId}] Starting ModelsLab video generation, mode: ${mode}`)
+
+  const isImg2Video = mode === 'img2video'
+
+  // Validate img2video requires imageUrl
+  if (isImg2Video && !imageUrl) {
+    throw new Error('imageUrl is required for img2video mode')
+  }
+
+  const endpoint = isImg2Video
+    ? 'https://modelslab.com/api/v6/video/img2video'
+    : 'https://modelslab.com/api/v6/video/text2video'
+
+  const requestBody: Record<string, unknown> = {
+    key: apiKey,
+    prompt,
+    output_type: 'mp4',
+    width,
+    height,
+    num_frames,
+  }
+
+  if (isImg2Video && imageUrl) {
+    requestBody.init_image = imageUrl
+  }
+
+  const createResponse = await fetch(endpoint, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(requestBody),
+  })
+
+  if (!createResponse.ok) {
+    const error = await createResponse.text()
+    throw new Error(`ModelsLab API error: ${createResponse.status} - ${error}`)
+  }
+
+  const createData = await createResponse.json()
+
+  logger.info(`[${requestId}] ModelsLab response status: ${createData.status}`)
+
+  // Handle immediate success
+  if (createData.status === 'success' && createData.output) {
+    const videoUrl = Array.isArray(createData.output) ? createData.output[0] : createData.output
+    const videoResponse = await fetch(videoUrl)
+    if (!videoResponse.ok) {
+      throw new Error(`Failed to download ModelsLab video: ${videoResponse.status}`)
+    }
+    const arrayBuffer = await videoResponse.arrayBuffer()
+    return {
+      buffer: Buffer.from(arrayBuffer),
+      width,
+      height,
+      jobId: String(createData.id || 'modelslab'),
+      duration: Math.round(num_frames / 8), // approximate: 8fps
+    }
+  }
+
+  // Handle async processing
+  if (createData.status === 'processing' && createData.id) {
+    const jobId = String(createData.id)
+    logger.info(`[${requestId}] ModelsLab job created: ${jobId}`)
+
+    const pollIntervalMs = 5000
+    const maxAttempts = 60
+    let attempts = 0
+
+    while (attempts < maxAttempts) {
+      await sleep(pollIntervalMs)
+
+      const fetchResponse = await fetch(
+        `https://modelslab.com/api/v6/video/fetch/${jobId}`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ key: apiKey }),
+        }
+      )
+
+      if (!fetchResponse.ok) {
+        throw new Error(`ModelsLab fetch error: ${fetchResponse.status}`)
+      }
+
+      const fetchData = await fetchResponse.json()
+
+      if (fetchData.status === 'success' && fetchData.output) {
+        logger.info(`[${requestId}] ModelsLab generation completed after ${attempts * 5}s`)
+
+        const videoUrl = Array.isArray(fetchData.output) ? fetchData.output[0] : fetchData.output
+        const videoResponse = await fetch(videoUrl)
+        if (!videoResponse.ok) {
+          throw new Error(`Failed to download ModelsLab video: ${videoResponse.status}`)
+        }
+        const arrayBuffer = await videoResponse.arrayBuffer()
+        return {
+          buffer: Buffer.from(arrayBuffer),
+          width,
+          height,
+          jobId,
+          duration: Math.round(num_frames / 8),
+        }
+      }
+
+      if (fetchData.status === 'error' || fetchData.status === 'failed') {
+        throw new Error(`ModelsLab generation failed: ${fetchData.message || 'Unknown error'}`)
+      }
+
+      attempts++
+    }
+
+    throw new Error('ModelsLab video generation timed out')
+  }
+
+  throw new Error(`ModelsLab API error: ${createData.message || 'Unexpected response format'}`)
+}
+
 function getVideoDimensions(
   aspectRatio: string,
   resolution: string