fix: tts stream error

src/pages/benchmark/config/detail-types.ts

@@ -26,6 +26,7 @@ export interface InstancesData {
   backend_version: any;
   api_detected_backend_version: any;
   backend_parameters: any;
+  injected_backend_parameters: string[];
   image_name: any;
   run_command: any;
   env: any;

src/pages/playground/speech/audio/pcm-player-workerlet.ts

@@ -1,94 +0,0 @@
-const workletCode = `class PCMPlayerProcessor extends AudioWorkletProcessor {
-  constructor() {
-    super();
-
-    this.capacity = 24000 * 2; // 2 seconds buffer
-    this.jitterThreshold = 24000 * 0.1; // 100ms
-
-    this.buffer = new Float32Array(this.capacity);
-
-    this.readIndex = 0;
-    this.writeIndex = 0;
-    this.size = 0;
-
-    this.streamEnded = false;
-    this.completionNotified = false;
-
-    this.port.onmessage = (event) => {
-      const { type, data } = event.data || {};
-
-      if (type === 'push' && data) {
-        this.push(data);
-      }
-
-      if (type === 'clear') {
-        this.readIndex = 0;
-        this.writeIndex = 0;
-        this.size = 0;
-        this.streamEnded = false;
-        this.completionNotified = false;
-      }
-
-      if (type === 'end-stream') {
-        this.streamEnded = true;
-      }
-    };
-  }
-
-  push(data) {
-    for (let i = 0; i < data.length; i++) {
-      if (this.size >= this.capacity) {
-        // backpressure: drop oldest data
-        this.readIndex = (this.readIndex + 1) % this.capacity;
-        this.size--;
-      }
-
-      this.buffer[this.writeIndex] = data[i];
-
-      this.writeIndex = (this.writeIndex + 1) % this.capacity;
-      this.size++;
-    }
-  }
-
-  process(inputs, outputs) {
-    const output = outputs[0];
-    if (!output) return true;
-
-    const channel = output[0];
-
-    for (let i = 0; i < channel.length; i++) {
-      // When stream has ended, ignore jitter threshold and play remaining data
-      if (!this.streamEnded && this.size < this.jitterThreshold) {
-        channel[i] = 0;
-        continue;
-      }
-
-      if (this.size === 0) {
-        channel[i] = 0;
-        continue;
-      }
-
-      channel[i] = this.buffer[this.readIndex];
-
-      this.readIndex = (this.readIndex + 1) % this.capacity;
-      this.size--;
-    }
-
-    // Check if playback is complete
-    if (this.streamEnded && this.size === 0 && !this.completionNotified) {
-      console.log('PCM playback complete, notifying main thread');
-      this.completionNotified = true;
-      this.port.postMessage({ type: 'playback-complete' });
-    }
-
-    return true;
-  }
-}
-
-registerProcessor('pcm-player', PCMPlayerProcessor);`;
-
-export const workerletUrl = (): string => {
-  const blob = new Blob([workletCode], { type: 'application/javascript' });
-  const workletUrl = URL.createObjectURL(blob);
-  return workletUrl;
-};

src/pages/playground/speech/hooks/use-pcm-stream-player.ts

@@ -1,6 +1,5 @@
 import { useMemoizedFn } from 'ahooks';
 import { useRef, useState } from 'react';
-import { workerletUrl } from '../audio/pcm-player-workerlet';
 
 interface Params {
   sampleRate?: number;
@@ -11,6 +10,8 @@ interface Params {
   onPlaybackComplete?: () => void;
 }
 
+const JITTER_DELAY = 0.1;
+
 export const usePCMStreamPlayer = (params?: Params) => {
   const {
     sampleRate = 24000,
@@ -24,58 +25,51 @@ export const usePCMStreamPlayer = (params?: Params) => {
   const [isPlaying, setIsPlaying] = useState(false);
 
   const audioContextRef = useRef<AudioContext | null>(null);
-  const workletNodeRef = useRef<AudioWorkletNode | null>(null);
   const analyserRef = useRef<AnalyserNode | null>(null);
 
   const leftoverRef = useRef<Uint8Array | null>(null);
   const streamEndedRef = useRef(false);
+  const nextStartTimeRef = useRef(0);
+  const pendingSourcesRef = useRef(0);
 
   const [audioChunks, setAudioChunks] = useState<any>({
     data: new Uint8Array(128),
     analyser: null
   });
 
+  const checkComplete = useMemoizedFn(() => {
+    if (streamEndedRef.current && pendingSourcesRef.current === 0) {
+      setIsPlaying(false);
+      onPlaybackComplete?.();
+    }
+  });
+
   const initialize = useMemoizedFn(async () => {
     if (audioContextRef.current) return;
 
     try {
       const ctx = new AudioContext({ sampleRate });
 
-      const workletUrl = workerletUrl();
-
-      await ctx.audioWorklet.addModule(workletUrl);
-
-      URL.revokeObjectURL(workletUrl);
-      const node = new AudioWorkletNode(ctx, 'pcm-player');
-
-      // Listen for messages from the worklet
-      node.port.onmessage = (event) => {
-        if (event.data.type === 'playback-complete') {
-          setIsPlaying(false);
-          onPlaybackComplete?.();
-        }
-      };
-
       const analyser = ctx.createAnalyser();
       analyser.fftSize = 512;
-
-      node.connect(analyser);
       analyser.connect(ctx.destination);
 
       audioContextRef.current = ctx;
-      workletNodeRef.current = node;
       analyserRef.current = analyser;
 
+      leftoverRef.current = null;
+      streamEndedRef.current = false;
+      nextStartTimeRef.current = 0;
+      pendingSourcesRef.current = 0;
+
       setAudioChunks({
         data: new Uint8Array(analyser.frequencyBinCount),
         analyser: analyserRef
       });
 
-      streamEndedRef.current = false;
-
       onReady?.();
-    } catch (err) {
+    } catch (err: any) {
-      onError?.(err);
+      onError?.(err?.message ?? String(err));
     }
   });
 
@@ -84,7 +78,6 @@ export const usePCMStreamPlayer = (params?: Params) => {
     const bytesPerSample = bitsPerSample / 8;
     const frameSize = bytesPerSample * numChannels;
 
-    // If there's leftover data from the previous chunk, prepend it to the current chunk
     if (leftoverRef.current) {
       const merged = new Uint8Array(leftoverRef.current.length + pcm.length);
       merged.set(leftoverRef.current);
@@ -116,32 +109,67 @@ export const usePCMStreamPlayer = (params?: Params) => {
   };
 
   const addChunk = useMemoizedFn((pcmChunk: Uint8Array) => {
-    const node = workletNodeRef.current;
+    const ctx = audioContextRef.current;
-    if (!node) return;
+    const analyser = analyserRef.current;
+    if (!ctx || !analyser) return;
 
-    setIsPlaying(true);
     const floatData = convertPCM(pcmChunk);
-
     if (!floatData.length) return;
 
-    node.port.postMessage({
+    setIsPlaying(true);
-      type: 'push',
+
-      data: floatData
+    const frames = floatData.length / numChannels;
-    });
+    const buffer = ctx.createBuffer(numChannels, frames, sampleRate);
+
+    if (numChannels === 1) {
+      buffer.copyToChannel(floatData, 0);
+    } else {
+      for (let ch = 0; ch < numChannels; ch++) {
+        const channelData = new Float32Array(frames);
+        for (let i = 0; i < frames; i++) {
+          channelData[i] = floatData[i * numChannels + ch];
+        }
+        buffer.copyToChannel(channelData, ch);
+      }
+    }
+
+    const source = ctx.createBufferSource();
+    source.buffer = buffer;
+    source.connect(analyser);
+
+    const now = ctx.currentTime;
+    if (nextStartTimeRef.current < now) {
+      nextStartTimeRef.current = now + JITTER_DELAY;
+    }
+
+    const startAt = nextStartTimeRef.current;
+    source.start(startAt);
+    nextStartTimeRef.current = startAt + buffer.duration;
+
+    pendingSourcesRef.current++;
+    source.onended = () => {
+      try {
+        source.disconnect();
+      } catch {
+        // ignore
+      }
+      pendingSourcesRef.current--;
+      checkComplete();
+    };
   });
 
   const stop = useMemoizedFn(() => {
     const ctx = audioContextRef.current;
     if (!ctx) return;
 
-    workletNodeRef.current?.port.postMessage({ type: 'clear' });
-
     ctx.close();
 
     audioContextRef.current = null;
-    workletNodeRef.current = null;
     analyserRef.current = null;
+    leftoverRef.current = null;
     streamEndedRef.current = false;
+    nextStartTimeRef.current = 0;
+    pendingSourcesRef.current = 0;
 
     setIsPlaying(false);
   });
@@ -151,11 +179,10 @@ export const usePCMStreamPlayer = (params?: Params) => {
   });
 
   const endStream = useMemoizedFn(() => {
-    const node = workletNodeRef.current;
+    if (!audioContextRef.current) return;
-    if (!node) return;
 
     streamEndedRef.current = true;
-    node.port.postMessage({ type: 'end-stream' });
+    checkComplete();
   });
 
   return {

src/pages/playground/speech/stt.tsx

@@ -15,6 +15,7 @@ import {
 } from '@gpustack/core-ui';
 import { useIntl } from '@umijs/max';
 import { Button, Spin, Tooltip } from 'antd';
+import _ from 'lodash';
 import React, {
   forwardRef,
   useCallback,
@@ -163,10 +164,9 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
         await nonStreamSTT.generate(params, getCanceltToken());
       }
     } catch (error: any) {
-      console.log('error:', error);
       setTokenResult({
         error: true,
-        errorMessage: error?.message || 'Unknown error'
+        errorMessage: error?.message || _.toString(error)
       });
     } finally {
       setLoading(false);

src/pages/playground/speech/tts.tsx

@@ -122,7 +122,7 @@ const GroundTTS: React.FC<MessageProps> = forwardRef((props, ref) => {
     onError: (error) => {
       setTokenResult({
         error: true,
-        errorMessage: error
+        errorMessage: error.message || _.toString(error)
       });
       setMessageList([]);
       setPlayingStream(false);

src/pages/playground/view-code/image.ts

@@ -22,6 +22,7 @@ ${formatCurlArgs(parameters, isFormdata)}`.trim();
   if (edit) {
     curlCode = `
 curl ${host}${api} \\
+-H "Content-Type: multipart/form-data" \\
 -H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\${modelProxy ? `\n-H "X-GPUStack-Model: ${parameters.model}" \\` : ''}
 -F image="@image.png" \\
 -F mask="@mask.png" \\
@@ -72,7 +73,6 @@ print(response.json()['data'][0]['b64_json'])`.trim();
 import requests\n
 url="${host}${api}"
 headers = {
-  "Content-type": "multipart/form-data",
   "Authorization": "Bearer $\{YOUR_GPUSTACK_API_KEY}"
 }
 data = ${JSON.stringify(_.omit(parameters, ['mask', 'image']), null, 2).replace(/null/g, 'None')}