root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: tts stream error

Browse Source
This commit is contained in:
jialin
2026-04-29 20:51:48 +08:00
committed by jialin
parent 7794e57bb1
commit ab89ddffc8
6 changed files with 71 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pages/benchmark/config/detail-types.ts
+1
View File
@@ -26,6 +26,7 @@ export interface InstancesData {
backend_version: any;
api_detected_backend_version: any;
backend_parameters: any;
injected_backend_parameters: string[];
image_name: any;
run_command: any;
env: any;
src/pages/playground/speech/audio/pcm-player-workerlet.ts
-94
View File
@@ -1,94 +0,0 @@
const workletCode = `class PCMPlayerProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.capacity = 24000 * 2; // 2 seconds buffer
this.jitterThreshold = 24000 * 0.1; // 100ms
this.buffer = new Float32Array(this.capacity);
this.readIndex = 0;
this.writeIndex = 0;
this.size = 0;
this.streamEnded = false;
this.completionNotified = false;
this.port.onmessage = (event) => {
const { type, data } = event.data || {};
if (type === 'push' && data) {
this.push(data);
}
if (type === 'clear') {
this.readIndex = 0;
this.writeIndex = 0;
this.size = 0;
this.streamEnded = false;
this.completionNotified = false;
}
if (type === 'end-stream') {
this.streamEnded = true;
}
};
}
push(data) {
for (let i = 0; i < data.length; i++) {
if (this.size >= this.capacity) {
// backpressure: drop oldest data
this.readIndex = (this.readIndex + 1) % this.capacity;
this.size--;
}
this.buffer[this.writeIndex] = data[i];
this.writeIndex = (this.writeIndex + 1) % this.capacity;
this.size++;
}
}
process(inputs, outputs) {
const output = outputs[0];
if (!output) return true;
const channel = output[0];
for (let i = 0; i < channel.length; i++) {
// When stream has ended, ignore jitter threshold and play remaining data
if (!this.streamEnded && this.size < this.jitterThreshold) {
channel[i] = 0;
continue;
}
if (this.size === 0) {
channel[i] = 0;
continue;
}
channel[i] = this.buffer[this.readIndex];
this.readIndex = (this.readIndex + 1) % this.capacity;
this.size--;
}
// Check if playback is complete
if (this.streamEnded && this.size === 0 && !this.completionNotified) {
console.log('PCM playback complete, notifying main thread');
this.completionNotified = true;
this.port.postMessage({ type: 'playback-complete' });
}
return true;
}
}
registerProcessor('pcm-player', PCMPlayerProcessor);`;
export const workerletUrl = (): string => {
const blob = new Blob([workletCode], { type: 'application/javascript' });
const workletUrl = URL.createObjectURL(blob);
return workletUrl;
};
src/pages/playground/speech/hooks/use-pcm-stream-player.ts
+66 -39
View File
@@ -1,6 +1,5 @@
import { useMemoizedFn } from 'ahooks';
import { useRef, useState } from 'react';
import { workerletUrl } from '../audio/pcm-player-workerlet';
interface Params {
sampleRate?: number;
@@ -11,6 +10,8 @@ interface Params {
onPlaybackComplete?: () => void;
}
const JITTER_DELAY = 0.1;
export const usePCMStreamPlayer = (params?: Params) => {
const {
sampleRate = 24000,
@@ -24,58 +25,51 @@ export const usePCMStreamPlayer = (params?: Params) => {
const [isPlaying, setIsPlaying] = useState(false);
const audioContextRef = useRef<AudioContext | null>(null);
const workletNodeRef = useRef<AudioWorkletNode | null>(null);
const analyserRef = useRef<AnalyserNode | null>(null);
const leftoverRef = useRef<Uint8Array | null>(null);
const streamEndedRef = useRef(false);
const nextStartTimeRef = useRef(0);
const pendingSourcesRef = useRef(0);
const [audioChunks, setAudioChunks] = useState<any>({
data: new Uint8Array(128),
analyser: null
});
const checkComplete = useMemoizedFn(() => {
if (streamEndedRef.current && pendingSourcesRef.current === 0) {
setIsPlaying(false);
onPlaybackComplete?.();
}
});
const initialize = useMemoizedFn(async () => {
if (audioContextRef.current) return;
try {
const ctx = new AudioContext({ sampleRate });
const workletUrl = workerletUrl();
await ctx.audioWorklet.addModule(workletUrl);
URL.revokeObjectURL(workletUrl);
const node = new AudioWorkletNode(ctx, 'pcm-player');
// Listen for messages from the worklet
node.port.onmessage = (event) => {
if (event.data.type === 'playback-complete') {
setIsPlaying(false);
onPlaybackComplete?.();
}
};
const analyser = ctx.createAnalyser();
analyser.fftSize = 512;
node.connect(analyser);
analyser.connect(ctx.destination);
audioContextRef.current = ctx;
workletNodeRef.current = node;
analyserRef.current = analyser;
leftoverRef.current = null;
streamEndedRef.current = false;
nextStartTimeRef.current = 0;
pendingSourcesRef.current = 0;
setAudioChunks({
data: new Uint8Array(analyser.frequencyBinCount),
analyser: analyserRef
});
streamEndedRef.current = false;
onReady?.();
} catch (err) {
onError?.(err);
} catch (err: any) {
onError?.(err?.message ?? String(err));
}
});
@@ -84,7 +78,6 @@ export const usePCMStreamPlayer = (params?: Params) => {
const bytesPerSample = bitsPerSample / 8;
const frameSize = bytesPerSample * numChannels;
// If there's leftover data from the previous chunk, prepend it to the current chunk
if (leftoverRef.current) {
const merged = new Uint8Array(leftoverRef.current.length + pcm.length);
merged.set(leftoverRef.current);
@@ -116,32 +109,67 @@ export const usePCMStreamPlayer = (params?: Params) => {
};
const addChunk = useMemoizedFn((pcmChunk: Uint8Array) => {
const node = workletNodeRef.current;
if (!node) return;
const ctx = audioContextRef.current;
const analyser = analyserRef.current;
if (!ctx || !analyser) return;
setIsPlaying(true);
const floatData = convertPCM(pcmChunk);
if (!floatData.length) return;
node.port.postMessage({
type: 'push',
data: floatData
});
setIsPlaying(true);
const frames = floatData.length / numChannels;
const buffer = ctx.createBuffer(numChannels, frames, sampleRate);
if (numChannels === 1) {
buffer.copyToChannel(floatData, 0);
} else {
for (let ch = 0; ch < numChannels; ch++) {
const channelData = new Float32Array(frames);
for (let i = 0; i < frames; i++) {
channelData[i] = floatData[i * numChannels + ch];
}
buffer.copyToChannel(channelData, ch);
}
}
const source = ctx.createBufferSource();
source.buffer = buffer;
source.connect(analyser);
const now = ctx.currentTime;
if (nextStartTimeRef.current < now) {
nextStartTimeRef.current = now + JITTER_DELAY;
}
const startAt = nextStartTimeRef.current;
source.start(startAt);
nextStartTimeRef.current = startAt + buffer.duration;
pendingSourcesRef.current++;
source.onended = () => {
try {
source.disconnect();
} catch {
// ignore
}
pendingSourcesRef.current--;
checkComplete();
};
});
const stop = useMemoizedFn(() => {
const ctx = audioContextRef.current;
if (!ctx) return;
workletNodeRef.current?.port.postMessage({ type: 'clear' });
ctx.close();
audioContextRef.current = null;
workletNodeRef.current = null;
analyserRef.current = null;
leftoverRef.current = null;
streamEndedRef.current = false;
nextStartTimeRef.current = 0;
pendingSourcesRef.current = 0;
setIsPlaying(false);
});
@@ -151,11 +179,10 @@ export const usePCMStreamPlayer = (params?: Params) => {
});
const endStream = useMemoizedFn(() => {
const node = workletNodeRef.current;
if (!node) return;
if (!audioContextRef.current) return;
streamEndedRef.current = true;
node.port.postMessage({ type: 'end-stream' });
checkComplete();
});
return {
src/pages/playground/speech/stt.tsx
+2 -2
View File
@@ -15,6 +15,7 @@ import {
} from '@gpustack/core-ui';
import { useIntl } from '@umijs/max';
import { Button, Spin, Tooltip } from 'antd';
import _ from 'lodash';
import React, {
forwardRef,
useCallback,
@@ -163,10 +164,9 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
await nonStreamSTT.generate(params, getCanceltToken());
}
} catch (error: any) {
console.log('error:', error);
setTokenResult({
error: true,
errorMessage: error?.message || 'Unknown error'
errorMessage: error?.message || _.toString(error)
});
} finally {
setLoading(false);
src/pages/playground/speech/tts.tsx
+1 -1
View File
@@ -122,7 +122,7 @@ const GroundTTS: React.FC<MessageProps> = forwardRef((props, ref) => {
onError: (error) => {
setTokenResult({
error: true,
errorMessage: error
errorMessage: error.message || _.toString(error)
});
setMessageList([]);
setPlayingStream(false);
src/pages/playground/view-code/image.ts
+1 -1
View File
@@ -22,6 +22,7 @@ ${formatCurlArgs(parameters, isFormdata)}`.trim();
if (edit) {
curlCode = `
curl ${host}${api} \\
-H "Content-Type: multipart/form-data" \\
-H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\${modelProxy ? `\n-H "X-GPUStack-Model: ${parameters.model}" \\` : ''}
-F image="@image.png" \\
-F mask="@mask.png" \\
@@ -72,7 +73,6 @@ print(response.json()['data'][0]['b64_json'])`.trim();
import requests\n
url="${host}${api}"
headers = {
"Content-type": "multipart/form-data",
"Authorization": "Bearer $\{YOUR_GPUSTACK_API_KEY}"
}
data = ${JSON.stringify(_.omit(parameters, ['mask', 'image']), null, 2).replace(/null/g, 'None')}