diff --git a/src/pages/benchmark/config/detail-types.ts b/src/pages/benchmark/config/detail-types.ts index 723f6a86..73460376 100644 --- a/src/pages/benchmark/config/detail-types.ts +++ b/src/pages/benchmark/config/detail-types.ts @@ -26,6 +26,7 @@ export interface InstancesData { backend_version: any; api_detected_backend_version: any; backend_parameters: any; + injected_backend_parameters: string[]; image_name: any; run_command: any; env: any; diff --git a/src/pages/playground/speech/audio/pcm-player-workerlet.ts b/src/pages/playground/speech/audio/pcm-player-workerlet.ts deleted file mode 100644 index a4bb57d4..00000000 --- a/src/pages/playground/speech/audio/pcm-player-workerlet.ts +++ /dev/null @@ -1,94 +0,0 @@ -const workletCode = `class PCMPlayerProcessor extends AudioWorkletProcessor { - constructor() { - super(); - - this.capacity = 24000 * 2; // 2 seconds buffer - this.jitterThreshold = 24000 * 0.1; // 100ms - - this.buffer = new Float32Array(this.capacity); - - this.readIndex = 0; - this.writeIndex = 0; - this.size = 0; - - this.streamEnded = false; - this.completionNotified = false; - - this.port.onmessage = (event) => { - const { type, data } = event.data || {}; - - if (type === 'push' && data) { - this.push(data); - } - - if (type === 'clear') { - this.readIndex = 0; - this.writeIndex = 0; - this.size = 0; - this.streamEnded = false; - this.completionNotified = false; - } - - if (type === 'end-stream') { - this.streamEnded = true; - } - }; - } - - push(data) { - for (let i = 0; i < data.length; i++) { - if (this.size >= this.capacity) { - // backpressure: drop oldest data - this.readIndex = (this.readIndex + 1) % this.capacity; - this.size--; - } - - this.buffer[this.writeIndex] = data[i]; - - this.writeIndex = (this.writeIndex + 1) % this.capacity; - this.size++; - } - } - - process(inputs, outputs) { - const output = outputs[0]; - if (!output) return true; - - const channel = output[0]; - - for (let i = 0; i < channel.length; i++) { - // When stream has ended, ignore jitter threshold and play remaining data - if (!this.streamEnded && this.size < this.jitterThreshold) { - channel[i] = 0; - continue; - } - - if (this.size === 0) { - channel[i] = 0; - continue; - } - - channel[i] = this.buffer[this.readIndex]; - - this.readIndex = (this.readIndex + 1) % this.capacity; - this.size--; - } - - // Check if playback is complete - if (this.streamEnded && this.size === 0 && !this.completionNotified) { - console.log('PCM playback complete, notifying main thread'); - this.completionNotified = true; - this.port.postMessage({ type: 'playback-complete' }); - } - - return true; - } -} - -registerProcessor('pcm-player', PCMPlayerProcessor);`; - -export const workerletUrl = (): string => { - const blob = new Blob([workletCode], { type: 'application/javascript' }); - const workletUrl = URL.createObjectURL(blob); - return workletUrl; -}; diff --git a/src/pages/playground/speech/hooks/use-pcm-stream-player.ts b/src/pages/playground/speech/hooks/use-pcm-stream-player.ts index 7f2496bc..c82e9594 100644 --- a/src/pages/playground/speech/hooks/use-pcm-stream-player.ts +++ b/src/pages/playground/speech/hooks/use-pcm-stream-player.ts @@ -1,6 +1,5 @@ import { useMemoizedFn } from 'ahooks'; import { useRef, useState } from 'react'; -import { workerletUrl } from '../audio/pcm-player-workerlet'; interface Params { sampleRate?: number; @@ -11,6 +10,8 @@ interface Params { onPlaybackComplete?: () => void; } +const JITTER_DELAY = 0.1; + export const usePCMStreamPlayer = (params?: Params) => { const { sampleRate = 24000, @@ -24,58 +25,51 @@ export const usePCMStreamPlayer = (params?: Params) => { const [isPlaying, setIsPlaying] = useState(false); const audioContextRef = useRef(null); - const workletNodeRef = useRef(null); const analyserRef = useRef(null); const leftoverRef = useRef(null); const streamEndedRef = useRef(false); + const nextStartTimeRef = useRef(0); + const pendingSourcesRef = useRef(0); const [audioChunks, setAudioChunks] = useState({ data: new Uint8Array(128), analyser: null }); + const checkComplete = useMemoizedFn(() => { + if (streamEndedRef.current && pendingSourcesRef.current === 0) { + setIsPlaying(false); + onPlaybackComplete?.(); + } + }); + const initialize = useMemoizedFn(async () => { if (audioContextRef.current) return; try { const ctx = new AudioContext({ sampleRate }); - const workletUrl = workerletUrl(); - - await ctx.audioWorklet.addModule(workletUrl); - - URL.revokeObjectURL(workletUrl); - const node = new AudioWorkletNode(ctx, 'pcm-player'); - - // Listen for messages from the worklet - node.port.onmessage = (event) => { - if (event.data.type === 'playback-complete') { - setIsPlaying(false); - onPlaybackComplete?.(); - } - }; - const analyser = ctx.createAnalyser(); analyser.fftSize = 512; - - node.connect(analyser); analyser.connect(ctx.destination); audioContextRef.current = ctx; - workletNodeRef.current = node; analyserRef.current = analyser; + leftoverRef.current = null; + streamEndedRef.current = false; + nextStartTimeRef.current = 0; + pendingSourcesRef.current = 0; + setAudioChunks({ data: new Uint8Array(analyser.frequencyBinCount), analyser: analyserRef }); - streamEndedRef.current = false; - onReady?.(); - } catch (err) { - onError?.(err); + } catch (err: any) { + onError?.(err?.message ?? String(err)); } }); @@ -84,7 +78,6 @@ export const usePCMStreamPlayer = (params?: Params) => { const bytesPerSample = bitsPerSample / 8; const frameSize = bytesPerSample * numChannels; - // If there's leftover data from the previous chunk, prepend it to the current chunk if (leftoverRef.current) { const merged = new Uint8Array(leftoverRef.current.length + pcm.length); merged.set(leftoverRef.current); @@ -116,32 +109,67 @@ export const usePCMStreamPlayer = (params?: Params) => { }; const addChunk = useMemoizedFn((pcmChunk: Uint8Array) => { - const node = workletNodeRef.current; - if (!node) return; + const ctx = audioContextRef.current; + const analyser = analyserRef.current; + if (!ctx || !analyser) return; - setIsPlaying(true); const floatData = convertPCM(pcmChunk); - if (!floatData.length) return; - node.port.postMessage({ - type: 'push', - data: floatData - }); + setIsPlaying(true); + + const frames = floatData.length / numChannels; + const buffer = ctx.createBuffer(numChannels, frames, sampleRate); + + if (numChannels === 1) { + buffer.copyToChannel(floatData, 0); + } else { + for (let ch = 0; ch < numChannels; ch++) { + const channelData = new Float32Array(frames); + for (let i = 0; i < frames; i++) { + channelData[i] = floatData[i * numChannels + ch]; + } + buffer.copyToChannel(channelData, ch); + } + } + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(analyser); + + const now = ctx.currentTime; + if (nextStartTimeRef.current < now) { + nextStartTimeRef.current = now + JITTER_DELAY; + } + + const startAt = nextStartTimeRef.current; + source.start(startAt); + nextStartTimeRef.current = startAt + buffer.duration; + + pendingSourcesRef.current++; + source.onended = () => { + try { + source.disconnect(); + } catch { + // ignore + } + pendingSourcesRef.current--; + checkComplete(); + }; }); const stop = useMemoizedFn(() => { const ctx = audioContextRef.current; if (!ctx) return; - workletNodeRef.current?.port.postMessage({ type: 'clear' }); - ctx.close(); audioContextRef.current = null; - workletNodeRef.current = null; analyserRef.current = null; + leftoverRef.current = null; streamEndedRef.current = false; + nextStartTimeRef.current = 0; + pendingSourcesRef.current = 0; setIsPlaying(false); }); @@ -151,11 +179,10 @@ export const usePCMStreamPlayer = (params?: Params) => { }); const endStream = useMemoizedFn(() => { - const node = workletNodeRef.current; - if (!node) return; + if (!audioContextRef.current) return; streamEndedRef.current = true; - node.port.postMessage({ type: 'end-stream' }); + checkComplete(); }); return { diff --git a/src/pages/playground/speech/stt.tsx b/src/pages/playground/speech/stt.tsx index f320e10d..5ea5b7f0 100644 --- a/src/pages/playground/speech/stt.tsx +++ b/src/pages/playground/speech/stt.tsx @@ -15,6 +15,7 @@ import { } from '@gpustack/core-ui'; import { useIntl } from '@umijs/max'; import { Button, Spin, Tooltip } from 'antd'; +import _ from 'lodash'; import React, { forwardRef, useCallback, @@ -163,10 +164,9 @@ const GroundSTT: React.FC = forwardRef((props, ref) => { await nonStreamSTT.generate(params, getCanceltToken()); } } catch (error: any) { - console.log('error:', error); setTokenResult({ error: true, - errorMessage: error?.message || 'Unknown error' + errorMessage: error?.message || _.toString(error) }); } finally { setLoading(false); diff --git a/src/pages/playground/speech/tts.tsx b/src/pages/playground/speech/tts.tsx index 81971165..57d7aa52 100644 --- a/src/pages/playground/speech/tts.tsx +++ b/src/pages/playground/speech/tts.tsx @@ -122,7 +122,7 @@ const GroundTTS: React.FC = forwardRef((props, ref) => { onError: (error) => { setTokenResult({ error: true, - errorMessage: error + errorMessage: error.message || _.toString(error) }); setMessageList([]); setPlayingStream(false); diff --git a/src/pages/playground/view-code/image.ts b/src/pages/playground/view-code/image.ts index a40ecb91..1134f5ec 100644 --- a/src/pages/playground/view-code/image.ts +++ b/src/pages/playground/view-code/image.ts @@ -22,6 +22,7 @@ ${formatCurlArgs(parameters, isFormdata)}`.trim(); if (edit) { curlCode = ` curl ${host}${api} \\ +-H "Content-Type: multipart/form-data" \\ -H "Authorization: Bearer $\{YOUR_GPUSTACK_API_KEY}" \\${modelProxy ? `\n-H "X-GPUStack-Model: ${parameters.model}" \\` : ''} -F image="@image.png" \\ -F mask="@mask.png" \\ @@ -72,7 +73,6 @@ print(response.json()['data'][0]['b64_json'])`.trim(); import requests\n url="${host}${api}" headers = { - "Content-type": "multipart/form-data", "Authorization": "Bearer $\{YOUR_GPUSTACK_API_KEY}" } data = ${JSON.stringify(_.omit(parameters, ['mask', 'image']), null, 2).replace(/null/g, 'None')}