root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add streaming for stt

Browse Source
This commit is contained in:
jialin
2026-03-17 16:35:55 +08:00
committed by jialin
parent 39bcc158bb
commit 49a60c25af
12 changed files with 311 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/locales/en-US/playground.ts
+3 -1
View File
@@ -173,5 +173,7 @@ export default {
'Enter a reference audio URL, or upload an audio file.',
'playground.params.refAudio.text':
'Transcript of Reference Audio (for ICL mode)',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)'
'playground.params.refAudio.vectorMode':
'Use Speaker Embedding Only (no ICL)',
'playground.params.streamMode': 'Enable Streaming'
};
src/locales/ja-JP/playground.ts
+4 -1
View File
@@ -176,7 +176,9 @@ export default {
'Enter a reference audio URL, or upload an audio file.',
'playground.params.refAudio.text':
'Transcript of Reference Audio (for ICL mode)',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)'
'playground.params.refAudio.vectorMode':
'Use Speaker Embedding Only (no ICL)',
'playground.params.streamMode': 'Enable Streaming'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
@@ -188,4 +190,5 @@ export default {
// 6. 'playground.uploadImage.url.button': 'Add Image from URL',
// 7. 'playground.params.duration': 'Duration (seconds)',
// 8. 'playground.params.resolution': 'Resolution'
// 9. 'playground.params.streamMode': 'Enable Streaming'
// ========== End of To-Do List ==========
src/locales/ru-RU/playground.ts
+5 -2
View File
@@ -170,7 +170,9 @@ export default {
'Enter a reference audio URL, or upload an audio file.',
'playground.params.refAudio.text':
'Transcript of Reference Audio (for ICL mode)',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)'
'playground.params.refAudio.vectorMode':
'Use Speaker Embedding Only (no ICL)',
'playground.params.streamMode': 'Enable Streaming'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
@@ -178,5 +180,6 @@ export default {
// 2. 'playground.uploadImage.url.holder': 'Enter an image URL',
// 3. 'playground.uploadImage.url.button': 'Add Image from URL',
// 4. 'playground.params.duration': 'Duration (seconds)',
// 5. 'playground.params.resolution': 'Resolution'
// 5. 'playground.params.resolution': 'Resolution',
// 6. 'playground.params.streamMode': 'Enable Streaming'
// ========== End of To-Do List ==========
src/locales/zh-CN/playground.ts
+2 -1
View File
@@ -166,5 +166,6 @@ export default {
'输入一个参考音频的 URL，或上传一个音频文件。',
'playground.params.refAudio.text': '参考音频文本（用于上下文学习模式）',
'playground.params.refAudio.vectorMode':
'仅使用说话人嵌入（不使用上下文学习）'
'仅使用说话人嵌入（不使用上下文学习）',
'playground.params.streamMode': '启用流式输出'
};
src/pages/playground/speech/forms/stt-form.tsx
+12
View File
@@ -1,3 +1,4 @@
import CheckboxField from '@/components/seal-form/checkbox-field';
import SealSelect from '@/components/seal-form/seal-select';
import { useIntl, useSearchParams } from '@umijs/max';
import { Form } from 'antd';
@@ -123,6 +124,17 @@ const STTForm: React.FC<ParamsSettingsProps> = forwardRef(
options={languageOptions}
></SealSelect>
</Form.Item>
<Form.Item
name="stream"
valuePropName="checked"
style={{ marginBottom: 8 }}
>
<CheckboxField
label={intl.formatMessage({
id: 'playground.params.streamMode'
})}
></CheckboxField>
</Form.Item>
</Form>
</FormContext.Provider>
);
src/pages/playground/speech/forms/tts-advance.tsx
+17 -2
View File
@@ -110,20 +110,35 @@ const TTSAdvanceConfig: React.FC = () => {
></SealInput.Input>
</Form.Item>
</Container>
<Form.Item name="ref_text" style={{ marginBottom: 8 }}>
<Form.Item name="ref_text">
<SealInput.TextArea
allowClear
scaleSize={true}
label={intl.formatMessage({ id: 'playground.params.refAudio.text' })}
></SealInput.TextArea>
</Form.Item>
<Form.Item name="x_vector_only_mode" valuePropName="checked">
<Form.Item
name="x_vector_only_mode"
valuePropName="checked"
style={{ marginBottom: 8 }}
>
<CheckboxField
label={intl.formatMessage({
id: 'playground.params.refAudio.vectorMode'
})}
></CheckboxField>
</Form.Item>
<Form.Item
name="stream"
valuePropName="checked"
style={{ marginBottom: 8 }}
>
<CheckboxField
label={intl.formatMessage({
id: 'playground.params.streamMode'
})}
></CheckboxField>
</Form.Item>
</>
);
};
src/pages/playground/speech/hooks/index.ts
+4 -2
View File
@@ -1,2 +1,4 @@
export { useNonStreamTTS } from './useNonStreamTTS';
export { useStreamTTS } from './useStreamTTS';
export { useNonStreamSTT } from './use-non-stream-stt';
export { useNonStreamTTS } from './use-non-stream-tts';
export { useStreamSTT } from './use-stream-stt';
export { useStreamTTS } from './use-stream-tts';
src/pages/playground/speech/hooks/use-non-stream-stt.ts
+77
View File
@@ -0,0 +1,77 @@
import { useCallback, useRef, useState } from 'react';
import { speechToText } from '../../apis';
import { extractErrorMessage } from '../../config';
interface UseNonStreamSTTParams {
onSuccess?: (result: { text: string }) => void;
onError?: (error: any) => void;
}
interface STTParams {
model: string;
language?: string;
file: File;
[key: string]: any;
}
export const useNonStreamSTT = (params?: UseNonStreamSTTParams) => {
const [loading, setLoading] = useState(false);
const [error, setError] = useState<any>(null);
const cancelTokenRef = useRef<any>(null);
const generate = useCallback(
async (sttParams: STTParams, cancelToken?: any) => {
try {
setLoading(true);
setError(null);
cancelTokenRef.current = cancelToken;
const result: any = await speechToText(
{
data: sttParams
},
{
cancelToken
}
);
if (
(result?.status_code && result?.status_code !== 200) ||
result?.error
) {
const errorMessage = extractErrorMessage(result);
setError({
error: true,
errorMessage
});
params?.onError?.(errorMessage);
return null;
}
params?.onSuccess?.(result);
return result;
} catch (err: any) {
const res = err?.response?.data;
if (res?.error || (res?.status_code && res?.status_code !== 200)) {
const errorMessage = extractErrorMessage(res);
setError({
error: true,
errorMessage
});
params?.onError?.(errorMessage);
}
return null;
} finally {
setLoading(false);
}
},
[params]
);
return {
generate,
loading,
error
};
};
src/pages/playground/speech/hooks/useNonStreamTTS.ts → src/pages/playground/speech/hooks/use-non-stream-tts.ts
View File
src/pages/playground/speech/hooks/use-stream-stt.ts
+131
View File
@@ -0,0 +1,131 @@
import {
fetchChunkedDataPostFormData,
readStreamData
} from '@/utils/fetch-chunk-data';
import { useCallback, useRef, useState } from 'react';
import { AUDIO_SPEECH_TO_TEXT_API } from '../../apis';
import { extractErrorMessage } from '../../config';
interface UseStreamSTTParams {
onChunk?: (text: string) => void;
onComplete?: (fullText: string) => void;
onError?: (error: any) => void;
}
interface STTParams {
model: string;
language?: string;
file: File;
stream?: boolean;
[key: string]: any;
}
export const useStreamSTT = (params?: UseStreamSTTParams) => {
const [loading, setLoading] = useState(false);
const [error, setError] = useState<any>(null);
const [progress, setProgress] = useState(0);
const controllerRef = useRef<AbortController | null>(null);
const generate = useCallback(
async (sttParams: STTParams) => {
try {
setLoading(true);
setError(null);
setProgress(0);
// Abort previous request if exists
controllerRef.current?.abort();
controllerRef.current = new AbortController();
const signal = controllerRef.current.signal;
// Add stream parameter
const streamParams = {
...sttParams,
stream: true
};
const result = await fetchChunkedDataPostFormData({
url: AUDIO_SPEECH_TO_TEXT_API,
data: streamParams,
signal
});
if ('error' in result) {
const errorMessage = extractErrorMessage(result.data);
setError({
error: true,
errorMessage
});
params?.onError?.(errorMessage);
return;
}
const { reader, decoder } = result;
if (!reader || !decoder) {
throw new Error('Failed to get reader from response');
}
// Collect all text chunks
let fullText = '';
await readStreamData(
reader,
decoder,
(chunks: any[]) => {
chunks.forEach((chunk) => {
if (chunk.error) {
const errorMessage = extractErrorMessage(chunk.error);
setError({
error: true,
errorMessage
});
params?.onError?.(errorMessage);
return;
}
// STT stream response format: { text: "..." }
if (chunk.text) {
fullText += chunk.text;
params?.onChunk?.(fullText);
setProgress((prev) => prev + 1);
}
});
},
100 // throttle delay
);
// Stream completed
params?.onComplete?.(fullText);
} catch (err: any) {
if (err.name === 'AbortError') {
console.log('Stream aborted');
return;
}
const errorMessage = err?.message || 'Stream processing failed';
setError({
error: true,
errorMessage
});
params?.onError?.(errorMessage);
} finally {
setLoading(false);
}
},
[params]
);
const abort = useCallback(() => {
controllerRef.current?.abort();
setLoading(false);
}, []);
return {
generate,
abort,
loading,
error,
progress
};
};
src/pages/playground/speech/hooks/useStreamTTS.ts → src/pages/playground/speech/hooks/use-stream-tts.ts
View File
src/pages/playground/speech/stt.tsx
+56 -33
View File
@@ -22,16 +22,17 @@ import React, {
useRef,
useState
} from 'react';
import { AUDIO_SPEECH_TO_TEXT_API, speechToText } from '../apis';
import { AUDIO_SPEECH_TO_TEXT_API } from '../apis';
import AudioInput from '../components/audio-input';
import RightContainer from '../components/right-container';
import ViewCommonCode from '../components/view-common-code';
import { SpeechToTextFormat, extractErrorMessage } from '../config';
import { SpeechToTextFormat } from '../config';
import '../style/ground-llm.less';
import '../style/speech-to-text.less';
import '../style/system-message-wrap.less';
import { speechToTextCode } from '../view-code/audio';
import STTForm from './forms/stt-form';
import { useNonStreamSTT, useStreamSTT } from './hooks';
interface MessageProps {
modelList: Global.BaseOption<string>[];
@@ -67,6 +68,46 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
const { initialize, updateScrollerPosition } = useOverlayScroller();
// Initialize non-stream STT hook
const nonStreamSTT = useNonStreamSTT({
onSuccess: (result) => {
setMessageList([
{
content: result.text,
uid: messageId.current
}
]);
},
onError: (error) => {
setTokenResult({
error: true,
errorMessage: error
});
}
});
// Initialize stream STT hook
const streamSTT = useStreamSTT({
onChunk: (text) => {
// Update message list with streaming text
setMessageList([
{
content: text,
uid: messageId.current
}
]);
},
onComplete: () => {
console.log('Stream transcription completed');
},
onError: (error) => {
setTokenResult({
error: true,
errorMessage: error
});
}
});
useImperativeHandle(ref, () => {
return {
viewCode() {
@@ -94,6 +135,7 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
const handleStopConversation = () => {
cancelRequest();
streamSTT.abort();
setLoading(false);
};
@@ -115,40 +157,21 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
type: audioData.type
})
};
const result: any = await speechToText(
{
data: params
},
{
cancelToken: getCanceltToken()
}
);
if (
(result?.status_code && result?.status_code !== 200) ||
result?.error
) {
setTokenResult({
error: true,
errorMessage: extractErrorMessage(result)
});
return;
// Choose stream or non-stream based on parameters
if (parameters.stream) {
// Stream mode: text will be updated in real-time
await streamSTT.generate(params);
} else {
// Non-stream mode: get complete text at once
await nonStreamSTT.generate(params, getCanceltToken());
}
setMessageList([
{
content: result.text,
uid: messageId.current
}
]);
} catch (error: any) {
console.log('error:', error);
const res = error?.response?.data;
if (res?.error || (res?.status_code && res?.status_code !== 200)) {
setTokenResult({
error: true,
errorMessage: extractErrorMessage(res)
});
}
setTokenResult({
error: true,
errorMessage: error?.message || 'Unknown error'
});
} finally {
setLoading(false);
setIsRecording(false);
@@ -202,7 +225,7 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
);
const handleOnAnalyse = useCallback((data: any, analyser: any) => {
setAudioChunks((pre: any) => {
setAudioChunks(() => {
return {
data: data,
analyser: analyser