diff --git a/src/locales/en-US/playground.ts b/src/locales/en-US/playground.ts index 31fd04bf..f0312ec9 100644 --- a/src/locales/en-US/playground.ts +++ b/src/locales/en-US/playground.ts @@ -173,5 +173,7 @@ export default { 'Enter a reference audio URL, or upload an audio file.', 'playground.params.refAudio.text': 'Transcript of Reference Audio (for ICL mode)', - 'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)' + 'playground.params.refAudio.vectorMode': + 'Use Speaker Embedding Only (no ICL)', + 'playground.params.streamMode': 'Enable Streaming' }; diff --git a/src/locales/ja-JP/playground.ts b/src/locales/ja-JP/playground.ts index e31d1d07..e5c3ff3c 100644 --- a/src/locales/ja-JP/playground.ts +++ b/src/locales/ja-JP/playground.ts @@ -176,7 +176,9 @@ export default { 'Enter a reference audio URL, or upload an audio file.', 'playground.params.refAudio.text': 'Transcript of Reference Audio (for ICL mode)', - 'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)' + 'playground.params.refAudio.vectorMode': + 'Use Speaker Embedding Only (no ICL)', + 'playground.params.streamMode': 'Enable Streaming' }; // ========== To-Do: Translate Keys (Remove After Translation) ========== @@ -188,4 +190,5 @@ export default { // 6. 'playground.uploadImage.url.button': 'Add Image from URL', // 7. 'playground.params.duration': 'Duration (seconds)', // 8. 'playground.params.resolution': 'Resolution' +// 9. 'playground.params.streamMode': 'Enable Streaming' // ========== End of To-Do List ========== diff --git a/src/locales/ru-RU/playground.ts b/src/locales/ru-RU/playground.ts index 6200fd44..54146ca6 100644 --- a/src/locales/ru-RU/playground.ts +++ b/src/locales/ru-RU/playground.ts @@ -170,7 +170,9 @@ export default { 'Enter a reference audio URL, or upload an audio file.', 'playground.params.refAudio.text': 'Transcript of Reference Audio (for ICL mode)', - 'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)' + 'playground.params.refAudio.vectorMode': + 'Use Speaker Embedding Only (no ICL)', + 'playground.params.streamMode': 'Enable Streaming' }; // ========== To-Do: Translate Keys (Remove After Translation) ========== @@ -178,5 +180,6 @@ export default { // 2. 'playground.uploadImage.url.holder': 'Enter an image URL', // 3. 'playground.uploadImage.url.button': 'Add Image from URL', // 4. 'playground.params.duration': 'Duration (seconds)', -// 5. 'playground.params.resolution': 'Resolution' +// 5. 'playground.params.resolution': 'Resolution', +// 6. 'playground.params.streamMode': 'Enable Streaming' // ========== End of To-Do List ========== diff --git a/src/locales/zh-CN/playground.ts b/src/locales/zh-CN/playground.ts index a78a94d4..95bca29b 100644 --- a/src/locales/zh-CN/playground.ts +++ b/src/locales/zh-CN/playground.ts @@ -166,5 +166,6 @@ export default { '输入一个参考音频的 URL，或上传一个音频文件。', 'playground.params.refAudio.text': '参考音频文本（用于上下文学习模式）', 'playground.params.refAudio.vectorMode': - '仅使用说话人嵌入（不使用上下文学习）' + '仅使用说话人嵌入（不使用上下文学习）', + 'playground.params.streamMode': '启用流式输出' }; diff --git a/src/pages/playground/speech/forms/stt-form.tsx b/src/pages/playground/speech/forms/stt-form.tsx index 83c510bf..dacba76f 100644 --- a/src/pages/playground/speech/forms/stt-form.tsx +++ b/src/pages/playground/speech/forms/stt-form.tsx @@ -1,3 +1,4 @@ +import CheckboxField from '@/components/seal-form/checkbox-field'; import SealSelect from '@/components/seal-form/seal-select'; import { useIntl, useSearchParams } from '@umijs/max'; import { Form } from 'antd'; @@ -123,6 +124,17 @@ const STTForm: React.FC = forwardRef( options={languageOptions} > + + + ); diff --git a/src/pages/playground/speech/forms/tts-advance.tsx b/src/pages/playground/speech/forms/tts-advance.tsx index 081a6501..b1fb2cb3 100644 --- a/src/pages/playground/speech/forms/tts-advance.tsx +++ b/src/pages/playground/speech/forms/tts-advance.tsx @@ -110,20 +110,35 @@ const TTSAdvanceConfig: React.FC = () => { > - + - + + + + ); }; diff --git a/src/pages/playground/speech/hooks/index.ts b/src/pages/playground/speech/hooks/index.ts index 361c5ec9..5909809b 100644 --- a/src/pages/playground/speech/hooks/index.ts +++ b/src/pages/playground/speech/hooks/index.ts @@ -1,2 +1,4 @@ -export { useNonStreamTTS } from './useNonStreamTTS'; -export { useStreamTTS } from './useStreamTTS'; +export { useNonStreamSTT } from './use-non-stream-stt'; +export { useNonStreamTTS } from './use-non-stream-tts'; +export { useStreamSTT } from './use-stream-stt'; +export { useStreamTTS } from './use-stream-tts'; diff --git a/src/pages/playground/speech/hooks/use-non-stream-stt.ts b/src/pages/playground/speech/hooks/use-non-stream-stt.ts new file mode 100644 index 00000000..7db28061 --- /dev/null +++ b/src/pages/playground/speech/hooks/use-non-stream-stt.ts @@ -0,0 +1,77 @@ +import { useCallback, useRef, useState } from 'react'; +import { speechToText } from '../../apis'; +import { extractErrorMessage } from '../../config'; + +interface UseNonStreamSTTParams { + onSuccess?: (result: { text: string }) => void; + onError?: (error: any) => void; +} + +interface STTParams { + model: string; + language?: string; + file: File; + [key: string]: any; +} + +export const useNonStreamSTT = (params?: UseNonStreamSTTParams) => { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const cancelTokenRef = useRef(null); + + const generate = useCallback( + async (sttParams: STTParams, cancelToken?: any) => { + try { + setLoading(true); + setError(null); + + cancelTokenRef.current = cancelToken; + + const result: any = await speechToText( + { + data: sttParams + }, + { + cancelToken + } + ); + + if ( + (result?.status_code && result?.status_code !== 200) || + result?.error + ) { + const errorMessage = extractErrorMessage(result); + setError({ + error: true, + errorMessage + }); + params?.onError?.(errorMessage); + return null; + } + + params?.onSuccess?.(result); + return result; + } catch (err: any) { + const res = err?.response?.data; + if (res?.error || (res?.status_code && res?.status_code !== 200)) { + const errorMessage = extractErrorMessage(res); + setError({ + error: true, + errorMessage + }); + params?.onError?.(errorMessage); + } + return null; + } finally { + setLoading(false); + } + }, + [params] + ); + + return { + generate, + loading, + error + }; +}; diff --git a/src/pages/playground/speech/hooks/useNonStreamTTS.ts b/src/pages/playground/speech/hooks/use-non-stream-tts.ts similarity index 100% rename from src/pages/playground/speech/hooks/useNonStreamTTS.ts rename to src/pages/playground/speech/hooks/use-non-stream-tts.ts diff --git a/src/pages/playground/speech/hooks/use-stream-stt.ts b/src/pages/playground/speech/hooks/use-stream-stt.ts new file mode 100644 index 00000000..af2552fa --- /dev/null +++ b/src/pages/playground/speech/hooks/use-stream-stt.ts @@ -0,0 +1,131 @@ +import { + fetchChunkedDataPostFormData, + readStreamData +} from '@/utils/fetch-chunk-data'; +import { useCallback, useRef, useState } from 'react'; +import { AUDIO_SPEECH_TO_TEXT_API } from '../../apis'; +import { extractErrorMessage } from '../../config'; + +interface UseStreamSTTParams { + onChunk?: (text: string) => void; + onComplete?: (fullText: string) => void; + onError?: (error: any) => void; +} + +interface STTParams { + model: string; + language?: string; + file: File; + stream?: boolean; + [key: string]: any; +} + +export const useStreamSTT = (params?: UseStreamSTTParams) => { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [progress, setProgress] = useState(0); + const controllerRef = useRef(null); + + const generate = useCallback( + async (sttParams: STTParams) => { + try { + setLoading(true); + setError(null); + setProgress(0); + + // Abort previous request if exists + controllerRef.current?.abort(); + controllerRef.current = new AbortController(); + const signal = controllerRef.current.signal; + + // Add stream parameter + const streamParams = { + ...sttParams, + stream: true + }; + + const result = await fetchChunkedDataPostFormData({ + url: AUDIO_SPEECH_TO_TEXT_API, + data: streamParams, + signal + }); + + if ('error' in result) { + const errorMessage = extractErrorMessage(result.data); + setError({ + error: true, + errorMessage + }); + params?.onError?.(errorMessage); + return; + } + + const { reader, decoder } = result; + + if (!reader || !decoder) { + throw new Error('Failed to get reader from response'); + } + + // Collect all text chunks + let fullText = ''; + + await readStreamData( + reader, + decoder, + (chunks: any[]) => { + chunks.forEach((chunk) => { + if (chunk.error) { + const errorMessage = extractErrorMessage(chunk.error); + setError({ + error: true, + errorMessage + }); + params?.onError?.(errorMessage); + return; + } + + // STT stream response format: { text: "..." } + if (chunk.text) { + fullText += chunk.text; + params?.onChunk?.(fullText); + setProgress((prev) => prev + 1); + } + }); + }, + 100 // throttle delay + ); + + // Stream completed + params?.onComplete?.(fullText); + } catch (err: any) { + if (err.name === 'AbortError') { + console.log('Stream aborted'); + return; + } + + const errorMessage = err?.message || 'Stream processing failed'; + setError({ + error: true, + errorMessage + }); + params?.onError?.(errorMessage); + } finally { + setLoading(false); + } + }, + [params] + ); + + const abort = useCallback(() => { + controllerRef.current?.abort(); + setLoading(false); + }, []); + + return { + generate, + abort, + loading, + error, + progress + }; +}; diff --git a/src/pages/playground/speech/hooks/useStreamTTS.ts b/src/pages/playground/speech/hooks/use-stream-tts.ts similarity index 100% rename from src/pages/playground/speech/hooks/useStreamTTS.ts rename to src/pages/playground/speech/hooks/use-stream-tts.ts diff --git a/src/pages/playground/speech/stt.tsx b/src/pages/playground/speech/stt.tsx index a4659ea6..31fa5180 100644 --- a/src/pages/playground/speech/stt.tsx +++ b/src/pages/playground/speech/stt.tsx @@ -22,16 +22,17 @@ import React, { useRef, useState } from 'react'; -import { AUDIO_SPEECH_TO_TEXT_API, speechToText } from '../apis'; +import { AUDIO_SPEECH_TO_TEXT_API } from '../apis'; import AudioInput from '../components/audio-input'; import RightContainer from '../components/right-container'; import ViewCommonCode from '../components/view-common-code'; -import { SpeechToTextFormat, extractErrorMessage } from '../config'; +import { SpeechToTextFormat } from '../config'; import '../style/ground-llm.less'; import '../style/speech-to-text.less'; import '../style/system-message-wrap.less'; import { speechToTextCode } from '../view-code/audio'; import STTForm from './forms/stt-form'; +import { useNonStreamSTT, useStreamSTT } from './hooks'; interface MessageProps { modelList: Global.BaseOption[]; @@ -67,6 +68,46 @@ const GroundSTT: React.FC = forwardRef((props, ref) => { const { initialize, updateScrollerPosition } = useOverlayScroller(); + // Initialize non-stream STT hook + const nonStreamSTT = useNonStreamSTT({ + onSuccess: (result) => { + setMessageList([ + { + content: result.text, + uid: messageId.current + } + ]); + }, + onError: (error) => { + setTokenResult({ + error: true, + errorMessage: error + }); + } + }); + + // Initialize stream STT hook + const streamSTT = useStreamSTT({ + onChunk: (text) => { + // Update message list with streaming text + setMessageList([ + { + content: text, + uid: messageId.current + } + ]); + }, + onComplete: () => { + console.log('Stream transcription completed'); + }, + onError: (error) => { + setTokenResult({ + error: true, + errorMessage: error + }); + } + }); + useImperativeHandle(ref, () => { return { viewCode() { @@ -94,6 +135,7 @@ const GroundSTT: React.FC = forwardRef((props, ref) => { const handleStopConversation = () => { cancelRequest(); + streamSTT.abort(); setLoading(false); }; @@ -115,40 +157,21 @@ const GroundSTT: React.FC = forwardRef((props, ref) => { type: audioData.type }) }; - const result: any = await speechToText( - { - data: params - }, - { - cancelToken: getCanceltToken() - } - ); - if ( - (result?.status_code && result?.status_code !== 200) || - result?.error - ) { - setTokenResult({ - error: true, - errorMessage: extractErrorMessage(result) - }); - return; + // Choose stream or non-stream based on parameters + if (parameters.stream) { + // Stream mode: text will be updated in real-time + await streamSTT.generate(params); + } else { + // Non-stream mode: get complete text at once + await nonStreamSTT.generate(params, getCanceltToken()); } - setMessageList([ - { - content: result.text, - uid: messageId.current - } - ]); } catch (error: any) { console.log('error:', error); - const res = error?.response?.data; - if (res?.error || (res?.status_code && res?.status_code !== 200)) { - setTokenResult({ - error: true, - errorMessage: extractErrorMessage(res) - }); - } + setTokenResult({ + error: true, + errorMessage: error?.message || 'Unknown error' + }); } finally { setLoading(false); setIsRecording(false); @@ -202,7 +225,7 @@ const GroundSTT: React.FC = forwardRef((props, ref) => { ); const handleOnAnalyse = useCallback((data: any, analyser: any) => { - setAudioChunks((pre: any) => { + setAudioChunks(() => { return { data: data, analyser: analyser