import { setRouteCache } from '@/atoms/route-cache'; import AlertInfo from '@/components/alert-info'; import AudioAnimation from '@/components/audio-animation'; import AudioPlayer from '@/components/audio-player'; import CopyButton from '@/components/copy-button'; import IconFont from '@/components/icon-font'; import UploadAudio from '@/components/upload-audio'; import routeCachekey from '@/config/route-cachekey'; import useOverlayScroller from '@/hooks/use-overlay-scroller'; import { useCancelToken } from '@/hooks/use-request-token'; import { readAudioFile } from '@/utils/load-audio-file'; import { SendOutlined } from '@ant-design/icons'; import { useIntl, useSearchParams } from '@umijs/max'; import { Button, Spin, Tooltip } from 'antd'; import classNames from 'classnames'; import 'overlayscrollbars/overlayscrollbars.css'; import React, { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; import { AUDIO_SPEECH_TO_TEXT_API, speechToText } from '../apis'; import { SpeechToTextFormat, extractErrorMessage } from '../config'; import { RealtimeParamsConfig as paramsConfig } from '../config/params-config'; import '../style/ground-left.less'; import '../style/speech-to-text.less'; import '../style/system-message-wrap.less'; import { speechToTextCode } from '../view-code/audio'; import AudioInput from './audio-input'; import DynamicParams from './dynamic-params'; import ViewCommonCode from './view-common-code'; interface MessageProps { modelList: Global.BaseOption[]; ref?: any; } const GroundSTT: React.FC = forwardRef((props, ref) => { const intl = useIntl(); const { modelList } = props; const messageId = useRef(0); const [messageList, setMessageList] = useState< { uid: number; content: string }[] >([]); const [searchParams] = useSearchParams(); const modelType = searchParams.get('type') || ''; const selectModel = searchParams.get('model') ? modelType === 'stt' && searchParams.get('model') : ''; const defaultModel = selectModel || modelList[0]?.value || ''; const [parameters, setParams] = useState({ model: defaultModel, language: 'auto' }); const [show, setShow] = useState(false); const [loading, setLoading] = useState(false); const [tokenResult, setTokenResult] = useState(null); const [collapse, setCollapse] = useState(false); const scroller = useRef(null); const paramsRef = useRef(null); const [audioPermissionOn, setAudioPermissionOn] = useState(true); const [audioData, setAudioData] = useState(null); const [audioChunks, setAudioChunks] = useState({ data: [], analyser: null }); const [isRecording, setIsRecording] = useState(false); const formRef = useRef(null); const { updateCancelToken, getCanceltToken, cancelRequest } = useCancelToken(); const { initialize, updateScrollerPosition } = useOverlayScroller(); const { initialize: innitializeParams } = useOverlayScroller(); useImperativeHandle(ref, () => { return { viewCode() { setShow(true); }, setCollapse() { setCollapse(!collapse); }, collapse: collapse }; }); const setMessageId = () => { messageId.current = messageId.current + 1; }; const viewCodeContent = useMemo(() => { return speechToTextCode({ api: AUDIO_SPEECH_TO_TEXT_API, parameters: { ...parameters } }); }, [parameters]); const handleStopConversation = () => { cancelRequest(); setLoading(false); }; const submitMessage = async () => { try { await formRef.current?.form.validateFields(); if (!parameters.model) return; setLoading(true); setMessageId(); setTokenResult(null); setMessageList([]); cancelRequest(); updateCancelToken(); setRouteCache(routeCachekey['/playground/speech'], true); const params = { ...parameters, file: new File([audioData.data], audioData.name, { type: audioData.type }) }; const result: any = await speechToText( { data: params }, { cancelToken: getCanceltToken() } ); if ( (result?.status_code && result?.status_code !== 200) || result?.error ) { setTokenResult({ error: true, errorMessage: extractErrorMessage(result) }); return; } setMessageList([ { content: result.text, uid: messageId.current } ]); } catch (error: any) { console.log('error:', error); const res = error?.response?.data; if (res?.error || (res?.status_code && res?.status_code !== 200)) { setTokenResult({ error: true, errorMessage: extractErrorMessage(res) }); } } finally { setLoading(false); setIsRecording(false); setRouteCache(routeCachekey['/playground/speech'], false); } }; const handleClear = () => { setMessageId(); setMessageList([]); setTokenResult(null); }; const handleCloseViewCode = () => { setShow(false); }; const handleOnAudioData = useCallback( (data: { chunks: Blob[]; url: string; name: string; duration: number; type: string; }) => { setAudioData(() => { return { url: data.url, name: data.name, data: data.chunks, type: data.type, duration: data.duration }; }); }, [] ); const handleOnAudioPermission = useCallback((permission: boolean) => { setAudioPermissionOn(permission); }, []); const handleUploadChange = useCallback( async (data: { file: any; fileList: any }) => { const res = await readAudioFile(data.file); setAudioData(res); setTokenResult(null); }, [] ); const handleOnAnalyse = useCallback((data: any, analyser: any) => { setAudioChunks((pre: any) => { return { data: data, analyser: analyser }; }); }, []); const handleOnRecord = useCallback((val: boolean) => { setIsRecording(val); setAudioData(null); setTokenResult(null); setMessageList([]); }, []); const handleOnGenerate = async () => { if (loading) { handleStopConversation(); return; } submitMessage(); }; const renderAniamtion = () => { if (!audioPermissionOn) { return (
{intl.formatMessage({ id: 'playground.audio.enablemic' })}
); } if (isRecording) { return ( ); } return (
{intl.formatMessage({ id: 'playground.audio.speechtotext.tips' })}
); }; const handleOnValuesChange = (changedValues: any, allValues: any) => { setParams(allValues); }; useEffect(() => { if (scroller.current) { initialize(scroller.current); } }, [initialize]); useEffect(() => { if (paramsRef.current) { innitializeParams(paramsRef.current); } }, [innitializeParams]); useEffect(() => { if (loading) { updateScrollerPosition(); } }, [messageList, loading]); return (
{!isRecording && ( )}
{audioData ? (
{ } } >
) : ( renderAniamtion() )}
{messageList?.length > 0 && ( )}
{!tokenResult && (
{messageList.length ? ( messageList[0]?.content ) : ( {intl.formatMessage({ id: 'playground.audio.generating.tips' })} )}
)} {tokenResult && (
)}
{loading && (
)}
); }); export default GroundSTT;