import { setRouteCache } from '@/atoms/route-cache'; import AlertInfo from '@/components/alert-info'; import IconFont from '@/components/icon-font'; import AutoComplete from '@/components/seal-form/auto-complete'; import FieldComponent from '@/components/seal-form/field-component'; import SealSelect from '@/components/seal-form/seal-select'; import SpeechContent from '@/components/speech-content'; import routeCachekey from '@/config/route-cachekey'; import useOverlayScroller from '@/hooks/use-overlay-scroller'; import CollapsePanel from '@/pages/_components/collapse-panel'; import { getLocale, useIntl, useSearchParams } from '@umijs/max'; import { Form, Spin } from 'antd'; import classNames from 'classnames'; import _ from 'lodash'; import 'overlayscrollbars/overlayscrollbars.css'; import React, { forwardRef, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; import { AUDIO_TEXT_TO_SPEECH_API, CHAT_API, textToSpeech } from '../apis'; import { RefAudioFormItem } from '../audio/form'; import { TTSParamsConfig as paramsConfig, TTSAdvancedParamsConfig } from '../audio/params-config'; import { extractErrorMessage } from '../config'; import { MessageItem, ParamsSchema } from '../config/types'; import '../style/ground-llm.less'; import '../style/system-message-wrap.less'; import { TextToSpeechCode } from '../view-code/audio'; import DynamicParams from './dynamic-params'; import MessageInput from './message-input'; import ViewCommonCode from './view-common-code'; const MetaFields = [ 'task_type', 'language', 'instructions', 'max_new_tokens', 'ref_audio' ]; interface MessageProps { modelList: Global.BaseOption[]; loaded?: boolean; ref?: any; } const GroundTTS: React.FC = forwardRef((props, ref) => { const { modelList } = props; const messageId = useRef(0); const [messageList, setMessageList] = useState< { input: string; voice: string; format: string; speed: number; uid: number; autoplay: boolean; audioUrl: string; }[] >([]); const locale = getLocale(); const intl = useIntl(); const [searchParams] = useSearchParams(); const modelType = searchParams.get('type') || ''; const selectModel = searchParams.get('model') ? modelType === 'tts' && searchParams.get('model') : ''; const [parameters, setParams] = useState({ model: selectModel, voice: '', response_format: 'mp3' }); const [show, setShow] = useState(false); const [loading, setLoading] = useState(false); const [tokenResult, setTokenResult] = useState(null); const [collapse, setCollapse] = useState(false); const controllerRef = useRef(null); const scroller = useRef(null); const paramsRef = useRef(null); const checkvalueRef = useRef(true); const [currentPrompt, setCurrentPrompt] = useState(''); const [voiceDataList, setVoiceList] = useState[]>( [] ); const [modelMeta, setModelMeta] = useState({}); const formRef = useRef(null); const { initialize } = useOverlayScroller(); const { initialize: innitializeParams } = useOverlayScroller(); const [activeKey, setActiveKey] = useState( 'advanced_config' ); useImperativeHandle(ref, () => { return { viewCode() { setShow(true); }, setCollapse() { setCollapse(!collapse); }, collapse: collapse }; }); const defaultModel = useMemo(() => { return selectModel || modelList[0]?.value || ''; }, [modelList]); const dropEmptyFields = (parameters: Record) => { const fields = [ 'task_type', 'instructions', 'max_new_tokens', 'ref_audio', 'ref_text', 'language', 'x_vector_only_mode' ]; const newParams = { ...parameters }; return _.omitBy(newParams, (value: any, key: string) => { return fields.includes(key) && !value; }); }; const viewCodeContent = useMemo(() => { return TextToSpeechCode({ api: AUDIO_TEXT_TO_SPEECH_API, parameters: { ...dropEmptyFields(parameters), input: currentPrompt } }); }, [parameters, currentPrompt]); const sortVoiceList = useCallback( (locale: string, voiceDataList: Global.BaseOption[]) => { const lang = locale === 'en-US' ? 'english' : 'chinese'; const list = voiceDataList.sort((a, b) => { const aContains = a.value.toLowerCase().includes(lang) ? 1 : 0; const bContains = b.value.toLowerCase().includes(lang) ? 1 : 0; return bContains - aContains; }); return list; }, [] ); const voiceList = useMemo(() => { if (!voiceDataList.length) return []; const newList = sortVoiceList(locale, voiceDataList); return newList; }, [locale, voiceDataList, sortVoiceList]); useEffect(() => { const newList = sortVoiceList(locale, voiceDataList); setParams((pre: any) => { return { ...pre, voice: newList[0]?.value }; }); formRef.current?.form.setFieldValue('voice', newList[0]?.value); }, [locale, voiceDataList, sortVoiceList]); const setMessageId = () => { messageId.current = messageId.current + 1; }; const handleStopConversation = () => { controllerRef.current?.abort?.(); setLoading(false); }; const handleInputChange = (e: any) => { setCurrentPrompt(e.target.value); }; const submitMessage = async (current?: { role: string; content: string }) => { try { await formRef.current?.form.validateFields(); if (!parameters.model) return; setLoading(true); setMessageId(); setTokenResult(null); setCurrentPrompt(current?.content || ''); setMessageList([]); setRouteCache(routeCachekey['/playground/speech'], true); controllerRef.current?.abort?.(); controllerRef.current = new AbortController(); const signal = controllerRef.current.signal; const params = { ...dropEmptyFields(parameters), input: current?.content || currentPrompt }; const res: any = await textToSpeech({ data: params, url: CHAT_API, signal }); setParams(params); console.log('result:', res); if ((res?.status_code && res?.status_code !== 200) || res?.error) { setTokenResult({ error: true, errorMessage: extractErrorMessage(res) }); setMessageList([]); return; } setMessageList([ { input: current?.content || currentPrompt, voice: parameters.voice, format: parameters.response_format, speed: parameters.speed, uid: messageId.current, autoplay: checkvalueRef.current, audioUrl: res.url } ]); } catch (error: any) { const res = error?.response?.data; console.log('error:', error); if (res?.error) { setTokenResult({ error: true, errorMessage: extractErrorMessage(res) }); } } finally { setLoading(false); setRouteCache(routeCachekey['/playground/speech'], false); } }; const handleClear = () => { setMessageId(); setMessageList([]); setTokenResult(null); }; const handleSendMessage = (message: Omit) => { submitMessage(message); }; const handleCloseViewCode = () => { setShow(false); }; const handleSelectModel = async (value: string) => { if (!value) { return; } const model = modelList.find((item) => item.value === value); const list = _.map(model?.meta?.voices || [], (item: any) => { return { label: item, value: item }; }); const newList = sortVoiceList(locale, list); setVoiceList(newList); setModelMeta(model?.meta || {}); setParams((pre: any) => { return { ...pre, ..._.pick(model?.meta || {}, MetaFields), task_type: model?.meta?.task_type, model: value, language: model?.meta?.languages?.[0] || '', voice: newList[0]?.value }; }); }; const handleOnValuesChange = useCallback( (changeValues: Record, allValues: Record) => { if (changeValues.model) { handleSelectModel(changeValues.model); } else { setParams(allValues); } }, [handleSelectModel] ); useEffect(() => { if (paramsRef.current) { innitializeParams(paramsRef.current); } }, [innitializeParams]); const handleOnCheckChange = (e: any) => { checkvalueRef.current = e.target.checked; }; const handleOnCollapse = (keys: string | string[]) => { setActiveKey(keys); }; const renderAdvancedFields = () => { const formItems = TTSAdvancedParamsConfig.map((item: ParamsSchema) => { const comProps = { ...item.attrs, label: item.label.isLocalized ? intl.formatMessage({ id: item.label.text }) : item.label.text }; return ( <> ); }); return ( {formItems} ) } ]} > ); }; const renderExtra = () => { return paramsConfig.map((item: ParamsSchema) => { const comProps = { ...item.attrs, options: item.name === 'voice' ? voiceList : item.options, label: item.label.isLocalized ? intl.formatMessage({ id: item.label.text }) : item.label.text }; return ( <> {item.type === 'AutoComplete' ? ( ) : ( )} ); }); }; useEffect(() => { if (defaultModel && modelList.length) { handleSelectModel(defaultModel); } }, [defaultModel, modelList.length]); useEffect(() => { if (scroller.current) { initialize(scroller.current); } }, [initialize]); useEffect(() => { if (paramsRef.current) { innitializeParams(paramsRef.current); } }, [innitializeParams]); return (
{messageList.length ? ( ) : (
{intl.formatMessage({ id: 'playground.audio.texttospeech.tips' })}
)} {loading && (
)}
{tokenResult && (
)}
{renderExtra()} {renderAdvancedFields()} ]} />
); }); export default GroundTTS;