chore: tts, stt meta

This commit is contained in:
jialin
2026-02-05 20:35:07 +08:00
parent a1a7979fb3
commit 2765d5edeb
11 changed files with 422 additions and 13 deletions
+3 -2
View File
@@ -166,10 +166,11 @@ export default {
'playground.params.resolution': 'Resolution',
'playground.params.taskType': 'Task Type',
'playground.params.instructions': 'Instructions',
'playground.params.instructions.tips': 'Voice style/emotion instructions',
'playground.params.maxTokens': 'Max New Tokens',
'playground.params.refAudio': 'Reference Audio',
'playground.params.refAudio.tips':
'Enter a reference audio URL, or upload an audio file.',
'playground.params.refAudio.text': 'Transcript of Reference Audio',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding'
'playground.params.refAudio.text': 'Transcript of Reference Audio (no ICL)',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)'
};
+3 -2
View File
@@ -169,12 +169,13 @@ export default {
'playground.params.resolution': 'Resolution',
'playground.params.taskType': 'Task Type',
'playground.params.instructions': 'Instructions',
'playground.params.instructions.tips': 'Voice style/emotion instructions',
'playground.params.maxTokens': 'Max New Tokens',
'playground.params.refAudio': 'Reference Audio',
'playground.params.refAudio.tips':
'Enter a reference audio URL, or upload an audio file.',
'playground.params.refAudio.text': 'Transcript of Reference Audio',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding'
'playground.params.refAudio.text': 'Transcript of Reference Audio (no ICL)',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
+3 -2
View File
@@ -163,12 +163,13 @@ export default {
'playground.params.resolution': 'Resolution',
'playground.params.taskType': 'Task Type',
'playground.params.instructions': 'Instructions',
'playground.params.instructions.tips': 'Voice style/emotion instructions',
'playground.params.maxTokens': 'Max New Tokens',
'playground.params.refAudio': 'Reference Audio',
'playground.params.refAudio.tips':
'Enter a reference audio URL, or upload an audio file.',
'playground.params.refAudio.text': 'Transcript of Reference Audio',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding'
'playground.params.refAudio.text': 'Transcript of Reference Audio (no ICL)',
'playground.params.refAudio.vectorMode': 'Use Speaker Embedding Only (no ICL)'
};
// ========== To-Do: Translate Keys (Remove After Translation) ==========
+4 -2
View File
@@ -159,10 +159,12 @@ export default {
'playground.params.resolution': '分辨率',
'playground.params.taskType': '任务类型',
'playground.params.instructions': '说明',
'playground.params.instructions.tips': '语音风格/情感说明',
'playground.params.maxTokens': '最大生成 Token 数',
'playground.params.refAudio': '参考音频',
'playground.params.refAudio.tips':
'输入一个参考音频的 URL,或上传一个音频文件。',
'playground.params.refAudio.text': '参考音频文本',
'playground.params.refAudio.vectorMode': '使用扬声器嵌入'
'playground.params.refAudio.text': '参考音频文本(用于上下文学习模式)',
'playground.params.refAudio.vectorMode':
'仅使用说话人嵌入(不使用上下文学习)'
};
@@ -25,7 +25,8 @@ export default function useCheckBackend() {
}) => {
const { isAudio, gpuOptions, defaultBackend } = data;
if (isAudio) {
return backendOptionsMap.voxBox;
// audio model use vllm backend by default
return backendOptionsMap.vllm;
}
if (checkOnlyAscendNPU(gpuOptions)) {
+11
View File
@@ -1,6 +1,7 @@
import CheckboxField from '@/components/seal-form/checkbox-field';
import SealInput from '@/components/seal-form/seal-input';
import UploadAudio from '@/components/upload-audio';
import useAppUtils from '@/hooks/use-app-utils';
import { convertFileToBase64 } from '@/utils/load-audio-file';
import { CloseCircleFilled } from '@ant-design/icons';
import { useIntl } from '@umijs/max';
@@ -36,10 +37,12 @@ const Container = styled.div`
`;
export const RefAudioFormItem: React.FC = () => {
const { getRuleMessage } = useAppUtils();
const { meta, onValuesChange } = useFormContext();
const form = Form.useFormInstance();
const intl = useIntl();
const [fileName, setFileName] = React.useState<string>('');
const taskType = Form.useWatch('task_type', form);
// handle upload audio file, transfer to a base64 url
const handleUploadChange = async (data: { file: any; fileList: any }) => {
@@ -73,10 +76,18 @@ export const RefAudioFormItem: React.FC = () => {
<Form.Item
name="ref_audio"
getValueProps={(value) => ({ value: fileName ? fileName : value })}
rules={[
{
required: taskType === 'Base',
message: getRuleMessage('input', 'playground.params.refAudio')
}
]}
dependencies={['task_type']}
>
<SealInput.Input
allowClear
readOnly={!!fileName}
required={taskType === 'Base'}
suffix={
<SuffixWrapper>
{fileName ? (
@@ -51,6 +51,10 @@ export const TTSAdvancedParamsConfig: ParamsSchema[] = [
text: 'playground.params.instructions',
isLocalized: true
},
description: {
text: 'playground.params.instructions.tips',
isLocalized: true
},
attrs: {
allowClear: true
},
+55 -3
View File
@@ -14,6 +14,7 @@ import { SendOutlined } from '@ant-design/icons';
import { useIntl, useSearchParams } from '@umijs/max';
import { Button, Spin, Tooltip } from 'antd';
import classNames from 'classnames';
import _ from 'lodash';
import React, {
forwardRef,
useCallback,
@@ -24,8 +25,14 @@ import React, {
useState
} from 'react';
import { AUDIO_SPEECH_TO_TEXT_API, speechToText } from '../apis';
import { SpeechToTextFormat, extractErrorMessage } from '../config';
import {
SpeechToTextFormat,
defaultLanguages,
extractErrorMessage
} from '../config';
import { allLanguages } from '../config/languages';
import { RealtimeParamsConfig as paramsConfig } from '../config/params-config';
import { ParamsSchema } from '../config/types';
import '../style/ground-llm.less';
import '../style/speech-to-text.less';
import '../style/system-message-wrap.less';
@@ -75,6 +82,9 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
const { initialize, updateScrollerPosition } = useOverlayScroller();
const { initialize: innitializeParams } = useOverlayScroller();
const [modelMeta, setModelMeta] = useState<any>(null);
const [fieldsConfig, setFieldsConfig] =
useState<ParamsSchema[]>(paramsConfig);
useImperativeHandle(ref, () => {
return {
@@ -265,8 +275,45 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
);
};
const handleSelectModel = (model: string) => {
if (!model) return;
const selected = modelList.find((item) => item.value === model);
setModelMeta(selected?.meta || {});
const languages = selected?.meta?.languages || [];
let currentLanguage = [...defaultLanguages];
if (languages.length > 0) {
currentLanguage = allLanguages.filter(
(lang) =>
languages.includes(lang.value) || languages.includes(lang.label)
);
const newConfig = paramsConfig.map((item) => {
const oItem = _.cloneDeep(item);
if (item.name === 'language') {
return {
...oItem,
options: currentLanguage
};
}
return oItem;
});
setFieldsConfig(newConfig);
}
setParams((pre: any) => {
return {
...pre,
language:
selected?.meta?.language || currentLanguage[0]?.value || 'auto',
model: model
};
});
};
const handleOnValuesChange = (changedValues: any, allValues: any) => {
setParams(allValues);
if (changedValues.model) {
handleSelectModel(changedValues.model);
} else {
setParams(allValues);
}
};
useEffect(() => {
@@ -287,6 +334,11 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
}
}, [messageList, loading]);
useEffect(() => {
const defaultModel = selectModel || modelList[0]?.value || '';
handleSelectModel(defaultModel);
}, [modelList, selectModel]);
return (
<div
className="ground-left-wrapper"
@@ -447,7 +499,7 @@ const GroundSTT: React.FC<MessageProps> = forwardRef((props, ref) => {
<DynamicParams
ref={formRef}
onValuesChange={handleOnValuesChange}
paramsConfig={paramsConfig}
paramsConfig={fieldsConfig}
initialValues={parameters}
modelList={modelList}
/>
@@ -265,6 +265,7 @@ const GroundTTS: React.FC<MessageProps> = forwardRef((props, ref) => {
..._.pick(model?.meta || {}, MetaFields),
task_type: model?.meta?.task_type,
model: value,
language: model?.meta?.language?.[0] || '',
voice: newList[0]?.value
};
});
@@ -272,7 +273,6 @@ const GroundTTS: React.FC<MessageProps> = forwardRef((props, ref) => {
const handleOnValuesChange = useCallback(
(changeValues: Record<string, any>, allValues: Record<string, any>) => {
console.log('handleOnValuesChange', changeValues, allValues);
if (changeValues.model) {
handleSelectModel(changeValues.model);
} else {
+9
View File
@@ -182,3 +182,12 @@ export const scaleImageSize = (size: { width: number; height: number }) => {
// png, jpg, jpeg
const acceptImageType = ['image/png', 'image/jpg', 'image/jpeg'];
export const EDIT_IMAGE_ACCEPT = acceptImageType.join(',');
export const defaultLanguages = [
{ label: 'Auto', value: 'auto' },
{ label: 'English', value: 'en' },
{ label: '中文', value: 'zh' },
{ label: '日本語', value: 'ja' },
{ label: 'Français', value: 'fr' },
{ label: 'Deutsch', value: 'de' }
];
+327
View File
@@ -0,0 +1,327 @@
export const allLanguages = [
{ label: 'Afaraf', value: 'aa' },
{ label: 'Afrikaans', value: 'af' },
{ label: 'Akan', value: 'ak' },
{ label: 'Shqip', value: 'sq' },
{ label: 'አማርኛ', value: 'am' },
{ label: 'العربية', value: 'ar' },
{ label: 'Aragonés', value: 'an' },
{ label: 'Հայերեն', value: 'hy' },
{ label: 'অসমীয়া', value: 'as' },
{ label: 'Avarca', value: 'av' },
{ label: 'Aymar aru', value: 'ay' },
{ label: 'Azərbaycan dili', value: 'az' },
{ label: 'Bashkir', value: 'ba' },
{ label: 'Български', value: 'bg' },
{ label: 'भोजपुरी', value: 'bh' },
{ label: 'Bislama', value: 'bi' },
{ label: 'বাংলা', value: 'bn' },
{ label: 'བོད་ཡིག', value: 'bo' },
{ label: 'Bosanski', value: 'bs' },
{ label: 'Brezhoneg', value: 'br' },
{ label: 'Català', value: 'ca' },
{ label: 'Čeština', value: 'cs' },
{ label: 'Чӑвашла', value: 'cv' },
{ label: 'Cymraeg', value: 'cy' },
{ label: 'Dansk', value: 'da' },
{ label: 'Deutsch', value: 'de' },
{ label: 'ދިވެހި', value: 'dv' },
{ label: 'Ελληνικά', value: 'el' },
{ label: 'English', value: 'en' },
{ label: 'Esperanto', value: 'eo' },
{ label: 'Español', value: 'es' },
{ label: 'Eesti', value: 'et' },
{ label: 'Euskara', value: 'eu' },
{ label: 'فارسی', value: 'fa' },
{ label: 'Suomi', value: 'fi' },
{ label: 'Fijian', value: 'fj' },
{ label: 'Føroyskt', value: 'fo' },
{ label: 'Français', value: 'fr' },
{ label: 'Frysk', value: 'fy' },
{ label: 'Gaeilge', value: 'ga' },
{ label: 'Gàidhlig', value: 'gd' },
{ label: 'Galego', value: 'gl' },
{ label: 'ગુજરાતી', value: 'gu' },
{ label: 'Hausa', value: 'ha' },
{ label: 'עברית', value: 'he' },
{ label: 'हिन्दी', value: 'hi' },
{ label: 'Hrvatski', value: 'hr' },
{ label: 'Magyar', value: 'hu' },
{ label: 'Հայերեն', value: 'hy' },
{ label: 'Bahasa Indonesia', value: 'id' },
{ label: 'Íslenska', value: 'is' },
{ label: 'Italiano', value: 'it' },
{ label: '日本語', value: 'ja' },
{ label: 'ქართული', value: 'ka' },
{ label: 'Қазақ тілі', value: 'kk' },
{ label: 'ភាសាខ្មែរ', value: 'km' },
{ label: '한국어', value: 'ko' },
{ label: 'Kurdî', value: 'ku' },
{ label: 'Кыргызча', value: 'ky' },
{ label: 'Latina', value: 'la' },
{ label: 'Lëtzebuergesch', value: 'lb' },
{ label: 'Lietuvių', value: 'lt' },
{ label: 'Latviešu', value: 'lv' },
{ label: 'Македонски', value: 'mk' },
{ label: 'മലയാളം', value: 'ml' },
{ label: 'Монгол', value: 'mn' },
{ label: 'मराठी', value: 'mr' },
{ label: 'Bahasa Melayu', value: 'ms' },
{ label: 'Malti', value: 'mt' },
{ label: 'မြန်မာဘာသာ', value: 'my' },
{ label: 'Norsk Bokmål', value: 'nb' },
{ label: 'नेपाली', value: 'ne' },
{ label: 'Nederlands', value: 'nl' },
{ label: 'Norsk Nynorsk', value: 'nn' },
{ label: 'Occitan', value: 'oc' },
{ label: 'ਪੰਜਾਬੀ', value: 'pa' },
{ label: 'Polski', value: 'pl' },
{ label: 'Português', value: 'pt' },
{ label: 'Română', value: 'ro' },
{ label: 'Русский', value: 'ru' },
{ label: 'संस्कृतम्', value: 'sa' },
{ label: 'Slovenčina', value: 'sk' },
{ label: 'Slovenščina', value: 'sl' },
{ label: 'Shqip', value: 'sq' },
{ label: 'Српски', value: 'sr' },
{ label: 'Svenska', value: 'sv' },
{ label: 'Kiswahili', value: 'sw' },
{ label: 'தமிழ்', value: 'ta' },
{ label: 'తెలుగు', value: 'te' },
{ label: 'ไทย', value: 'th' },
{ label: 'Türkçe', value: 'tr' },
{ label: 'Українська', value: 'uk' },
{ label: 'اردو', value: 'ur' },
{ label: 'Tiếng Việt', value: 'vi' },
{ label: '中文', value: 'zh' },
{ label: 'isiZulu', value: 'zu' }
];
export const languageList = [
{ label: 'Afar', value: 'aa' },
{ label: 'Abkhazian', value: 'ab' },
{ label: 'Avestan', value: 'ae' },
{ label: 'Afrikaans', value: 'af' },
{ label: 'Akan', value: 'ak' },
{ label: 'Amharic', value: 'am' },
{ label: 'Aragonese', value: 'an' },
{ label: 'Arabic', value: 'ar' },
{ label: 'Assamese', value: 'as' },
{ label: 'Avaric', value: 'av' },
{ label: 'Aymara', value: 'ay' },
{ label: 'Azerbaijani', value: 'az' },
{ label: 'Bashkir', value: 'ba' },
{ label: 'Belarusian', value: 'be' },
{ label: 'Bulgarian', value: 'bg' },
{ label: 'Bihari', value: 'bh' },
{ label: 'Bislama', value: 'bi' },
{ label: 'Bambara', value: 'bm' },
{ label: 'Bengali', value: 'bn' },
{ label: 'Tibetan', value: 'bo' },
{ label: 'Breton', value: 'br' },
{ label: 'Bosnian', value: 'bs' },
{ label: 'Catalan', value: 'ca' },
{ label: 'Chechen', value: 'ce' },
{ label: 'Chamorro', value: 'ch' },
{ label: 'Corsican', value: 'co' },
{ label: 'Cree', value: 'cr' },
{ label: 'Czech', value: 'cs' },
{ label: 'Church Slavic', value: 'cu' },
{ label: 'Chuvash', value: 'cv' },
{ label: 'Welsh', value: 'cy' },
{ label: 'Danish', value: 'da' },
{ label: 'German', value: 'de' },
{ label: 'Divehi', value: 'dv' },
{ label: 'Dzongkha', value: 'dz' },
{ label: 'Greek', value: 'el' },
{ label: 'English', value: 'en' },
{ label: 'Esperanto', value: 'eo' },
{ label: 'Spanish', value: 'es' },
{ label: 'Estonian', value: 'et' },
{ label: 'Basque', value: 'eu' },
{ label: 'Persian', value: 'fa' },
{ label: 'Fulah', value: 'ff' },
{ label: 'Finnish', value: 'fi' },
{ label: 'Fijian', value: 'fj' },
{ label: 'Faroese', value: 'fo' },
{ label: 'French', value: 'fr' },
{ label: 'Western Frisian', value: 'fy' },
{ label: 'Irish', value: 'ga' },
{ label: 'Scottish Gaelic', value: 'gd' },
{ label: 'Galician', value: 'gl' },
{ label: 'Guarani', value: 'gn' },
{ label: 'Gujarati', value: 'gu' },
{ label: 'Hausa', value: 'ha' },
{ label: 'Hebrew', value: 'he' },
{ label: 'Hindi', value: 'hi' },
{ label: 'Hiri Motu', value: 'ho' },
{ label: 'Croatian', value: 'hr' },
{ label: 'Haitian', value: 'ht' },
{ label: 'Hungarian', value: 'hu' },
{ label: 'Armenian', value: 'hy' },
{ label: 'Herero', value: 'hz' },
{ label: 'Interlingua', value: 'ia' },
{ label: 'Indonesian', value: 'id' },
{ label: 'Interlingue', value: 'ie' },
{ label: 'Igbo', value: 'ig' },
{ label: 'Inupiaq', value: 'ik' },
{ label: 'Ido', value: 'io' },
{ label: 'Icelandic', value: 'is' },
{ label: 'Italian', value: 'it' },
{ label: 'Inuktitut', value: 'iu' },
{ label: 'Japanese', value: 'ja' },
{ label: 'Javanese', value: 'jv' },
{ label: 'Georgian', value: 'ka' },
{ label: 'Kongo', value: 'kg' },
{ label: 'Kikuyu', value: 'ki' },
{ label: 'Kwanyama', value: 'kj' },
{ label: 'Kazakh', value: 'kk' },
{ label: 'Kalaallisut', value: 'kl' },
{ label: 'Khmer', value: 'km' },
{ label: 'Kannada', value: 'kn' },
{ label: 'Korean', value: 'ko' },
{ label: 'Kanuri', value: 'kr' },
{ label: 'Kashmiri', value: 'ks' },
{ label: 'Kurdish', value: 'ku' },
{ label: 'Komi', value: 'kv' },
{ label: 'Cornish', value: 'kw' },
{ label: 'Kirghiz', value: 'ky' },
{ label: 'Latin', value: 'la' },
{ label: 'Luxembourgish', value: 'lb' },
{ label: 'Ganda', value: 'lg' },
{ label: 'Limburgish', value: 'li' },
{ label: 'Lingala', value: 'ln' },
{ label: 'Lao', value: 'lo' },
{ label: 'Lithuanian', value: 'lt' },
{ label: 'Luba-Katanga', value: 'lu' },
{ label: 'Latvian', value: 'lv' },
{ label: 'Manx', value: 'gv' },
{ label: 'Macedonian', value: 'mk' },
{ label: 'Malayalam', value: 'ml' },
{ label: 'Mongolian', value: 'mn' },
{ label: 'Marathi', value: 'mr' },
{ label: 'Malay', value: 'ms' },
{ label: 'Maltese', value: 'mt' },
{ label: 'Burmese', value: 'my' },
{ label: 'Nauru', value: 'na' },
{ label: 'Norwegian Bokmål', value: 'nb' },
{ label: 'North Ndebele', value: 'nd' },
{ label: 'Nepali', value: 'ne' },
{ label: 'Ndonga', value: 'ng' },
{ label: 'Dutch', value: 'nl' },
{ label: 'Norwegian Nynorsk', value: 'nn' },
{ label: 'Norwegian', value: 'no' },
{ label: 'Occitan', value: 'oc' },
{ label: 'Ojibwa', value: 'oj' },
{ label: 'Oromo', value: 'om' },
{ label: 'Oriya', value: 'or' },
{ label: 'Ossetian', value: 'os' },
{ label: 'Punjabi', value: 'pa' },
{ label: 'Pali', value: 'pi' },
{ label: 'Polish', value: 'pl' },
{ label: 'Pashto', value: 'ps' },
{ label: 'Portuguese', value: 'pt' },
{ label: 'Quechua', value: 'qu' },
{ label: 'Romansh', value: 'rm' },
{ label: 'Kirundi', value: 'rn' },
{ label: 'Romanian', value: 'ro' },
{ label: 'Russian', value: 'ru' },
{ label: 'Sanskrit', value: 'sa' },
{ label: 'Sardinian', value: 'sc' },
{ label: 'Sindhi', value: 'sd' },
{ label: 'Northern Sami', value: 'se' },
{ label: 'Sango', value: 'sg' },
{ label: 'Sinhala', value: 'si' },
{ label: 'Slovak', value: 'sk' },
{ label: 'Slovenian', value: 'sl' },
{ label: 'Samoan', value: 'sm' },
{ label: 'Shona', value: 'sn' },
{ label: 'Somali', value: 'so' },
{ label: 'Albanian', value: 'sq' },
{ label: 'Serbian', value: 'sr' },
{ label: 'Swati', value: 'ss' },
{ label: 'Southern Sotho', value: 'st' },
{ label: 'Sundanese', value: 'su' },
{ label: 'Swedish', value: 'sv' },
{ label: 'Swahili', value: 'sw' },
{ label: 'Tamil', value: 'ta' },
{ label: 'Telugu', value: 'te' },
{ label: 'Tajik', value: 'tg' },
{ label: 'Thai', value: 'th' },
{ label: 'Tigrinya', value: 'ti' },
{ label: 'Turkmen', value: 'tk' },
{ label: 'Tagalog', value: 'tl' },
{ label: 'Tswana', value: 'tn' },
{ label: 'Tonga', value: 'to' },
{ label: 'Turkish', value: 'tr' },
{ label: 'Tsonga', value: 'ts' },
{ label: 'Tatar', value: 'tt' },
{ label: 'Twi', value: 'tw' },
{ label: 'Tahitian', value: 'ty' },
{ label: 'Uighur', value: 'ug' },
{ label: 'Ukrainian', value: 'uk' },
{ label: 'Urdu', value: 'ur' },
{ label: 'Uzbek', value: 'uz' },
{ label: 'Venda', value: 've' },
{ label: 'Vietnamese', value: 'vi' },
{ label: 'Volapük', value: 'vo' },
{ label: 'Walloon', value: 'wa' },
{ label: 'Wolof', value: 'wo' },
{ label: 'Xhosa', value: 'xh' },
{ label: 'Yiddish', value: 'yi' },
{ label: 'Yoruba', value: 'yo' },
{ label: 'Zhuang', value: 'za' },
{ label: 'Chinese', value: 'zh' },
{ label: 'Zulu', value: 'zu' }
];