feat: playground tts

This commit is contained in:
jialin
2024-11-22 10:00:34 +08:00
parent 16e2ac5b6c
commit 84e0c46711
10 changed files with 706 additions and 42 deletions
+8
View File
@@ -25,6 +25,14 @@ export default [
icon: 'Comment',
component: './playground/index'
},
{
name: 'speech',
title: 'Speech',
path: '/playground/speech',
key: 'speech',
icon: 'Comment',
component: './playground/speech'
},
{
name: 'embedding',
title: 'embedding',
+2
View File
@@ -2,7 +2,9 @@ export default {
'menu.dashboard': 'Dashboard',
'menu.playground': 'Playground',
'menu.playground.rerank': 'Rerank',
'menu.playground.embedding': 'Embedding',
'menu.playground.chat': 'Chat',
'menu.playground.speech': 'Speech',
'menu.compare': 'Compare',
'menu.models': 'Models',
'menu.resources': 'Resources',
+1
View File
@@ -4,6 +4,7 @@ export default {
'menu.playground.rerank': '重排',
'menu.playground.embedding': '文本嵌入',
'menu.playground.chat': '对话',
'menu.playground.speech': '语音',
'menu.compare': '多模型对比',
'menu.models': '模型',
'menu.resources': '资源',
@@ -23,6 +23,10 @@ const options = [
{
label: '--ubatch-size',
value: '--ubatch-size'
},
{
label: '--images',
value: '--images'
}
];
@@ -0,0 +1,7 @@
import React from 'react';
const GroundStt = () => {
return <div>STT</div>;
};
export default React.memo(GroundStt);
@@ -0,0 +1,353 @@
import useOverlayScroller from '@/hooks/use-overlay-scroller';
import { fetchChunkedData, readStreamData } from '@/utils/fetch-chunk-data';
import { useIntl, useSearchParams } from '@umijs/max';
import { Spin } from 'antd';
import classNames from 'classnames';
import _ from 'lodash';
import 'overlayscrollbars/overlayscrollbars.css';
import {
forwardRef,
memo,
useEffect,
useImperativeHandle,
useMemo,
useRef,
useState
} from 'react';
import { CHAT_API } from '../apis';
import { Roles, generateMessages } from '../config';
import { TTSParamsConfig as paramsConfig } from '../config/params-config';
import { MessageItem } from '../config/types';
import '../style/ground-left.less';
import '../style/system-message-wrap.less';
import MessageInput from './message-input';
import MessageContent from './multiple-chat/message-content';
import SystemMessage from './multiple-chat/system-message';
import ReferenceParams from './reference-params';
import RerankerParams from './reranker-params';
import ViewCodeModal from './view-code-modal';
interface MessageProps {
modelList: Global.BaseOption<string>[];
loaded?: boolean;
ref?: any;
}
const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
const { modelList } = props;
const messageId = useRef<number>(0);
const [messageList, setMessageList] = useState<MessageItem[]>([]);
const intl = useIntl();
const [searchParams] = useSearchParams();
const selectModel = searchParams.get('model') || '';
const [parameters, setParams] = useState<any>({});
const [systemMessage, setSystemMessage] = useState('');
const [show, setShow] = useState(false);
const [loading, setLoading] = useState(false);
const [tokenResult, setTokenResult] = useState<any>(null);
const [collapse, setCollapse] = useState(false);
const contentRef = useRef<any>('');
const controllerRef = useRef<any>(null);
const scroller = useRef<any>(null);
const currentMessageRef = useRef<any>(null);
const paramsRef = useRef<any>(null);
const messageListLengthCache = useRef<number>(0);
const { initialize, updateScrollerPosition } = useOverlayScroller();
const { initialize: innitializeParams } = useOverlayScroller();
const initialValues = {
voice: 'Alloy',
response_format: 'mp3',
speed: 1
};
useImperativeHandle(ref, () => {
return {
viewCode() {
setShow(true);
},
setCollapse() {
setCollapse(!collapse);
},
collapse: collapse
};
});
const viewCodeMessage = useMemo(() => {
return generateMessages([
{ role: Roles.System, content: systemMessage },
...messageList
]);
}, [messageList, systemMessage]);
const setMessageId = () => {
messageId.current = messageId.current + 1;
};
const handleNewMessage = (message?: { role: string; content: string }) => {
const newMessage = message || {
role:
_.last(messageList)?.role === Roles.User ? Roles.Assistant : Roles.User,
content: ''
};
messageList.push({
...newMessage,
uid: messageId.current + 1
});
setMessageId();
setMessageList([...messageList]);
};
const joinMessage = (chunk: any) => {
setTokenResult({
...(chunk?.usage ?? {})
});
if (!chunk || !_.get(chunk, 'choices', []).length) {
return;
}
contentRef.current =
contentRef.current + _.get(chunk, 'choices.0.delta.content', '');
setMessageList([
...messageList,
...currentMessageRef.current,
{
role: Roles.Assistant,
content: contentRef.current,
uid: messageId.current
}
]);
};
const handleStopConversation = () => {
controllerRef.current?.abort?.();
setLoading(false);
};
const submitMessage = async (current?: { role: string; content: string }) => {
if (!parameters.model) return;
try {
setLoading(true);
setMessageId();
setTokenResult(null);
controllerRef.current?.abort?.();
controllerRef.current = new AbortController();
const signal = controllerRef.current.signal;
currentMessageRef.current = current
? [
{
...current,
uid: messageId.current
}
]
: [];
contentRef.current = '';
setMessageList((pre) => {
return [...pre, ...currentMessageRef.current];
});
const messageParams = [
{ role: Roles.System, content: systemMessage },
...messageList,
...currentMessageRef.current
];
const messages = generateMessages(messageParams);
const chatParams = {
messages: messages,
...parameters,
stream: true,
stream_options: {
include_usage: true
}
};
const result: any = await fetchChunkedData({
data: chatParams,
url: CHAT_API,
signal
});
if (result?.error) {
setTokenResult({
error: true,
errorMessage:
result?.data?.error?.message || result?.data?.message || ''
});
return;
}
setMessageId();
const { reader, decoder } = result;
await readStreamData(reader, decoder, (chunk: any) => {
if (chunk?.error) {
setTokenResult({
error: true,
errorMessage: chunk?.error?.message || chunk?.message || ''
});
return;
}
joinMessage(chunk);
});
} catch (error) {
// console.log('error:', error);
} finally {
setLoading(false);
}
};
const handleClear = () => {
if (!messageList.length) {
return;
}
setMessageId();
setMessageList([]);
setTokenResult(null);
};
const handleSendMessage = (message: Omit<MessageItem, 'uid'>) => {
console.log('message:', message);
const currentMessage =
message.content || message.imgs?.length ? message : undefined;
submitMessage(currentMessage);
};
const handleCloseViewCode = () => {
setShow(false);
};
const handleSelectModel = () => {};
const handlePresetPrompt = (list: { role: string; content: string }[]) => {
const sysMsg = list.filter((item) => item.role === 'system');
const userMsg = list
.filter((item) => item.role === 'user')
.map((item) => {
setMessageId();
return {
...item,
uid: messageId.current
};
});
setSystemMessage(sysMsg[0]?.content || '');
setMessageList(userMsg);
};
useEffect(() => {
if (scroller.current) {
initialize(scroller.current);
}
}, [scroller.current, initialize]);
useEffect(() => {
if (paramsRef.current) {
innitializeParams(paramsRef.current);
}
}, [paramsRef.current, innitializeParams]);
useEffect(() => {
if (loading) {
updateScrollerPosition();
}
}, [messageList, loading]);
useEffect(() => {
if (messageList.length > messageListLengthCache.current) {
updateScrollerPosition();
}
messageListLengthCache.current = messageList.length;
}, [messageList.length]);
return (
<div className="ground-left-wrapper">
<div className="ground-left">
<div className="message-list-wrap" ref={scroller}>
<>
<div
style={{
marginBottom: 20
}}
>
<SystemMessage
style={{
borderRadius: 'var(--border-radius-mini)',
overflow: 'hidden'
}}
systemMessage={systemMessage}
setSystemMessage={setSystemMessage}
></SystemMessage>
</div>
<div className="content">
<MessageContent
spans={{
span: 24,
count: 1
}}
messageList={messageList}
setMessageList={setMessageList}
editable={true}
loading={loading}
/>
{loading && (
<Spin size="small">
<div style={{ height: '46px' }}></div>
</Spin>
)}
</div>
</>
</div>
{tokenResult && (
<div style={{ height: 40 }}>
<ReferenceParams usage={tokenResult}></ReferenceParams>
</div>
)}
<div className="ground-left-footer">
<MessageInput
scope="chat"
loading={loading}
disabled={!parameters.model}
isEmpty={!messageList.length}
handleSubmit={handleSendMessage}
addMessage={handleNewMessage}
handleAbortFetch={handleStopConversation}
clearAll={handleClear}
setModelSelections={handleSelectModel}
presetPrompt={handlePresetPrompt}
modelList={modelList}
/>
</div>
</div>
<div
className={classNames('params-wrapper', {
collapsed: collapse
})}
ref={paramsRef}
>
<div className="box">
<RerankerParams
setParams={setParams}
paramsConfig={paramsConfig}
initialValues={initialValues}
params={parameters}
selectedModel={selectModel}
modelList={modelList}
/>
</div>
</div>
<ViewCodeModal
open={show}
payLoad={{
messages: viewCodeMessage
}}
parameters={parameters}
onCancel={handleCloseViewCode}
title={intl.formatMessage({ id: 'playground.viewcode' })}
></ViewCodeModal>
</div>
);
});
export default memo(GroundLeft);
@@ -1,9 +1,10 @@
import FieldWrapper from '@/components/seal-form/field-wrapper';
import SealInput from '@/components/seal-form/seal-input';
import SealSelect from '@/components/seal-form/seal-select';
import { INPUT_WIDTH } from '@/constants';
import { InfoCircleOutlined } from '@ant-design/icons';
import { useIntl } from '@umijs/max';
import { Form, InputNumber, Tooltip } from 'antd';
import { Form, InputNumber, Slider, Tooltip } from 'antd';
import _ from 'lodash';
import { memo, useCallback, useEffect, useId } from 'react';
import { ParamsSchema } from '../config/types';
@@ -104,45 +105,6 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = ({
form.setFieldsValue(globalParams);
}, [globalParams]);
const renderFields = useCallback(() => {
console.log('paramsConfig:', paramsConfig);
if (!paramsConfig?.length) {
return null;
}
return paramsConfig.map((item: ParamsSchema) => {
if (item.type === 'InputNumber') {
return (
<Form.Item name={item.name} rules={item.rules} key={item.name}>
<SealInput.Number
{...item.attrs}
style={{ width: '100%' }}
label={
item.label.isLocalized
? intl.formatMessage({ id: item.label.text })
: item.label.text
}
></SealInput.Number>
</Form.Item>
);
}
if (item.type === 'Select') {
return (
<Form.Item name={item.name} rules={item.rules} key={item.name}>
<SealSelect
options={item.options}
label={
item.label.isLocalized
? intl.formatMessage({ id: item.label.text })
: item.label.text
}
></SealSelect>
</Form.Item>
);
}
return null;
});
}, [paramsConfig]);
const renderLabel = (args: {
field: string;
label: string;
@@ -178,6 +140,73 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = ({
);
};
const renderFields = useCallback(() => {
console.log('paramsConfig:', paramsConfig);
if (!paramsConfig?.length) {
return null;
}
return paramsConfig.map((item: ParamsSchema) => {
if (item.type === 'InputNumber') {
return (
<Form.Item name={item.name} rules={item.rules} key={item.name}>
<SealInput.Number
{...item.attrs}
style={{ width: '100%' }}
label={
item.label.isLocalized
? intl.formatMessage({ id: item.label.text })
: item.label.text
}
></SealInput.Number>
</Form.Item>
);
}
if (item.type === 'Select') {
return (
<Form.Item name={item.name} rules={item.rules} key={item.name}>
<SealSelect
{...item.attrs}
options={item.options}
label={
item.label.isLocalized
? intl.formatMessage({ id: item.label.text })
: item.label.text
}
></SealSelect>
</Form.Item>
);
}
if (item.type === 'Slider') {
return (
<Form.Item name={item.name} rules={item.rules} key={item.name}>
<FieldWrapper
label={renderLabel({
field: item.name,
label: item.label.isLocalized
? intl.formatMessage({ id: item.label.text })
: item.label.text,
description: item.description?.isLocalized
? intl.formatMessage({ id: item.description?.text })
: item.description?.text || ''
})}
style={{ padding: '20px 2px 0' }}
variant="borderless"
>
<Slider
{...item.attrs}
style={{ marginBottom: 0, marginTop: 16, marginInline: 0 }}
tooltip={{ open: false }}
value={form.getFieldValue(item.name) || undefined}
onChange={(val) => handleFieldValueChange(val, item.name)}
></Slider>
</FieldWrapper>
</Form.Item>
);
}
return null;
});
}, [paramsConfig]);
return (
<Form
name={formId}
@@ -0,0 +1,67 @@
import { ParamsSchema } from './types';
export const TTSParamsConfig: ParamsSchema[] = [
{
type: 'Select',
name: 'voice',
options: [
{ label: 'Alloy', value: 'Alloy' },
{ label: 'Echo', value: 'Echo' },
{ label: 'Fable', value: 'Fable' },
{ label: 'Onyx', value: 'Onyx' },
{ label: 'Nova', value: 'Nova' },
{ label: 'Shimmer', value: 'Shimmer' }
],
label: {
text: 'Voice',
isLocalized: false
},
rules: [
{
required: true,
message: 'Voice is required'
}
]
},
{
type: 'Select',
name: 'response_format',
options: [
{ label: 'mp3', value: 'mp3' },
{ label: 'opus', value: 'opus' },
{ label: 'aac', value: 'aac' },
{ label: 'flac', value: 'flac' },
{ label: 'wav', value: 'wav' },
{ label: 'pcm', value: 'pcm' }
],
label: {
text: 'Response Format',
isLocalized: false
},
rules: [
{
required: false
}
]
},
{
type: 'Select',
name: 'speed',
options: [
{ label: '0.25x', value: 0.25 },
{ label: '0.5x', value: 0.5 },
{ label: '1x', value: 1 },
{ label: '2x', value: 2 },
{ label: '4x', value: 4 }
],
label: {
text: 'Speed',
isLocalized: false
},
rules: [
{
required: false
}
]
}
];
+6 -2
View File
@@ -27,14 +27,18 @@ export interface ParamsSchema {
text: string;
isLocalized?: boolean;
};
options?: Global.BaseOption<string>[];
options?: Global.BaseOption<string | number>[];
value?: string | number | boolean | string[];
min?: number;
max?: number;
step?: number;
disabled?: boolean;
defaultValue?: string | number | boolean;
rules: { required: boolean; message: string }[];
rules: { required: boolean; message?: string }[];
placeholder?: string;
attrs?: Record<string, any>;
description?: {
text: string;
isLocalized?: boolean;
};
}
+189
View File
@@ -0,0 +1,189 @@
import IconFont from '@/components/icon-font';
import breakpoints from '@/config/breakpoints';
import HotKeys from '@/config/hotkeys';
import useWindowResize from '@/hooks/use-window-resize';
import { AudioOutlined } from '@ant-design/icons';
import { PageContainer } from '@ant-design/pro-components';
import { useIntl } from '@umijs/max';
import { Button, Segmented, Space, Tabs, TabsProps } from 'antd';
import classNames from 'classnames';
import _ from 'lodash';
import { useCallback, useEffect, useRef, useState } from 'react';
import { useHotkeys } from 'react-hotkeys-hook';
import { queryModelsList } from './apis';
import GroundSTT from './components/ground-stt';
import GroundTTS from './components/ground-tts';
import './style/play-ground.less';
const TabsValueMap = {
Tab1: 'tts',
Tab2: 'stt'
};
const Playground: React.FC = () => {
const intl = useIntl();
const { size } = useWindowResize();
const [activeKey, setActiveKey] = useState(TabsValueMap.Tab1);
const groundTabRef1 = useRef<any>(null);
const groundTabRef2 = useRef<any>(null);
const [modelList, setModelList] = useState<Global.BaseOption<string>[]>([]);
const [loaded, setLoaded] = useState(false);
const optionsList = [
{
label: 'TTS',
value: TabsValueMap.Tab1,
icon: <AudioOutlined />
},
{
label: 'Realtime',
value: TabsValueMap.Tab2,
icon: <IconFont type={'icon-audio'}></IconFont>
}
];
const handleViewCode = useCallback(() => {
if (activeKey === TabsValueMap.Tab1) {
groundTabRef1.current?.viewCode?.();
} else if (activeKey === TabsValueMap.Tab2) {
groundTabRef2.current?.viewCode?.();
}
}, [activeKey]);
const handleToggleCollapse = useCallback(() => {
if (activeKey === TabsValueMap.Tab1) {
groundTabRef1.current?.setCollapse?.();
return;
}
groundTabRef2.current?.setCollapse?.();
}, [activeKey]);
const items: TabsProps['items'] = [
{
key: 'tts',
label: 'TTS',
children: (
<GroundTTS ref={groundTabRef1} modelList={modelList}></GroundTTS>
)
},
{
key: 'stt',
label: 'Realtime',
children: (
<GroundSTT modelList={modelList} loaded={loaded} ref={groundTabRef2} />
)
}
];
useEffect(() => {
if (size.width < breakpoints.lg) {
if (!groundTabRef1.current?.collapse) {
groundTabRef1.current?.setCollapse?.();
}
}
}, [size.width]);
useEffect(() => {
const getModelList = async () => {
try {
const params = {
embedding_only: false
};
const res = await queryModelsList(params);
const list = _.map(res.data || [], (item: any) => {
return {
value: item.id,
label: item.id
};
}) as Global.BaseOption<string>[];
return list;
} catch (error) {
console.error(error);
return [];
}
};
const fetchData = async () => {
try {
const modelist = await getModelList();
setModelList(modelist);
} catch (error) {
setLoaded(true);
}
};
fetchData();
}, []);
const renderExtra = () => {
if (activeKey === 'compare') {
return false;
}
return (
<Space key="buttons">
<Button
size="middle"
onClick={handleViewCode}
icon={<IconFont type="icon-code" className="font-size-16"></IconFont>}
>
{intl.formatMessage({ id: 'playground.viewcode' })}
</Button>
<Button
size="middle"
onClick={handleToggleCollapse}
icon={
<IconFont
type="icon-a-layout6-line"
className="font-size-16"
></IconFont>
}
></Button>
</Space>
);
};
useHotkeys(
HotKeys.RIGHT.join(','),
() => {
groundTabRef1.current?.setCollapse?.();
},
{
preventDefault: true
}
);
return (
<PageContainer
ghost
header={{
title: (
<div className="flex items-center">
<span className="font-600">
{intl.formatMessage({ id: 'menu.playground.speech' })}
</span>
{
<Segmented
options={optionsList}
size="middle"
className="m-l-40"
onChange={(key) => setActiveKey(key)}
></Segmented>
}
</div>
),
breadcrumb: {}
}}
extra={renderExtra()}
className={classNames('playground-container', {
compare: activeKey === 'compare',
chat: activeKey !== 'compare'
})}
>
<div className="play-ground">
<div className="chat">
<Tabs items={items} activeKey={activeKey}></Tabs>
</div>
</div>
</PageContainer>
);
};
export default Playground;