root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: audio model deploy

Browse Source
This commit is contained in:
jialin
2024-11-27 11:04:31 +08:00
parent 5e0772fae9
commit ba76498acc
31 changed files with 708 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pages/playground/apis/index.ts
+29 -8
View File
@@ -1,3 +1,4 @@
import { MODELS_API } from '@/pages/llmodels/apis';
import { request } from '@umijs/max';
export const CHAT_API = '/v1-openai/chat/completions';
@@ -96,17 +97,37 @@ export const textToSpeech = async (params: any, options?: any) => {
if (!res.ok) {
throw new Error('Network response was not ok');
}
return res.json();
const audioBlob = await res.blob();
const audioUrl = URL.createObjectURL(audioBlob);
return audioUrl;
};
// export const speechToText = async (params: any, options?: any) => {
// const res = await fetch(AUDIO_SPEECH_TO_TEXT_API, {
// method: 'POST',
// body: JSON.stringify(params.data),
// signal: params.signal
// });
// if (!res.ok) {
// throw new Error('Network response was not ok');
// }
// return res.json();
// };
export const speechToText = async (params: any, options?: any) => {
const res = await fetch(AUDIO_SPEECH_TO_TEXT_API, {
return request(AUDIO_SPEECH_TO_TEXT_API, {
method: 'POST',
body: JSON.stringify(params.data),
signal: params.signal
data: params.data,
headers: {
'Content-Type': 'multipart/form-data'
}
});
};
export const queryModelVoices = async (params: { name: string }) => {
return request(`${MODELS_API}/${params.name}/voices`, {
method: 'GET',
skipErrorHandler: true
});
if (!res.ok) {
throw new Error('Network response was not ok');
}
return res.json();
};
src/pages/playground/components/audio-input.tsx
+13 -15
View File
@@ -1,4 +1,5 @@
import { AudioOutlined } from '@ant-design/icons';
import { useIntl } from '@umijs/max';
import { Button, Space, Tooltip } from 'antd';
import React, { useCallback, useEffect, useRef, useState } from 'react';
// import '../style/audio-input.less';
@@ -19,6 +20,7 @@ interface AudioInputProps {
}
const AudioInput: React.FC<AudioInputProps> = (props) => {
const intl = useIntl();
const [audioOn, setAudioOn] = useState(false);
const [isRecording, setIsRecording] = useState(false);
const [audioPermission, setAudioPermission] = useState(true);
@@ -150,9 +152,9 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
stopRecording();
return;
}
try {
await EnableAudio();
console.log('audioStream:', audioStream.current);
audioRecorder.current = new MediaRecorder(audioStream.current);
const audioChunks: any[] = [];
@@ -172,7 +174,7 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
const audioUrl = URL.createObjectURL(audioBlob);
handleAudioData({
chunks: audioChunks,
chunks: audioBlob,
size: audioBlob.size,
type: audioBlob.type,
url: audioUrl,
@@ -188,8 +190,9 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
startTime.current = Date.now();
audioRecorder.current.start(1000);
generateVisualData();
console.log('start recording');
} catch (error) {
// console.log(error);
console.log('error====', error);
}
};
@@ -208,7 +211,13 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
<div className="audio-input">
<Space size={40} className="btns">
{
<Tooltip title="Start Recording">
<Tooltip
title={
isRecording
? intl.formatMessage({ id: 'playground.audio.stoprecord' })
: intl.formatMessage({ id: 'playground.audio.startrecord' })
}
>
<div
style={{
display: 'flex',
@@ -229,17 +238,6 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
</div>
</Tooltip>
}
{/* {isRecording && (
<Tooltip title="Stop Recording">
<Button
shape="circle"
icon={<IconFont type="icon-stop2"></IconFont>}
size="middle"
type={props.type ?? 'text'}
onClick={stopRecording}
></Button>
</Tooltip>
)} */}
</Space>
</div>
);
src/pages/playground/components/dynamic-params.tsx
+11 -1
View File
@@ -32,6 +32,7 @@ type ParamsSettingsProps = {
modelList: Global.BaseOption<string>[];
onValuesChange?: (changeValues: any, value: Record<string, any>) => void;
setParams: (params: any) => void;
onModelChange?: (model: string) => void;
globalParams?: Record<string, any>;
paramsConfig?: ParamsSchema[];
initialValues?: Record<string, any>;
@@ -43,6 +44,7 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = forwardRef(
{
setParams,
onValuesChange,
onModelChange,
selectedModel,
globalParams,
initialValues,
@@ -84,6 +86,13 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = forwardRef(
}
}, [modelList, showModelSelector, selectedModel, initialValues]);
const handleModelChange = useCallback(
(value: string) => {
onModelChange?.(value);
},
[onModelChange]
);
const handleOnFinish = (values: any) => {
console.log('handleOnFinish', values);
};
@@ -239,7 +248,7 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = forwardRef(
}
return null;
});
}, [paramsConfig, params]);
}, [paramsConfig, params, intl]);
return (
<Form
@@ -272,6 +281,7 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = forwardRef(
]}
>
<SealSelect
onChange={handleModelChange}
showSearch={true}
options={modelList}
label={intl.formatMessage({ id: 'playground.model' })}
src/pages/playground/components/ground-embedding.tsx
+74 -5
View File
@@ -11,7 +11,7 @@ import {
SendOutlined
} from '@ant-design/icons';
import { useIntl, useSearchParams } from '@umijs/max';
import { Button, Segmented, Tabs } from 'antd';
import { Button, Checkbox, Segmented, Tabs } from 'antd';
import classNames from 'classnames';
import { PCA } from 'ml-pca';
import 'overlayscrollbars/overlayscrollbars.css';
@@ -72,8 +72,15 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
>([]);
const [outputType, setOutputType] = useState<string>('chart');
const [outputHeight, setOutputHeight] = useState<number>(180);
const [embeddingData, setEmbeddingData] = useState<string>('');
const [embeddingData, setEmbeddingData] = useState<{
code: string;
copyValue: string;
}>({
code: '',
copyValue: ''
});
const [lessTwoInput, setLessTwoInput] = useState<boolean>(false);
const multiplePasteEnable = useRef<boolean>(true);
const [textList, setTextList] = useState<
{ text: string; uid: number | string; name: string }[]
@@ -97,6 +104,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
const { initialize: innitializeParams, updateScrollerPosition } =
useOverlayScroller();
const formRef = useRef<any>(null);
useImperativeHandle(ref, () => {
return {
@@ -137,7 +145,15 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
};
});
setScatterData(list);
setEmbeddingData(JSON.stringify(embeddings, null, 2));
const embeddingJson = embeddings.map((item, index) => {
item.embedding = item.embedding.slice(0, 5);
item.embedding.push(null);
return item;
});
setEmbeddingData({
code: JSON.stringify(embeddingJson, null, 2).replace(/null/g, '...'),
copyValue: JSON.stringify(embeddings, null, 2)
});
} catch (e) {
console.log('error:', e);
}
@@ -155,6 +171,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
};
const submitMessage = async (current?: { role: string; content: string }) => {
await formRef.current?.form.validateFields();
if (!parameters.model) return;
try {
@@ -255,6 +272,25 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
setTextList(list);
};
const handleOnPaste = useCallback(
(e: any, index: number) => {
if (!multiplePasteEnable.current) return;
const text = e.clipboardData.getData('text');
if (text) {
console.log('text:', text);
const dataLlist = text.split('\n').map((item: string) => {
return {
text: item,
uid: inputListRef.current?.setMessageId(),
name: ''
};
});
setTextList([...textList.slice(0, index), ...dataLlist]);
}
},
[textList]
);
const handleClearDocuments = () => {
setTextList([
{
@@ -300,7 +336,8 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
<HighlightCode
height={outputHeight - 20}
theme="light"
code={embeddingData}
code={embeddingData.code}
copyValue={embeddingData.copyValue}
lang="json"
copyable={true}
style={{ marginBottom: 0 }}
@@ -355,7 +392,37 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
</span>
</div>
</h3>
<div className="flex gap-10">
<div className="flex-center gap-10">
<Button className="flex-center" size="middle">
<Checkbox
defaultChecked={multiplePasteEnable.current}
onChange={(e: any) => {
multiplePasteEnable.current = e.target.checked;
}}
>
{intl.formatMessage({
id: 'playground.input.multiplePaste'
})}
</Checkbox>
</Button>
{/* <Tooltip
title={intl.formatMessage({
id: 'playground.input.multiplePaste'
})}
>
<Switch
checkedChildren={intl.formatMessage({
id: 'playground.multiple.on'
})}
unCheckedChildren={intl.formatMessage({
id: 'playground.multiple.off'
})}
defaultChecked={multiplePasteEnable.current}
onChange={(checked) => {
multiplePasteEnable.current = checked;
}}
/>
</Tooltip> */}
<Button size="middle" onClick={handleAddText}>
<PlusOutlined />
{intl.formatMessage({ id: 'playground.embedding.addtext' })}
@@ -399,6 +466,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
ref={inputListRef}
textList={textList}
onChange={handleTextListChange}
onPaste={handleOnPaste}
></InputList>
<div style={{ marginTop: 8 }}>
<FileList
@@ -505,6 +573,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
>
<div className="box">
<DynamicParams
ref={formRef}
setParams={setParams}
paramsConfig={paramsConfig}
initialValues={initialValues}
src/pages/playground/components/ground-images.tsx
+3 -2
View File
@@ -283,6 +283,7 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
if (item.b64_json) {
imgItem.dataUrl += item.b64_json;
}
const progress = _.round(item.progress, 0);
newImageList[item.index] = {
dataUrl: imgItem.dataUrl,
height: '100%',
@@ -291,8 +292,8 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
maxWidth: `${imgSize[0]}px`,
uid: imgItem.uid,
span: imgItem.span,
loading: _.round(item.progress, 0) < 100,
progress: _.round(item.progress, 0)
loading: progress < 100,
progress: progress
};
});
setImageList([...newImageList]);
src/pages/playground/components/ground-reranker.tsx
+57 -2
View File
@@ -2,7 +2,7 @@ import useOverlayScroller from '@/hooks/use-overlay-scroller';
import useRequestToken from '@/hooks/use-request-token';
import { ClearOutlined, PlusOutlined, SendOutlined } from '@ant-design/icons';
import { useIntl, useSearchParams } from '@umijs/max';
import { Button, Input, Spin, Tag } from 'antd';
import { Button, Checkbox, Input, Spin, Tag } from 'antd';
import classNames from 'classnames';
import _ from 'lodash';
import 'overlayscrollbars/overlayscrollbars.css';
@@ -76,6 +76,8 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
const paramsRef = useRef<any>(null);
const messageListLengthCache = useRef<number>(0);
const requestToken = useRef<any>(null);
const formRef = useRef<any>(null);
const multiplePasteEnable = useRef<boolean>(true);
const [fileList, setFileList] = useState<
{
text: string;
@@ -186,6 +188,7 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
};
const submitMessage = async (current?: { content: string }) => {
await formRef.current?.form.validateFields();
if (!parameters.model) return;
try {
setLoading(true);
@@ -340,6 +343,26 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
},
[]
);
const handleOnPaste = useCallback(
(e: any, index: number) => {
if (!multiplePasteEnable.current) return;
const text = e.clipboardData.getData('text');
if (text) {
console.log('text:', text);
const dataLlist = text.split('\n').map((item: string) => {
return {
text: item,
uid: inputListRef.current?.setMessageId(),
name: ''
};
});
setTextList([...textList.slice(0, index), ...dataLlist]);
}
},
[textList]
);
const handleOnSort = useCallback(
(list: { text: string; uid: number | string; name: string }[]) => {
const newList = list?.map((item) => {
@@ -435,7 +458,37 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
</span>
)}
</span>
<div className="flex gap-10">
<div className="flex-center gap-10">
<Button className="flex-center" size="middle">
<Checkbox
defaultChecked={multiplePasteEnable.current}
onChange={(e: any) => {
multiplePasteEnable.current = e.target.checked;
}}
>
{intl.formatMessage({
id: 'playground.input.multiplePaste'
})}
</Checkbox>
</Button>
{/* <Tooltip
title={intl.formatMessage({
id: 'playground.input.multiplePaste'
})}
>
<Switch
checkedChildren={intl.formatMessage({
id: 'playground.multiple.on'
})}
unCheckedChildren={intl.formatMessage({
id: 'playground.multiple.off'
})}
defaultChecked={multiplePasteEnable.current}
onChange={(checked) => {
multiplePasteEnable.current = checked;
}}
/>
</Tooltip> */}
<Button size="middle" onClick={handleAddText}>
<PlusOutlined />
{intl.formatMessage({ id: 'playground.embedding.addtext' })}
@@ -460,6 +513,7 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
onChange={handleTextListChange}
onSort={handleOnSort}
extra={renderPercent}
onPaste={handleOnPaste}
></InputList>
</div>
</div>
@@ -488,6 +542,7 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
>
<div className="box">
<DynamicParams
ref={formRef}
setParams={setParams}
params={parameters}
paramsConfig={paramsConfig}
src/pages/playground/components/ground-stt.tsx
+65 -104
View File
@@ -4,7 +4,7 @@ import IconFont from '@/components/icon-font';
import UploadAudio from '@/components/upload-audio';
import useOverlayScroller from '@/hooks/use-overlay-scroller';
import { readAudioFile } from '@/utils/load-audio-file';
import { AudioOutlined, ThunderboltOutlined } from '@ant-design/icons';
import { AudioOutlined, SendOutlined } from '@ant-design/icons';
import { useIntl, useSearchParams } from '@umijs/max';
import { Button, Spin, Tag, Tooltip } from 'antd';
import classNames from 'classnames';
@@ -18,7 +18,7 @@ import {
useRef,
useState
} from 'react';
import { CHAT_API, speechToText } from '../apis';
import { speechToText } from '../apis';
import { RealtimeParamsConfig as paramsConfig } from '../config/params-config';
import { MessageItem } from '../config/types';
import '../style/ground-left.less';
@@ -26,7 +26,6 @@ import '../style/speech-to-text.less';
import '../style/system-message-wrap.less';
import AudioInput from './audio-input';
import DynamicParams from './dynamic-params';
import MessageContent from './multiple-chat/message-content';
import ViewCodeModal from './view-code-modal';
interface MessageProps {
@@ -40,17 +39,17 @@ const initialValues = {
};
const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
const intl = useIntl();
const { modelList } = props;
const messageId = useRef<number>(0);
const [messageList, setMessageList] = useState<MessageItem[]>([
{
content: 'Generating text content...',
content: '',
title: '',
role: '',
uid: messageId.current
}
]);
const intl = useIntl();
const [searchParams] = useSearchParams();
const selectModel = searchParams.get('model') || '';
const [parameters, setParams] = useState<any>({});
@@ -70,6 +69,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
});
const [isRecording, setIsRecording] = useState(false);
const [recordEnd, setRecordEnd] = useState(false);
const formRef = useRef<any>(null);
const { initialize, updateScrollerPosition } = useOverlayScroller();
const { initialize: innitializeParams } = useOverlayScroller();
@@ -95,7 +95,8 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
setLoading(false);
};
const submitMessage = async (current?: { role: string; content: string }) => {
const submitMessage = async () => {
await formRef.current?.form.validateFields();
if (!parameters.model) return;
try {
setLoading(true);
@@ -106,17 +107,12 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
controllerRef.current = new AbortController();
const signal = controllerRef.current.signal;
const chatParams = {
const params = {
...parameters,
stream: true,
stream_options: {
include_usage: true
}
file: new File([audioData.data], audioData.name)
};
const result: any = await speechToText({
data: chatParams,
url: CHAT_API,
signal
data: params
});
if (result?.error) {
@@ -129,16 +125,18 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
}
setMessageList([
{
content: 'Generating text content...',
content: result.text,
title: '',
role: '',
uid: messageId.current
}
]);
} catch (error) {
// console.log('error:', error);
console.log('error:', error);
} finally {
setLoading(false);
setRecordEnd(false);
setIsRecording(false);
}
};
const handleClear = () => {
@@ -160,6 +158,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
return {
url: data.url,
name: data.name,
data: data.chunks,
duration: data.duration
};
});
@@ -177,7 +176,6 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
const handleUploadChange = useCallback(
async (data: { file: any; fileList: any }) => {
const res = await readAudioFile(data.file.originFileObj);
console.log('res=======', res);
setAudioData(res);
setRecordEnd(true);
},
@@ -195,20 +193,12 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
const handleOnRecord = useCallback((val: boolean) => {
setIsRecording(val);
setAudioData(null);
console.log('data===', val);
}, []);
const handleOnGenerate = useCallback(() => {
setMessageList([
{
content: 'Generating text content...',
title: '',
role: '',
uid: messageId.current
}
]);
setRecordEnd(false);
setIsRecording(false);
}, []);
const handleOnGenerate = async () => {
submitMessage();
};
const handleOnDiscard = useCallback(() => {
setRecordEnd(false);
@@ -229,6 +219,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
></AudioAnimation>
);
}
return (
<div className="tips-text">
<IconFont type={'icon-audio'} style={{ fontSize: 20 }}></IconFont>
@@ -238,7 +229,9 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
</div>
);
};
useEffect(() => {
console.log('parameters:', parameters);
}, [parameters]);
useEffect(() => {}, [messageList]);
useEffect(() => {
if (scroller.current) {
@@ -271,51 +264,21 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
<div className="ground-left-footer" style={{ flex: 1 }}>
<div className="speech-to-text">
<div className="speech-box">
{isRecording ? (
<>
<AudioInput
type="default"
voiceActivity={true}
onAudioData={handleOnAudioData}
onAudioPermission={handleOnAudioPermission}
onAnalyse={handleOnAnalyse}
onRecord={handleOnRecord}
></AudioInput>
</>
) : (
<>
{/* <Tooltip title="discard">
<Button
onClick={handleOnDiscard}
icon={<DeleteRowOutlined />}
shape="circle"
></Button>
</Tooltip>
<Tooltip title="generate text content">
<Button
type="primary"
onClick={handleOnGenerate}
shape="circle"
icon={<ThunderboltOutlined></ThunderboltOutlined>}
></Button>
</Tooltip> */}
<Tooltip title="Upload an audio file">
<UploadAudio
type="default"
accept=".mp3,.mp4,.wav"
onChange={handleUploadChange}
></UploadAudio>
</Tooltip>
<AudioInput
type="default"
voiceActivity={true}
onAudioData={handleOnAudioData}
onAudioPermission={handleOnAudioPermission}
onAnalyse={handleOnAnalyse}
onRecord={handleOnRecord}
></AudioInput>
</>
{!isRecording && (
<UploadAudio
type="default"
accept=".mp3,.mp4,.wav"
onChange={handleUploadChange}
></UploadAudio>
)}
<AudioInput
type="default"
voiceActivity={true}
onAudioData={handleOnAudioData}
onAudioPermission={handleOnAudioPermission}
onAnalyse={handleOnAnalyse}
onRecord={handleOnRecord}
></AudioInput>
</div>
{audioData ? (
@@ -326,24 +289,6 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
name={audioData.name}
duration={audioData.duration}
></AudioPlayer>
{/* <div
style={{
paddingRight: 5,
display: 'flex',
justifyContent: 'flex-end',
marginTop: 30
}}
>
<Tooltip title="generate text content">
<Button
size="middle"
type="primary"
icon={<ThunderboltOutlined></ThunderboltOutlined>}
>
Generata Text Content
</Button>
</Tooltip>
</div> */}
</div>
</div>
) : (
@@ -399,20 +344,29 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
className="message-list-wrap"
ref={scroller}
style={{
borderTop: messageList.length
? '1px solid var(--ant-color-split)'
: '1px solid var(--ant-color-split)'
borderTop: '1px solid var(--ant-color-split)'
}}
>
<div className="content" style={{ height: '100%' }}>
<>
<MessageContent
actions={[]}
messageList={messageList[0] ? [messageList[0]] : []}
editable={false}
showTitle={false}
loading={true}
/>
<div
style={{
padding: '8px 14px',
lineHeight: '20px',
display: 'flex',
justifyContent: 'center'
}}
>
{audioData ? (
messageList[0]?.content
) : (
<span className="text-tertiary">
{intl.formatMessage({
id: 'playground.audio.generating.tips'
})}
</span>
)}
</div>
{loading && (
<Spin size="small">
<div style={{ height: '46px' }}></div>
@@ -422,12 +376,18 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
</div>
</div>
<div style={{ padding: '16px 32px', textAlign: 'right' }}>
<Tooltip title="generate text content">
<Tooltip
title={intl.formatMessage({
id: 'playground.audio.button.generate'
})}
>
<Button
style={{ width: 46 }}
size="middle"
disabled={!audioData}
type="primary"
onClick={handleOnGenerate}
icon={<ThunderboltOutlined></ThunderboltOutlined>}
icon={<SendOutlined></SendOutlined>}
></Button>
</Tooltip>
</div>
@@ -441,6 +401,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
>
<div className="box">
<DynamicParams
ref={formRef}
setParams={setParams}
paramsConfig={paramsConfig}
initialValues={initialValues}
src/pages/playground/components/ground-tts.tsx
+87 -22
View File
@@ -1,25 +1,29 @@
import IconFont from '@/components/icon-font';
import SealSelect from '@/components/seal-form/seal-select';
import SpeechContent from '@/components/speech-content';
import useOverlayScroller from '@/hooks/use-overlay-scroller';
import { ThunderboltOutlined } from '@ant-design/icons';
import { SendOutlined } from '@ant-design/icons';
import { useIntl, useSearchParams } from '@umijs/max';
import { Spin } from 'antd';
import { Form, Spin } from 'antd';
import classNames from 'classnames';
import _ from 'lodash';
import 'overlayscrollbars/overlayscrollbars.css';
import {
forwardRef,
memo,
useCallback,
useEffect,
useImperativeHandle,
useMemo,
useRef,
useState
} from 'react';
import { CHAT_API, textToSpeech } from '../apis';
import { CHAT_API, queryModelVoices, textToSpeech } from '../apis';
import { TTSParamsConfig as paramsConfig } from '../config/params-config';
import { MessageItem } from '../config/types';
import { MessageItem, ParamsSchema } from '../config/types';
import '../style/ground-left.less';
import '../style/system-message-wrap.less';
import RerankerParams from './dynamic-params';
import DynamicParams from './dynamic-params';
import MessageInput from './message-input';
import ReferenceParams from './reference-params';
import ViewCodeModal from './view-code-modal';
@@ -31,7 +35,7 @@ interface MessageProps {
}
const initialValues = {
voice: 'Alloy',
voice: '',
response_format: 'mp3',
speed: 1
};
@@ -41,12 +45,13 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
const messageId = useRef<number>(0);
const [messageList, setMessageList] = useState<
{
prompt: string;
input: string;
voice: string;
format: string;
speed: number;
uid: number;
autoplay: boolean;
audioUrl: string;
}[]
>([]);
@@ -54,19 +59,18 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
const [searchParams] = useSearchParams();
const selectModel = searchParams.get('model') || '';
const [parameters, setParams] = useState<any>({});
const [systemMessage, setSystemMessage] = useState('');
const [show, setShow] = useState(false);
const [loading, setLoading] = useState(false);
const [tokenResult, setTokenResult] = useState<any>(null);
const [collapse, setCollapse] = useState(false);
const contentRef = useRef<any>('');
const controllerRef = useRef<any>(null);
const scroller = useRef<any>(null);
const currentMessageRef = useRef<any>(null);
const paramsRef = useRef<any>(null);
const messageListLengthCache = useRef<number>(0);
const checkvalueRef = useRef<any>(true);
const [currentPrompt, setCurrentPrompt] = useState<string>('');
const [voiceList, setVoiceList] = useState<Global.BaseOption<string>[]>([]);
const formRef = useRef<any>(null);
const { initialize, updateScrollerPosition } = useOverlayScroller();
const { initialize: innitializeParams } = useOverlayScroller();
@@ -93,6 +97,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
};
const submitMessage = async (current?: { role: string; content: string }) => {
await formRef.current?.form.validateFields();
if (!parameters.model) return;
try {
setLoading(true);
@@ -104,24 +109,27 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
controllerRef.current = new AbortController();
const signal = controllerRef.current.signal;
const chatParams = {
const params = {
...parameters,
prompt: current?.content || currentPrompt
input: current?.content || currentPrompt
};
const result: any = await textToSpeech({
data: chatParams,
const audioUrl: any = await textToSpeech({
data: params,
url: CHAT_API,
signal
});
console.log('result:', parameters, audioUrl);
setMessageList([
{
prompt: current?.content || currentPrompt,
input: current?.content || currentPrompt,
voice: parameters.voice,
format: parameters.response_format,
speed: parameters.speed,
uid: messageId.current,
autoplay: checkvalueRef.current
autoplay: checkvalueRef.current,
audioUrl: audioUrl
}
]);
} catch (error) {
@@ -147,11 +155,68 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
setShow(false);
};
const handleSelectModel = () => {};
const handleSelectModel = useCallback(
async (value: string) => {
const data: any = modelList.find((item) => item.value === value);
if (!data) return;
try {
const res = await queryModelVoices({
name: data?.modelId as string
});
const voiceList = _.map(res.voices || [], (item: any) => {
return {
label: item,
value: item
};
});
setVoiceList(voiceList);
setParams((pre: any) => {
return {
...pre,
voice: voiceList[0]?.value
};
});
formRef.current?.form.setFieldValue('voice', voiceList[0]?.value);
} catch (error) {
setVoiceList([]);
formRef.current?.form.setFieldValue('voice', '');
setParams((pre: any) => {
return {
...pre,
voice: ''
};
});
}
},
[modelList]
);
const handleOnCheckChange = (e: any) => {
checkvalueRef.current = e.target.checked;
};
const renderExtra = useMemo(() => {
return paramsConfig.map((item: ParamsSchema) => {
return (
<Form.Item name={item.name} rules={item.rules} key={item.name}>
<SealSelect
{...item.attrs}
options={item.name === 'voice' ? voiceList : item.options}
label={
item.label.isLocalized
? intl.formatMessage({ id: item.label.text })
: item.label.text
}
></SealSelect>
</Form.Item>
);
});
}, [paramsConfig, intl, voiceList]);
useEffect(() => {
handleSelectModel(parameters.model);
}, [parameters.model, handleSelectModel]);
useEffect(() => {
if (scroller.current) {
initialize(scroller.current);
@@ -233,9 +298,8 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
handleSubmit={handleSendMessage}
handleAbortFetch={handleStopConversation}
clearAll={handleClear}
setModelSelections={handleSelectModel}
shouldResetMessage={false}
submitIcon={<ThunderboltOutlined></ThunderboltOutlined>}
submitIcon={<SendOutlined></SendOutlined>}
modelList={modelList}
/>
</div>
@@ -247,13 +311,14 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
ref={paramsRef}
>
<div className="box">
<RerankerParams
<DynamicParams
ref={formRef}
setParams={setParams}
paramsConfig={paramsConfig}
initialValues={initialValues}
params={parameters}
selectedModel={selectModel}
modelList={modelList}
extra={renderExtra}
/>
</div>
</div>
@@ -261,7 +326,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
<ViewCodeModal
open={show}
payLoad={{
prompt: currentPrompt
input: currentPrompt
}}
api="audio/speech"
clientType="audio.speech"
src/pages/playground/components/input-list.tsx
+7 -2
View File
@@ -27,6 +27,7 @@ interface InputListProps {
onChange?: (
textList: { text: string; uid: number | string; name: string }[]
) => void;
onPaste?: (e: any, index: number) => void;
onSort?: (
textList: { text: string; uid: number | string; name: string }[]
) => void;
@@ -42,7 +43,8 @@ const InputList: React.FC<InputListProps> = forwardRef(
height,
onSort,
onChange,
extra
extra,
onPaste
},
ref
) => {
@@ -181,6 +183,7 @@ const InputList: React.FC<InputListProps> = forwardRef(
const setMessageId = () => {
messageId.current = messageId.current + 1;
return messageId.current;
};
const handleAdd = () => {
@@ -222,7 +225,8 @@ const InputList: React.FC<InputListProps> = forwardRef(
useImperativeHandle(ref, () => ({
handleAdd,
handleDelete,
handleTextChange
handleTextChange,
setMessageId
}));
return (
@@ -239,6 +243,7 @@ const InputList: React.FC<InputListProps> = forwardRef(
id: 'playground.embedding.inputyourtext'
})}
onChange={(e) => handleTextChange(e.target.value, text)}
onPaste={(e) => onPaste?.(e, index)}
></RowTextarea>
</div>
<span className="btn-group">
src/pages/playground/config/params-config.ts
+6 -25
View File
@@ -5,12 +5,12 @@ export const TTSParamsConfig: ParamsSchema[] = [
type: 'Select',
name: 'voice',
options: [
{ label: 'Alloy', value: 'Alloy' },
{ label: 'Echo', value: 'Echo' },
{ label: 'Fable', value: 'Fable' },
{ label: 'Onyx', value: 'Onyx' },
{ label: 'Nova', value: 'Nova' },
{ label: 'Shimmer', value: 'Shimmer' }
// { label: 'Alloy', value: 'Alloy' },
// { label: 'Echo', value: 'Echo' },
// { label: 'Fable', value: 'Fable' },
// { label: 'Onyx', value: 'Onyx' },
// { label: 'Nova', value: 'Nova' },
// { label: 'Shimmer', value: 'Shimmer' }
],
label: {
text: 'playground.params.voice',
@@ -64,25 +64,6 @@ export const TTSParamsConfig: ParamsSchema[] = [
}
]
}
// {
// type: 'TextArea',
// name: 'prompt',
// label: {
// text: 'Prompt',
// isLocalized: false
// },
// attrs: {
// autoSize: {
// minRows: 2,
// maxRows: 3
// }
// },
// rules: [
// {
// required: false
// }
// ]
// }
];
export const RealtimeParamsConfig: ParamsSchema[] = [
src/pages/playground/speech.tsx
+23 -2
View File
@@ -2,6 +2,7 @@ import IconFont from '@/components/icon-font';
import breakpoints from '@/config/breakpoints';
import HotKeys from '@/config/hotkeys';
import useWindowResize from '@/hooks/use-window-resize';
import { queryModelsList as queryGPUStackModels } from '@/pages/llmodels/apis';
import { AudioOutlined } from '@ant-design/icons';
import { PageContainer } from '@ant-design/pro-components';
import { useIntl } from '@umijs/max';
@@ -102,10 +103,30 @@ const Playground: React.FC = () => {
}
};
const getGpuStackModels = async () => {
try {
const res: any = await queryGPUStackModels({ page: 1, perPage: 100 });
return res.items || [];
} catch (error) {
return [];
}
};
const fetchData = async () => {
try {
const modelist = await getModelList();
setModelList(modelist);
const [modelist, list] = await Promise.all([
getModelList(),
getGpuStackModels()
]);
const dataMap = list.reduce((acc: any, cur: any) => {
acc[cur.name] = cur;
return acc;
}, {});
const dataList = modelist.map((item: any) => {
item.modelId = dataMap[item.value]?.id;
return item;
});
setModelList(dataList);
} catch (error) {
setLoaded(true);
}
src/pages/playground/style/ground-left.less
+1 -1
View File
@@ -86,7 +86,7 @@
height: fit-content;
top: -10px;
font-size: var(--font-size-middle);
left: calc(50% + 18px);
left: calc(50% + 19px);
transform: translateX(-50%);
background-color: transparent;
}