diff --git a/config/config.ts b/config/config.ts index 6b51745c..154126d8 100644 --- a/config/config.ts +++ b/config/config.ts @@ -14,7 +14,7 @@ const isProduction = env === 'production'; const t = Date.now(); export default defineConfig({ proxy: { - ...proxy() + ...proxy('http://192.168.50.2') }, history: { type: 'hash' diff --git a/src/assets/styles/common.less b/src/assets/styles/common.less index 12d2f01a..5d30537e 100644 --- a/src/assets/styles/common.less +++ b/src/assets/styles/common.less @@ -148,6 +148,10 @@ gap: 20px; } +.gap-6 { + gap: 6px; +} + .line-24 { line-height: 24px; } diff --git a/src/components/highlight-code/code-viewer-dark.tsx b/src/components/highlight-code/code-viewer-dark.tsx index 65ee712d..789418d9 100644 --- a/src/components/highlight-code/code-viewer-dark.tsx +++ b/src/components/highlight-code/code-viewer-dark.tsx @@ -4,6 +4,7 @@ import './styles/dark.less'; interface CodeViewerProps { code: string; + copyValue?: string; lang: string; autodetect?: boolean; ignoreIllegals?: boolean; @@ -14,6 +15,7 @@ interface CodeViewerProps { const DarkViewer: React.FC = (props) => { const { code, + copyValue, lang, autodetect, ignoreIllegals, @@ -26,6 +28,7 @@ const DarkViewer: React.FC = (props) => { style={props.style} height={height} code={code} + copyValue={copyValue} lang={lang} theme="dark" autodetect={autodetect} diff --git a/src/components/highlight-code/code-viewer-light.tsx b/src/components/highlight-code/code-viewer-light.tsx index 1a6485eb..60d7e523 100644 --- a/src/components/highlight-code/code-viewer-light.tsx +++ b/src/components/highlight-code/code-viewer-light.tsx @@ -4,6 +4,7 @@ import './styles/light.less'; interface CodeViewerProps { code: string; + copyValue?: string; lang: string; autodetect?: boolean; ignoreIllegals?: boolean; @@ -14,6 +15,7 @@ interface CodeViewerProps { const LightViewer: React.FC = (props) => { const { code, + copyValue, lang, autodetect, ignoreIllegals, @@ -27,6 +29,7 @@ const LightViewer: React.FC = (props) => { style={style} height={height} code={code} + copyValue={copyValue} lang={lang} theme="light" autodetect={autodetect} diff --git a/src/components/highlight-code/code-viewer.tsx b/src/components/highlight-code/code-viewer.tsx index 92ca9258..d817509d 100644 --- a/src/components/highlight-code/code-viewer.tsx +++ b/src/components/highlight-code/code-viewer.tsx @@ -6,6 +6,7 @@ import { escapeHtml } from './utils'; interface CodeViewerProps { code: string; + copyValue?: string; lang: string; autodetect?: boolean; ignoreIllegals?: boolean; @@ -17,6 +18,7 @@ interface CodeViewerProps { const CodeViewer: React.FC = (props) => { const { code, + copyValue, lang, autodetect = true, ignoreIllegals = true, @@ -87,7 +89,7 @@ const CodeViewer: React.FC = (props) => { > {copyable && ( diff --git a/src/components/highlight-code/index.tsx b/src/components/highlight-code/index.tsx index aa742c23..a8a23d70 100644 --- a/src/components/highlight-code/index.tsx +++ b/src/components/highlight-code/index.tsx @@ -10,10 +10,12 @@ const HighlightCode: React.FC<{ theme?: 'light' | 'dark'; height?: string | number; style?: React.CSSProperties; + copyValue?: string; }> = (props) => { const { style, code, + copyValue, lang = 'bash', copyable = true, theme = 'dark', @@ -26,6 +28,7 @@ const HighlightCode: React.FC<{ diff --git a/src/components/seal-form/row-textarea.tsx b/src/components/seal-form/row-textarea.tsx index f6ec114e..87fc1eb5 100644 --- a/src/components/seal-form/row-textarea.tsx +++ b/src/components/seal-form/row-textarea.tsx @@ -12,6 +12,7 @@ interface SystemMessageProps { label?: React.ReactNode; height?: number; onChange: (e: any) => void; + onPaste?: (e: any) => void; } const RowTextarea: React.FC = (props) => { @@ -52,6 +53,9 @@ const RowTextarea: React.FC = (props) => { const handleClear = () => { onChange?.({ target: { value: '' } }); }; + const handleOnPaste = (e: any) => { + props.onPaste?.(e); + }; return (
= (props) => { onBlur={handleBlur} allowClear={false} onChange={handleOnChange} + onPaste={handleOnPaste} >
} diff --git a/src/components/speech-content/speech-item.tsx b/src/components/speech-content/speech-item.tsx index bb2e78b4..f06460a7 100644 --- a/src/components/speech-content/speech-item.tsx +++ b/src/components/speech-content/speech-item.tsx @@ -9,7 +9,7 @@ import React, { useRef, useState } from 'react'; import AudioPlayer from './audio-player'; import './styles/index.less'; -const audioUrl = require('./ih.mp4'); +// const audioUrl = require('./ih.mp4'); interface SpeechContentProps { prompt: string; @@ -17,6 +17,7 @@ interface SpeechContentProps { voice: string; format: string; speed: number; + audioUrl: string; } const SpeechItem: React.FC = (props) => { console.log('porps=======', props); @@ -34,13 +35,16 @@ const SpeechItem: React.FC = (props) => { return (
- {/* */}
{props.voice}
- +
@@ -75,11 +79,11 @@ const SpeechItem: React.FC = (props) => {
- {collapsed && ( + {/* {collapsed && (
{props.prompt}
- )} + )} */} ); }; diff --git a/src/components/upload-audio/index.tsx b/src/components/upload-audio/index.tsx index 958438f4..fd5b13d9 100644 --- a/src/components/upload-audio/index.tsx +++ b/src/components/upload-audio/index.tsx @@ -1,4 +1,5 @@ import { UploadOutlined } from '@ant-design/icons'; +import { useIntl } from '@umijs/max'; import { Button, Tooltip, Upload } from 'antd'; import React from 'react'; @@ -10,6 +11,7 @@ interface UploadAudioProps { } const UploadAudio: React.FC = (props) => { + const intl = useIntl(); const beforeUpload = (file: any) => { return true; }; @@ -22,7 +24,13 @@ const UploadAudio: React.FC = (props) => { [] ); return ( - + { + return { category: 'tags', predicate: 'contains', values: [tag] }; + }); + const tasksCriterion = params.tasks?.map((task: string) => { + return { category: 'tasks', predicate: 'contains', values: [task] }; + }); + + const Criterion = + tagsCriterion?.length || tasksCriterion?.length + ? { + Criterion: [...(tagsCriterion || []), ...(tasksCriterion || [])] + } + : {}; const res = await fetch(`${MODEL_SCOPE_LIST_MODEL_API}`, { method: 'PUT', signal: config?.signal, @@ -299,3 +306,19 @@ export async function downloadModelFile( )?.text(); return res; } +export async function downloadModelScopeModelfile( + params: { name: string }, + options?: any +) { + const res = await fetch( + `${MODE_SCOPE_MODEL_FIELS_API}${params.name}/resolve/master/config.json`, + { + method: 'GET', + signal: options?.signal + } + ); + if (!res.ok) { + throw new Error('Network response was not ok'); + } + return res.json(); +} diff --git a/src/pages/llmodels/components/advance-config.tsx b/src/pages/llmodels/components/advance-config.tsx index 3777fd53..1ea80b06 100644 --- a/src/pages/llmodels/components/advance-config.tsx +++ b/src/pages/llmodels/components/advance-config.tsx @@ -35,6 +35,7 @@ interface AdvanceConfigProps { gpuOptions: Array; action: PageActionType; source: string; + modelTask: string; } const AdvanceConfig: React.FC = (props) => { diff --git a/src/pages/llmodels/components/data-form.tsx b/src/pages/llmodels/components/data-form.tsx index 377c90c4..6d4c84b6 100644 --- a/src/pages/llmodels/components/data-form.tsx +++ b/src/pages/llmodels/components/data-form.tsx @@ -16,8 +16,11 @@ import React, { } from 'react'; import { queryGPUList } from '../apis'; import { + HuggingFaceTaskMap, + ModelscopeTaskMap, backendOptionsMap, modelSourceMap, + modelTaskMap, ollamaModelOptions } from '../config'; import { FormData, GPUListItem } from '../config/types'; @@ -45,6 +48,12 @@ const DataForm: React.FC = forwardRef((props, ref) => { const [gpuOptions, setGpuOptions] = useState< Array >([]); + const [modelTask, setModelTask] = useState>({ + type: '', + value: '', + text2speech: false, + speech2text: false + }); const sourceOptions = [ { @@ -116,6 +125,27 @@ const DataForm: React.FC = forwardRef((props, ref) => { const reg = /(-gguf)$/i; name = _.toLower(name).replace(reg, ''); + const modelTask = + HuggingFaceTaskMap.audio.includes(props.selectedModel.task) || + ModelscopeTaskMap.audio.includes(props.selectedModel.task) + ? modelTaskMap.audio + : ''; + + setModelTask({ + value: props.selectedModel.task, + type: modelTask, + text2speech: + HuggingFaceTaskMap[modelTaskMap.textToSpeech] === + props.selectedModel.task || + ModelscopeTaskMap[modelTaskMap.textToSpeech] === + props.selectedModel.task, + speech2text: + HuggingFaceTaskMap[modelTaskMap.speechToText] === + props.selectedModel.task || + ModelscopeTaskMap[modelTaskMap.speechToText] === + props.selectedModel.task + }); + if (SEARCH_SOURCE.includes(props.source)) { form.setFieldsValue({ repo_id: props.selectedModel.name, @@ -321,6 +351,8 @@ const DataForm: React.FC = forwardRef((props, ref) => { if (gpu) { onOk({ ..._.omit(formdata, ['scheduleType']), + speech_to_text: modelTask.speech2text, + text_to_speech: modelTask.text2speech, gpu_selector: { gpu_name: gpu.name, gpu_index: gpu.index, @@ -329,19 +361,24 @@ const DataForm: React.FC = forwardRef((props, ref) => { }); } else { onOk({ - ..._.omit(formdata, ['scheduleType']) + ..._.omit(formdata, ['scheduleType']), + speech_to_text: modelTask.speech2text, + text_to_speech: modelTask.text2speech }); } }; useEffect(() => { - if (action === PageAction.CREATE) { + if (action === PageAction.EDIT) return; + if (modelTask.type === modelTaskMap.audio) { + form.setFieldValue('backend', backendOptionsMap.voxBox); + } else { form.setFieldValue( 'backend', isGGUF ? backendOptionsMap.llamaBox : backendOptionsMap.vllm ); } - }, [isGGUF]); + }, [isGGUF, modelTask]); useEffect(() => { handleOnSelectModel(); }, [props.selectedModel.name]); @@ -449,6 +486,7 @@ const DataForm: React.FC = forwardRef((props, ref) => { form={form} gpuOptions={gpuOptions} isGGUF={isGGUF} + modelTask={modelTask} action={action} source={props.source} > diff --git a/src/pages/llmodels/components/hf-model-item.tsx b/src/pages/llmodels/components/hf-model-item.tsx index 1bb2fb06..be45ce4a 100644 --- a/src/pages/llmodels/components/hf-model-item.tsx +++ b/src/pages/llmodels/components/hf-model-item.tsx @@ -23,7 +23,7 @@ interface HFModelItemProps { source?: string; tags?: string[]; } -const warningTask = ['audio', 'video']; +const warningTask = ['video']; const SUPPORTEDSOURCE = [ modelSourceMap.huggingface_value, diff --git a/src/pages/llmodels/components/model-card.tsx b/src/pages/llmodels/components/model-card.tsx index 5f7bffbe..0146a680 100644 --- a/src/pages/llmodels/components/model-card.tsx +++ b/src/pages/llmodels/components/model-card.tsx @@ -10,11 +10,18 @@ import { useIntl } from '@umijs/max'; import { Button, Empty, Spin, Tag, Tooltip } from 'antd'; import { some } from 'lodash'; import 'overlayscrollbars/overlayscrollbars.css'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState +} from 'react'; import SimpleBar from 'simplebar-react'; import 'simplebar-react/dist/simplebar.min.css'; import { downloadModelFile, + downloadModelScopeModelfile, queryHuggingfaceModelDetail, queryModelScopeModelDetail } from '../apis'; @@ -37,10 +44,32 @@ const ModelCard: React.FC<{ const [readmeText, setReadmeText] = useState(null); const requestToken = useRef(null); const axiosTokenRef = useRef(null); + const loadConfigTokenRef = useRef(null); + const loadConfigJsonTokenRef = useRef(null); const [isGGUFModel, setIsGGUFModel] = useState(false); const [loading, setLoading] = useState(false); - const loadFile = async (repo: string, sha: string) => { + const modelTags = useMemo(() => { + if (modelSource === modelSourceMap.huggingface_value) { + return modelData?.pipeline_tag ? [modelData?.pipeline_tag] : []; + } + if (modelSource === modelSourceMap.modelscope_value) { + return modelData?.Tasks?.map((task: any) => task?.Name)?.filter( + (val: string) => val + ); + } + return []; + }, [modelSource, modelData]); + + const modelType = useMemo(() => { + if (modelSource === modelSourceMap.huggingface_value) { + return modelData?.config?.model_type || modelData?.ModelType?.[0]; + } + if (modelSource === modelSourceMap.modelscope_value) { + return modelData?.ModelType?.[0]; + } + }, [modelData, modelSource]); + const loadFile = useCallback(async (repo: string, sha: string) => { try { axiosTokenRef.current?.abort?.(); axiosTokenRef.current = new AbortController(); @@ -54,12 +83,32 @@ const ModelCard: React.FC<{ signal: axiosTokenRef.current.signal } ); - console.log('readme++++++++', res); return res || ''; } catch (error) { return ''; } - }; + }, []); + + const loadConfig = useCallback(async (repo: string, sha: string) => { + try { + loadConfigTokenRef.current?.abort?.(); + loadConfigTokenRef.current = new AbortController(); + const res = await downloadModelFile( + { + repo, + revision: sha, + path: 'config.json' + }, + { + signal: loadConfigTokenRef.current.signal + } + ); + return res || null; + } catch (error) { + console.log('error======', error); + return null; + } + }, []); const removeMetadata = useCallback((str: string) => { let indexes = []; @@ -81,6 +130,12 @@ const ModelCard: React.FC<{ // huggingface model card data const getHuggingfaceModelDetail = async () => { try { + const configjson = await loadConfig( + props.selectedModel.name, + 'main' + ).catch(() => { + return null; + }); const [modelcard, readme] = await Promise.all([ queryHuggingfaceModelDetail( { repo: props.selectedModel.name }, @@ -92,11 +147,13 @@ const ModelCard: React.FC<{ ]); setModelData(modelcard); + // remove the meta data from readme const newReadme = removeMetadata(readme); setReadmeText(newReadme); const isGGUF = modelcard.tags?.includes('gguf'); + console.log('modelData++++++++++++', isGGUF); setIsGGUF(isGGUF); setIsGGUFModel(isGGUF); } catch (error) { @@ -107,8 +164,30 @@ const ModelCard: React.FC<{ } }; + const loadModelscopeModelConfig = useCallback(async (name: string) => { + try { + loadConfigJsonTokenRef.current?.abort?.(); + loadConfigJsonTokenRef.current = new AbortController(); + return await downloadModelScopeModelfile( + { + name: name + }, + { + signal: loadConfigJsonTokenRef.current.token + } + ); + } catch (error) { + return null; + } + }, []); + const getModelScopeModelDetail = async () => { try { + const configjson = await loadModelscopeModelConfig( + props.selectedModel.name + ).catch(() => { + return null; + }); const data = await queryModelScopeModelDetail( { name: props.selectedModel.name @@ -121,6 +200,7 @@ const ModelCard: React.FC<{ ...data?.Data, name: `${data.Data?.Path}/${data.Data?.Name}` }); + console.log('modelData++++++++++++', configjson, data?.Data); setReadmeText(data?.Data?.ReadMeContent); const isGGUF = some( data?.Data?.Tags, @@ -230,6 +310,8 @@ const ModelCard: React.FC<{ return () => { requestToken.current?.cancel?.(); axiosTokenRef.current?.abort?.(); + loadConfigTokenRef.current?.abort?.(); + loadConfigJsonTokenRef.current?.abort?.(); }; }, []); @@ -243,13 +325,13 @@ const ModelCard: React.FC<{ {modelData ? (
- {modelData.config?.model_type && ( + {modelType && ( {intl.formatMessage({ id: 'models.architecture' })}: - {modelData.config?.model_type} + {modelType} )} @@ -258,6 +340,14 @@ const ModelCard: React.FC<{ GGUF )} + {!!modelTags.length && + modelTags.map((tag: string, index: number) => { + return ( + + {tag} + + ); + })}
{readmeText && isGGUFModel && (
= (props) => { const intl = useIntl(); - const { modelSource, setLoadingModel, onSourceChange, onSelectModel } = props; + const { modelSource, setLoadingModel, onSelectModel } = props; const [dataSource, setDataSource] = useState<{ repoOptions: any[]; loading: boolean; @@ -44,6 +47,7 @@ const SearchModel: React.FC = (props) => { const axiosTokenRef = useRef(null); const searchInputRef = useRef(''); const filterGGUFRef = useRef(); + const filterTaskRef = useRef(''); const modelFilesSortOptions = useRef([ { label: intl.formatMessage({ id: 'models.sort.trending' }), @@ -64,7 +68,6 @@ const SearchModel: React.FC = (props) => { ]); const handleOnSelectModel = useCallback((item: any) => { - console.log('handleOnSelectModel', item); onSelectModel(item); setCurrent(item.id); }, []); @@ -78,7 +81,7 @@ const SearchModel: React.FC = (props) => { query: searchInputRef.current || '', sort: sort, tags: filterGGUFRef.current ? ['gguf'] : [], - task + task: HuggingFaceTaskMap[filterTaskRef.current] || task } }; const data = await queryHuggingfaceModels(params, { @@ -102,7 +105,10 @@ const SearchModel: React.FC = (props) => { try { const params = { Name: `${searchInputRef.current}`, - filterGGUF: filterGGUFRef.current, + tags: filterGGUFRef.current ? ['gguf'] : [], + tasks: filterTaskRef.current + ? ([ModelscopeTaskMap[filterTaskRef.current]] as string[]) + : [], SortBy: ModelScopeSortType[sort] }; const data = await queryModelScopeModels(params, { @@ -213,6 +219,11 @@ const SearchModel: React.FC = (props) => { handleOnSearchRepo(); }; + const handleFilterTaskChange = useCallback((value: string) => { + filterTaskRef.current = value; + handleOnSearchRepo(); + }, []); + const renderHFSearch = () => { return ( <> @@ -222,15 +233,22 @@ const SearchModel: React.FC = (props) => { modelSource={modelSource} >
- + {/* {intl.formatMessage( { id: 'models.search.result' }, { count: dataSource.repoOptions.length } )} - - + */} + = (props) => { - + + + +
diff --git a/src/pages/llmodels/config/index.ts b/src/pages/llmodels/config/index.ts index b2dd28ed..0c48775d 100644 --- a/src/pages/llmodels/config/index.ts +++ b/src/pages/llmodels/config/index.ts @@ -73,6 +73,38 @@ export const backendOptionsMap = { voxBox: 'vox-box' }; +export const modelTaskMap = { + textToSpeech: 'text-to-speech', + speechToText: 'speech-to-text', + textToText: 'text-to-text', + textToImage: 'text-to-image', + audio: 'audio', + image: 'image' +}; + +export const ModelscopeTaskMap = { + [modelTaskMap.textToSpeech]: 'text-to-speech', + [modelTaskMap.speechToText]: 'auto-speech-recognition', + [modelTaskMap.textToText]: 'TextToText', + [modelTaskMap.textToImage]: 'text-to-image', + audio: ['text-to-speech', 'auto-speech-recognition'] +}; + +export const HuggingFaceTaskMap = { + [modelTaskMap.textToSpeech]: 'text-to-speech', + [modelTaskMap.speechToText]: 'automatic-speech-recognition', + [modelTaskMap.textToText]: 'text-2-text', + [modelTaskMap.textToImage]: 'text-to-image', + audio: ['text-to-speech', 'automatic-speech-recognition'] +}; + +export const AudioModeTypeMap = { + FunASR: ['FunASR', 'funasr', 'fun-asr', 'fun_asr'], + Bark: ['Bark', 'bark'], + Whisper: ['Whisper', 'whisper'], + CosyVoice: ['CosyVoice', 'cosyvoice', 'cosy-voice', 'cosy_voice'] +}; + export const modelSourceMap: Record = { huggingface: 'Hugging Face', ollama_library: 'Ollama Library', diff --git a/src/pages/playground/apis/index.ts b/src/pages/playground/apis/index.ts index 5b0851e3..77ec32c6 100644 --- a/src/pages/playground/apis/index.ts +++ b/src/pages/playground/apis/index.ts @@ -1,3 +1,4 @@ +import { MODELS_API } from '@/pages/llmodels/apis'; import { request } from '@umijs/max'; export const CHAT_API = '/v1-openai/chat/completions'; @@ -96,17 +97,37 @@ export const textToSpeech = async (params: any, options?: any) => { if (!res.ok) { throw new Error('Network response was not ok'); } - return res.json(); + + const audioBlob = await res.blob(); + const audioUrl = URL.createObjectURL(audioBlob); + return audioUrl; }; +// export const speechToText = async (params: any, options?: any) => { +// const res = await fetch(AUDIO_SPEECH_TO_TEXT_API, { +// method: 'POST', +// body: JSON.stringify(params.data), +// signal: params.signal +// }); +// if (!res.ok) { +// throw new Error('Network response was not ok'); +// } +// return res.json(); +// }; + export const speechToText = async (params: any, options?: any) => { - const res = await fetch(AUDIO_SPEECH_TO_TEXT_API, { + return request(AUDIO_SPEECH_TO_TEXT_API, { method: 'POST', - body: JSON.stringify(params.data), - signal: params.signal + data: params.data, + headers: { + 'Content-Type': 'multipart/form-data' + } + }); +}; + +export const queryModelVoices = async (params: { name: string }) => { + return request(`${MODELS_API}/${params.name}/voices`, { + method: 'GET', + skipErrorHandler: true }); - if (!res.ok) { - throw new Error('Network response was not ok'); - } - return res.json(); }; diff --git a/src/pages/playground/components/audio-input.tsx b/src/pages/playground/components/audio-input.tsx index aabc3390..4f6c0e03 100644 --- a/src/pages/playground/components/audio-input.tsx +++ b/src/pages/playground/components/audio-input.tsx @@ -1,4 +1,5 @@ import { AudioOutlined } from '@ant-design/icons'; +import { useIntl } from '@umijs/max'; import { Button, Space, Tooltip } from 'antd'; import React, { useCallback, useEffect, useRef, useState } from 'react'; // import '../style/audio-input.less'; @@ -19,6 +20,7 @@ interface AudioInputProps { } const AudioInput: React.FC = (props) => { + const intl = useIntl(); const [audioOn, setAudioOn] = useState(false); const [isRecording, setIsRecording] = useState(false); const [audioPermission, setAudioPermission] = useState(true); @@ -150,9 +152,9 @@ const AudioInput: React.FC = (props) => { stopRecording(); return; } + try { await EnableAudio(); - console.log('audioStream:', audioStream.current); audioRecorder.current = new MediaRecorder(audioStream.current); const audioChunks: any[] = []; @@ -172,7 +174,7 @@ const AudioInput: React.FC = (props) => { const audioUrl = URL.createObjectURL(audioBlob); handleAudioData({ - chunks: audioChunks, + chunks: audioBlob, size: audioBlob.size, type: audioBlob.type, url: audioUrl, @@ -188,8 +190,9 @@ const AudioInput: React.FC = (props) => { startTime.current = Date.now(); audioRecorder.current.start(1000); generateVisualData(); + console.log('start recording'); } catch (error) { - // console.log(error); + console.log('error====', error); } }; @@ -208,7 +211,13 @@ const AudioInput: React.FC = (props) => {
{ - +
= (props) => {
} - {/* {isRecording && ( - - - - )} */}
); diff --git a/src/pages/playground/components/dynamic-params.tsx b/src/pages/playground/components/dynamic-params.tsx index ca5de27d..a1148c4e 100644 --- a/src/pages/playground/components/dynamic-params.tsx +++ b/src/pages/playground/components/dynamic-params.tsx @@ -32,6 +32,7 @@ type ParamsSettingsProps = { modelList: Global.BaseOption[]; onValuesChange?: (changeValues: any, value: Record) => void; setParams: (params: any) => void; + onModelChange?: (model: string) => void; globalParams?: Record; paramsConfig?: ParamsSchema[]; initialValues?: Record; @@ -43,6 +44,7 @@ const ParamsSettings: React.FC = forwardRef( { setParams, onValuesChange, + onModelChange, selectedModel, globalParams, initialValues, @@ -84,6 +86,13 @@ const ParamsSettings: React.FC = forwardRef( } }, [modelList, showModelSelector, selectedModel, initialValues]); + const handleModelChange = useCallback( + (value: string) => { + onModelChange?.(value); + }, + [onModelChange] + ); + const handleOnFinish = (values: any) => { console.log('handleOnFinish', values); }; @@ -239,7 +248,7 @@ const ParamsSettings: React.FC = forwardRef( } return null; }); - }, [paramsConfig, params]); + }, [paramsConfig, params, intl]); return (
= forwardRef( ]} > = forwardRef((props, ref) => { >([]); const [outputType, setOutputType] = useState('chart'); const [outputHeight, setOutputHeight] = useState(180); - const [embeddingData, setEmbeddingData] = useState(''); + const [embeddingData, setEmbeddingData] = useState<{ + code: string; + copyValue: string; + }>({ + code: '', + copyValue: '' + }); const [lessTwoInput, setLessTwoInput] = useState(false); + const multiplePasteEnable = useRef(true); const [textList, setTextList] = useState< { text: string; uid: number | string; name: string }[] @@ -97,6 +104,7 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { const { initialize: innitializeParams, updateScrollerPosition } = useOverlayScroller(); + const formRef = useRef(null); useImperativeHandle(ref, () => { return { @@ -137,7 +145,15 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { }; }); setScatterData(list); - setEmbeddingData(JSON.stringify(embeddings, null, 2)); + const embeddingJson = embeddings.map((item, index) => { + item.embedding = item.embedding.slice(0, 5); + item.embedding.push(null); + return item; + }); + setEmbeddingData({ + code: JSON.stringify(embeddingJson, null, 2).replace(/null/g, '...'), + copyValue: JSON.stringify(embeddings, null, 2) + }); } catch (e) { console.log('error:', e); } @@ -155,6 +171,7 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { }; const submitMessage = async (current?: { role: string; content: string }) => { + await formRef.current?.form.validateFields(); if (!parameters.model) return; try { @@ -255,6 +272,25 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { setTextList(list); }; + const handleOnPaste = useCallback( + (e: any, index: number) => { + if (!multiplePasteEnable.current) return; + const text = e.clipboardData.getData('text'); + if (text) { + console.log('text:', text); + const dataLlist = text.split('\n').map((item: string) => { + return { + text: item, + uid: inputListRef.current?.setMessageId(), + name: '' + }; + }); + setTextList([...textList.slice(0, index), ...dataLlist]); + } + }, + [textList] + ); + const handleClearDocuments = () => { setTextList([ { @@ -300,7 +336,8 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { = forwardRef((props, ref) => {
-
+
+ + {/* + { + multiplePasteEnable.current = checked; + }} + /> + */} + {/* + { + multiplePasteEnable.current = checked; + }} + /> + */}
@@ -488,6 +542,7 @@ const GroundReranker: React.FC = forwardRef((props, ref) => { >
= forwardRef((props, ref) => { + const intl = useIntl(); const { modelList } = props; const messageId = useRef(0); const [messageList, setMessageList] = useState([ { - content: 'Generating text content...', + content: '', title: '', role: '', uid: messageId.current } ]); - const intl = useIntl(); const [searchParams] = useSearchParams(); const selectModel = searchParams.get('model') || ''; const [parameters, setParams] = useState({}); @@ -70,6 +69,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { }); const [isRecording, setIsRecording] = useState(false); const [recordEnd, setRecordEnd] = useState(false); + const formRef = useRef(null); const { initialize, updateScrollerPosition } = useOverlayScroller(); const { initialize: innitializeParams } = useOverlayScroller(); @@ -95,7 +95,8 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { setLoading(false); }; - const submitMessage = async (current?: { role: string; content: string }) => { + const submitMessage = async () => { + await formRef.current?.form.validateFields(); if (!parameters.model) return; try { setLoading(true); @@ -106,17 +107,12 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { controllerRef.current = new AbortController(); const signal = controllerRef.current.signal; - const chatParams = { + const params = { ...parameters, - stream: true, - stream_options: { - include_usage: true - } + file: new File([audioData.data], audioData.name) }; const result: any = await speechToText({ - data: chatParams, - url: CHAT_API, - signal + data: params }); if (result?.error) { @@ -129,16 +125,18 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { } setMessageList([ { - content: 'Generating text content...', + content: result.text, title: '', role: '', uid: messageId.current } ]); } catch (error) { - // console.log('error:', error); + console.log('error:', error); } finally { setLoading(false); + setRecordEnd(false); + setIsRecording(false); } }; const handleClear = () => { @@ -160,6 +158,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { return { url: data.url, name: data.name, + data: data.chunks, duration: data.duration }; }); @@ -177,7 +176,6 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { const handleUploadChange = useCallback( async (data: { file: any; fileList: any }) => { const res = await readAudioFile(data.file.originFileObj); - console.log('res=======', res); setAudioData(res); setRecordEnd(true); }, @@ -195,20 +193,12 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { const handleOnRecord = useCallback((val: boolean) => { setIsRecording(val); setAudioData(null); + console.log('data===', val); }, []); - const handleOnGenerate = useCallback(() => { - setMessageList([ - { - content: 'Generating text content...', - title: '', - role: '', - uid: messageId.current - } - ]); - setRecordEnd(false); - setIsRecording(false); - }, []); + const handleOnGenerate = async () => { + submitMessage(); + }; const handleOnDiscard = useCallback(() => { setRecordEnd(false); @@ -229,6 +219,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { > ); } + return (
@@ -238,7 +229,9 @@ const GroundLeft: React.FC = forwardRef((props, ref) => {
); }; - + useEffect(() => { + console.log('parameters:', parameters); + }, [parameters]); useEffect(() => {}, [messageList]); useEffect(() => { if (scroller.current) { @@ -271,51 +264,21 @@ const GroundLeft: React.FC = forwardRef((props, ref) => {
- {isRecording ? ( - <> - - - ) : ( - <> - {/* - - - - - */} - - - - - + {!isRecording && ( + )} +
{audioData ? ( @@ -326,24 +289,6 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { name={audioData.name} duration={audioData.duration} > - {/*
- - - -
*/}
) : ( @@ -399,20 +344,29 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { className="message-list-wrap" ref={scroller} style={{ - borderTop: messageList.length - ? '1px solid var(--ant-color-split)' - : '1px solid var(--ant-color-split)' + borderTop: '1px solid var(--ant-color-split)' }} >
<> - +
+ {audioData ? ( + messageList[0]?.content + ) : ( + + {intl.formatMessage({ + id: 'playground.audio.generating.tips' + })} + + )} +
{loading && (
@@ -422,12 +376,18 @@ const GroundLeft: React.FC = forwardRef((props, ref) => {
- +
@@ -441,6 +401,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { >
= forwardRef((props, ref) => { const messageId = useRef(0); const [messageList, setMessageList] = useState< { - prompt: string; + input: string; voice: string; format: string; speed: number; uid: number; autoplay: boolean; + audioUrl: string; }[] >([]); @@ -54,19 +59,18 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { const [searchParams] = useSearchParams(); const selectModel = searchParams.get('model') || ''; const [parameters, setParams] = useState({}); - const [systemMessage, setSystemMessage] = useState(''); const [show, setShow] = useState(false); const [loading, setLoading] = useState(false); const [tokenResult, setTokenResult] = useState(null); const [collapse, setCollapse] = useState(false); - const contentRef = useRef(''); const controllerRef = useRef(null); const scroller = useRef(null); - const currentMessageRef = useRef(null); const paramsRef = useRef(null); const messageListLengthCache = useRef(0); const checkvalueRef = useRef(true); const [currentPrompt, setCurrentPrompt] = useState(''); + const [voiceList, setVoiceList] = useState[]>([]); + const formRef = useRef(null); const { initialize, updateScrollerPosition } = useOverlayScroller(); const { initialize: innitializeParams } = useOverlayScroller(); @@ -93,6 +97,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { }; const submitMessage = async (current?: { role: string; content: string }) => { + await formRef.current?.form.validateFields(); if (!parameters.model) return; try { setLoading(true); @@ -104,24 +109,27 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { controllerRef.current = new AbortController(); const signal = controllerRef.current.signal; - const chatParams = { + const params = { ...parameters, - prompt: current?.content || currentPrompt + input: current?.content || currentPrompt }; - const result: any = await textToSpeech({ - data: chatParams, + const audioUrl: any = await textToSpeech({ + data: params, url: CHAT_API, signal }); + console.log('result:', parameters, audioUrl); + setMessageList([ { - prompt: current?.content || currentPrompt, + input: current?.content || currentPrompt, voice: parameters.voice, format: parameters.response_format, speed: parameters.speed, uid: messageId.current, - autoplay: checkvalueRef.current + autoplay: checkvalueRef.current, + audioUrl: audioUrl } ]); } catch (error) { @@ -147,11 +155,68 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { setShow(false); }; - const handleSelectModel = () => {}; + const handleSelectModel = useCallback( + async (value: string) => { + const data: any = modelList.find((item) => item.value === value); + if (!data) return; + try { + const res = await queryModelVoices({ + name: data?.modelId as string + }); + const voiceList = _.map(res.voices || [], (item: any) => { + return { + label: item, + value: item + }; + }); + setVoiceList(voiceList); + setParams((pre: any) => { + return { + ...pre, + voice: voiceList[0]?.value + }; + }); + formRef.current?.form.setFieldValue('voice', voiceList[0]?.value); + } catch (error) { + setVoiceList([]); + formRef.current?.form.setFieldValue('voice', ''); + setParams((pre: any) => { + return { + ...pre, + voice: '' + }; + }); + } + }, + [modelList] + ); const handleOnCheckChange = (e: any) => { checkvalueRef.current = e.target.checked; }; + + const renderExtra = useMemo(() => { + return paramsConfig.map((item: ParamsSchema) => { + return ( + + + + ); + }); + }, [paramsConfig, intl, voiceList]); + + useEffect(() => { + handleSelectModel(parameters.model); + }, [parameters.model, handleSelectModel]); + useEffect(() => { if (scroller.current) { initialize(scroller.current); @@ -233,9 +298,8 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { handleSubmit={handleSendMessage} handleAbortFetch={handleStopConversation} clearAll={handleClear} - setModelSelections={handleSelectModel} shouldResetMessage={false} - submitIcon={} + submitIcon={} modelList={modelList} />
@@ -247,13 +311,14 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { ref={paramsRef} >
-
@@ -261,7 +326,7 @@ const GroundLeft: React.FC = forwardRef((props, ref) => { void; + onPaste?: (e: any, index: number) => void; onSort?: ( textList: { text: string; uid: number | string; name: string }[] ) => void; @@ -42,7 +43,8 @@ const InputList: React.FC = forwardRef( height, onSort, onChange, - extra + extra, + onPaste }, ref ) => { @@ -181,6 +183,7 @@ const InputList: React.FC = forwardRef( const setMessageId = () => { messageId.current = messageId.current + 1; + return messageId.current; }; const handleAdd = () => { @@ -222,7 +225,8 @@ const InputList: React.FC = forwardRef( useImperativeHandle(ref, () => ({ handleAdd, handleDelete, - handleTextChange + handleTextChange, + setMessageId })); return ( @@ -239,6 +243,7 @@ const InputList: React.FC = forwardRef( id: 'playground.embedding.inputyourtext' })} onChange={(e) => handleTextChange(e.target.value, text)} + onPaste={(e) => onPaste?.(e, index)} > diff --git a/src/pages/playground/config/params-config.ts b/src/pages/playground/config/params-config.ts index ed36b6c8..ed6bf975 100644 --- a/src/pages/playground/config/params-config.ts +++ b/src/pages/playground/config/params-config.ts @@ -5,12 +5,12 @@ export const TTSParamsConfig: ParamsSchema[] = [ type: 'Select', name: 'voice', options: [ - { label: 'Alloy', value: 'Alloy' }, - { label: 'Echo', value: 'Echo' }, - { label: 'Fable', value: 'Fable' }, - { label: 'Onyx', value: 'Onyx' }, - { label: 'Nova', value: 'Nova' }, - { label: 'Shimmer', value: 'Shimmer' } + // { label: 'Alloy', value: 'Alloy' }, + // { label: 'Echo', value: 'Echo' }, + // { label: 'Fable', value: 'Fable' }, + // { label: 'Onyx', value: 'Onyx' }, + // { label: 'Nova', value: 'Nova' }, + // { label: 'Shimmer', value: 'Shimmer' } ], label: { text: 'playground.params.voice', @@ -64,25 +64,6 @@ export const TTSParamsConfig: ParamsSchema[] = [ } ] } - // { - // type: 'TextArea', - // name: 'prompt', - // label: { - // text: 'Prompt', - // isLocalized: false - // }, - // attrs: { - // autoSize: { - // minRows: 2, - // maxRows: 3 - // } - // }, - // rules: [ - // { - // required: false - // } - // ] - // } ]; export const RealtimeParamsConfig: ParamsSchema[] = [ diff --git a/src/pages/playground/speech.tsx b/src/pages/playground/speech.tsx index c3147ac6..3bc83561 100644 --- a/src/pages/playground/speech.tsx +++ b/src/pages/playground/speech.tsx @@ -2,6 +2,7 @@ import IconFont from '@/components/icon-font'; import breakpoints from '@/config/breakpoints'; import HotKeys from '@/config/hotkeys'; import useWindowResize from '@/hooks/use-window-resize'; +import { queryModelsList as queryGPUStackModels } from '@/pages/llmodels/apis'; import { AudioOutlined } from '@ant-design/icons'; import { PageContainer } from '@ant-design/pro-components'; import { useIntl } from '@umijs/max'; @@ -102,10 +103,30 @@ const Playground: React.FC = () => { } }; + const getGpuStackModels = async () => { + try { + const res: any = await queryGPUStackModels({ page: 1, perPage: 100 }); + return res.items || []; + } catch (error) { + return []; + } + }; + const fetchData = async () => { try { - const modelist = await getModelList(); - setModelList(modelist); + const [modelist, list] = await Promise.all([ + getModelList(), + getGpuStackModels() + ]); + const dataMap = list.reduce((acc: any, cur: any) => { + acc[cur.name] = cur; + return acc; + }, {}); + const dataList = modelist.map((item: any) => { + item.modelId = dataMap[item.value]?.id; + return item; + }); + setModelList(dataList); } catch (error) { setLoaded(true); } diff --git a/src/pages/playground/style/ground-left.less b/src/pages/playground/style/ground-left.less index 6371b863..76b222ee 100644 --- a/src/pages/playground/style/ground-left.less +++ b/src/pages/playground/style/ground-left.less @@ -86,7 +86,7 @@ height: fit-content; top: -10px; font-size: var(--font-size-middle); - left: calc(50% + 18px); + left: calc(50% + 19px); transform: translateX(-50%); background-color: transparent; } diff --git a/src/utils/load-audio-file.ts b/src/utils/load-audio-file.ts index 155262db..28ee76bc 100644 --- a/src/utils/load-audio-file.ts +++ b/src/utils/load-audio-file.ts @@ -12,13 +12,20 @@ export const loadAudioData = async (data: any, type: string) => { audio.addEventListener('loadedmetadata', () => { const duration = audio.duration; - resolve({ size: fileSize, duration: Math.ceil(duration), url: url }); + resolve({ + data: audioBlob, + size: fileSize, + type: type, + duration: Math.ceil(duration), + url: url + }); }); audio.addEventListener('ended', () => { URL.revokeObjectURL(audio.src); }); } catch (error) { + console.log('error====', error); reject(error); } }); @@ -29,7 +36,6 @@ export const readAudioFile = async (file: File) => { const reader = new FileReader(); reader.onload = async function (e: any) { try { - // const size = convertFileSize(file.size); console.log('file====', file); const arrayBuffer = e.target.result; const audioData = await loadAudioData(arrayBuffer, file.type); @@ -38,6 +44,7 @@ export const readAudioFile = async (file: File) => { name: file.name }); } catch (error) { + console.log('error====', error); reject(error); } };