From c74118e9134b0966a09d0f654fddc028124e0765 Mon Sep 17 00:00:00 2001 From: jialin Date: Wed, 25 Mar 2026 20:43:25 +0800 Subject: [PATCH] refactor: embedding messages format --- src/components/seal-form/row-textarea.tsx | 35 ++-- src/pages/playground/apis/index.ts | 10 ++ .../playground/components/input-list.tsx | 34 ++-- .../config/embedding-worker.worker.ts | 26 ++- src/pages/playground/config/index.ts | 26 ++- src/pages/playground/embedding/page.tsx | 153 +++++++++--------- .../playground/hooks/use-embedding-worker.ts | 14 +- 7 files changed, 186 insertions(+), 112 deletions(-) diff --git a/src/components/seal-form/row-textarea.tsx b/src/components/seal-form/row-textarea.tsx index e8c51c1e..7121b700 100644 --- a/src/components/seal-form/row-textarea.tsx +++ b/src/components/seal-form/row-textarea.tsx @@ -11,16 +11,19 @@ import './styles/row-textarea.less'; interface SystemMessageProps { style?: React.CSSProperties; data: { - text: string; + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; uid: number | string; name: string; - dataUrl?: string; + role?: string; }; placeholder?: string; label?: React.ReactNode; height?: number; onUploadImage?: (list: { uid: number | string; dataUrl: string }[]) => void; - onDeleteImage?: () => void; + onDeleteImage?: ( + updatedImgs: { uid: number | string; dataUrl: string }[] + ) => void; onChange: (e: any) => void; onPaste?: (e: any) => void; onDelete?: () => void; @@ -90,8 +93,8 @@ const RowTextarea: React.FC = (props) => { const start = e.target.selectionStart; const end = e.target.selectionEnd; - const beforeText = data.text.substring(0, start); - const afterText = data.text.substring(end, data.text.length); + const beforeText = data.content.substring(0, start); + const afterText = data.content.substring(end, data.content.length); onSelect?.({ start, end, @@ -116,8 +119,12 @@ const RowTextarea: React.FC = (props) => { updateUidCount: () => `img-${Date.now()}` }); - const expanded = autoSize.focus || !!data.dataUrl || isFromUrl; - // const expanded = true; + const handleDeleteImage = (uid: number) => { + const updatedImgs = (data.imgs || []).filter((img) => img.uid !== uid); + onDeleteImage?.(updatedImgs); + }; + + const expanded = autoSize.focus || !!data.imgs?.length || isFromUrl; console.log('expanded===========', expanded); @@ -141,14 +148,12 @@ const RowTextarea: React.FC = (props) => { className="textarea-wrapper" > {label && {label}} - {data.dataUrl && ( + {!!data.imgs?.length && (
)} @@ -164,7 +169,7 @@ const RowTextarea: React.FC = (props) => { width: '100%', boxShadow: 'none' }} - value={data.text} + value={data.content} autoSize={ expanded ? { @@ -192,7 +197,7 @@ const RowTextarea: React.FC = (props) => { {
{label && {label}} - {data.text || ( + {data.content || ( {placeholder} @@ -208,7 +213,7 @@ const RowTextarea: React.FC = (props) => { > {ImageURLInput}
- {!expanded && data.text && ( + {!expanded && data.content && ( )} {UploadImageButton} diff --git a/src/pages/playground/apis/index.ts b/src/pages/playground/apis/index.ts index 5d8717a7..b2f9517d 100644 --- a/src/pages/playground/apis/index.ts +++ b/src/pages/playground/apis/index.ts @@ -58,6 +58,16 @@ export const handleEmbedding = async ( encoding_format?: string; dimensions?: number; input: string[]; + messages?: { + role: string; + content: Array<{ + type: string; + text?: string; + image_url?: { + url: string; + }; + }>; + }[]; }, options?: any ) => { diff --git a/src/pages/playground/components/input-list.tsx b/src/pages/playground/components/input-list.tsx index 4663ec34..607bfc5e 100644 --- a/src/pages/playground/components/input-list.tsx +++ b/src/pages/playground/components/input-list.tsx @@ -9,25 +9,26 @@ interface InputListProps { extra?: (data: any) => React.ReactNode; showLabel?: boolean; textList: { - text: string; + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; uid: number | string; name: string; - dataUrl?: string; + role: string; }[]; onChange?: ( textList: { - text: string; + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; uid: number | string; name: string; - dataUrl?: string; + role: string; }[] ) => void; onPaste?: (e: any, index: number) => void; - onDeleteImage?: (dataItem: { - text: string; - uid: number | string; - name: string; - }) => void; + onDeleteImage?: ( + itemUid: number | string, + updatedImgs: { uid: number | string; dataUrl: string }[] + ) => void; onUploadImage?: ( list: { uid: number | string; dataUrl: string }[], index: number @@ -69,14 +70,15 @@ const InputList: React.FC = forwardRef( setMessageId(); const dataList = [...textList]; dataList.push({ - text: '', + content: '', uid: messageId.current, - name: `Text ${dataList.length + 1}` + name: `Text ${dataList.length + 1}`, + role: 'user' }); onChange?.(dataList); }; - const handleDelete = (dataItem: { text: string; uid: number | string }) => { + const handleDelete = (dataItem: { uid: number | string }) => { const dataList = [...textList]; const index = dataList.findIndex((item) => item.uid === dataItem.uid); dataList.splice(index, 1); @@ -85,11 +87,11 @@ const InputList: React.FC = forwardRef( const handleTextChange = ( value: string, - dataItem: { text: string; uid: number | string } + dataItem: { uid: number | string } ) => { const dataList = [...textList]; const index = dataList.findIndex((item) => item.uid === dataItem.uid); - dataList[index].text = value; + dataList[index].content = value; onChange?.(dataList); }; @@ -117,7 +119,9 @@ const InputList: React.FC = forwardRef( onPaste={(e) => onPaste?.(e, index)} onSelect={(data) => onSelect?.({ ...data, index })} onUploadImage={(list) => onUploadImage?.(list, index)} - onDeleteImage={() => onDeleteImage?.(item)} + onDeleteImage={(updatedImgs) => + onDeleteImage?.(item.uid, updatedImgs) + } onDelete={() => handleDelete(item)} >
diff --git a/src/pages/playground/config/embedding-worker.worker.ts b/src/pages/playground/config/embedding-worker.worker.ts index 2135e35d..d72e484d 100644 --- a/src/pages/playground/config/embedding-worker.worker.ts +++ b/src/pages/playground/config/embedding-worker.worker.ts @@ -50,11 +50,21 @@ const normalizeEmbeddingToCanvas = ( self.onmessage = ( event: MessageEvent<{ embeddings: any[]; - fileList: { text: string; name: string; uid: number | string }[]; - textList: { text: string; name: string; uid: number | string }[]; + fileList: { + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; + name: string; + uid: number | string; + }[]; + textList: { + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; + name: string; + uid: number | string; + }[]; }> ) => { - const { embeddings, fileList, textList } = event.data; + const { embeddings, fileList = [], textList } = event.data; try { const dataList = embeddings.map((item) => { @@ -71,8 +81,12 @@ self.onmessage = ( const pcadata = pca.predict(dataList, { nComponents: 2 }).to2DArray(); const input = [ - ...textList.map((item) => item.text).filter((item) => item), - ...fileList.map((item) => item.text).filter((item) => item) + ...textList + .map((item) => item.content || (item.imgs?.length ? `[image]` : '')) + .filter((item) => item), + ...fileList + .map((item) => item.content || (item.imgs?.length ? `[image]` : '')) + .filter((item) => item) ]; const list = pcadata.map((item: number[], index: number) => { @@ -83,7 +97,7 @@ self.onmessage = ( }; }); - const embeddingJson = embeddings.map((o, index) => { + const embeddingJson = embeddings.map((o) => { const item = _.cloneDeep(o); item.embedding = item.embedding.slice(0, 5); item.embedding.push(null); diff --git a/src/pages/playground/config/index.ts b/src/pages/playground/config/index.ts index e72f90ee..c3959f12 100644 --- a/src/pages/playground/config/index.ts +++ b/src/pages/playground/config/index.ts @@ -49,9 +49,13 @@ export const formatMessageParams = (messageList: any[]) => { return result; }; -export const generateMessagesByListContent = (messageList: any[]) => { +export const generateMessagesByListContent = ( + messageList: any[], + multimodal?: boolean +) => { if (!messageList.length) return []; return messageList.map((item: MessageItem) => { + // image if (item.imgs?.length || item.audio?.length) { const content: any[] = []; // image @@ -90,7 +94,25 @@ export const generateMessagesByListContent = (messageList: any[]) => { }; } - return _.omit(item, ['uid', 'imgs', 'audio']); + // for multimodal model, if there is text content but no image or audio, also send as multimodal format + if ( + multimodal && + item.content && + !item.imgs?.length && + !item.audio?.length + ) { + return { + role: item.role, + content: [ + { + type: 'text', + text: item.content + } + ] + }; + } + + return _.omit(item, ['uid', 'imgs', 'audio', 'name']); }); }; diff --git a/src/pages/playground/embedding/page.tsx b/src/pages/playground/embedding/page.tsx index 0cafcfbb..e28abba1 100644 --- a/src/pages/playground/embedding/page.tsx +++ b/src/pages/playground/embedding/page.tsx @@ -27,12 +27,11 @@ import React, { useState } from 'react'; import { EMBEDDING_API, handleEmbedding } from '../apis'; -import FileList from '../components/file-list'; import InputList from '../components/input-list'; import RightContainer from '../components/right-container'; import TokenUsage from '../components/token-usage'; import ViewCommonCode from '../components/view-common-code'; -import { extractErrorMessage } from '../config'; +import { extractErrorMessage, generateMessagesByListContent } from '../config'; import { embeddingSamples } from '../config/samples'; import { LLM_METAKEYS } from '../hooks/config'; import useEmbeddingWorker from '../hooks/use-embedding-worker'; @@ -64,9 +63,6 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { const inputListRef = useRef(null); const messageListLengthCache = useRef(0); const requestToken = useRef(null); - const [fileList, setFileList] = useState< - { text: string; name: string; uid: number | string }[] - >([]); const [outputType, setOutputType] = useState('chart'); const [outputHeight, setOutputHeight] = useState(180); const [embeddingData, setEmbeddingData] = useState<{ @@ -81,19 +77,25 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { const selectionTextRef = useRef(null); const [textList, setTextList] = useState< - { text: string; dataUrl?: string; uid: number | string; name: string }[] + { + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; + uid: number | string; + name: string; + role: string; + }[] >([ { - text: '', + content: '', uid: -1, name: '', - dataUrl: '' + role: 'user' }, { - text: '', + content: '', uid: -2, name: '', - dataUrl: '' + role: 'user' } ]); @@ -132,24 +134,47 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { }; }); + const formatInputs = ( + list: { + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; + uid: number | string; + name: string; + role: string; + }[] + ) => { + const validTextList = textList.filter( + (item) => item.content || item.imgs?.length + ); + + const hasImages = validTextList.some((item) => item.imgs?.length); + + const mutipleInput = generateMessagesByListContent(validTextList, true); + + // const firstInput = mutipleInput?.[0]; + // mutipleInput.forEach((item: any) => { + // firstInput.content = firstInput.content.concat(item.content); + // }); + + return hasImages + ? { messages: mutipleInput } + : { input: [...validTextList.map((item) => item.content || '')] }; + }; + const viewCodeContent = useMemo(() => { - console.log('viewCodeContent:', embeddingData.copyValue); return generateEmbeddingCode({ api: EMBEDDING_API, parameters: { ...parameters, - input: [ - ...textList.map((item) => item.text).filter((item) => item), - ...fileList.map((item) => item.text).filter((item) => item) - ] + ...formatInputs(textList) } }); - }, [parameters, textList, fileList]); + }, [parameters, textList]); const inputEmpty = useMemo(() => { - const list = [...textList, ...fileList]; + const list = [...textList]; return list.length < 2; - }, [textList, fileList]); + }, [textList]); const setMessageId = () => { return inputListRef.current?.setMessageId?.(); @@ -165,23 +190,18 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { await formRef.current?.form.validateFields(); if (!parameters.model) return; const validTextList = textList.filter( - (item) => item.text || item.dataUrl + (item) => item.content || item.imgs?.length ); - const validFileList = fileList.filter((item) => item.text); - const inputList = [ - ...validTextList.map((item) => item.text || item.dataUrl || ''), - ...validFileList.map((item) => item.text || '') - ]; + const inputList = formatInputs(textList); - if (inputList.length < 2) { + if (validTextList.length < 2) { setLessTwoInput(true); return; } setTextList(validTextList); - setFileList(validFileList); setLessTwoInput(false); setLoading(true); @@ -197,7 +217,7 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { { model: parameters.model, encoding_format: 'float', - input: inputList + ...inputList }, { token: requestToken.current.token @@ -220,8 +240,7 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { postMessage({ embeddings: embeddingsList, - textList: textList, - fileList: fileList + textList: textList }); } catch (error: any) { setTokenResult({ @@ -262,17 +281,18 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { } }; - const handleDeleteFile = (uid: number | string) => { - setFileList((preList) => { - return preList.filter((item) => item.uid !== uid); - }); - }; const handleAddText = () => { inputListRef.current?.handleAdd(); }; const handleTextListChange = ( - list: { text: string; uid: number | string; name: string }[] + list: { + content: string; + imgs?: { uid: number | string; dataUrl: string }[]; + uid: number | string; + name: string; + role: string; + }[] ) => { setTextList(list); }; @@ -281,35 +301,29 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { list: { uid: number | string; dataUrl: string }[], index: number ) => { - // replace the text with dataUrl in the textList setTextList((preList) => { const newList = [...preList]; const current = newList[index]; if (current) { newList[index] = { ...current, - text: '', - uid: list[0].uid, - dataUrl: list[0].dataUrl + content: '', + imgs: list }; } return newList; }); }; - const handleOnDeleteImage = (item: { - text: string; - uid: number | string; - name: string; - dataUrl?: string; - }) => { - // replace the dataUrl with empty text in the textList + const handleOnDeleteImage = ( + itemUid: number | string, + updatedImgs: { uid: number | string; dataUrl: string }[] + ) => { setTextList((preList) => { const newList = [...preList]; - const current = newList.find((i) => i.uid === item.uid); + const current = newList.find((i) => i.uid === itemUid); if (current) { - current.text = ''; - current.dataUrl = ''; + current.imgs = updatedImgs; } return newList; }); @@ -335,17 +349,18 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { if (text) { const dataLlist = text.split('\n').map((item: string) => { return { - text: item?.trim(), + content: item?.trim(), name: '', - uid: setMessageId() + uid: setMessageId(), + role: 'user' }; }); - dataLlist[0].text = `${selectionTextRef.current?.beforeText || ''}${dataLlist[0].text}${selectionTextRef.current?.afterText || ''}`; + dataLlist[0].content = `${selectionTextRef.current?.beforeText || ''}${dataLlist[0].content}${selectionTextRef.current?.afterText || ''}`; const result = [ ...textList.slice(0, index), ...dataLlist, ...textList.slice(index + 1) - ].filter((item) => item.text); + ].filter((item) => item.content || item.imgs?.length); setTextList(result); } @@ -356,17 +371,18 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { const handleClearDocuments = () => { setTextList([ { - text: '', + content: '', uid: -1, - name: '' + name: '', + role: 'user' }, { - text: '', + content: '', uid: -2, - name: '' + name: '', + role: 'user' } ]); - setFileList([]); setScatterData([]); setTokenResult(null); setLessTwoInput(false); @@ -452,11 +468,11 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { }, [initialize]); useEffect(() => { - if (textList.length + fileList.length > messageListLengthCache.current) { + if (textList.length > messageListLengthCache.current) { updateDocumentScrollerPosition(); } - messageListLengthCache.current = textList.length + fileList.length; - }, [textList.length, fileList.length]); + messageListLengthCache.current = textList.length; + }, [textList.length]); useEffect(() => { if (intl.locale || 'en-US') { @@ -465,9 +481,10 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { if (sample) { setTextList( sample.map((item: string, index: number) => ({ - text: item, + content: item, uid: setMessageId(), - name: `Document ${index + 1}` + name: `Document ${index + 1}`, + role: 'user' })) ); } @@ -577,15 +594,6 @@ const GroundEmbedding: React.FC = forwardRef((props, ref) => { onUploadImage={handleOnUploadImage} onDeleteImage={handleOnDeleteImage} > - {fileList.length > 0 && ( -
- -
- )} {lessTwoInput && (
= forwardRef((props, ref) => { { if (workerRef.current) { workerRef.current.postMessage(params);