refactor: embedding messages format

src/components/seal-form/row-textarea.tsx

@@ -11,16 +11,19 @@ import './styles/row-textarea.less';
 interface SystemMessageProps {
   style?: React.CSSProperties;
   data: {
-    text: string;
+    content: string;
+    imgs?: { uid: number | string; dataUrl: string }[];
     uid: number | string;
     name: string;
-    dataUrl?: string;
+    role?: string;
   };
   placeholder?: string;
   label?: React.ReactNode;
   height?: number;
   onUploadImage?: (list: { uid: number | string; dataUrl: string }[]) => void;
-  onDeleteImage?: () => void;
+  onDeleteImage?: (
+    updatedImgs: { uid: number | string; dataUrl: string }[]
+  ) => void;
   onChange: (e: any) => void;
   onPaste?: (e: any) => void;
   onDelete?: () => void;
@@ -90,8 +93,8 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
     const start = e.target.selectionStart;
     const end = e.target.selectionEnd;
 
-    const beforeText = data.text.substring(0, start);
+    const beforeText = data.content.substring(0, start);
-    const afterText = data.text.substring(end, data.text.length);
+    const afterText = data.content.substring(end, data.content.length);
     onSelect?.({
       start,
       end,
@@ -116,8 +119,12 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
       updateUidCount: () => `img-${Date.now()}`
     });
 
-  const expanded = autoSize.focus || !!data.dataUrl || isFromUrl;
+  const handleDeleteImage = (uid: number) => {
-  // const expanded = true;
+    const updatedImgs = (data.imgs || []).filter((img) => img.uid !== uid);
+    onDeleteImage?.(updatedImgs);
+  };
+
+  const expanded = autoSize.focus || !!data.imgs?.length || isFromUrl;
 
   console.log('expanded===========', expanded);
 
@@ -141,14 +148,12 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
           className="textarea-wrapper"
         >
           {label && <span className="textarea-label">{label}</span>}
-          {data.dataUrl && (
+          {!!data.imgs?.length && (
             <div style={{ padding: 8 }}>
               <ThumbImg
                 editable
-                dataList={
+                dataList={data.imgs}
-                  data.dataUrl ? [{ uid: data.uid, dataUrl: data.dataUrl }] : []
+                onDelete={handleDeleteImage}
-                }
-                onDelete={onDeleteImage}
               />
             </div>
           )}
@@ -164,7 +169,7 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
               width: '100%',
               boxShadow: 'none'
             }}
-            value={data.text}
+            value={data.content}
             autoSize={
               expanded
                 ? {
@@ -192,7 +197,7 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
             {
               <div className="content" style={{ height: height }}>
                 {label && <span className="title">{label}</span>}
-                {data.text || (
+                {data.content || (
                   <span style={{ color: 'var(--ant-color-text-tertiary)' }}>
                     {placeholder}
                   </span>
@@ -208,7 +213,7 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
         >
           {ImageURLInput}
           <div className={'actions'}>
-            {!expanded && data.text && (
+            {!expanded && data.content && (
               <SmallCloseButton onClick={handleClear}></SmallCloseButton>
             )}
             {UploadImageButton}

src/pages/playground/apis/index.ts

@@ -58,6 +58,16 @@ export const handleEmbedding = async (
     encoding_format?: string;
     dimensions?: number;
     input: string[];
+    messages?: {
+      role: string;
+      content: Array<{
+        type: string;
+        text?: string;
+        image_url?: {
+          url: string;
+        };
+      }>;
+    }[];
   },
   options?: any
 ) => {

src/pages/playground/components/input-list.tsx

@@ -9,25 +9,26 @@ interface InputListProps {
   extra?: (data: any) => React.ReactNode;
   showLabel?: boolean;
   textList: {
-    text: string;
+    content: string;
+    imgs?: { uid: number | string; dataUrl: string }[];
     uid: number | string;
     name: string;
-    dataUrl?: string;
+    role: string;
   }[];
   onChange?: (
     textList: {
-      text: string;
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
       uid: number | string;
       name: string;
-      dataUrl?: string;
+      role: string;
     }[]
   ) => void;
   onPaste?: (e: any, index: number) => void;
-  onDeleteImage?: (dataItem: {
+  onDeleteImage?: (
-    text: string;
+    itemUid: number | string,
-    uid: number | string;
+    updatedImgs: { uid: number | string; dataUrl: string }[]
-    name: string;
+  ) => void;
-  }) => void;
   onUploadImage?: (
     list: { uid: number | string; dataUrl: string }[],
     index: number
@@ -69,14 +70,15 @@ const InputList: React.FC<InputListProps> = forwardRef(
       setMessageId();
       const dataList = [...textList];
       dataList.push({
-        text: '',
+        content: '',
         uid: messageId.current,
-        name: `Text ${dataList.length + 1}`
+        name: `Text ${dataList.length + 1}`,
+        role: 'user'
       });
       onChange?.(dataList);
     };
 
-    const handleDelete = (dataItem: { text: string; uid: number | string }) => {
+    const handleDelete = (dataItem: { uid: number | string }) => {
       const dataList = [...textList];
       const index = dataList.findIndex((item) => item.uid === dataItem.uid);
       dataList.splice(index, 1);
@@ -85,11 +87,11 @@ const InputList: React.FC<InputListProps> = forwardRef(
 
     const handleTextChange = (
       value: string,
-      dataItem: { text: string; uid: number | string }
+      dataItem: { uid: number | string }
     ) => {
       const dataList = [...textList];
       const index = dataList.findIndex((item) => item.uid === dataItem.uid);
-      dataList[index].text = value;
+      dataList[index].content = value;
       onChange?.(dataList);
     };
 
@@ -117,7 +119,9 @@ const InputList: React.FC<InputListProps> = forwardRef(
                   onPaste={(e) => onPaste?.(e, index)}
                   onSelect={(data) => onSelect?.({ ...data, index })}
                   onUploadImage={(list) => onUploadImage?.(list, index)}
-                  onDeleteImage={() => onDeleteImage?.(item)}
+                  onDeleteImage={(updatedImgs) =>
+                    onDeleteImage?.(item.uid, updatedImgs)
+                  }
                   onDelete={() => handleDelete(item)}
                 ></RowTextarea>
               </div>

src/pages/playground/config/embedding-worker.worker.ts

@@ -50,11 +50,21 @@ const normalizeEmbeddingToCanvas = (
 self.onmessage = (
   event: MessageEvent<{
     embeddings: any[];
-    fileList: { text: string; name: string; uid: number | string }[];
+    fileList: {
-    textList: { text: string; name: string; uid: number | string }[];
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      name: string;
+      uid: number | string;
+    }[];
+    textList: {
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      name: string;
+      uid: number | string;
+    }[];
   }>
 ) => {
-  const { embeddings, fileList, textList } = event.data;
+  const { embeddings, fileList = [], textList } = event.data;
 
   try {
     const dataList = embeddings.map((item) => {
@@ -71,8 +81,12 @@ self.onmessage = (
     const pcadata = pca.predict(dataList, { nComponents: 2 }).to2DArray();
 
     const input = [
-      ...textList.map((item) => item.text).filter((item) => item),
+      ...textList
-      ...fileList.map((item) => item.text).filter((item) => item)
+        .map((item) => item.content || (item.imgs?.length ? `[image]` : ''))
+        .filter((item) => item),
+      ...fileList
+        .map((item) => item.content || (item.imgs?.length ? `[image]` : ''))
+        .filter((item) => item)
     ];
 
     const list = pcadata.map((item: number[], index: number) => {
@@ -83,7 +97,7 @@ self.onmessage = (
       };
     });
 
-    const embeddingJson = embeddings.map((o, index) => {
+    const embeddingJson = embeddings.map((o) => {
       const item = _.cloneDeep(o);
       item.embedding = item.embedding.slice(0, 5);
       item.embedding.push(null);

src/pages/playground/config/index.ts

@@ -49,9 +49,13 @@ export const formatMessageParams = (messageList: any[]) => {
   return result;
 };
 
-export const generateMessagesByListContent = (messageList: any[]) => {
+export const generateMessagesByListContent = (
+  messageList: any[],
+  multimodal?: boolean
+) => {
   if (!messageList.length) return [];
   return messageList.map((item: MessageItem) => {
+    // image
     if (item.imgs?.length || item.audio?.length) {
       const content: any[] = [];
       // image
@@ -90,7 +94,25 @@ export const generateMessagesByListContent = (messageList: any[]) => {
       };
     }
 
-    return _.omit(item, ['uid', 'imgs', 'audio']);
+    // for multimodal model, if there is text content but no image or audio, also send as multimodal format
+    if (
+      multimodal &&
+      item.content &&
+      !item.imgs?.length &&
+      !item.audio?.length
+    ) {
+      return {
+        role: item.role,
+        content: [
+          {
+            type: 'text',
+            text: item.content
+          }
+        ]
+      };
+    }
+
+    return _.omit(item, ['uid', 'imgs', 'audio', 'name']);
   });
 };
 

src/pages/playground/embedding/page.tsx

@@ -27,12 +27,11 @@ import React, {
   useState
 } from 'react';
 import { EMBEDDING_API, handleEmbedding } from '../apis';
-import FileList from '../components/file-list';
 import InputList from '../components/input-list';
 import RightContainer from '../components/right-container';
 import TokenUsage from '../components/token-usage';
 import ViewCommonCode from '../components/view-common-code';
-import { extractErrorMessage } from '../config';
+import { extractErrorMessage, generateMessagesByListContent } from '../config';
 import { embeddingSamples } from '../config/samples';
 import { LLM_METAKEYS } from '../hooks/config';
 import useEmbeddingWorker from '../hooks/use-embedding-worker';
@@ -64,9 +63,6 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
   const inputListRef = useRef<any>(null);
   const messageListLengthCache = useRef<number>(0);
   const requestToken = useRef<any>(null);
-  const [fileList, setFileList] = useState<
-    { text: string; name: string; uid: number | string }[]
-  >([]);
   const [outputType, setOutputType] = useState<string>('chart');
   const [outputHeight, setOutputHeight] = useState<number>(180);
   const [embeddingData, setEmbeddingData] = useState<{
@@ -81,19 +77,25 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
   const selectionTextRef = useRef<any>(null);
 
   const [textList, setTextList] = useState<
-    { text: string; dataUrl?: string; uid: number | string; name: string }[]
+    {
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      uid: number | string;
+      name: string;
+      role: string;
+    }[]
   >([
     {
-      text: '',
+      content: '',
       uid: -1,
       name: '',
-      dataUrl: ''
+      role: 'user'
     },
     {
-      text: '',
+      content: '',
       uid: -2,
       name: '',
-      dataUrl: ''
+      role: 'user'
     }
   ]);
 
@@ -132,24 +134,47 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
     };
   });
 
+  const formatInputs = (
+    list: {
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      uid: number | string;
+      name: string;
+      role: string;
+    }[]
+  ) => {
+    const validTextList = textList.filter(
+      (item) => item.content || item.imgs?.length
+    );
+
+    const hasImages = validTextList.some((item) => item.imgs?.length);
+
+    const mutipleInput = generateMessagesByListContent(validTextList, true);
+
+    // const firstInput = mutipleInput?.[0];
+    // mutipleInput.forEach((item: any) => {
+    //   firstInput.content = firstInput.content.concat(item.content);
+    // });
+
+    return hasImages
+      ? { messages: mutipleInput }
+      : { input: [...validTextList.map((item) => item.content || '')] };
+  };
+
   const viewCodeContent = useMemo(() => {
-    console.log('viewCodeContent:', embeddingData.copyValue);
     return generateEmbeddingCode({
       api: EMBEDDING_API,
       parameters: {
         ...parameters,
-        input: [
+        ...formatInputs(textList)
-          ...textList.map((item) => item.text).filter((item) => item),
-          ...fileList.map((item) => item.text).filter((item) => item)
-        ]
       }
     });
-  }, [parameters, textList, fileList]);
+  }, [parameters, textList]);
 
   const inputEmpty = useMemo(() => {
-    const list = [...textList, ...fileList];
+    const list = [...textList];
     return list.length < 2;
-  }, [textList, fileList]);
+  }, [textList]);
 
   const setMessageId = () => {
     return inputListRef.current?.setMessageId?.();
@@ -165,23 +190,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
       await formRef.current?.form.validateFields();
       if (!parameters.model) return;
       const validTextList = textList.filter(
-        (item) => item.text || item.dataUrl
+        (item) => item.content || item.imgs?.length
       );
-      const validFileList = fileList.filter((item) => item.text);
 
-      const inputList = [
+      const inputList = formatInputs(textList);
-        ...validTextList.map((item) => item.text || item.dataUrl || ''),
-        ...validFileList.map((item) => item.text || '')
-      ];
 
-      if (inputList.length < 2) {
+      if (validTextList.length < 2) {
         setLessTwoInput(true);
 
         return;
       }
 
       setTextList(validTextList);
-      setFileList(validFileList);
 
       setLessTwoInput(false);
       setLoading(true);
@@ -197,7 +217,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
         {
           model: parameters.model,
           encoding_format: 'float',
-          input: inputList
+          ...inputList
         },
         {
           token: requestToken.current.token
@@ -220,8 +240,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
 
       postMessage({
         embeddings: embeddingsList,
-        textList: textList,
+        textList: textList
-        fileList: fileList
       });
     } catch (error: any) {
       setTokenResult({
@@ -262,17 +281,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
     }
   };
 
-  const handleDeleteFile = (uid: number | string) => {
-    setFileList((preList) => {
-      return preList.filter((item) => item.uid !== uid);
-    });
-  };
   const handleAddText = () => {
     inputListRef.current?.handleAdd();
   };
 
   const handleTextListChange = (
-    list: { text: string; uid: number | string; name: string }[]
+    list: {
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      uid: number | string;
+      name: string;
+      role: string;
+    }[]
   ) => {
     setTextList(list);
   };
@@ -281,35 +301,29 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
     list: { uid: number | string; dataUrl: string }[],
     index: number
   ) => {
-    // replace the text with dataUrl in the textList
     setTextList((preList) => {
       const newList = [...preList];
       const current = newList[index];
       if (current) {
         newList[index] = {
           ...current,
-          text: '',
+          content: '',
-          uid: list[0].uid,
+          imgs: list
-          dataUrl: list[0].dataUrl
         };
       }
       return newList;
     });
   };
 
-  const handleOnDeleteImage = (item: {
+  const handleOnDeleteImage = (
-    text: string;
+    itemUid: number | string,
-    uid: number | string;
+    updatedImgs: { uid: number | string; dataUrl: string }[]
-    name: string;
+  ) => {
-    dataUrl?: string;
-  }) => {
-    // replace the dataUrl with empty text in the textList
     setTextList((preList) => {
       const newList = [...preList];
-      const current = newList.find((i) => i.uid === item.uid);
+      const current = newList.find((i) => i.uid === itemUid);
       if (current) {
-        current.text = '';
+        current.imgs = updatedImgs;
-        current.dataUrl = '';
       }
       return newList;
     });
@@ -335,17 +349,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
       if (text) {
         const dataLlist = text.split('\n').map((item: string) => {
           return {
-            text: item?.trim(),
+            content: item?.trim(),
             name: '',
-            uid: setMessageId()
+            uid: setMessageId(),
+            role: 'user'
           };
         });
-        dataLlist[0].text = `${selectionTextRef.current?.beforeText || ''}${dataLlist[0].text}${selectionTextRef.current?.afterText || ''}`;
+        dataLlist[0].content = `${selectionTextRef.current?.beforeText || ''}${dataLlist[0].content}${selectionTextRef.current?.afterText || ''}`;
         const result = [
           ...textList.slice(0, index),
           ...dataLlist,
           ...textList.slice(index + 1)
-        ].filter((item) => item.text);
+        ].filter((item) => item.content || item.imgs?.length);
 
         setTextList(result);
       }
@@ -356,17 +371,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
   const handleClearDocuments = () => {
     setTextList([
       {
-        text: '',
+        content: '',
         uid: -1,
-        name: ''
+        name: '',
+        role: 'user'
       },
       {
-        text: '',
+        content: '',
         uid: -2,
-        name: ''
+        name: '',
+        role: 'user'
       }
     ]);
-    setFileList([]);
     setScatterData([]);
     setTokenResult(null);
     setLessTwoInput(false);
@@ -452,11 +468,11 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
   }, [initialize]);
 
   useEffect(() => {
-    if (textList.length + fileList.length > messageListLengthCache.current) {
+    if (textList.length > messageListLengthCache.current) {
       updateDocumentScrollerPosition();
     }
-    messageListLengthCache.current = textList.length + fileList.length;
+    messageListLengthCache.current = textList.length;
-  }, [textList.length, fileList.length]);
+  }, [textList.length]);
 
   useEffect(() => {
     if (intl.locale || 'en-US') {
@@ -465,9 +481,10 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
       if (sample) {
         setTextList(
           sample.map((item: string, index: number) => ({
-            text: item,
+            content: item,
             uid: setMessageId(),
-            name: `Document ${index + 1}`
+            name: `Document ${index + 1}`,
+            role: 'user'
           }))
         );
       }
@@ -577,15 +594,6 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
                 onUploadImage={handleOnUploadImage}
                 onDeleteImage={handleOnDeleteImage}
               ></InputList>
-              {fileList.length > 0 && (
-                <div style={{ marginTop: 8 }}>
-                  <FileList
-                    fileList={fileList}
-                    textListCount={textList.length || 0}
-                    onDelete={handleDeleteFile}
-                  ></FileList>
-                </div>
-              )}
               {lessTwoInput && (
                 <div className="m-t-16">
                   <AlertInfo
@@ -645,6 +653,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
             <Segmented
               onChange={handleOutputTypeChange}
               value={outputType}
+              size="middle"
               options={[
                 {
                   label: intl.formatMessage({

src/pages/playground/hooks/use-embedding-worker.ts

@@ -17,8 +17,18 @@ export default function useEmbeddingWorker() {
 
   const postMessage = (params: {
     embeddings: any[];
-    textList: { text: string; name: string; uid: number | string }[];
+    textList: {
-    fileList: { text: string; name: string; uid: number | string }[];
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      name: string;
+      uid: number | string;
+    }[];
+    fileList?: {
+      content: string;
+      imgs?: { uid: number | string; dataUrl: string }[];
+      name: string;
+      uid: number | string;
+    }[];
   }) => {
     if (workerRef.current) {
       workerRef.current.postMessage(params);