root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: embedding messages format

Browse Source
This commit is contained in:
jialin
2026-03-26 17:59:33 +08:00
committed by jialin
parent bb3cef89c1
commit c74118e913
7 changed files with 186 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/seal-form/row-textarea.tsx
+20 -15
View File
@@ -11,16 +11,19 @@ import './styles/row-textarea.less';
interface SystemMessageProps {
style?: React.CSSProperties;
data: {
text: string;
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
uid: number | string;
name: string;
dataUrl?: string;
role?: string;
};
placeholder?: string;
label?: React.ReactNode;
height?: number;
onUploadImage?: (list: { uid: number | string; dataUrl: string }[]) => void;
onDeleteImage?: () => void;
onDeleteImage?: (
updatedImgs: { uid: number | string; dataUrl: string }[]
) => void;
onChange: (e: any) => void;
onPaste?: (e: any) => void;
onDelete?: () => void;
@@ -90,8 +93,8 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
const start = e.target.selectionStart;
const end = e.target.selectionEnd;
const beforeText = data.text.substring(0, start);
const afterText = data.text.substring(end, data.text.length);
const beforeText = data.content.substring(0, start);
const afterText = data.content.substring(end, data.content.length);
onSelect?.({
start,
end,
@@ -116,8 +119,12 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
updateUidCount: () => `img-${Date.now()}`
});
const expanded = autoSize.focus || !!data.dataUrl || isFromUrl;
// const expanded = true;
const handleDeleteImage = (uid: number) => {
const updatedImgs = (data.imgs || []).filter((img) => img.uid !== uid);
onDeleteImage?.(updatedImgs);
};
const expanded = autoSize.focus || !!data.imgs?.length || isFromUrl;
console.log('expanded===========', expanded);
@@ -141,14 +148,12 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
className="textarea-wrapper"
>
{label && <span className="textarea-label">{label}</span>}
{data.dataUrl && (
{!!data.imgs?.length && (
<div style={{ padding: 8 }}>
<ThumbImg
editable
dataList={
data.dataUrl ? [{ uid: data.uid, dataUrl: data.dataUrl }] : []
}
onDelete={onDeleteImage}
dataList={data.imgs}
onDelete={handleDeleteImage}
/>
</div>
)}
@@ -164,7 +169,7 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
width: '100%',
boxShadow: 'none'
}}
value={data.text}
value={data.content}
autoSize={
expanded
? {
@@ -192,7 +197,7 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
{
<div className="content" style={{ height: height }}>
{label && <span className="title">{label}</span>}
{data.text || (
{data.content || (
<span style={{ color: 'var(--ant-color-text-tertiary)' }}>
{placeholder}
</span>
@@ -208,7 +213,7 @@ const RowTextarea: React.FC<SystemMessageProps> = (props) => {
>
{ImageURLInput}
<div className={'actions'}>
{!expanded && data.text && (
{!expanded && data.content && (
<SmallCloseButton onClick={handleClear}></SmallCloseButton>
)}
{UploadImageButton}
src/pages/playground/apis/index.ts
+10
View File
@@ -58,6 +58,16 @@ export const handleEmbedding = async (
encoding_format?: string;
dimensions?: number;
input: string[];
messages?: {
role: string;
content: Array<{
type: string;
text?: string;
image_url?: {
url: string;
};
}>;
}[];
},
options?: any
) => {
src/pages/playground/components/input-list.tsx
+19 -15
View File
@@ -9,25 +9,26 @@ interface InputListProps {
extra?: (data: any) => React.ReactNode;
showLabel?: boolean;
textList: {
text: string;
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
uid: number | string;
name: string;
dataUrl?: string;
role: string;
}[];
onChange?: (
textList: {
text: string;
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
uid: number | string;
name: string;
dataUrl?: string;
role: string;
}[]
) => void;
onPaste?: (e: any, index: number) => void;
onDeleteImage?: (dataItem: {
text: string;
uid: number | string;
name: string;
}) => void;
onDeleteImage?: (
itemUid: number | string,
updatedImgs: { uid: number | string; dataUrl: string }[]
) => void;
onUploadImage?: (
list: { uid: number | string; dataUrl: string }[],
index: number
@@ -69,14 +70,15 @@ const InputList: React.FC<InputListProps> = forwardRef(
setMessageId();
const dataList = [...textList];
dataList.push({
text: '',
content: '',
uid: messageId.current,
name: `Text ${dataList.length + 1}`
name: `Text ${dataList.length + 1}`,
role: 'user'
});
onChange?.(dataList);
};
const handleDelete = (dataItem: { text: string; uid: number | string }) => {
const handleDelete = (dataItem: { uid: number | string }) => {
const dataList = [...textList];
const index = dataList.findIndex((item) => item.uid === dataItem.uid);
dataList.splice(index, 1);
@@ -85,11 +87,11 @@ const InputList: React.FC<InputListProps> = forwardRef(
const handleTextChange = (
value: string,
dataItem: { text: string; uid: number | string }
dataItem: { uid: number | string }
) => {
const dataList = [...textList];
const index = dataList.findIndex((item) => item.uid === dataItem.uid);
dataList[index].text = value;
dataList[index].content = value;
onChange?.(dataList);
};
@@ -117,7 +119,9 @@ const InputList: React.FC<InputListProps> = forwardRef(
onPaste={(e) => onPaste?.(e, index)}
onSelect={(data) => onSelect?.({ ...data, index })}
onUploadImage={(list) => onUploadImage?.(list, index)}
onDeleteImage={() => onDeleteImage?.(item)}
onDeleteImage={(updatedImgs) =>
onDeleteImage?.(item.uid, updatedImgs)
}
onDelete={() => handleDelete(item)}
></RowTextarea>
</div>
src/pages/playground/config/embedding-worker.worker.ts
+20 -6
View File
@@ -50,11 +50,21 @@ const normalizeEmbeddingToCanvas = (
self.onmessage = (
event: MessageEvent<{
embeddings: any[];
fileList: { text: string; name: string; uid: number | string }[];
textList: { text: string; name: string; uid: number | string }[];
fileList: {
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
name: string;
uid: number | string;
}[];
textList: {
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
name: string;
uid: number | string;
}[];
}>
) => {
const { embeddings, fileList, textList } = event.data;
const { embeddings, fileList = [], textList } = event.data;
try {
const dataList = embeddings.map((item) => {
@@ -71,8 +81,12 @@ self.onmessage = (
const pcadata = pca.predict(dataList, { nComponents: 2 }).to2DArray();
const input = [
...textList.map((item) => item.text).filter((item) => item),
...fileList.map((item) => item.text).filter((item) => item)
...textList
.map((item) => item.content || (item.imgs?.length ? `[image]` : ''))
.filter((item) => item),
...fileList
.map((item) => item.content || (item.imgs?.length ? `[image]` : ''))
.filter((item) => item)
];
const list = pcadata.map((item: number[], index: number) => {
@@ -83,7 +97,7 @@ self.onmessage = (
};
});
const embeddingJson = embeddings.map((o, index) => {
const embeddingJson = embeddings.map((o) => {
const item = _.cloneDeep(o);
item.embedding = item.embedding.slice(0, 5);
item.embedding.push(null);
src/pages/playground/config/index.ts
+24 -2
View File
@@ -49,9 +49,13 @@ export const formatMessageParams = (messageList: any[]) => {
return result;
};
export const generateMessagesByListContent = (messageList: any[]) => {
export const generateMessagesByListContent = (
messageList: any[],
multimodal?: boolean
) => {
if (!messageList.length) return [];
return messageList.map((item: MessageItem) => {
// image
if (item.imgs?.length || item.audio?.length) {
const content: any[] = [];
// image
@@ -90,7 +94,25 @@ export const generateMessagesByListContent = (messageList: any[]) => {
};
}
return _.omit(item, ['uid', 'imgs', 'audio']);
// for multimodal model, if there is text content but no image or audio, also send as multimodal format
if (
multimodal &&
item.content &&
!item.imgs?.length &&
!item.audio?.length
) {
return {
role: item.role,
content: [
{
type: 'text',
text: item.content
}
]
};
}
return _.omit(item, ['uid', 'imgs', 'audio', 'name']);
});
};
src/pages/playground/embedding/page.tsx
+81 -72
View File
@@ -27,12 +27,11 @@ import React, {
useState
} from 'react';
import { EMBEDDING_API, handleEmbedding } from '../apis';
import FileList from '../components/file-list';
import InputList from '../components/input-list';
import RightContainer from '../components/right-container';
import TokenUsage from '../components/token-usage';
import ViewCommonCode from '../components/view-common-code';
import { extractErrorMessage } from '../config';
import { extractErrorMessage, generateMessagesByListContent } from '../config';
import { embeddingSamples } from '../config/samples';
import { LLM_METAKEYS } from '../hooks/config';
import useEmbeddingWorker from '../hooks/use-embedding-worker';
@@ -64,9 +63,6 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
const inputListRef = useRef<any>(null);
const messageListLengthCache = useRef<number>(0);
const requestToken = useRef<any>(null);
const [fileList, setFileList] = useState<
{ text: string; name: string; uid: number | string }[]
>([]);
const [outputType, setOutputType] = useState<string>('chart');
const [outputHeight, setOutputHeight] = useState<number>(180);
const [embeddingData, setEmbeddingData] = useState<{
@@ -81,19 +77,25 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
const selectionTextRef = useRef<any>(null);
const [textList, setTextList] = useState<
{ text: string; dataUrl?: string; uid: number | string; name: string }[]
{
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
uid: number | string;
name: string;
role: string;
}[]
>([
{
text: '',
content: '',
uid: -1,
name: '',
dataUrl: ''
role: 'user'
},
{
text: '',
content: '',
uid: -2,
name: '',
dataUrl: ''
role: 'user'
}
]);
@@ -132,24 +134,47 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
};
});
const formatInputs = (
list: {
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
uid: number | string;
name: string;
role: string;
}[]
) => {
const validTextList = textList.filter(
(item) => item.content || item.imgs?.length
);
const hasImages = validTextList.some((item) => item.imgs?.length);
const mutipleInput = generateMessagesByListContent(validTextList, true);
// const firstInput = mutipleInput?.[0];
// mutipleInput.forEach((item: any) => {
// firstInput.content = firstInput.content.concat(item.content);
// });
return hasImages
? { messages: mutipleInput }
: { input: [...validTextList.map((item) => item.content || '')] };
};
const viewCodeContent = useMemo(() => {
console.log('viewCodeContent:', embeddingData.copyValue);
return generateEmbeddingCode({
api: EMBEDDING_API,
parameters: {
...parameters,
input: [
...textList.map((item) => item.text).filter((item) => item),
...fileList.map((item) => item.text).filter((item) => item)
]
...formatInputs(textList)
}
});
}, [parameters, textList, fileList]);
}, [parameters, textList]);
const inputEmpty = useMemo(() => {
const list = [...textList, ...fileList];
const list = [...textList];
return list.length < 2;
}, [textList, fileList]);
}, [textList]);
const setMessageId = () => {
return inputListRef.current?.setMessageId?.();
@@ -165,23 +190,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
await formRef.current?.form.validateFields();
if (!parameters.model) return;
const validTextList = textList.filter(
(item) => item.text || item.dataUrl
(item) => item.content || item.imgs?.length
);
const validFileList = fileList.filter((item) => item.text);
const inputList = [
...validTextList.map((item) => item.text || item.dataUrl || ''),
...validFileList.map((item) => item.text || '')
];
const inputList = formatInputs(textList);
if (inputList.length < 2) {
if (validTextList.length < 2) {
setLessTwoInput(true);
return;
}
setTextList(validTextList);
setFileList(validFileList);
setLessTwoInput(false);
setLoading(true);
@@ -197,7 +217,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
{
model: parameters.model,
encoding_format: 'float',
input: inputList
...inputList
},
{
token: requestToken.current.token
@@ -220,8 +240,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
postMessage({
embeddings: embeddingsList,
textList: textList,
fileList: fileList
textList: textList
});
} catch (error: any) {
setTokenResult({
@@ -262,17 +281,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
}
};
const handleDeleteFile = (uid: number | string) => {
setFileList((preList) => {
return preList.filter((item) => item.uid !== uid);
});
};
const handleAddText = () => {
inputListRef.current?.handleAdd();
};
const handleTextListChange = (
list: { text: string; uid: number | string; name: string }[]
list: {
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
uid: number | string;
name: string;
role: string;
}[]
) => {
setTextList(list);
};
@@ -281,35 +301,29 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
list: { uid: number | string; dataUrl: string }[],
index: number
) => {
// replace the text with dataUrl in the textList
setTextList((preList) => {
const newList = [...preList];
const current = newList[index];
if (current) {
newList[index] = {
...current,
text: '',
uid: list[0].uid,
dataUrl: list[0].dataUrl
content: '',
imgs: list
};
}
return newList;
});
};
const handleOnDeleteImage = (item: {
text: string;
uid: number | string;
name: string;
dataUrl?: string;
}) => {
// replace the dataUrl with empty text in the textList
const handleOnDeleteImage = (
itemUid: number | string,
updatedImgs: { uid: number | string; dataUrl: string }[]
) => {
setTextList((preList) => {
const newList = [...preList];
const current = newList.find((i) => i.uid === item.uid);
const current = newList.find((i) => i.uid === itemUid);
if (current) {
current.text = '';
current.dataUrl = '';
current.imgs = updatedImgs;
}
return newList;
});
@@ -335,17 +349,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
if (text) {
const dataLlist = text.split('\n').map((item: string) => {
return {
text: item?.trim(),
content: item?.trim(),
name: '',
uid: setMessageId()
uid: setMessageId(),
role: 'user'
};
});
dataLlist[0].text = `${selectionTextRef.current?.beforeText || ''}${dataLlist[0].text}${selectionTextRef.current?.afterText || ''}`;
dataLlist[0].content = `${selectionTextRef.current?.beforeText || ''}${dataLlist[0].content}${selectionTextRef.current?.afterText || ''}`;
const result = [
...textList.slice(0, index),
...dataLlist,
...textList.slice(index + 1)
].filter((item) => item.text);
].filter((item) => item.content || item.imgs?.length);
setTextList(result);
}
@@ -356,17 +371,18 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
const handleClearDocuments = () => {
setTextList([
{
text: '',
content: '',
uid: -1,
name: ''
name: '',
role: 'user'
},
{
text: '',
content: '',
uid: -2,
name: ''
name: '',
role: 'user'
}
]);
setFileList([]);
setScatterData([]);
setTokenResult(null);
setLessTwoInput(false);
@@ -452,11 +468,11 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
}, [initialize]);
useEffect(() => {
if (textList.length + fileList.length > messageListLengthCache.current) {
if (textList.length > messageListLengthCache.current) {
updateDocumentScrollerPosition();
}
messageListLengthCache.current = textList.length + fileList.length;
}, [textList.length, fileList.length]);
messageListLengthCache.current = textList.length;
}, [textList.length]);
useEffect(() => {
if (intl.locale || 'en-US') {
@@ -465,9 +481,10 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
if (sample) {
setTextList(
sample.map((item: string, index: number) => ({
text: item,
content: item,
uid: setMessageId(),
name: `Document ${index + 1}`
name: `Document ${index + 1}`,
role: 'user'
}))
);
}
@@ -577,15 +594,6 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
onUploadImage={handleOnUploadImage}
onDeleteImage={handleOnDeleteImage}
></InputList>
{fileList.length > 0 && (
<div style={{ marginTop: 8 }}>
<FileList
fileList={fileList}
textListCount={textList.length || 0}
onDelete={handleDeleteFile}
></FileList>
</div>
)}
{lessTwoInput && (
<div className="m-t-16">
<AlertInfo
@@ -645,6 +653,7 @@ const GroundEmbedding: React.FC<MessageProps> = forwardRef((props, ref) => {
<Segmented
onChange={handleOutputTypeChange}
value={outputType}
size="middle"
options={[
{
label: intl.formatMessage({
src/pages/playground/hooks/use-embedding-worker.ts
+12 -2
View File
@@ -17,8 +17,18 @@ export default function useEmbeddingWorker() {
const postMessage = (params: {
embeddings: any[];
textList: { text: string; name: string; uid: number | string }[];
fileList: { text: string; name: string; uid: number | string }[];
textList: {
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
name: string;
uid: number | string;
}[];
fileList?: {
content: string;
imgs?: { uid: number | string; dataUrl: string }[];
name: string;
uid: number | string;
}[];
}) => {
if (workerRef.current) {
workerRef.current.postMessage(params);