root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: upload audio file in chat

Browse Source
This commit is contained in:
jialin
2025-06-03 19:58:08 +08:00
parent ed28bcdfb0
commit 471e1ae96f
8 changed files with 187 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/audio-player/audio-element.tsx
+20
View File
@@ -0,0 +1,20 @@
import React from 'react';
import styled from 'styled-components';
const AudioWrapper = styled.div`
width: 100%;
height: 100%;
display: flex;
align-items: center;
justify-content: flex-start;
`;
const AudioElement: React.FC<any> = (props) => {
return (
<AudioWrapper>
<audio {...props}></audio>
</AudioWrapper>
);
};
export default AudioElement;
src/locales/en-US/playground.ts
+1 -1
View File
@@ -90,7 +90,7 @@ export default {
'playground.audio.stoprecord': 'Stop Recording',
'playground.audio.generating.tips': 'Generated text will appear here.',
'playground.audio.uploadfile.tips':
'Please upload an audio file, supported formats: {formats}',
'Upload an audio file, supported formats: {formats}',
'playground.input.multiplePaste': 'Batch Input Mode',
'playground.input.multiplePaste.tips':
'When enabled, pasted multi-line text will be automatically split by newline into separate entries in the form.',
src/locales/zh-CN/playground.ts
+1 -1
View File
@@ -87,7 +87,7 @@ export default {
'playground.audio.startrecord': '开始录音',
'playground.audio.stoprecord': '停止录音',
'playground.audio.generating.tips': '生成的文本将出现在这里',
'playground.audio.uploadfile.tips': '请上传音频文件，支持格式：{formats}',
'playground.audio.uploadfile.tips': '上传音频文件，支持格式：{formats}',
'playground.audio.button.generate': '生成文本',
'playground.input.multiplePaste': '批量输入',
'playground.input.multiplePaste.tips':
src/pages/playground/components/message-input.tsx
+63 -33
View File
@@ -1,7 +1,14 @@
import AudioElement from '@/components/audio-player/audio-element';
import IconFont from '@/components/icon-font';
import UploadAudio from '@/components/upload-audio';
import HotKeys, { KeyMap } from '@/config/hotkeys';
import { convertFileToBase64 } from '@/utils/load-audio-file';
import { ClearOutlined, SendOutlined, SwapOutlined } from '@ant-design/icons';
import { convertFileToBase64, readAudioFile } from '@/utils/load-audio-file';
import {
ClearOutlined,
CustomerServiceOutlined,
SendOutlined,
SwapOutlined
} from '@ant-design/icons';
import { useIntl } from '@umijs/max';
import { Button, Checkbox, Divider, Input, Tooltip } from 'antd';
import _ from 'lodash';
@@ -14,8 +21,9 @@ import React, {
useState
} from 'react';
import { useHotkeys } from 'react-hotkeys-hook';
import styled from 'styled-components';
import { Roles } from '../config';
import { MessageItem } from '../config/types';
import { AudioFormat, MessageItem } from '../config/types';
import '../style/message-input.less';
import ThumbImg from './thumb-img';
import UploadImg from './upload-img';
@@ -26,6 +34,12 @@ const audioTypeMap: Record<string, string> = {
'audio/mpeg': 'mp3'
};
const AudioWrapper = styled.div`
audio {
padding-top: 10px;
}
`;
type CurrentMessage = Omit<MessageItem, 'uid'>;
type ActionType =
@@ -139,9 +153,14 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
content: '',
imgs: []
});
const imgCountRef = useRef(0);
const uidCountRef = useRef(0);
const inputRef = useRef<any>(null);
const updateUidCount = () => {
uidCountRef.current += 1;
return uidCountRef.current;
};
const isDisabled = useMemo(() => {
return disabled
? true
@@ -239,9 +258,9 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
if (imgs.length) {
const list = _.map(imgs, (img: string) => {
imgCountRef.current += 1;
updateUidCount();
return {
uid: imgCountRef.current,
uid: uidCountRef.current,
dataUrl: img
};
});
@@ -273,14 +292,19 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
}) => {
// convert audio file to base64
try {
console.log('audio file====', data.file);
const base64Audio = await convertFileToBase64(data.file);
const audioData = await readAudioFile(data.file);
console.log('audioData====', audioData);
setMessage({
...message,
audio: {
format: audioTypeMap[data.file.type],
dataUrl: base64Audio
}
audio: [
{
uid: updateUidCount(),
format: audioTypeMap[data.file.type] as AudioFormat,
base64: base64Audio.split(',')[1],
data: _.pick(audioData, ['url', 'name', 'duration'])
}
]
});
} catch (error) {
console.error('Error converting audio to Base64:', error);
@@ -298,6 +322,17 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
});
};
const handleDeleteAudio = (uid: number | string) => {
const list = _.filter(
message.audio,
(item: MessageItem) => item.uid !== uid
);
setMessage({
...message,
audio: list
});
};
const handleOnPaste = (e: any) => {
const text = e.clipboardData.getData('text');
if (!text) {
@@ -318,16 +353,6 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
}
};
const handleDeleteLastImage = useCallback(() => {
if (message.imgs && message.imgs?.length > 0) {
const newImgList = [...(message.imgs || [])];
const lastImage = newImgList.pop();
if (lastImage) {
handleDeleteImg(lastImage.uid);
}
}
}, [message.imgs, handleDeleteImg]);
useImperativeHandle(ref, () => ({
handleInputChange: handleInputChange
}));
@@ -387,7 +412,7 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
size="middle"
></UploadImg>
)}
{/* {actions.includes('upload') && message.role === Roles.User && (
{actions.includes('upload') && message.role === Roles.User && (
<UploadAudio
type="text"
accept={'.mp3,.wav'}
@@ -396,12 +421,6 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
icon={<CustomerServiceOutlined />}
onChange={handleUploadAudioChange}
></UploadAudio>
)} */}
{actions.includes('upload') && message.role === Roles.User && (
<UploadImg
handleUpdateImgList={handleUpdateImgList}
size="middle"
></UploadImg>
)}
{tools}
{actions.includes('clear') && (
@@ -490,11 +509,22 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
)}
</div>
</div>
<ThumbImg
dataList={message.imgs || []}
onDelete={handleDeleteImg}
editable={true}
></ThumbImg>
<div className="flex">
<ThumbImg
dataList={message.imgs || []}
onDelete={handleDeleteImg}
editable={true}
></ThumbImg>
{message.audio && message.audio.length > 0 && (
<AudioWrapper>
<AudioElement
src={message.audio?.[0].data?.url}
onDelete={handleDeleteAudio}
controls
></AudioElement>
</AudioWrapper>
)}
</div>
<div className="input-box">
{actions.includes('paste') ? (
<TextArea
src/pages/playground/components/multiple-chat/message-body.tsx
+47 -12
View File
@@ -1,3 +1,4 @@
import AudioElement from '@/components/audio-player/audio-element';
import FullMarkdown from '@/components/markdown-viewer/full-markdown';
import { Input } from 'antd';
import classNames from 'classnames';
@@ -8,11 +9,19 @@ import React, {
useImperativeHandle,
useRef
} from 'react';
import styled from 'styled-components';
import { Roles } from '../../config';
import { MessageItem, MessageItemAction } from '../../config/types';
import ThumbImg from '../thumb-img';
import ThinkContent from './think-content';
const AudioWrapper = styled.div`
padding-left: 10px;
audio {
padding-top: 10px;
}
`;
interface MessageBodyProps {
ref?: any;
data: MessageItem;
@@ -93,6 +102,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
role: data.role,
content: data.content,
uid: data.uid,
audio: data.audio || [],
imgs: [...(data.imgs || []), ...list]
});
}
@@ -117,6 +127,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
role: data.role,
content: data.content,
uid: data.uid,
audio: data.audio || [],
imgs: list
});
};
@@ -124,6 +135,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
const handleMessageChange = (e: any) => {
updateMessage?.({
imgs: data.imgs || [],
audio: data.audio || [],
role: data.role,
content: e.target.value,
uid: data.uid
@@ -143,6 +155,7 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
role: data.role,
content: editContent,
uid: data.uid,
audio: data.audio || [],
imgs: data.imgs
});
};
@@ -182,14 +195,25 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
return (
<div
className={classNames('content-item-content', {
'has-img': data.imgs?.length
'has-img':
data.imgs?.length || (data.audio && data.audio?.length > 0)
})}
>
<ThumbImg
editable={editable}
dataList={data.imgs || []}
onDelete={handleDeleteImg}
/>
<div className="justify-start">
<ThumbImg
editable={editable}
dataList={data.imgs || []}
onDelete={handleDeleteImg}
/>
{data.audio && data.audio.length > 0 && (
<AudioWrapper>
<AudioElement
src={data.audio?.[0]?.data.url}
controls
></AudioElement>
</AudioWrapper>
)}
</div>
{data.content && <div className="text">{data.content}</div>}
</div>
);
@@ -199,15 +223,26 @@ const MessageBody: React.FC<MessageBodyProps> = forwardRef(
return (
<div
className={classNames('message-content-input', {
'has-img': data.imgs?.length
'has-img':
data.imgs?.length || (data.audio && data.audio?.length > 0)
})}
onClick={handleClickWrapper}
>
<ThumbImg
editable={editable}
dataList={data.imgs || []}
onDelete={handleDeleteImg}
/>
<div className="justify-start">
<ThumbImg
editable={editable}
dataList={data.imgs || []}
onDelete={handleDeleteImg}
/>
{data.audio && data.audio.length > 0 && (
<AudioWrapper>
<AudioElement
src={data.audio?.[0]?.data.url}
controls
></AudioElement>
</AudioWrapper>
)}
</div>
<>
{data.role === Roles.User ? (
<Input.TextArea
src/pages/playground/config/index.ts
+28 -2
View File
@@ -1,6 +1,6 @@
import _, { map } from 'lodash';
import { CREAT_IMAGE_API } from '../apis';
import { MessageItem } from './types';
import { AudioData, MessageItem } from './types';
export const Roles = {
User: 'user',
@@ -76,7 +76,33 @@ export const generateMessagesByListContent = (messageList: any[]) => {
content: content
};
}
return _.omit(item, ['uid', 'imgs']);
// audio
if (item.audio?.length) {
const content = map(item.audio, (item: AudioData) => {
return {
type: 'input_audio',
input_audio: {
data: item.base64,
format: item.format
}
};
});
if (item.content) {
content.push({
type: 'text',
text: item.content
});
}
return {
role: item.role,
content: content
};
}
return _.omit(item, ['uid', 'imgs', 'audio']);
});
};
src/pages/playground/config/types.ts
+14 -1
View File
@@ -13,10 +13,23 @@ export type MessageItemAction =
| 'markdown'
| 'edit';
export type AudioFormat = 'wav' | 'mp3';
export interface AudioData {
uid: string | number;
base64: string;
format: AudioFormat;
data: {
url: string;
name: string;
duration: number;
};
}
export interface MessageItem {
content: string;
imgs?: { uid: string | number; dataUrl: string }[];
audio?: { uid: string | number; dataUrl: string; format: 'wav' | 'mp3' };
audio?: AudioData[];
role: string;
title?: React.ReactNode;
uid: number;
src/utils/load-audio-file.ts
+13 -2
View File
@@ -9,7 +9,16 @@ export const convertFileToBase64 = (file: File): Promise<string> => {
});
};
export const loadAudioData = async (data: any, type: string) => {
export const loadAudioData = async (
data: any,
type: string
): Promise<{
data: Blob;
size: number | string;
type: string;
duration: number;
url: string;
}> => {
return new Promise((resolve, reject) => {
try {
const audioBlob = new Blob([data], { type: type });
@@ -40,7 +49,9 @@ export const loadAudioData = async (data: any, type: string) => {
});
};
export const readAudioFile = async (file: File) => {
export const readAudioFile = async (
file: File
): Promise<{ url: string; name: string; duration: number }> => {
console.log('file====', file);
return new Promise((resolve, reject) => {
const reader = new FileReader();