root/gpustack-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: audio, recording format error

Browse Source
This commit is contained in:
jialin
2024-12-11 18:39:36 +08:00
parent f64e696f7a
commit fef5d0e688
10 changed files with 131 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+1
View File
@@ -17,6 +17,7 @@
"dependencies": {
"@ant-design/icons": "^5.5.1",
"@ant-design/pro-components": "^2.7.18",
"@ffmpeg/ffmpeg": "^0.12.10",
"@huggingface/gguf": "^0.1.7",
"@huggingface/hub": "^0.15.1",
"@huggingface/tasks": "^0.11.6",
pnpm-lock.yaml
Generated
+19 -28
View File
@@ -14,6 +14,9 @@ dependencies:
'@ant-design/pro-components':
specifier: ^2.7.18
version: 2.7.19(antd@5.21.6)(rc-field-form@2.4.0)(react-dom@18.2.0)(react@18.2.0)
'@ffmpeg/ffmpeg':
specifier: ^0.12.10
version: 0.12.10
'@huggingface/gguf':
specifier: ^0.1.7
version: 0.1.7
@@ -4129,6 +4132,18 @@ packages:
resolution: {integrity: sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==, tarball: https://registry.npmjs.org/@exodus/schemasafe/-/schemasafe-1.3.0.tgz}
dev: false
/@ffmpeg/ffmpeg@0.12.10:
resolution: {integrity: sha512-lVtk8PW8e+NUzGZhPTWj2P1J4/NyuCrbDD3O9IGpSeLYtUZKBqZO8CNj1WYGghep/MXoM8e1qVY1GztTkf8YYQ==, tarball: https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.12.10.tgz}
engines: {node: '>=18.x'}
dependencies:
'@ffmpeg/types': 0.12.2
dev: false
/@ffmpeg/types@0.12.2:
resolution: {integrity: sha512-NJtxwPoLb60/z1Klv0ueshguWQ/7mNm106qdHkB4HL49LXszjhjCCiL+ldHJGQ9ai2Igx0s4F24ghigy//ERdA==, tarball: https://registry.npmjs.org/@ffmpeg/types/-/types-0.12.2.tgz}
engines: {node: '>=16.x'}
dev: false
/@floating-ui/core@0.6.2:
resolution: {integrity: sha512-jktYRmZwmau63adUG3GKOAVCofBXkk55S/zQ94XOorAHhwqFIOFAy1rSp2N0Wp6/tGbe9V3u/ExlGZypyY17rg==, tarball: https://registry.npmjs.org/@floating-ui/core/-/core-0.6.2.tgz}
dev: false
@@ -4995,7 +5010,7 @@ packages:
dependencies:
'@babel/core': 7.24.5
postcss: 7.0.39
postcss-syntax: 0.36.2(postcss@7.0.39)
postcss-syntax: 0.36.2(postcss@8.4.38)
transitivePeerDependencies:
- supports-color
dev: false
@@ -5036,7 +5051,7 @@ packages:
postcss-syntax: '>=0.36.2'
dependencies:
postcss: 7.0.39
postcss-syntax: 0.36.2(postcss@7.0.39)
postcss-syntax: 0.36.2(postcss@8.4.38)
remark: 13.0.0
unist-util-find-all-after: 3.0.2
transitivePeerDependencies:
@@ -14430,7 +14445,7 @@ packages:
dependencies:
htmlparser2: 3.10.1
postcss: 7.0.39
postcss-syntax: 0.36.2(postcss@7.0.39)
postcss-syntax: 0.36.2(postcss@8.4.38)
dev: false
/postcss-image-set-function@4.0.7(postcss@8.4.38):
@@ -14904,30 +14919,6 @@ packages:
lodash: 4.17.21
postcss: 8.4.38
/postcss-syntax@0.36.2(postcss@7.0.39):
resolution: {integrity: sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w==}
peerDependencies:
postcss: '>=5.0.0'
postcss-html: '*'
postcss-jsx: '*'
postcss-less: '*'
postcss-markdown: '*'
postcss-scss: '*'
peerDependenciesMeta:
postcss-html:
optional: true
postcss-jsx:
optional: true
postcss-less:
optional: true
postcss-markdown:
optional: true
postcss-scss:
optional: true
dependencies:
postcss: 7.0.39
dev: false
/postcss-syntax@0.36.2(postcss@8.4.38):
resolution: {integrity: sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w==}
peerDependencies:
@@ -18198,7 +18189,7 @@ packages:
postcss-sass: 0.4.4
postcss-scss: 2.1.1
postcss-selector-parser: 6.0.16
postcss-syntax: 0.36.2(postcss@7.0.39)
postcss-syntax: 0.36.2(postcss@8.4.38)
postcss-value-parser: 4.2.0
resolve-from: 5.0.0
slash: 3.0.0
src/components/audio-player/index.tsx
+12
View File
@@ -175,6 +175,18 @@ const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
console.log('onload', e);
};
const onDownload = useCallback(() => {
const url = props.url || '';
const filename = props.name;
const link = document.createElement('a');
link.href = url;
link.download = filename;
document.body.appendChild(link);
link.click();
link.remove();
}, [props.url, props.name]);
useEffect(() => {
if (audioRef.current) {
initPlayerConfig();
src/pages/llmodels/components/deploy-modal.tsx
+5 -7
View File
@@ -94,17 +94,15 @@ const AddModal: React.FC<AddModalProps> = (props) => {
if (!open) {
setIsGGUF(false);
form.current?.setFieldValue?.('backend', backendOptionsMap.vllm);
} else if (source === modelSourceMap.ollama_library_value) {
form.current?.setFieldValue?.('backend', backendOptionsMap.llamaBox);
setIsGGUF(true);
}
return () => {
setSelectedModel({});
};
}, [open]);
useEffect(() => {
if (source === modelSourceMap.ollama_library_value) {
setIsGGUF(true);
}
}, [source]);
}, [open, source]);
return (
<Drawer
src/pages/playground/components/audio-input.tsx
+78 -28
View File
@@ -1,5 +1,6 @@
import { externalRefer } from '@/constants/external-links';
import { AudioOutlined } from '@ant-design/icons';
import { FFmpeg } from '@ffmpeg/ffmpeg';
import { useIntl } from '@umijs/max';
import { Button, Space, Tooltip } from 'antd';
import dayjs from 'dayjs';
@@ -43,8 +44,8 @@ const audioFormat: Record<string, string> = {
};
const recordingFormat = {
type: 'audio/wav',
suffix: '.wav'
type: 'audio/mpeg',
suffix: '.mp3'
};
const AudioInput: React.FC<AudioInputProps> = (props) => {
@@ -59,6 +60,7 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
const analyser = useRef<any>(null);
const dataArray = useRef<any>(null);
const audioUrl = useRef<string>('');
const ffmpeg = useRef<any>(null);
const initAudioContext = useCallback(() => {
audioContext.current = new (window.AudioContext ||
@@ -90,6 +92,53 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
// props.onRecord?.(false);
};
const getAudioFormat = (type?: string) => {
const mimeType = type || audioRecorder.current?.mimeType;
let resultFormat = recordingFormat;
Object.keys(audioFormat).forEach((key: string) => {
if (mimeType.includes(key)) {
resultFormat = {
type: mimeType,
suffix: `.${audioFormat[key]}`
};
}
});
return resultFormat;
};
const convertAudioBlob2Mp3 = useCallback(
async (audioBlob: any, mimeType: string): Promise<Blob> => {
try {
if (!ffmpeg.current.loaded) {
await ffmpeg.current.load();
}
const format = getAudioFormat(mimeType);
const recordFile = new Uint8Array(await audioBlob.arrayBuffer());
await ffmpeg.current.writeFile(`input${format.suffix}`, recordFile);
await ffmpeg.current.exec([
'-i',
`input${format.suffix}`,
'-f',
'mp3',
'output.mp3'
]);
const mp3Data = await ffmpeg.current.readFile('output.mp3');
const mp3Blob = new Blob([mp3Data.buffer], { type: 'audio/mpeg' });
return mp3Blob;
} catch (error) {
return new Blob();
}
},
[]
);
// get all audio tracks
const getAudioTracks = () => {
const audioTracks = audioStream.current.getAudioTracks();
@@ -174,26 +223,11 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
const handleAudioData = (audioData: any) => {
props.onAudioData?.(audioData);
};
const getAudioFormat = (type?: string) => {
const mimeType = type || audioRecorder.current?.mimeType;
let resultFormat = recordingFormat;
Object.keys(audioFormat).forEach((key: string) => {
if (mimeType.includes(key)) {
resultFormat = {
type: mimeType,
suffix: `.${audioFormat[key]}`
};
}
});
return resultFormat;
};
const generateFileNameByTime = (type?: string) => {
const res = getAudioFormat(type);
const format = getAudioFormat(type);
// format: recording-YYYY-MM-DD-HH_mm_ss.wav
return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${res.suffix}`;
return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${format.suffix}`;
};
// start recording
const StartRecording = async () => {
@@ -209,6 +243,8 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
audioRecorder.current = new MediaRecorder(audioStream.current);
console.log('audioRecorder:', audioRecorder.current);
const audioChunks: any[] = [];
audioRecorder.current.ondataavailable = (event: any) => {
@@ -221,20 +257,27 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
};
// stop recording
audioRecorder.current.onstop = () => {
const res = getAudioFormat();
audioRecorder.current.onstop = async () => {
const audioBlob = new Blob(audioChunks, {
type: 'audio/mpeg'
type: audioRecorder.current.mimeType
});
console.log('audioBlob:', res, audioBlob);
audioUrl.current = URL.createObjectURL(audioBlob);
// convert audio blob to mp3
const mp3Blob = await convertAudioBlob2Mp3(
audioBlob,
audioRecorder.current.mimeType
);
console.log('mp3Blob:', mp3Blob, audioBlob);
audioUrl.current = mp3Blob.size ? URL.createObjectURL(mp3Blob) : '';
handleAudioData({
chunks: audioBlob,
size: audioBlob.size,
type: audioBlob.type,
chunks: mp3Blob,
size: mp3Blob.size,
type: mp3Blob.type,
url: audioUrl.current,
name: generateFileNameByTime(audioBlob.type),
name: generateFileNameByTime(mp3Blob.type),
duration: Math.floor((Date.now() - startTime.current) / 1000)
});
@@ -302,6 +345,13 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
checkMicrophonePermission();
}, []);
useEffect(() => {
ffmpeg.current = new FFmpeg();
return () => {
ffmpeg.current = null;
};
}, []);
return (
<div className="audio-input">
<Space size={40} className="btns">
src/pages/playground/components/ground-images.tsx
+7 -6
View File
@@ -11,7 +11,7 @@ import {
} from '@/utils/fetch-chunk-data';
import { FileImageOutlined, SwapOutlined } from '@ant-design/icons';
import { useIntl, useSearchParams } from '@umijs/max';
import { Button, Divider, Form, Tooltip } from 'antd';
import { Button, Form, Tooltip } from 'antd';
import classNames from 'classnames';
import _ from 'lodash';
import 'overlayscrollbars/overlayscrollbars.css';
@@ -540,11 +540,16 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
placeholer={intl.formatMessage({
id: 'playground.input.prompt.holder'
})}
actions={['clear']}
actions={['clear', 'upload']}
defaultSize={{
minRows: 5,
maxRows: 5
}}
title={
<span className="font-600">
{intl.formatMessage({ id: 'playground.image.prompt' })}
</span>
}
loading={loading}
disabled={!parameters.model}
isEmpty={!imageList.length}
@@ -555,10 +560,6 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
clearAll={handleClear}
tools={
<>
<span className="font-600">
{intl.formatMessage({ id: 'playground.image.prompt' })}
</span>
<Divider type="vertical" />
<Tooltip
title={intl.formatMessage({
id: 'playground.image.prompt.random'
src/pages/playground/components/message-input.tsx
+1
View File
@@ -355,6 +355,7 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
<div className="messageInput" style={{ ...style }}>
<div className="tool-bar">
<div className="actions">
{title}
{
<>
{actions.includes('role') && (
src/pages/playground/components/thumb-img.tsx
+1
View File
@@ -169,6 +169,7 @@ const ThumbImg: React.FC<{
return (
<SingleImage
{...item}
key={item.uid}
autoSize={autoSize}
editable={editable}
autoBgColor={autoBgColor}
src/pages/playground/config/params-config.ts
+3
View File
@@ -319,6 +319,9 @@ export const ImageAdvancedParamsConfig: ParamsSchema[] = [
text: 'playground.image.params.negativePrompt',
isLocalized: true
},
attrs: {
trim: false
},
rules: [
{
required: false
src/pages/playground/style/thumb-img.less
+4
View File
@@ -34,6 +34,10 @@
width: 16px;
overflow: hidden;
pointer-events: all;
&:hover {
transform: scale(1.1);
}
}
&:hover {