From fef5d0e68810300f949cb734c2660f6a47dbc446 Mon Sep 17 00:00:00 2001 From: jialin Date: Wed, 11 Dec 2024 18:37:59 +0800 Subject: [PATCH] fix: audio, recording format error --- package.json | 1 + pnpm-lock.yaml | 47 ++++---- src/components/audio-player/index.tsx | 12 ++ .../llmodels/components/deploy-modal.tsx | 12 +- .../playground/components/audio-input.tsx | 106 +++++++++++++----- .../playground/components/ground-images.tsx | 13 ++- .../playground/components/message-input.tsx | 1 + src/pages/playground/components/thumb-img.tsx | 1 + src/pages/playground/config/params-config.ts | 3 + src/pages/playground/style/thumb-img.less | 4 + 10 files changed, 131 insertions(+), 69 deletions(-) diff --git a/package.json b/package.json index 1f3865f8..ef9b92df 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dependencies": { "@ant-design/icons": "^5.5.1", "@ant-design/pro-components": "^2.7.18", + "@ffmpeg/ffmpeg": "^0.12.10", "@huggingface/gguf": "^0.1.7", "@huggingface/hub": "^0.15.1", "@huggingface/tasks": "^0.11.6", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57bc4b71..d38ec150 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ dependencies: '@ant-design/pro-components': specifier: ^2.7.18 version: 2.7.19(antd@5.21.6)(rc-field-form@2.4.0)(react-dom@18.2.0)(react@18.2.0) + '@ffmpeg/ffmpeg': + specifier: ^0.12.10 + version: 0.12.10 '@huggingface/gguf': specifier: ^0.1.7 version: 0.1.7 @@ -4129,6 +4132,18 @@ packages: resolution: {integrity: sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==, tarball: https://registry.npmjs.org/@exodus/schemasafe/-/schemasafe-1.3.0.tgz} dev: false + /@ffmpeg/ffmpeg@0.12.10: + resolution: {integrity: sha512-lVtk8PW8e+NUzGZhPTWj2P1J4/NyuCrbDD3O9IGpSeLYtUZKBqZO8CNj1WYGghep/MXoM8e1qVY1GztTkf8YYQ==, tarball: https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.12.10.tgz} + engines: {node: '>=18.x'} + dependencies: + '@ffmpeg/types': 0.12.2 + dev: false + + /@ffmpeg/types@0.12.2: + resolution: {integrity: sha512-NJtxwPoLb60/z1Klv0ueshguWQ/7mNm106qdHkB4HL49LXszjhjCCiL+ldHJGQ9ai2Igx0s4F24ghigy//ERdA==, tarball: https://registry.npmjs.org/@ffmpeg/types/-/types-0.12.2.tgz} + engines: {node: '>=16.x'} + dev: false + /@floating-ui/core@0.6.2: resolution: {integrity: sha512-jktYRmZwmau63adUG3GKOAVCofBXkk55S/zQ94XOorAHhwqFIOFAy1rSp2N0Wp6/tGbe9V3u/ExlGZypyY17rg==, tarball: https://registry.npmjs.org/@floating-ui/core/-/core-0.6.2.tgz} dev: false @@ -4995,7 +5010,7 @@ packages: dependencies: '@babel/core': 7.24.5 postcss: 7.0.39 - postcss-syntax: 0.36.2(postcss@7.0.39) + postcss-syntax: 0.36.2(postcss@8.4.38) transitivePeerDependencies: - supports-color dev: false @@ -5036,7 +5051,7 @@ packages: postcss-syntax: '>=0.36.2' dependencies: postcss: 7.0.39 - postcss-syntax: 0.36.2(postcss@7.0.39) + postcss-syntax: 0.36.2(postcss@8.4.38) remark: 13.0.0 unist-util-find-all-after: 3.0.2 transitivePeerDependencies: @@ -14430,7 +14445,7 @@ packages: dependencies: htmlparser2: 3.10.1 postcss: 7.0.39 - postcss-syntax: 0.36.2(postcss@7.0.39) + postcss-syntax: 0.36.2(postcss@8.4.38) dev: false /postcss-image-set-function@4.0.7(postcss@8.4.38): @@ -14904,30 +14919,6 @@ packages: lodash: 4.17.21 postcss: 8.4.38 - /postcss-syntax@0.36.2(postcss@7.0.39): - resolution: {integrity: sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w==} - peerDependencies: - postcss: '>=5.0.0' - postcss-html: '*' - postcss-jsx: '*' - postcss-less: '*' - postcss-markdown: '*' - postcss-scss: '*' - peerDependenciesMeta: - postcss-html: - optional: true - postcss-jsx: - optional: true - postcss-less: - optional: true - postcss-markdown: - optional: true - postcss-scss: - optional: true - dependencies: - postcss: 7.0.39 - dev: false - /postcss-syntax@0.36.2(postcss@8.4.38): resolution: {integrity: sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w==} peerDependencies: @@ -18198,7 +18189,7 @@ packages: postcss-sass: 0.4.4 postcss-scss: 2.1.1 postcss-selector-parser: 6.0.16 - postcss-syntax: 0.36.2(postcss@7.0.39) + postcss-syntax: 0.36.2(postcss@8.4.38) postcss-value-parser: 4.2.0 resolve-from: 5.0.0 slash: 3.0.0 diff --git a/src/components/audio-player/index.tsx b/src/components/audio-player/index.tsx index 735b9c48..6cd52d89 100644 --- a/src/components/audio-player/index.tsx +++ b/src/components/audio-player/index.tsx @@ -175,6 +175,18 @@ const AudioPlayer: React.FC = forwardRef((props, ref) => { console.log('onload', e); }; + const onDownload = useCallback(() => { + const url = props.url || ''; + const filename = props.name; + + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + link.remove(); + }, [props.url, props.name]); + useEffect(() => { if (audioRef.current) { initPlayerConfig(); diff --git a/src/pages/llmodels/components/deploy-modal.tsx b/src/pages/llmodels/components/deploy-modal.tsx index 36f99502..59f2e625 100644 --- a/src/pages/llmodels/components/deploy-modal.tsx +++ b/src/pages/llmodels/components/deploy-modal.tsx @@ -94,17 +94,15 @@ const AddModal: React.FC = (props) => { if (!open) { setIsGGUF(false); form.current?.setFieldValue?.('backend', backendOptionsMap.vllm); + } else if (source === modelSourceMap.ollama_library_value) { + form.current?.setFieldValue?.('backend', backendOptionsMap.llamaBox); + setIsGGUF(true); } + return () => { setSelectedModel({}); }; - }, [open]); - - useEffect(() => { - if (source === modelSourceMap.ollama_library_value) { - setIsGGUF(true); - } - }, [source]); + }, [open, source]); return ( = { }; const recordingFormat = { - type: 'audio/wav', - suffix: '.wav' + type: 'audio/mpeg', + suffix: '.mp3' }; const AudioInput: React.FC = (props) => { @@ -59,6 +60,7 @@ const AudioInput: React.FC = (props) => { const analyser = useRef(null); const dataArray = useRef(null); const audioUrl = useRef(''); + const ffmpeg = useRef(null); const initAudioContext = useCallback(() => { audioContext.current = new (window.AudioContext || @@ -90,6 +92,53 @@ const AudioInput: React.FC = (props) => { // props.onRecord?.(false); }; + const getAudioFormat = (type?: string) => { + const mimeType = type || audioRecorder.current?.mimeType; + + let resultFormat = recordingFormat; + + Object.keys(audioFormat).forEach((key: string) => { + if (mimeType.includes(key)) { + resultFormat = { + type: mimeType, + suffix: `.${audioFormat[key]}` + }; + } + }); + return resultFormat; + }; + + const convertAudioBlob2Mp3 = useCallback( + async (audioBlob: any, mimeType: string): Promise => { + try { + if (!ffmpeg.current.loaded) { + await ffmpeg.current.load(); + } + const format = getAudioFormat(mimeType); + const recordFile = new Uint8Array(await audioBlob.arrayBuffer()); + + await ffmpeg.current.writeFile(`input${format.suffix}`, recordFile); + + await ffmpeg.current.exec([ + '-i', + `input${format.suffix}`, + '-f', + 'mp3', + 'output.mp3' + ]); + + const mp3Data = await ffmpeg.current.readFile('output.mp3'); + + const mp3Blob = new Blob([mp3Data.buffer], { type: 'audio/mpeg' }); + + return mp3Blob; + } catch (error) { + return new Blob(); + } + }, + [] + ); + // get all audio tracks const getAudioTracks = () => { const audioTracks = audioStream.current.getAudioTracks(); @@ -174,26 +223,11 @@ const AudioInput: React.FC = (props) => { const handleAudioData = (audioData: any) => { props.onAudioData?.(audioData); }; - const getAudioFormat = (type?: string) => { - const mimeType = type || audioRecorder.current?.mimeType; - - let resultFormat = recordingFormat; - - Object.keys(audioFormat).forEach((key: string) => { - if (mimeType.includes(key)) { - resultFormat = { - type: mimeType, - suffix: `.${audioFormat[key]}` - }; - } - }); - return resultFormat; - }; const generateFileNameByTime = (type?: string) => { - const res = getAudioFormat(type); + const format = getAudioFormat(type); // format: recording-YYYY-MM-DD-HH_mm_ss.wav - return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${res.suffix}`; + return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${format.suffix}`; }; // start recording const StartRecording = async () => { @@ -209,6 +243,8 @@ const AudioInput: React.FC = (props) => { audioRecorder.current = new MediaRecorder(audioStream.current); + console.log('audioRecorder:', audioRecorder.current); + const audioChunks: any[] = []; audioRecorder.current.ondataavailable = (event: any) => { @@ -221,20 +257,27 @@ const AudioInput: React.FC = (props) => { }; // stop recording - audioRecorder.current.onstop = () => { - const res = getAudioFormat(); + audioRecorder.current.onstop = async () => { const audioBlob = new Blob(audioChunks, { - type: 'audio/mpeg' + type: audioRecorder.current.mimeType }); - console.log('audioBlob:', res, audioBlob); - audioUrl.current = URL.createObjectURL(audioBlob); + + // convert audio blob to mp3 + const mp3Blob = await convertAudioBlob2Mp3( + audioBlob, + audioRecorder.current.mimeType + ); + + console.log('mp3Blob:', mp3Blob, audioBlob); + + audioUrl.current = mp3Blob.size ? URL.createObjectURL(mp3Blob) : ''; handleAudioData({ - chunks: audioBlob, - size: audioBlob.size, - type: audioBlob.type, + chunks: mp3Blob, + size: mp3Blob.size, + type: mp3Blob.type, url: audioUrl.current, - name: generateFileNameByTime(audioBlob.type), + name: generateFileNameByTime(mp3Blob.type), duration: Math.floor((Date.now() - startTime.current) / 1000) }); @@ -302,6 +345,13 @@ const AudioInput: React.FC = (props) => { checkMicrophonePermission(); }, []); + useEffect(() => { + ffmpeg.current = new FFmpeg(); + return () => { + ffmpeg.current = null; + }; + }, []); + return (
diff --git a/src/pages/playground/components/ground-images.tsx b/src/pages/playground/components/ground-images.tsx index 13dde128..d47c2993 100644 --- a/src/pages/playground/components/ground-images.tsx +++ b/src/pages/playground/components/ground-images.tsx @@ -11,7 +11,7 @@ import { } from '@/utils/fetch-chunk-data'; import { FileImageOutlined, SwapOutlined } from '@ant-design/icons'; import { useIntl, useSearchParams } from '@umijs/max'; -import { Button, Divider, Form, Tooltip } from 'antd'; +import { Button, Form, Tooltip } from 'antd'; import classNames from 'classnames'; import _ from 'lodash'; import 'overlayscrollbars/overlayscrollbars.css'; @@ -540,11 +540,16 @@ const GroundImages: React.FC = forwardRef((props, ref) => { placeholer={intl.formatMessage({ id: 'playground.input.prompt.holder' })} - actions={['clear']} + actions={['clear', 'upload']} defaultSize={{ minRows: 5, maxRows: 5 }} + title={ + + {intl.formatMessage({ id: 'playground.image.prompt' })} + + } loading={loading} disabled={!parameters.model} isEmpty={!imageList.length} @@ -555,10 +560,6 @@ const GroundImages: React.FC = forwardRef((props, ref) => { clearAll={handleClear} tools={ <> - - {intl.formatMessage({ id: 'playground.image.prompt' })} - - = forwardRef(
+ {title} { <> {actions.includes('role') && ( diff --git a/src/pages/playground/components/thumb-img.tsx b/src/pages/playground/components/thumb-img.tsx index 63ed9c78..2e023083 100644 --- a/src/pages/playground/components/thumb-img.tsx +++ b/src/pages/playground/components/thumb-img.tsx @@ -169,6 +169,7 @@ const ThumbImg: React.FC<{ return (