fix: audio, recording format error

package.json

@@ -17,6 +17,7 @@
   "dependencies": {
     "@ant-design/icons": "^5.5.1",
     "@ant-design/pro-components": "^2.7.18",
+    "@ffmpeg/ffmpeg": "^0.12.10",
     "@huggingface/gguf": "^0.1.7",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/tasks": "^0.11.6",

pnpm-lock.yaml

@@ -14,6 +14,9 @@ dependencies:
   '@ant-design/pro-components':
     specifier: ^2.7.18
     version: 2.7.19(antd@5.21.6)(rc-field-form@2.4.0)(react-dom@18.2.0)(react@18.2.0)
+  '@ffmpeg/ffmpeg':
+    specifier: ^0.12.10
+    version: 0.12.10
   '@huggingface/gguf':
     specifier: ^0.1.7
     version: 0.1.7
@@ -4129,6 +4132,18 @@ packages:
     resolution: {integrity: sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==, tarball: https://registry.npmjs.org/@exodus/schemasafe/-/schemasafe-1.3.0.tgz}
     dev: false
 
+  /@ffmpeg/ffmpeg@0.12.10:
+    resolution: {integrity: sha512-lVtk8PW8e+NUzGZhPTWj2P1J4/NyuCrbDD3O9IGpSeLYtUZKBqZO8CNj1WYGghep/MXoM8e1qVY1GztTkf8YYQ==, tarball: https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.12.10.tgz}
+    engines: {node: '>=18.x'}
+    dependencies:
+      '@ffmpeg/types': 0.12.2
+    dev: false
+
+  /@ffmpeg/types@0.12.2:
+    resolution: {integrity: sha512-NJtxwPoLb60/z1Klv0ueshguWQ/7mNm106qdHkB4HL49LXszjhjCCiL+ldHJGQ9ai2Igx0s4F24ghigy//ERdA==, tarball: https://registry.npmjs.org/@ffmpeg/types/-/types-0.12.2.tgz}
+    engines: {node: '>=16.x'}
+    dev: false
+
   /@floating-ui/core@0.6.2:
     resolution: {integrity: sha512-jktYRmZwmau63adUG3GKOAVCofBXkk55S/zQ94XOorAHhwqFIOFAy1rSp2N0Wp6/tGbe9V3u/ExlGZypyY17rg==, tarball: https://registry.npmjs.org/@floating-ui/core/-/core-0.6.2.tgz}
     dev: false
@@ -4995,7 +5010,7 @@ packages:
     dependencies:
       '@babel/core': 7.24.5
       postcss: 7.0.39
-      postcss-syntax: 0.36.2(postcss@7.0.39)
+      postcss-syntax: 0.36.2(postcss@8.4.38)
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -5036,7 +5051,7 @@ packages:
       postcss-syntax: '>=0.36.2'
     dependencies:
       postcss: 7.0.39
-      postcss-syntax: 0.36.2(postcss@7.0.39)
+      postcss-syntax: 0.36.2(postcss@8.4.38)
       remark: 13.0.0
       unist-util-find-all-after: 3.0.2
     transitivePeerDependencies:
@@ -14430,7 +14445,7 @@ packages:
     dependencies:
       htmlparser2: 3.10.1
       postcss: 7.0.39
-      postcss-syntax: 0.36.2(postcss@7.0.39)
+      postcss-syntax: 0.36.2(postcss@8.4.38)
     dev: false
 
   /postcss-image-set-function@4.0.7(postcss@8.4.38):
@@ -14904,30 +14919,6 @@ packages:
       lodash: 4.17.21
       postcss: 8.4.38
 
-  /postcss-syntax@0.36.2(postcss@7.0.39):
-    resolution: {integrity: sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w==}
-    peerDependencies:
-      postcss: '>=5.0.0'
-      postcss-html: '*'
-      postcss-jsx: '*'
-      postcss-less: '*'
-      postcss-markdown: '*'
-      postcss-scss: '*'
-    peerDependenciesMeta:
-      postcss-html:
-        optional: true
-      postcss-jsx:
-        optional: true
-      postcss-less:
-        optional: true
-      postcss-markdown:
-        optional: true
-      postcss-scss:
-        optional: true
-    dependencies:
-      postcss: 7.0.39
-    dev: false
-
   /postcss-syntax@0.36.2(postcss@8.4.38):
     resolution: {integrity: sha512-nBRg/i7E3SOHWxF3PpF5WnJM/jQ1YpY9000OaVXlAQj6Zp/kIqJxEDWIZ67tAd7NLuk7zqN4yqe9nc0oNAOs1w==}
     peerDependencies:
@@ -18198,7 +18189,7 @@ packages:
       postcss-sass: 0.4.4
       postcss-scss: 2.1.1
       postcss-selector-parser: 6.0.16
-      postcss-syntax: 0.36.2(postcss@7.0.39)
+      postcss-syntax: 0.36.2(postcss@8.4.38)
       postcss-value-parser: 4.2.0
       resolve-from: 5.0.0
       slash: 3.0.0

src/components/audio-player/index.tsx

@@ -175,6 +175,18 @@ const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
     console.log('onload', e);
   };
 
+  const onDownload = useCallback(() => {
+    const url = props.url || '';
+    const filename = props.name;
+
+    const link = document.createElement('a');
+    link.href = url;
+    link.download = filename;
+    document.body.appendChild(link);
+    link.click();
+    link.remove();
+  }, [props.url, props.name]);
+
   useEffect(() => {
     if (audioRef.current) {
       initPlayerConfig();

src/pages/llmodels/components/deploy-modal.tsx

@@ -94,17 +94,15 @@ const AddModal: React.FC<AddModalProps> = (props) => {
     if (!open) {
       setIsGGUF(false);
       form.current?.setFieldValue?.('backend', backendOptionsMap.vllm);
+    } else if (source === modelSourceMap.ollama_library_value) {
+      form.current?.setFieldValue?.('backend', backendOptionsMap.llamaBox);
+      setIsGGUF(true);
     }
+
     return () => {
       setSelectedModel({});
     };
-  }, [open]);
+  }, [open, source]);
-
-  useEffect(() => {
-    if (source === modelSourceMap.ollama_library_value) {
-      setIsGGUF(true);
-    }
-  }, [source]);
 
   return (
     <Drawer

src/pages/playground/components/audio-input.tsx

@@ -1,5 +1,6 @@
 import { externalRefer } from '@/constants/external-links';
 import { AudioOutlined } from '@ant-design/icons';
+import { FFmpeg } from '@ffmpeg/ffmpeg';
 import { useIntl } from '@umijs/max';
 import { Button, Space, Tooltip } from 'antd';
 import dayjs from 'dayjs';
@@ -43,8 +44,8 @@ const audioFormat: Record<string, string> = {
 };
 
 const recordingFormat = {
-  type: 'audio/wav',
+  type: 'audio/mpeg',
-  suffix: '.wav'
+  suffix: '.mp3'
 };
 
 const AudioInput: React.FC<AudioInputProps> = (props) => {
@@ -59,6 +60,7 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
   const analyser = useRef<any>(null);
   const dataArray = useRef<any>(null);
   const audioUrl = useRef<string>('');
+  const ffmpeg = useRef<any>(null);
 
   const initAudioContext = useCallback(() => {
     audioContext.current = new (window.AudioContext ||
@@ -90,6 +92,53 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
     // props.onRecord?.(false);
   };
 
+  const getAudioFormat = (type?: string) => {
+    const mimeType = type || audioRecorder.current?.mimeType;
+
+    let resultFormat = recordingFormat;
+
+    Object.keys(audioFormat).forEach((key: string) => {
+      if (mimeType.includes(key)) {
+        resultFormat = {
+          type: mimeType,
+          suffix: `.${audioFormat[key]}`
+        };
+      }
+    });
+    return resultFormat;
+  };
+
+  const convertAudioBlob2Mp3 = useCallback(
+    async (audioBlob: any, mimeType: string): Promise<Blob> => {
+      try {
+        if (!ffmpeg.current.loaded) {
+          await ffmpeg.current.load();
+        }
+        const format = getAudioFormat(mimeType);
+        const recordFile = new Uint8Array(await audioBlob.arrayBuffer());
+
+        await ffmpeg.current.writeFile(`input${format.suffix}`, recordFile);
+
+        await ffmpeg.current.exec([
+          '-i',
+          `input${format.suffix}`,
+          '-f',
+          'mp3',
+          'output.mp3'
+        ]);
+
+        const mp3Data = await ffmpeg.current.readFile('output.mp3');
+
+        const mp3Blob = new Blob([mp3Data.buffer], { type: 'audio/mpeg' });
+
+        return mp3Blob;
+      } catch (error) {
+        return new Blob();
+      }
+    },
+    []
+  );
+
   // get all audio tracks
   const getAudioTracks = () => {
     const audioTracks = audioStream.current.getAudioTracks();
@@ -174,26 +223,11 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
   const handleAudioData = (audioData: any) => {
     props.onAudioData?.(audioData);
   };
-  const getAudioFormat = (type?: string) => {
-    const mimeType = type || audioRecorder.current?.mimeType;
-
-    let resultFormat = recordingFormat;
-
-    Object.keys(audioFormat).forEach((key: string) => {
-      if (mimeType.includes(key)) {
-        resultFormat = {
-          type: mimeType,
-          suffix: `.${audioFormat[key]}`
-        };
-      }
-    });
-    return resultFormat;
-  };
 
   const generateFileNameByTime = (type?: string) => {
-    const res = getAudioFormat(type);
+    const format = getAudioFormat(type);
     // format: recording-YYYY-MM-DD-HH_mm_ss.wav
-    return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${res.suffix}`;
+    return `recording-${dayjs().format('YYYY-MM-DD-HH_mm_ss')}${format.suffix}`;
   };
   // start recording
   const StartRecording = async () => {
@@ -209,6 +243,8 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
 
       audioRecorder.current = new MediaRecorder(audioStream.current);
 
+      console.log('audioRecorder:', audioRecorder.current);
+
       const audioChunks: any[] = [];
 
       audioRecorder.current.ondataavailable = (event: any) => {
@@ -221,20 +257,27 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
       };
 
       // stop recording
-      audioRecorder.current.onstop = () => {
+      audioRecorder.current.onstop = async () => {
-        const res = getAudioFormat();
         const audioBlob = new Blob(audioChunks, {
-          type: 'audio/mpeg'
+          type: audioRecorder.current.mimeType
         });
-        console.log('audioBlob:', res, audioBlob);
+
-        audioUrl.current = URL.createObjectURL(audioBlob);
+        // convert audio blob to mp3
+        const mp3Blob = await convertAudioBlob2Mp3(
+          audioBlob,
+          audioRecorder.current.mimeType
+        );
+
+        console.log('mp3Blob:', mp3Blob, audioBlob);
+
+        audioUrl.current = mp3Blob.size ? URL.createObjectURL(mp3Blob) : '';
 
         handleAudioData({
-          chunks: audioBlob,
+          chunks: mp3Blob,
-          size: audioBlob.size,
+          size: mp3Blob.size,
-          type: audioBlob.type,
+          type: mp3Blob.type,
           url: audioUrl.current,
-          name: generateFileNameByTime(audioBlob.type),
+          name: generateFileNameByTime(mp3Blob.type),
           duration: Math.floor((Date.now() - startTime.current) / 1000)
         });
 
@@ -302,6 +345,13 @@ const AudioInput: React.FC<AudioInputProps> = (props) => {
     checkMicrophonePermission();
   }, []);
 
+  useEffect(() => {
+    ffmpeg.current = new FFmpeg();
+    return () => {
+      ffmpeg.current = null;
+    };
+  }, []);
+
   return (
     <div className="audio-input">
       <Space size={40} className="btns">

src/pages/playground/components/ground-images.tsx

@@ -11,7 +11,7 @@ import {
 } from '@/utils/fetch-chunk-data';
 import { FileImageOutlined, SwapOutlined } from '@ant-design/icons';
 import { useIntl, useSearchParams } from '@umijs/max';
-import { Button, Divider, Form, Tooltip } from 'antd';
+import { Button, Form, Tooltip } from 'antd';
 import classNames from 'classnames';
 import _ from 'lodash';
 import 'overlayscrollbars/overlayscrollbars.css';
@@ -540,11 +540,16 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
             placeholer={intl.formatMessage({
               id: 'playground.input.prompt.holder'
             })}
-            actions={['clear']}
+            actions={['clear', 'upload']}
             defaultSize={{
               minRows: 5,
               maxRows: 5
             }}
+            title={
+              <span className="font-600">
+                {intl.formatMessage({ id: 'playground.image.prompt' })}
+              </span>
+            }
             loading={loading}
             disabled={!parameters.model}
             isEmpty={!imageList.length}
@@ -555,10 +560,6 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
             clearAll={handleClear}
             tools={
               <>
-                <span className="font-600">
-                  {intl.formatMessage({ id: 'playground.image.prompt' })}
-                </span>
-                <Divider type="vertical" />
                 <Tooltip
                   title={intl.formatMessage({
                     id: 'playground.image.prompt.random'

src/pages/playground/components/message-input.tsx

@@ -355,6 +355,7 @@ const MessageInput: React.FC<MessageInputProps> = forwardRef(
       <div className="messageInput" style={{ ...style }}>
         <div className="tool-bar">
           <div className="actions">
+            {title}
             {
               <>
                 {actions.includes('role') && (

src/pages/playground/components/thumb-img.tsx

@@ -169,6 +169,7 @@ const ThumbImg: React.FC<{
                 return (
                   <SingleImage
                     {...item}
+                    key={item.uid}
                     autoSize={autoSize}
                     editable={editable}
                     autoBgColor={autoBgColor}

src/pages/playground/config/params-config.ts

@@ -319,6 +319,9 @@ export const ImageAdvancedParamsConfig: ParamsSchema[] = [
       text: 'playground.image.params.negativePrompt',
       isLocalized: true
     },
+    attrs: {
+      trim: false
+    },
     rules: [
       {
         required: false

src/pages/playground/style/thumb-img.less

@@ -34,6 +34,10 @@
     width: 16px;
     overflow: hidden;
     pointer-events: all;
+
+    &:hover {
+      transform: scale(1.1);
+    }
   }
 
   &:hover {