chore: speech to text

package.json

@@ -28,7 +28,7 @@
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.5.0",
     "ansi-to-html": "^0.7.2",
-    "antd": "^5.20.6",
+    "antd": "^5.21.6",
     "antd-style": "^3.6.2",
     "axios": "^1.7.2",
     "classnames": "^2.5.1",
@@ -45,6 +45,8 @@
     "lodash": "^4.17.21",
     "mammoth": "^1.8.0",
     "marked": "^14.1.0",
+    "ml-dataset-iris": "^1.2.1",
+    "ml-pca": "^4.1.1",
     "numeral": "^2.0.6",
     "overlayscrollbars": "^2.10.0",
     "overlayscrollbars-react": "^0.5.6",
@@ -58,6 +60,7 @@
     "simplebar-react": "^3.2.6",
     "umap-js": "^1.4.0",
     "umi-presets-pro": "^2.0.3",
+    "wavesurfer.js": "^7.8.8",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
   },
   "devDependencies": {

src/assets/styles/common.less

@@ -166,6 +166,10 @@
   font-size: 20px;
 }
 
+.font-size-24 {
+  font-size: 24px;
+}
+
 .m-b-0 {
   margin-bottom: 0;
 }

src/components/audio-animation/index.less

@@ -0,0 +1,10 @@
+.canvas-wrap {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  text-align: center;
+
+  canvas {
+    display: block;
+  }
+}

src/components/audio-animation/index.tsx

@@ -0,0 +1,121 @@
+import React, { useEffect } from 'react';
+import './index.less';
+
+interface AudioAnimationProps {
+  width: number;
+  height: number;
+  analyserData: {
+    data: Uint8Array;
+    analyser: any;
+  };
+}
+
+const AudioAnimation: React.FC<AudioAnimationProps> = (props) => {
+  const { width, height, analyserData } = props;
+  const canvasRef = React.useRef<HTMLCanvasElement>(null);
+  const animationId = React.useRef<number>(0);
+  const isScaled = React.useRef<boolean>(false);
+  const oscillationOffset = React.useRef(0);
+  const direction = React.useRef(1);
+
+  const startAudioVisualization = () => {
+    if (!canvasRef.current || !analyserData.data?.length) return;
+
+    const canvas = canvasRef.current;
+    const canvasCtx = canvas.getContext('2d');
+    if (!canvasCtx) return;
+
+    const WIDTH = (canvas.width = width * 2);
+    const HEIGHT = (canvas.height = height * 2);
+
+    if (!isScaled.current) {
+      canvasCtx.scale(2, 2);
+      isScaled.current = true;
+    }
+
+    const barWidth = 3;
+    const barSpacing = 2;
+    const centerX = HEIGHT / 2;
+    const centerLine = Math.floor(HEIGHT / 2);
+    const jitterAmplitude = 60; // 最大抖动幅度
+    const minJitter = 15; // 最小抖动幅度
+
+    const frameInterval = 2;
+    let frameCount = 0;
+
+    canvasCtx.fillStyle = '#0073EF';
+
+    const draw = () => {
+      frameCount++;
+      if (frameCount % frameInterval !== 0) {
+        animationId.current = requestAnimationFrame(draw);
+        return;
+      }
+      analyserData.analyser?.current?.getByteFrequencyData(analyserData.data);
+      canvasCtx.clearRect(0, 0, WIDTH, HEIGHT);
+
+      const barCount = analyserData.data.length;
+      const totalWidth = barCount * (barWidth + barSpacing) - barSpacing;
+      let x = centerX - totalWidth / 2 + oscillationOffset.current;
+      oscillationOffset.current += direction.current * 0.5;
+
+      if (oscillationOffset.current > 20 || oscillationOffset.current < -20) {
+        direction.current *= -1;
+      }
+
+      for (let i = 0; i < barCount; i++) {
+        const baseHeight = Math.floor(analyserData.data[i] / 2);
+        const jitter =
+          minJitter +
+          Math.round((Math.random() - 0.5) * (jitterAmplitude - minJitter));
+        const barHeight = baseHeight + jitter;
+
+        const topY = Math.round(centerLine - barHeight / 2);
+        const bottomY = Math.round(centerLine + barHeight / 2);
+
+        canvasCtx.beginPath();
+        canvasCtx.moveTo(x, bottomY);
+        canvasCtx.lineTo(x, topY + 2);
+        canvasCtx.arcTo(x + barWidth, topY + 2, x + barWidth, bottomY, 2);
+        canvasCtx.lineTo(x + barWidth, bottomY);
+        canvasCtx.closePath();
+        canvasCtx.fill();
+
+        x += barWidth + barSpacing;
+      }
+
+      animationId.current = requestAnimationFrame(draw);
+    };
+
+    draw();
+  };
+
+  useEffect(() => {
+    if (!analyserData.data?.length || !analyserData.analyser.current) {
+      canvasRef.current
+        ?.getContext('2d')
+        ?.clearRect(0, 0, width * 2, height * 2);
+      cancelAnimationFrame(animationId.current);
+      animationId.current = 0;
+      return;
+    }
+    startAudioVisualization();
+    return () => {
+      if (animationId.current) cancelAnimationFrame(animationId.current);
+    };
+  }, [analyserData, width, height]);
+
+  return (
+    <div
+      className="canvas-wrap"
+      style={{
+        width: '100%',
+        height: height
+      }}
+    >
+      <canvas ref={canvasRef} style={{ width, height }}></canvas>
+    </div>
+  );
+};
+
+export default React.memo(AudioAnimation);

src/components/audio-player/index.less

@@ -0,0 +1,53 @@
+.player-wrap {
+  width: 100%;
+  display: flex;
+  background-color: var(--ant-color-fill-quaternary);
+  border-radius: 36px;
+
+  .player-ui {
+    padding: 5px 10px;
+    flex: 1;
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+  }
+
+  .play-content {
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+    flex: 1;
+  }
+
+  .time {
+    &.current {
+      margin-left: 6px;
+    }
+  }
+
+  .progress-bar {
+    margin-inline: 10px;
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+
+    .slider {
+      width: 100%;
+    }
+
+    .file-name {
+      line-height: 20px;
+      height: 20px;
+    }
+
+    .ant-slider-horizontal {
+      margin-block: 2px;
+    }
+  }
+
+  .speaker {
+    margin-left: 10px;
+  }
+}

src/components/audio-player/index.tsx

@@ -0,0 +1,191 @@
+import { formatTime } from '@/utils/index';
+import { PauseCircleFilled, PlayCircleFilled } from '@ant-design/icons';
+import { Button, Slider } from 'antd';
+import React, {
+  forwardRef,
+  useCallback,
+  useEffect,
+  useImperativeHandle
+} from 'react';
+import IconFont from '../icon-font';
+import './index.less';
+
+interface AudioPlayerProps {
+  autoplay?: boolean;
+  url: string;
+  speed?: number;
+  ref?: any;
+  name: string;
+  height?: number;
+  width?: number;
+  duration?: number;
+}
+
+const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
+  const { autoplay = false, speed: defaultSpeed = 1 } = props;
+  const audioRef = React.useRef<HTMLAudioElement>(null);
+  const [audioState, setAudioState] = React.useState<{
+    currentTime: number;
+    duration: number;
+  }>({
+    currentTime: 0,
+    duration: 0
+  });
+  const [playOn, setPlayOn] = React.useState<boolean>(false);
+  const [speakerOn, setSpeakerOn] = React.useState<boolean>(false);
+  const [volume, setVolume] = React.useState<number>(0.5);
+  const [speed, setSpeed] = React.useState<number>(defaultSpeed);
+  const timer = React.useRef<any>(null);
+
+  useImperativeHandle(ref, () => ({
+    play: () => {
+      audioRef.current?.play();
+    },
+    pause: () => {
+      audioRef.current?.pause();
+    }
+  }));
+
+  const handleAudioOnPlay = useCallback(() => {
+    timer.current = setInterval(() => {
+      setAudioState((prestate) => {
+        return {
+          currentTime: Math.ceil(audioRef.current?.currentTime || 0),
+          duration:
+            prestate.duration || Math.ceil(audioRef.current?.duration || 0)
+        };
+      });
+
+      if (audioRef.current?.paused || audioRef.current?.ended) {
+        clearInterval(timer.current);
+        setPlayOn(false);
+        setAudioState((prestate: any) => {
+          return {
+            currentTime: 0,
+            duration: prestate.duration
+          };
+        });
+      }
+    }, 500);
+  }, []);
+
+  const handlePlay = useCallback(() => {
+    if (playOn) {
+      audioRef.current?.pause();
+    } else {
+      audioRef.current?.play();
+    }
+    setPlayOn(!playOn);
+  }, [playOn]);
+
+  const initPlayerConfig = useCallback(() => {
+    // set volume
+    audioRef.current!.volume = volume;
+    // set playback rate
+    audioRef.current!.playbackRate = speed;
+  }, []);
+
+  const handleLoadedMetadata = useCallback(
+    (data: any) => {
+      const duration = Math.ceil(audioRef.current?.duration || 0);
+      setAudioState({
+        currentTime: 0,
+        duration:
+          duration && duration !== Infinity ? duration : props.duration || 0
+      });
+      setPlayOn(autoplay);
+    },
+    [autoplay, props.duration]
+  );
+
+  const handleCurrentChange = useCallback((val: number) => {
+    audioRef.current!.currentTime = val;
+    setAudioState((prestate) => {
+      return {
+        currentTime: val,
+        duration: prestate.duration
+      };
+    });
+  }, []);
+
+  useEffect(() => {
+    if (audioRef.current) {
+      initPlayerConfig();
+    }
+  }, [audioRef.current]);
+
+  useEffect(() => {
+    return () => {
+      clearInterval(timer.current);
+    };
+  }, []);
+
+  return (
+    <div className="player-wrap" style={{ width: props.width || '100%' }}>
+      <div className="player-ui">
+        <span className="play-btn">
+          <Button
+            size="middle"
+            type="text"
+            onClick={handlePlay}
+            icon={
+              !playOn ? (
+                <PlayCircleFilled
+                  style={{ fontSize: '22px' }}
+                ></PlayCircleFilled>
+              ) : (
+                <PauseCircleFilled
+                  style={{ fontSize: '22px' }}
+                ></PauseCircleFilled>
+              )
+            }
+          ></Button>
+        </span>
+        <div className="play-content">
+          <span className="time current">
+            {' '}
+            {formatTime(audioState.currentTime)}
+          </span>
+          <div className="progress-bar">
+            <span className="file-name">{props.name}</span>
+            <div className="slider">
+              <Slider
+                tooltip={{ open: false }}
+                min={0}
+                max={audioState.duration}
+                value={audioState.currentTime}
+                onChange={handleCurrentChange}
+              />
+            </div>
+            <span>{props.speed ? `${props.speed}x` : '1x'}</span>
+          </div>
+          <span className="time">{formatTime(audioState.duration)}</span>
+        </div>
+        <span className="speaker">
+          <Button
+            size="middle"
+            type="text"
+            icon={
+              <IconFont
+                type="icon-SpeakerHigh"
+                style={{ fontSize: '22px' }}
+              ></IconFont>
+            }
+          ></Button>
+        </span>
+      </div>
+      <audio
+        controls
+        autoPlay={autoplay}
+        src={props.url}
+        ref={audioRef}
+        preload="metadata"
+        style={{ display: 'none' }}
+        onPlay={handleAudioOnPlay}
+        onLoadedMetadata={handleLoadedMetadata}
+      ></audio>
+    </div>
+  );
+});
+
+export default React.memo(AudioPlayer);

src/components/echarts/config.ts

@@ -84,6 +84,7 @@ export const title = {
   textStyle: {
     fontSize: 12,
     color: '#000'
+    // fontWeight: 500
   },
   text: ''
 };

src/components/echarts/scatter.tsx

@@ -13,9 +13,11 @@ const options: any = {
     top: -1,
     bottom: -1,
     left: -1,
-    containLabel: true
+    containLabel: true,
+    borderRadius: 4
   },
   xAxis: {
+    scale: false,
     slient: true,
     splitNumber: 15,
     splitLine: {
@@ -34,6 +36,7 @@ const options: any = {
     }
   },
   yAxis: {
+    scale: false,
     slient: true,
     splitNumber: 10,
     splitLine: {

src/components/icon-font/index.tsx

@@ -1,7 +1,7 @@
 import { createFromIconfontCN } from '@ant-design/icons';
 
 const IconFont = createFromIconfontCN({
-  scriptUrl: '//at.alicdn.com/t/c/font_4613488_flbkvujyhg4.js'
+  scriptUrl: '//at.alicdn.com/t/c/font_4613488_f5wastzhj2w.js'
 });
 
 export default IconFont;

src/components/speech-content/audio-player.tsx

@@ -0,0 +1,47 @@
+import React, {
+  forwardRef,
+  useEffect,
+  useImperativeHandle,
+  useRef
+} from 'react';
+import useWavesurfer from './hooks/use-wavesurfer';
+
+interface AudioPlayerProps {
+  autoplay: boolean;
+  audioUrl: string;
+  speed: number;
+  ref?: any;
+  height?: number;
+  width?: number;
+}
+
+const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
+  const { autoplay, audioUrl, speed = 1, ...rest } = props;
+  const container = useRef<HTMLDivElement>(null);
+  const { createWavesurfer, play, pause, destroyWavesurfer } = useWavesurfer({
+    container,
+    autoplay: autoplay,
+    url: audioUrl,
+    audioRate: speed,
+    ...rest
+  });
+
+  useImperativeHandle(ref, () => {
+    return {
+      play,
+      pause
+    };
+  });
+
+  useEffect(() => {
+    if (container.current) {
+      createWavesurfer();
+    }
+    return () => {
+      destroyWavesurfer();
+    };
+  }, [container.current]);
+  return <div ref={container} className="audio-container"></div>;
+});
+
+export default React.memo(AudioPlayer);

src/components/speech-content/hooks/use-wavesurfer.ts

@@ -0,0 +1,68 @@
+import { useRef } from 'react';
+import WaveSurfer from 'wavesurfer.js';
+
+interface Options {
+  container: React.RefObject<HTMLDivElement>;
+  waveColor?: string;
+  progressColor?: string;
+  url: string;
+  barWidth?: number;
+  barGap?: number;
+  barRadius?: number;
+  autoplay?: boolean;
+  audioRate?: number;
+}
+const useWavesurfer = (options: Options) => {
+  const wavesurfer = useRef<WaveSurfer | null>(null);
+
+  const { container, url, ...rest } = options;
+
+  const createWavesurfer = () => {
+    if (!container.current) {
+      return;
+    }
+    if (wavesurfer.current) {
+      wavesurfer.current.destroy();
+    }
+    wavesurfer.current = WaveSurfer.create({
+      container: container.current,
+      waveColor: '#4096ff',
+      progressColor: 'rgb(100, 0, 100)',
+      url: url,
+      height: 60,
+      barWidth: 2,
+      barGap: 1,
+      barRadius: 2,
+      interact: true,
+      cursorWidth: 0,
+      ...rest
+    });
+  };
+
+  const destroyWavesurfer = () => {
+    if (wavesurfer.current) {
+      wavesurfer.current.destroy();
+    }
+  };
+
+  const play = () => {
+    if (wavesurfer.current) {
+      wavesurfer.current.play();
+    }
+  };
+
+  const pause = () => {
+    if (wavesurfer.current) {
+      wavesurfer.current.pause();
+    }
+  };
+
+  return {
+    createWavesurfer,
+    play,
+    pause,
+    destroyWavesurfer
+  };
+};
+
+export default useWavesurfer;

src/components/speech-content/index.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+import SpeechItem from './speech-item';
+
+interface SpeechContentProps {
+  dataList: any[];
+  loading?: boolean;
+}
+
+const SpeechContent: React.FC<SpeechContentProps> = (props) => {
+  console.log('SpeechContent', props);
+  return (
+    <div>
+      {props.dataList.map((item) => (
+        <SpeechItem key={item.uid} {...item} />
+      ))}
+    </div>
+  );
+};
+
+export default React.memo(SpeechContent);

src/components/speech-content/speech-item.tsx

@@ -0,0 +1,87 @@
+import IconFont from '@/components/icon-font';
+import {
+  DownloadOutlined,
+  FileTextOutlined,
+  PlayCircleOutlined
+} from '@ant-design/icons';
+import { Button, Tooltip } from 'antd';
+import React, { useRef, useState } from 'react';
+import AudioPlayer from './audio-player';
+import './styles/index.less';
+
+const audioUrl = require('./ih.mp4');
+
+interface SpeechContentProps {
+  prompt: string;
+  autoplay: boolean;
+  voice: string;
+  format: string;
+  speed: number;
+}
+const SpeechItem: React.FC<SpeechContentProps> = (props) => {
+  console.log('porps=======', props);
+  const [collapsed, setCollapsed] = useState(false);
+  const ref = useRef<HTMLAudioElement>(null);
+
+  const handlePlay = () => {
+    ref.current?.play();
+  };
+
+  const handleCollapse = () => {
+    setCollapsed(!collapsed);
+  };
+
+  return (
+    <div>
+      <div className="speech-item">
+        {/* <audio controls autoPlay={true} src={require('./ih.mp4')}></audio> */}
+        <div className="voice">
+          <IconFont type="icon-user_voice" className="font-size-16" />
+          <span className="text">{props.voice}</span>
+        </div>
+        <div className="wrapper">
+          <AudioPlayer {...props} audioUrl={audioUrl} ref={ref}></AudioPlayer>
+        </div>
+      </div>
+      <div className="speech-actions">
+        <span className="tags">
+          <span className="item">{props.format}</span>
+          <span className="item splitor"></span>
+          <span className="item">{props.speed}x</span>
+        </span>
+        <div className="actions">
+          <Tooltip title="Play">
+            <Button
+              onClick={handlePlay}
+              icon={<PlayCircleOutlined />}
+              type="text"
+              size="small"
+            ></Button>
+          </Tooltip>
+          <Tooltip title="Download">
+            <Button
+              icon={<DownloadOutlined />}
+              type="text"
+              size="small"
+            ></Button>
+          </Tooltip>
+          <Tooltip title="Show Prompt">
+            <Button
+              icon={<FileTextOutlined />}
+              type="text"
+              size="small"
+              onClick={handleCollapse}
+            ></Button>
+          </Tooltip>
+        </div>
+      </div>
+      {collapsed && (
+        <div className="prompt-box">
+          <div className="prompt">{props.prompt}</div>
+        </div>
+      )}
+    </div>
+  );
+};
+
+export default React.memo(SpeechItem);

src/components/speech-content/styles/index.less

@@ -0,0 +1,93 @@
+.speech-item {
+  display: flex;
+  justify-content: flex-start;
+  align-items: center;
+
+  .voice {
+    width: 80px;
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+    gap: 5px;
+
+    .text {
+      display: flex;
+      padding: 2px 4px;
+      border-radius: 4px;
+      border: 1px solid var(--ant-color-border);
+    }
+  }
+
+  .wrapper {
+    flex: 1;
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+
+    .audio-container {
+      flex: 1;
+    }
+
+    .format {
+      display: flex;
+      padding-left: 5px;
+    }
+  }
+
+  audio {
+    flex: 1;
+  }
+}
+
+.prompt-box {
+  padding-left: 80px;
+
+  .prompt {
+    margin-top: 16px;
+    padding: 10px;
+    border-radius: var(--border-radius-base);
+    background-color: var(--ant-color-fill-quaternary);
+  }
+}
+
+.speech-actions {
+  display: flex;
+  justify-content: space-between;
+  margin-top: 10px;
+  padding-left: 80px;
+
+  .actions {
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+    gap: 15px;
+
+    .anticon {
+      font-size: 14px;
+    }
+  }
+
+  .tags {
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+
+    .item {
+      display: flex;
+      justify-content: center;
+      align-items: center;
+      padding: 2px;
+      border-radius: 4px;
+      color: var(--ant-color-text-tertiary);
+    }
+
+    .splitor {
+      display: flex;
+      width: 1px;
+      height: 1px;
+      border-radius: 2px;
+      margin: 0 8px;
+      background-color: var(--ant-color-fill-content-hover);
+    }
+  }
+}

src/components/upload-audio/index.tsx

@@ -0,0 +1,54 @@
+import { UploadOutlined } from '@ant-design/icons';
+import { Button, Tooltip, Upload } from 'antd';
+import React from 'react';
+
+interface UploadAudioProps {
+  accept?: string;
+  maxCount?: number;
+  type?: 'text' | 'primary' | 'default';
+  onChange?: (data: { file: any; fileList: any[] }) => void;
+}
+
+const UploadAudio: React.FC<UploadAudioProps> = (props) => {
+  const beforeUpload = (file: any) => {
+    return true;
+  };
+
+  const handleOnChange = React.useCallback(
+    (data: { file: any; fileList: any }) => {
+      console.log('handleOnChange', data);
+      props.onChange?.(data);
+    },
+    []
+  );
+  return (
+    <Tooltip title={`Upload an audio file, support for ${props.accept}`}>
+      <Upload
+        beforeUpload={beforeUpload}
+        onChange={handleOnChange}
+        accept={props.accept}
+        multiple={false}
+        maxCount={1}
+        showUploadList={false}
+        fileList={[]}
+      >
+        <div
+          style={{
+            display: 'flex',
+            flexDirection: 'column',
+            alignItems: 'center',
+            gap: 16
+          }}
+        >
+          <Button
+            icon={<UploadOutlined />}
+            type={props.type ?? 'text'}
+            shape="circle"
+          ></Button>
+        </div>
+      </Upload>
+    </Tooltip>
+  );
+};
+
+export default React.memo(UploadAudio);

src/locales/en-US/menu.ts

@@ -4,8 +4,8 @@ export default {
   'menu.playground.rerank': 'Rerank',
   'menu.playground.embedding': 'Embedding',
   'menu.playground.chat': 'Chat',
-  'menu.playground.speech': 'Speech',
+  'menu.playground.speech': 'Audio',
-  'menu.playground.text2images': 'Text to Images',
+  'menu.playground.text2images': 'Image',
   'menu.compare': 'Compare',
   'menu.models': 'Models',
   'menu.resources': 'Resources',

src/locales/en-US/playground.ts

@@ -36,6 +36,7 @@ export default {
   'playground.img.upload.success': 'Upload Success',
   'playground.img.upload.error': 'Upload Error',
   'playground.toolbar.clearmsg': 'Clear Messages',
+  'playground.toolbar.autoplay': 'Autoplay',
   'playground.toolbar.prompts': 'Prompts',
   'playground.toolbar.compare2Model': '2-Model Comparison',
   'playground.toolbar.compare3Model': '3-Model Comparison',

src/locales/zh-CN/menu.ts

@@ -5,7 +5,7 @@ export default {
   'menu.playground.embedding': '文本嵌入',
   'menu.playground.chat': '对话',
   'menu.playground.speech': '语音',
-  'menu.playground.text2images': '文生图',
+  'menu.playground.text2images': '图片',
   'menu.compare': '多模型对比',
   'menu.models': '模型',
   'menu.resources': '资源',

src/locales/zh-CN/playground.ts

@@ -36,6 +36,7 @@ export default {
   'playground.img.upload.success': '上传成功',
   'playground.img.upload.error': '上传失败',
   'playground.toolbar.clearmsg': '清空消息',
+  'playground.toolbar.autoplay': '自动播放',
   'playground.toolbar.prompts': '提示词',
   'playground.toolbar.compare2Model': '2 模型对比',
   'playground.toolbar.compare3Model': '3 模型对比',

src/pages/playground/apis/index.ts

@@ -43,6 +43,8 @@ export const rerankerQuery = async (
 export const handleEmbedding = async (
   params: {
     model: string;
+    encoding_format?: string;
+    dimensions?: number;
     input: string[];
   },
   options?: any

src/pages/playground/components/audio-input.tsx

@@ -0,0 +1,246 @@
+import { AudioOutlined } from '@ant-design/icons';
+import { Button, Space, Tooltip } from 'antd';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+// import '../style/audio-input.less';
+
+interface AudioInputProps {
+  onAudioData: (audioData: {
+    chunks: any[];
+    url: string;
+    name: string;
+    duration: number;
+  }) => void;
+  onAnalyse?: (analyseData: any, frequencyBinCount: any) => void;
+  onAudioPermission: (audioPermission: boolean) => void;
+  onRecord?: (isRecording: boolean) => void;
+  voiceActivity?: boolean;
+  type?: 'text' | 'primary' | 'default';
+}
+
+const AudioInput: React.FC<AudioInputProps> = (props) => {
+  const [audioOn, setAudioOn] = useState(false);
+  const [isRecording, setIsRecording] = useState(false);
+  const [audioPermission, setAudioPermission] = useState(true);
+  const audioStream = useRef<any>(null);
+  const audioRecorder = useRef<any>(null);
+  const startTime = useRef<number>(0);
+  const audioContext = useRef<any>(null);
+  const analyser = useRef<any>(null);
+  const dataArray = useRef<any>(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+
+  const initAudioContext = useCallback(() => {
+    audioContext.current = new (window.AudioContext ||
+      window.webkitAudioContext)();
+
+    analyser.current = audioContext.current.createAnalyser();
+    analyser.current.fftSize = 256;
+    dataArray.current = new Uint8Array(analyser.current.frequencyBinCount);
+  }, []);
+
+  const generateVisualData = useCallback(() => {
+    const source = audioContext.current.createMediaStreamSource(
+      audioStream.current
+    );
+    source.connect(analyser.current);
+  }, []);
+
+  // stop all audio tracks
+  const stopAudioTracks = () => {
+    audioStream.current?.getTracks().forEach((track: any) => {
+      track.stop();
+    });
+  };
+
+  const handleStopRecording = () => {
+    setIsRecording(false);
+    audioRecorder.current?.stop();
+    props.onRecord?.(false);
+  };
+
+  // get all audio tracks
+  const getAudioTracks = () => {
+    const audioTracks = audioStream.current.getAudioTracks();
+    audioTracks.forEach((track: any) => {
+      track.onended = () => {
+        setAudioPermission(false);
+      };
+    });
+  };
+
+  // check if microphone is on
+  const isMicrophoneOn = () => {
+    return (
+      audioStream.current &&
+      audioStream.current
+        .getTracks()
+        .some((track: any) => track.readyState === 'live')
+    );
+  };
+
+  const microphonePermissionDenied = async () => {
+    const permissionStatus = await navigator.permissions.query({
+      name: 'microphone' as any
+    });
+
+    return permissionStatus.state === 'denied';
+  };
+
+  const checkMicrophonePermission = async () => {
+    try {
+      const permissionStatus = await navigator.permissions.query({
+        name: 'microphone' as any
+      });
+      console.log('permissionStatus:', permissionStatus);
+      if (permissionStatus.state === 'granted') {
+        setAudioPermission(true);
+        props.onAudioPermission(true);
+      } else if (permissionStatus.state === 'denied') {
+        setAudioPermission(false);
+        props.onAudioPermission(false);
+        handleStopRecording();
+      }
+
+      permissionStatus.onchange = () => {
+        console.log('permission changed');
+        checkMicrophonePermission();
+      };
+    } catch (error) {
+      // todo
+    }
+  };
+
+  // open audio
+  const EnableAudio = async () => {
+    try {
+      audioStream.current = await navigator.mediaDevices.getUserMedia({
+        audio: true
+      });
+      getAudioTracks();
+      setAudioOn(true);
+      setAudioPermission(true);
+      initAudioContext();
+    } catch (error) {
+      // console.log(error);
+    }
+  };
+
+  // close audio
+  const disableAudio = () => {
+    stopAudioTracks();
+    setAudioOn(false);
+    handleStopRecording();
+    audioStream.current = null;
+  };
+
+  const stopRecording = () => {
+    audioRecorder.current?.stop();
+    setIsRecording(false);
+  };
+
+  const handleAudioData = (audioData: any) => {
+    props.onAudioData?.(audioData);
+  };
+
+  // start recording
+  const StartRecording = async () => {
+    if (isRecording) {
+      stopRecording();
+      return;
+    }
+    try {
+      await EnableAudio();
+      console.log('audioStream:', audioStream.current);
+      audioRecorder.current = new MediaRecorder(audioStream.current);
+
+      const audioChunks: any[] = [];
+
+      audioRecorder.current.ondataavailable = (event: any) => {
+        audioChunks.push(event.data);
+        if (props.voiceActivity) {
+          analyser.current?.getByteFrequencyData(dataArray.current);
+
+          props.onAnalyse?.(dataArray.current, analyser);
+        }
+      };
+
+      // stop recording
+      audioRecorder.current.onstop = () => {
+        const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
+        const audioUrl = URL.createObjectURL(audioBlob);
+
+        handleAudioData({
+          chunks: audioChunks,
+          size: audioBlob.size,
+          type: audioBlob.type,
+          url: audioUrl,
+          name: `recording-${new Date().toISOString()}.wav`,
+          duration: Math.ceil((Date.now() - startTime.current) / 1000)
+        });
+
+        props.onAnalyse?.([], null);
+      };
+
+      setIsRecording(true);
+      props.onRecord?.(true);
+      startTime.current = Date.now();
+      audioRecorder.current.start(1000);
+      generateVisualData();
+    } catch (error) {
+      // console.log(error);
+    }
+  };
+
+  useEffect(() => {
+    return () => {
+      handleStopRecording();
+      stopAudioTracks();
+    };
+  }, []);
+
+  useEffect(() => {
+    checkMicrophonePermission();
+  }, []);
+
+  return (
+    <div className="audio-input">
+      <Space size={40} className="btns">
+        {
+          <Tooltip title="Start Recording">
+            <div
+              style={{
+                display: 'flex',
+                flexDirection: 'column',
+                alignItems: 'center',
+                gap: 16
+              }}
+            >
+              <Button
+                disabled={!audioPermission}
+                shape="circle"
+                icon={<AudioOutlined />}
+                size="middle"
+                type={props.type ?? 'text'}
+                danger={isRecording}
+                onClick={StartRecording}
+              ></Button>
+            </div>
+          </Tooltip>
+        }
+        {/* {isRecording && (
+          <Tooltip title="Stop Recording">
+            <Button
+              shape="circle"
+              icon={<IconFont type="icon-stop2"></IconFont>}
+              size="middle"
+              type={props.type ?? 'text'}
+              onClick={stopRecording}
+            ></Button>
+          </Tooltip>
+        )} */}
+      </Space>
+    </div>
+  );
+};
+
+export default React.memo(AudioInput);

src/pages/playground/components/ground-embedding.tsx

@@ -11,10 +11,12 @@ import {
 import { useIntl, useSearchParams } from '@umijs/max';
 import { Button, Tooltip } from 'antd';
 import classNames from 'classnames';
+import { PCA } from 'ml-pca';
 import 'overlayscrollbars/overlayscrollbars.css';
 import {
   forwardRef,
   memo,
+  useCallback,
   useEffect,
   useImperativeHandle,
   useMemo,
@@ -23,7 +25,7 @@ import {
 } from 'react';
 import { UMAP } from 'umap-js';
 import { handleEmbedding } from '../apis';
-import { MessageItem, ParamsSchema } from '../config/types';
+import { ParamsSchema } from '../config/types';
 import '../style/ground-left.less';
 import '../style/rerank.less';
 import '../style/system-message-wrap.less';
@@ -40,34 +42,34 @@ interface MessageProps {
 }
 
 const paramsConfig: ParamsSchema[] = [
-  {
+  // {
-    type: 'Select',
+  //   type: 'Select',
-    name: 'truncate',
+  //   name: 'truncate',
-    label: {
+  //   label: {
-      text: 'Truncate',
+  //     text: 'Truncate',
-      isLocalized: false
+  //     isLocalized: false
-    },
+  //   },
-    options: [
+  //   options: [
-      {
+  //     {
-        label: 'None',
+  //       label: 'None',
-        value: 'none'
+  //       value: 'none'
-      },
+  //     },
-      {
+  //     {
-        label: 'Start',
+  //       label: 'Start',
-        value: 'start'
+  //       value: 'start'
-      },
+  //     },
-      {
+  //     {
-        label: 'End',
+  //       label: 'End',
-        value: 'end'
+  //       value: 'end'
-      }
+  //     }
-    ],
+  //   ],
-    rules: [
+  //   rules: [
-      {
+  //     {
-        required: true,
+  //       required: true,
-        message: 'Please select truncate'
+  //       message: 'Please select truncate'
-      }
+  //     }
-    ]
+  //   ]
-  }
+  // }
 ];
 
 const initialValues = {
@@ -79,7 +81,6 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
   const acceptType =
     '.txt, .doc, .docx, .xls, .xlsx, .csv, .md, .pdf, .eml, .msg, .ppt, .pptx, .xml, .epub, .html';
   const messageId = useRef<number>(0);
-  const [messageList, setMessageList] = useState<MessageItem[]>([]);
 
   const intl = useIntl();
   const requestSource = useRequestToken();
@@ -117,11 +118,11 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
 
   const [scatterData, setScatterData] = useState<any[]>([]);
 
-  const { initialize, updateScrollerPosition } = useOverlayScroller();
+  const { initialize, updateScrollerPosition: updateDocumentScrollerPosition } =
-  const {
+    useOverlayScroller();
-    initialize: innitializeParams,
+
-    updateScrollerPosition: updateDocumentScrollerPosition
+  const { initialize: innitializeParams, updateScrollerPosition } =
-  } = useOverlayScroller();
+    useOverlayScroller();
 
   useImperativeHandle(ref, () => {
     return {
@@ -139,35 +140,74 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
     return list.length < 2;
   }, [textList, fileList]);
 
-  const generateEmbedding = (embeddings: any[]) => {
+  const cosine = useCallback((x: number[], y: number[]) => {
-    try {
+    let result = 0;
-      const umap = new UMAP({
+    let normX = 0;
-        // random() {
+    let normY = 0;
-        //   return 0.1;
-        // },
-        // minDist: 0.1,
-        nComponents: 2,
-        nNeighbors: 1
-      });
 
-      const dataList = embeddings.map((item) => {
+    for (let i = 0; i < x.length; i++) {
-        return item.embedding;
+      result += x[i] * y[i];
-      });
+      normX += x[i] ** 2;
-
+      normY += y[i] ** 2;
-      const embedding = umap.fit([...dataList, ...dataList]);
-
-      const list = embedding.map((item: number[], index: number) => {
-        return {
-          value: item,
-          name: index + 1,
-          text: `test test test test test`
-        };
-      });
-      setScatterData(list);
-    } catch (e) {
-      // console.log('error:', e);
     }
-  };
+
+    if (normX === 0 && normY === 0) {
+      return 0;
+    } else if (normX === 0 || normY === 0) {
+      return 1;
+    } else {
+      return 1 - result / Math.sqrt(normX * normY);
+    }
+  }, []);
+
+  const generateEmbedding = useCallback(
+    (embeddings: any[]) => {
+      try {
+        const umap = new UMAP({
+          // random: random,
+          minDist: 0,
+          nComponents: 3,
+          nEpochs: 200,
+          distanceFn: cosine,
+          nNeighbors: embeddings.length - 1
+        });
+
+        const dataList = embeddings.map((item) => {
+          return item.embedding;
+        });
+
+        const embedding = umap.fit([...dataList]);
+        console.log('embedding:----------------', embedding);
+        const pca = new PCA(dataList, {});
+        console.log('dataList====', dataList, embeddings);
+        const pcadata = pca.predict(dataList, { nComponents: 2 }).to2DArray();
+        console.log('pcadata++++++++++++++++', pcadata);
+
+        const input = [
+          ...textList.map((item) => item.text),
+          ...fileList.map((item) => item.text)
+        ];
+
+        const list = pcadata.map((item: number[], index: number) => {
+          return {
+            value: item,
+            name: index + 1,
+            text: input[index]
+          };
+        });
+        console.log('embedding____________:', {
+          list,
+          input,
+          textList,
+          fileList
+        });
+        setScatterData(list);
+      } catch (e) {
+        console.log('error:', e);
+      }
+    },
+    [textList, fileList]
+  );
 
   const setMessageId = () => {
     messageId.current = messageId.current + 1;
@@ -193,6 +233,7 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
       const result: any = await handleEmbedding(
         {
           model: parameters.model,
+          encoding_format: 'float',
           input: [
             ...textList.map((item) => item.text),
             ...fileList.map((item) => item.text)
@@ -221,14 +262,6 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
       setLoading(false);
     }
   };
-  const handleClear = () => {
-    if (!messageList.length) {
-      return;
-    }
-    setMessageId();
-    setScatterData([]);
-    setTokenResult(null);
-  };
 
   const handleSendMessage = () => {
     submitMessage();
@@ -297,10 +330,6 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
     }
   }, [paramsRef.current, innitializeParams]);
 
-  useEffect(() => {
-    updateScrollerPosition();
-  }, [messageList]);
-
   useEffect(() => {
     if (textList.length + fileList.length > messageListLengthCache.current) {
       updateDocumentScrollerPosition();

src/pages/playground/components/ground-images.tsx

@@ -1,27 +1,24 @@
 import useOverlayScroller from '@/hooks/use-overlay-scroller';
 import useRequestToken from '@/hooks/use-request-token';
+import ThumbImg from '@/pages/playground/components/thumb-img';
 import { useIntl, useSearchParams } from '@umijs/max';
-import { Spin } from 'antd';
 import classNames from 'classnames';
 import _ from 'lodash';
 import 'overlayscrollbars/overlayscrollbars.css';
-import {
+import React, {
   forwardRef,
   memo,
   useEffect,
   useImperativeHandle,
-  useMemo,
   useRef,
   useState
 } from 'react';
 import { createImages } from '../apis';
-import { Roles, generateMessages } from '../config';
 import { ImageParamsConfig as paramsConfig } from '../config/params-config';
 import { MessageItem } from '../config/types';
 import '../style/ground-left.less';
 import '../style/system-message-wrap.less';
 import MessageInput from './message-input';
-import MessageContent from './multiple-chat/message-content';
 import ReferenceParams from './reference-params';
 import RerankerParams from './reranker-params';
 import ViewCodeModal from './view-code-modal';
@@ -42,24 +39,24 @@ const initialValues = {
 const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
   const { modelList } = props;
   const messageId = useRef<number>(0);
-  const [messageList, setMessageList] = useState<MessageItem[]>([]);
+  const [messageList, setMessageList] = useState<
+    { dataUrl: string; height: number; uid: number }[]
+  >([]);
 
   const intl = useIntl();
   const requestSource = useRequestToken();
   const [searchParams] = useSearchParams();
   const selectModel = searchParams.get('model') || '';
   const [parameters, setParams] = useState<any>({});
-  const [systemMessage, setSystemMessage] = useState('');
   const [show, setShow] = useState(false);
   const [loading, setLoading] = useState(false);
   const [tokenResult, setTokenResult] = useState<any>(null);
   const [collapse, setCollapse] = useState(false);
-  const contentRef = useRef<any>('');
   const scroller = useRef<any>(null);
-  const currentMessageRef = useRef<any>(null);
   const paramsRef = useRef<any>(null);
   const messageListLengthCache = useRef<number>(0);
   const requestToken = useRef<any>(null);
+  const [currentPrompt, setCurrentPrompt] = useState<string>('');
 
   const { initialize, updateScrollerPosition } = useOverlayScroller();
   const { initialize: innitializeParams } = useOverlayScroller();
@@ -76,32 +73,11 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
     };
   });
 
-  const viewCodeMessage = useMemo(() => {
-    return generateMessages([
-      { role: Roles.System, content: systemMessage },
-      ...messageList
-    ]);
-  }, [messageList, systemMessage]);
-
   const setMessageId = () => {
     messageId.current = messageId.current + 1;
     return messageId.current;
   };
 
-  const handleNewMessage = (message?: { role: string; content: string }) => {
-    const newMessage = message || {
-      role:
-        _.last(messageList)?.role === Roles.User ? Roles.Assistant : Roles.User,
-      content: ''
-    };
-    messageList.push({
-      ...newMessage,
-      uid: messageId.current + 1
-    });
-    setMessageId();
-    setMessageList([...messageList]);
-  };
-
   const handleStopConversation = () => {
     requestToken.current?.cancel?.();
     setLoading(false);
@@ -112,26 +88,25 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
     try {
       setLoading(true);
       setMessageId();
+      setCurrentPrompt(current?.content || '');
+      setMessageList(
+        Array(parameters.n)
+          .fill({})
+          .map((item, index: number) => {
+            return {
+              dataUrl: '',
+              height: 256,
+              width: 256,
+              uid: index
+            };
+          })
+      );
 
       requestToken.current?.cancel?.();
       requestToken.current = requestSource();
 
-      currentMessageRef.current = current
-        ? [
-            {
-              ...current,
-              uid: messageId.current
-            }
-          ]
-        : [];
-
-      contentRef.current = '';
-      setMessageList((pre) => {
-        return [...pre, ...currentMessageRef.current];
-      });
-
       const params = {
-        prompt: current?.content || '',
+        prompt: current?.content || currentPrompt || '',
         ...parameters
       };
 
@@ -143,25 +118,19 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
         return {
           dataUrl: `data:image/png;base64,${item.b64_json}`,
           created: result.created,
+          height: 256,
+          width: 256,
           uid: index
         };
       });
-      setMessageList((pre) => {
+      setMessageList(imgList);
-        return [
-          ...pre,
-          {
-            content: '',
-            role: Roles.Assistant,
-            imgs: imgList,
-            uid: messageId.current
-          }
-        ];
-      });
       console.log('result:', imgList);
 
       setMessageId();
     } catch (error) {
       // console.log('error:', error);
+      requestToken.current?.cancel?.();
+      setMessageList([]);
     } finally {
       setLoading(false);
     }
@@ -177,8 +146,7 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
 
   const handleSendMessage = (message: Omit<MessageItem, 'uid'>) => {
     console.log('message:', message);
-    const currentMessage =
+    const currentMessage = message.content ? message : undefined;
-      message.content || message.imgs?.length ? message : undefined;
     submitMessage(currentMessage);
   };
 
@@ -186,23 +154,6 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
     setShow(false);
   };
 
-  const handleSelectModel = () => {};
-
-  const handlePresetPrompt = (list: { role: string; content: string }[]) => {
-    const sysMsg = list.filter((item) => item.role === 'system');
-    const userMsg = list
-      .filter((item) => item.role === 'user')
-      .map((item) => {
-        setMessageId();
-        return {
-          ...item,
-          uid: messageId.current
-        };
-      });
-    setSystemMessage(sysMsg[0]?.content || '');
-    setMessageList(userMsg);
-  };
-
   useEffect(() => {
     if (scroller.current) {
       initialize(scroller.current);
@@ -234,22 +185,12 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
         <div className="message-list-wrap" ref={scroller}>
           <>
             <div className="content">
-              <MessageContent
+              <ThumbImg
-                spans={{
+                style={{ paddingInline: 0 }}
-                  span: 24,
-                  count: 1
-                }}
-                actions={[]}
-                messageList={messageList}
-                setMessageList={setMessageList}
                 editable={false}
+                dataList={messageList}
                 loading={loading}
-              />
+              ></ThumbImg>
-              {loading && (
-                <Spin size="small">
-                  <div style={{ height: '46px' }}></div>
-                </Spin>
-              )}
             </div>
           </>
         </div>
@@ -260,19 +201,15 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
         )}
         <div className="ground-left-footer">
           <MessageInput
-            scope="chat"
             placeholer="Type <kbd>/</kbd> to input prompt"
             actions={['clear']}
             loading={loading}
             disabled={!parameters.model}
             isEmpty={!messageList.length}
             handleSubmit={handleSendMessage}
-            addMessage={handleNewMessage}
             handleAbortFetch={handleStopConversation}
+            shouldResetMessage={false}
             clearAll={handleClear}
-            setModelSelections={handleSelectModel}
-            presetPrompt={handlePresetPrompt}
-            modelList={modelList}
           />
         </div>
       </div>
@@ -297,7 +234,7 @@ const GroundImages: React.FC<MessageProps> = forwardRef((props, ref) => {
       <ViewCodeModal
         open={show}
         payLoad={{
-          prompt: currentMessageRef.current?.[0]?.content
+          prompt: currentPrompt
         }}
         parameters={parameters}
         onCancel={handleCloseViewCode}

src/pages/playground/components/ground-reranker.tsx

@@ -2,13 +2,14 @@ import useOverlayScroller from '@/hooks/use-overlay-scroller';
 import useRequestToken from '@/hooks/use-request-token';
 import { ClearOutlined, PlusOutlined, SearchOutlined } from '@ant-design/icons';
 import { useIntl, useSearchParams } from '@umijs/max';
-import { Button, Spin } from 'antd';
+import { Button, Progress, Spin, Tag } from 'antd';
 import classNames from 'classnames';
 import _ from 'lodash';
 import 'overlayscrollbars/overlayscrollbars.css';
 import {
   forwardRef,
   memo,
+  useCallback,
   useEffect,
   useImperativeHandle,
   useRef,
@@ -21,8 +22,6 @@ import '../style/rerank.less';
 import '../style/system-message-wrap.less';
 import InputList from './input-list';
 import MessageInput from './message-input';
-import ReferenceParams from './reference-params';
-import RerankMessage from './rerank-message';
 import RerankerParams from './reranker-params';
 import ViewRerankCode from './view-rerank-code';
 
@@ -73,19 +72,33 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
   const [tokenResult, setTokenResult] = useState<any>(null);
   const [collapse, setCollapse] = useState(false);
   const contentRef = useRef<any>('');
-  const controllerRef = useRef<any>(null);
   const scroller = useRef<any>(null);
-  const currentMessageRef = useRef<any>(null);
   const inputListRef = useRef<any>(null);
   const paramsRef = useRef<any>(null);
   const messageListLengthCache = useRef<number>(0);
   const requestToken = useRef<any>(null);
   const [fileList, setFileList] = useState<
-    { text: string; name: string; uid: number | string }[]
+    {
+      text: string;
+      name: string;
+      uid: number | string;
+      score?: number;
+      showExtra?: boolean;
+      percent?: number;
+      rank?: number;
+    }[]
   >([]);
 
   const [textList, setTextList] = useState<
-    { text: string; uid: number | string; name: string }[]
+    {
+      text: string;
+      uid: number | string;
+      name: string;
+      score?: number;
+      showExtra?: boolean;
+      percent?: number;
+      rank?: number;
+    }[]
   >([
     {
       text: '',
@@ -99,11 +112,10 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
     }
   ]);
 
-  const { initialize, updateScrollerPosition } = useOverlayScroller();
+  const { initialize, updateScrollerPosition: updateDocumentScrollerPosition } =
-  const {
+    useOverlayScroller();
-    initialize: innitializeParams,
+  const { initialize: innitializeParams, updateScrollerPosition } =
-    updateScrollerPosition: updateDocumentScrollerPosition
+    useOverlayScroller();
-  } = useOverlayScroller();
 
   useImperativeHandle(ref, () => {
     return {
@@ -116,8 +128,74 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
     };
   });
 
+  // [0.1, 1.0]
+  const normalizValue = (data: { min: number; max: number; value: number }) => {
+    const range = [0.5, 1.0];
+    const [a, b] = range;
+    const { min, max, value } = data;
+
+    if (isNaN(value) || isNaN(min) || isNaN(max) || min > max) {
+      return 0;
+    }
+    if (min === max) {
+      return 100;
+    }
+    const res = a + ((value - min) * (b - a)) / (max - min);
+    return res * 100;
+  };
+
+  const renderPercent = useCallback((data: any) => {
+    if (!data.showExtra) {
+      return null;
+    }
+    const percent = data.percent;
+    return (
+      <>
+        <Progress
+          size={{
+            height: 4
+          }}
+          type="line"
+          status="normal"
+          strokeLinecap={'square'}
+          showInfo={false}
+          percentPosition={{ align: 'end', type: 'outer' }}
+          strokeColor={`linear-gradient(90deg, #388bff 0%, rgba(255,255,255,1) ${percent}%)`}
+          trailColor="transparent"
+          percent={percent}
+          style={{
+            position: 'absolute',
+            left: 0,
+            bottom: -2,
+            width: 'calc(100% - 2px)',
+            lineHeight: '12px',
+            borderRadius: '0 0 0 6px',
+            overflow: 'hidden'
+          }}
+        ></Progress>
+        <span
+          className="flex-center hover-hidden"
+          style={{
+            position: 'absolute',
+            right: 10,
+            top: 8,
+            padding: '0 4px',
+            backgroundColor: 'transparent',
+            opacity: 0.7
+          }}
+        >
+          <Tag color={'geekblue'}>Rank: {data.rank}</Tag>
+          <Tag style={{ margin: 0 }} color={'cyan'}>
+            Score: {_.round(data.score, 2)}
+          </Tag>
+        </span>
+      </>
+    );
+  }, []);
+
   const setMessageId = () => {
     messageId.current = messageId.current + 1;
+    return messageId.current;
   };
 
   const handleStopConversation = () => {
@@ -153,28 +231,52 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
           token: requestToken.current.token
         }
       );
-      console.log('result:', result);
 
       setMessageId();
       setTokenResult(result.usage);
-      const maxItem = _.maxBy(result.results || [], (item: any) =>
+
-        Math.abs(item.relevance_score)
+      const sortList = _.sortBy(
+        result.results || [],
+        (item: any) => item.relevance_score
       );
-      const maxValue = _.ceil(maxItem?.relevance_score, 2);
+      const maxValue = sortList[sortList.length - 1].relevance_score;
+      const minValue = sortList[0].relevance_score;
+
+      let newTextList = [...textList];
+
+      result.results?.forEach((item: any, sIndex: number) => {
+        newTextList[item.index] = {
+          ...newTextList[item.index],
+          rank: sIndex + 1,
+          score: item.relevance_score,
+          showExtra: true,
+          percent: normalizValue({
+            min: minValue,
+            max: maxValue,
+            value: item.relevance_score
+          })
+        };
+      });
+      setTextList(newTextList);
 
       setMessageList([
         {
           title: 'Results',
           role: '',
           content: result.results?.map((item: any) => {
+            const percent: number = normalizValue({
+              min: minValue,
+              max: maxValue,
+              value: item.relevance_score
+            });
             return {
               uid: item.index,
               text: `${item.document?.text?.slice(0, 500) || ''}`,
               docIndex: item.index,
               title: documentList[item.index]?.name || '',
               score: item.relevance_score,
-              normalizValue:
+              extra: renderPercent(percent),
-                _.round(_.round(item.relevance_score, 2) / maxValue, 2) * 100
+              normalizValue: percent
             };
           }),
           uid: messageId.current
@@ -280,8 +382,11 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
       <div className="ground-left">
         <div className="ground-left-footer">
           <MessageInput
-            scope="reranker"
             actions={[]}
+            defaultSize={{
+              minRows: 1,
+              maxRows: 2
+            }}
             submitIcon={<SearchOutlined className="font-size-16" />}
             loading={loading}
             disabled={!parameters.model}
@@ -294,7 +399,11 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
             placeholer={intl.formatMessage({
               id: 'playground.input.keyword.holder'
             })}
-            tools={<span style={{ paddingLeft: 6 }}>Query</span>}
+            tools={
+              <span style={{ paddingLeft: 6, fontSize: 14, fontWeight: 500 }}>
+                Query
+              </span>
+            }
             style={{
               borderTop: 'none',
               width: 'unset',
@@ -313,19 +422,6 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
                 <span>Documents</span>
               </h3>
               <div className="flex gap-10">
-                {/* <UploadFile
-                  handleUpdateFileList={handleUpdateFileList}
-                  accept={acceptType}
-                >
-                  <Tooltip title={<span>Support: {acceptType}</span>}>
-                    <Button
-                      size="middle"
-                      icon={<UploadOutlined></UploadOutlined>}
-                    >
-                      Upload File
-                    </Button>
-                  </Tooltip>
-                </UploadFile> */}
                 <Button size="middle" onClick={handleAddText}>
                   <PlusOutlined />
                   Add Text
@@ -344,24 +440,20 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
                 ref={inputListRef}
                 textList={textList}
                 onChange={handleTextListChange}
+                extra={renderPercent}
               ></InputList>
-              {/* <div style={{ marginTop: 8 }}>
-                <FileList
-                  fileList={fileList}
-                  textListCount={textList.length || 0}
-                  onDelete={handleDeleteFile}
-                ></FileList>
-              </div> */}
             </div>
           </div>
           <div>
             {messageList.length ? (
               <div className="result-header flex-center">
                 <h3 className="font-size-14 m-b-0">Results</h3>
-                <ReferenceParams
+                {tokenResult?.total_tokens && (
-                  usage={tokenResult}
+                  <span style={{ color: 'var(--ant-orange)' }}>
-                  showOutput={false}
+                    {intl.formatMessage({ id: 'playground.tokenusage' })}:{' '}
-                ></ReferenceParams>
+                    {tokenResult?.total_tokens}
+                  </span>
+                )}
               </div>
             ) : null}
           </div>
@@ -371,7 +463,7 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
           >
             <>
               <div className="content">
-                <RerankMessage dataList={messageList} />
+                {/*<RerankMessage dataList={messageList} />*/}
                 {loading && (
                   <Spin size="small">
                     <div style={{ height: '46px' }}></div>
@@ -402,7 +494,9 @@ const GroundReranker: React.FC<MessageProps> = forwardRef((props, ref) => {
 
       <ViewRerankCode
         open={show}
-        documentList={[...textList, ...fileList].map((item) => item.text)}
+        documentList={[...textList, ...fileList]
+          .map((item) => item.text)
+          .filter((text) => text)}
         parameters={{
           ...parameters,
           query: contentRef.current

src/pages/playground/components/ground-stt.tsx

@@ -1,7 +1,475 @@
-import React from 'react';
+import AudioAnimation from '@/components/audio-animation';
+import AudioPlayer from '@/components/audio-player';
+import IconFont from '@/components/icon-font';
+import UploadAudio from '@/components/upload-audio';
+import useOverlayScroller from '@/hooks/use-overlay-scroller';
+import { fetchChunkedData, readStreamData } from '@/utils/fetch-chunk-data';
+import { readAudioFile } from '@/utils/load-audio-file';
+import { AudioOutlined, ThunderboltOutlined } from '@ant-design/icons';
+import { useIntl, useSearchParams } from '@umijs/max';
+import { Button, Spin, Tag, Tooltip } from 'antd';
+import classNames from 'classnames';
+import _ from 'lodash';
+import 'overlayscrollbars/overlayscrollbars.css';
+import {
+  forwardRef,
+  memo,
+  useCallback,
+  useEffect,
+  useImperativeHandle,
+  useMemo,
+  useRef,
+  useState
+} from 'react';
+import { CHAT_API } from '../apis';
+import { Roles, generateMessages } from '../config';
+import { RealtimeParamsConfig as paramsConfig } from '../config/params-config';
+import { MessageItem } from '../config/types';
+import '../style/ground-left.less';
+import '../style/speech-to-text.less';
+import '../style/system-message-wrap.less';
+import AudioInput from './audio-input';
+import MessageContent from './multiple-chat/message-content';
+import RerankerParams from './reranker-params';
+import ViewCodeModal from './view-code-modal';
 
-const GroundStt = () => {
+interface MessageProps {
-  return <div>STT</div>;
+  modelList: Global.BaseOption<string>[];
+  loaded?: boolean;
+  ref?: any;
+}
+
+const initialValues = {
+  language: 'auto'
 };
 
-export default React.memo(GroundStt);
+const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
+  const { modelList } = props;
+  const messageId = useRef<number>(0);
+  const [messageList, setMessageList] = useState<MessageItem[]>([]);
+
+  const intl = useIntl();
+  const [searchParams] = useSearchParams();
+  const selectModel = searchParams.get('model') || '';
+  const [parameters, setParams] = useState<any>({});
+  const [systemMessage, setSystemMessage] = useState('');
+  const [show, setShow] = useState(false);
+  const [loading, setLoading] = useState(false);
+  const [tokenResult, setTokenResult] = useState<any>(null);
+  const [collapse, setCollapse] = useState(false);
+  const contentRef = useRef<any>('');
+  const controllerRef = useRef<any>(null);
+  const scroller = useRef<any>(null);
+  const currentMessageRef = useRef<any>(null);
+  const paramsRef = useRef<any>(null);
+  const messageListLengthCache = useRef<number>(0);
+  const [audioPermissionOn, setAudioPermissionOn] = useState(true);
+  const [audioData, setAudioData] = useState<any>(null);
+  const [audioChunks, setAudioChunks] = useState<any>({
+    data: [],
+    analyser: null
+  });
+  const [isRecording, setIsRecording] = useState(false);
+
+  const { initialize, updateScrollerPosition } = useOverlayScroller();
+  const { initialize: innitializeParams } = useOverlayScroller();
+
+  useImperativeHandle(ref, () => {
+    return {
+      viewCode() {
+        setShow(true);
+      },
+      setCollapse() {
+        setCollapse(!collapse);
+      },
+      collapse: collapse
+    };
+  });
+
+  const viewCodeMessage = useMemo(() => {
+    return generateMessages([
+      { role: Roles.System, content: systemMessage },
+      ...messageList
+    ]);
+  }, [messageList, systemMessage]);
+
+  const setMessageId = () => {
+    messageId.current = messageId.current + 1;
+  };
+
+  const joinMessage = (chunk: any) => {
+    setTokenResult({
+      ...(chunk?.usage ?? {})
+    });
+
+    if (!chunk || !_.get(chunk, 'choices', []).length) {
+      return;
+    }
+    contentRef.current =
+      contentRef.current + _.get(chunk, 'choices.0.delta.content', '');
+    setMessageList([
+      ...messageList,
+      ...currentMessageRef.current,
+      {
+        role: Roles.Assistant,
+        content: contentRef.current,
+        uid: messageId.current
+      }
+    ]);
+  };
+  const handleStopConversation = () => {
+    controllerRef.current?.abort?.();
+    setLoading(false);
+  };
+
+  const submitMessage = async (current?: { role: string; content: string }) => {
+    if (!parameters.model) return;
+    try {
+      setLoading(true);
+      setMessageId();
+      setTokenResult(null);
+
+      controllerRef.current?.abort?.();
+      controllerRef.current = new AbortController();
+      const signal = controllerRef.current.signal;
+      currentMessageRef.current = current
+        ? [
+            {
+              ...current,
+              uid: messageId.current
+            }
+          ]
+        : [];
+
+      contentRef.current = '';
+      setMessageList((pre) => {
+        return [...pre, ...currentMessageRef.current];
+      });
+
+      const messageParams = [
+        { role: Roles.System, content: systemMessage },
+        ...messageList,
+        ...currentMessageRef.current
+      ];
+
+      const messages = generateMessages(messageParams);
+
+      const chatParams = {
+        messages: messages,
+        ...parameters,
+        stream: true,
+        stream_options: {
+          include_usage: true
+        }
+      };
+      const result: any = await fetchChunkedData({
+        data: chatParams,
+        url: CHAT_API,
+        signal
+      });
+
+      if (result?.error) {
+        setTokenResult({
+          error: true,
+          errorMessage:
+            result?.data?.error?.message || result?.data?.message || ''
+        });
+        return;
+      }
+      setMessageId();
+      const { reader, decoder } = result;
+      await readStreamData(reader, decoder, (chunk: any) => {
+        if (chunk?.error) {
+          setTokenResult({
+            error: true,
+            errorMessage: chunk?.error?.message || chunk?.message || ''
+          });
+          return;
+        }
+        joinMessage(chunk);
+      });
+    } catch (error) {
+      // console.log('error:', error);
+    } finally {
+      setLoading(false);
+    }
+  };
+  const handleClear = () => {
+    if (!messageList.length) {
+      return;
+    }
+    setMessageId();
+    setMessageList([]);
+    setTokenResult(null);
+  };
+
+  const renderTitle = useCallback((role: string) => {
+    return (
+      <span>
+        {intl.formatMessage({ id: `playground.${role}` })}
+        <span className="text-tertiary m-l-5">00:10</span>
+      </span>
+    );
+  }, []);
+
+  const handleSendMessage = (message: Omit<MessageItem, 'uid'>) => {
+    setLoading(true);
+    setMessageList([
+      ...messageList,
+      {
+        role: Roles.User,
+        title: renderTitle(Roles.User),
+        content: 'test data test data',
+        uid: messageId.current
+      }
+    ]);
+
+    setTimeout(() => {
+      setMessageList([
+        ...messageList,
+        {
+          role: Roles.Assistant,
+          title: renderTitle(Roles.Assistant),
+          content: 'generate by assistant',
+          uid: messageId.current
+        }
+      ]);
+      setLoading(false);
+    }, 1000);
+  };
+
+  const handleCloseViewCode = () => {
+    setShow(false);
+  };
+
+  const handleOnAudioData = useCallback(
+    (data: { chunks: Blob[]; url: string; name: string; duration: number }) => {
+      setAudioData(() => {
+        return {
+          url: data.url,
+          name: data.name,
+          duration: data.duration
+        };
+      });
+    },
+    []
+  );
+
+  const handleOnAudioPermission = useCallback((permission: boolean) => {
+    setAudioPermissionOn(permission);
+  }, []);
+
+  const handleUploadChange = useCallback(
+    async (data: { file: any; fileList: any }) => {
+      const res = await readAudioFile(data.file.originFileObj);
+      console.log('res=======', res);
+      setAudioData(res);
+    },
+    []
+  );
+
+  const handleOnAnalyse = useCallback((data: any, analyser: any) => {
+    setAudioChunks((pre: any) => {
+      return {
+        data: data,
+        analyser: analyser
+      };
+    });
+  }, []);
+  const handleOnRecord = useCallback((val: boolean) => {
+    setIsRecording(val);
+    setAudioData(null);
+  }, []);
+
+  const renderAniamtion = () => {
+    if (!audioPermissionOn) {
+      return null;
+    }
+    if (isRecording) {
+      return (
+        <AudioAnimation
+          height={66}
+          width={200}
+          analyserData={audioChunks}
+        ></AudioAnimation>
+      );
+    }
+    return (
+      <div className="tips-text">
+        <IconFont type={'icon-audio'} style={{ fontSize: 20 }}></IconFont>
+        <span>Upload an audio file or start recording</span>
+      </div>
+    );
+  };
+
+  useEffect(() => {}, [messageList]);
+  useEffect(() => {
+    if (scroller.current) {
+      initialize(scroller.current);
+    }
+  }, [scroller.current, initialize]);
+
+  useEffect(() => {
+    if (paramsRef.current) {
+      innitializeParams(paramsRef.current);
+    }
+  }, [paramsRef.current, innitializeParams]);
+
+  useEffect(() => {
+    if (loading) {
+      updateScrollerPosition();
+    }
+  }, [messageList, loading]);
+
+  useEffect(() => {
+    if (messageList.length > messageListLengthCache.current) {
+      updateScrollerPosition();
+    }
+    messageListLengthCache.current = messageList.length;
+  }, [messageList.length]);
+
+  return (
+    <div className="ground-left-wrapper">
+      <div className="ground-left">
+        <div className="ground-left-footer" style={{ flex: 1 }}>
+          <div className="speech-to-text">
+            <div className="speech-box">
+              <Tooltip title="Upload an audio file">
+                <UploadAudio
+                  type="default"
+                  accept=".mp3,.mp4,.wav"
+                  onChange={handleUploadChange}
+                ></UploadAudio>
+              </Tooltip>
+              <AudioInput
+                type="default"
+                voiceActivity={true}
+                onAudioData={handleOnAudioData}
+                onAudioPermission={handleOnAudioPermission}
+                onAnalyse={handleOnAnalyse}
+                onRecord={handleOnRecord}
+              ></AudioInput>
+            </div>
+
+            {audioData ? (
+              <div className="flex-between flex-center">
+                <div style={{ flex: 1 }}>
+                  <AudioPlayer
+                    url={audioData.url}
+                    name={audioData.name}
+                    duration={audioData.duration}
+                  ></AudioPlayer>
+                  <div
+                    style={{
+                      paddingRight: 5,
+                      display: 'flex',
+                      justifyContent: 'flex-end',
+                      marginTop: 30
+                    }}
+                  >
+                    <Tooltip title="generate text content">
+                      <Button
+                        size="middle"
+                        type="primary"
+                        icon={<ThunderboltOutlined></ThunderboltOutlined>}
+                      >
+                        Generata Text Content
+                      </Button>
+                    </Tooltip>
+                  </div>
+                </div>
+              </div>
+            ) : (
+              renderAniamtion()
+            )}
+          </div>
+          {!audioPermissionOn && (
+            <div
+              style={{
+                display: 'flex',
+                flexDirection: 'column',
+                justifyContent: 'center',
+                alignItems: 'center',
+                height: '100%'
+              }}
+            >
+              <span>
+                <Tag
+                  style={{
+                    width: 36,
+                    height: 36,
+                    lineHeight: '36px',
+                    textAlign: 'center'
+                  }}
+                  bordered={false}
+                  color="error"
+                  icon={
+                    <AudioOutlined className="font-size-16"></AudioOutlined>
+                  }
+                ></Tag>
+              </span>
+              <span
+                style={{
+                  marginTop: 10,
+                  fontSize: 14,
+                  fontWeight: 500
+                }}
+              >
+                Enable microphone access in your browser&rsquo;s settings.
+              </span>
+            </div>
+          )}
+        </div>
+        <div className="message-list-wrap" ref={scroller}>
+          <>
+            <div className="content" style={{ height: '100%' }}>
+              <>
+                <MessageContent
+                  actions={['copy']}
+                  messageList={messageList[0] ? [messageList[0]] : []}
+                  setMessageList={setMessageList}
+                  editable={false}
+                  loading={loading}
+                />
+                {loading && (
+                  <Spin size="small">
+                    <div style={{ height: '46px' }}></div>
+                  </Spin>
+                )}
+              </>
+            </div>
+          </>
+        </div>
+      </div>
+      <div
+        className={classNames('params-wrapper', {
+          collapsed: collapse
+        })}
+        ref={paramsRef}
+      >
+        <div className="box">
+          <RerankerParams
+            setParams={setParams}
+            paramsConfig={paramsConfig}
+            initialValues={initialValues}
+            params={parameters}
+            selectedModel={selectModel}
+            modelList={modelList}
+          />
+        </div>
+      </div>
+
+      <ViewCodeModal
+        open={show}
+        payLoad={{
+          messages: viewCodeMessage
+        }}
+        parameters={parameters}
+        onCancel={handleCloseViewCode}
+        title={intl.formatMessage({ id: 'playground.viewcode' })}
+      ></ViewCodeModal>
+    </div>
+  );
+});
+
+export default memo(GroundLeft);

src/pages/playground/components/ground-tts.tsx

@@ -1,3 +1,4 @@
+import SpeechContent from '@/components/speech-content';
 import useOverlayScroller from '@/hooks/use-overlay-scroller';
 import { fetchChunkedData, readStreamData } from '@/utils/fetch-chunk-data';
 import { useIntl, useSearchParams } from '@umijs/max';
@@ -21,8 +22,6 @@ import { MessageItem } from '../config/types';
 import '../style/ground-left.less';
 import '../style/system-message-wrap.less';
 import MessageInput from './message-input';
-import MessageContent from './multiple-chat/message-content';
-import SystemMessage from './multiple-chat/system-message';
 import ReferenceParams from './reference-params';
 import RerankerParams from './reranker-params';
 import ViewCodeModal from './view-code-modal';
@@ -59,6 +58,7 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
   const currentMessageRef = useRef<any>(null);
   const paramsRef = useRef<any>(null);
   const messageListLengthCache = useRef<number>(0);
+  const checkvalueRef = useRef<any>(true);
 
   const { initialize, updateScrollerPosition } = useOverlayScroller();
   const { initialize: innitializeParams } = useOverlayScroller();
@@ -207,10 +207,23 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
   };
 
   const handleSendMessage = (message: Omit<MessageItem, 'uid'>) => {
-    console.log('message:', message);
+    // submitMessage(currentMessage);
-    const currentMessage =
+    setMessageId();
-      message.content || message.imgs?.length ? message : undefined;
+    setLoading(true);
-    submitMessage(currentMessage);
+
+    setTimeout(() => {
+      setMessageList([
+        {
+          prompt: message.content,
+          voice: parameters.voice,
+          format: parameters.response_format,
+          speed: parameters.speed,
+          uid: messageId.current,
+          autoplay: checkvalueRef.current
+        }
+      ]);
+      setLoading(false);
+    }, 1000);
   };
 
   const handleCloseViewCode = () => {
@@ -233,7 +246,10 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
     setSystemMessage(sysMsg[0]?.content || '');
     setMessageList(userMsg);
   };
-
+  const handleOnCheckChange = (e: any) => {
+    console.log('handleOnCheckChange', e);
+    checkvalueRef.current = e.target.checked;
+  };
   useEffect(() => {
     if (scroller.current) {
       initialize(scroller.current);
@@ -264,32 +280,8 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
       <div className="ground-left">
         <div className="message-list-wrap" ref={scroller}>
           <>
-            <div
-              style={{
-                marginBottom: 20
-              }}
-            >
-              <SystemMessage
-                style={{
-                  borderRadius: 'var(--border-radius-mini)',
-                  overflow: 'hidden'
-                }}
-                systemMessage={systemMessage}
-                setSystemMessage={setSystemMessage}
-              ></SystemMessage>
-            </div>
-
             <div className="content">
-              <MessageContent
+              <SpeechContent dataList={messageList} loading={loading} />
-                spans={{
-                  span: 24,
-                  count: 1
-                }}
-                messageList={messageList}
-                setMessageList={setMessageList}
-                editable={true}
-                loading={loading}
-              />
               {loading && (
                 <Spin size="small">
                   <div style={{ height: '46px' }}></div>
@@ -306,9 +298,14 @@ const GroundLeft: React.FC<MessageProps> = forwardRef((props, ref) => {
         <div className="ground-left-footer">
           <MessageInput
             scope="chat"
+            actions={['clear', 'check']}
+            checkLabel={intl.formatMessage({
+              id: 'playground.toolbar.autoplay'
+            })}
+            onCheck={handleOnCheckChange}
             loading={loading}
             disabled={!parameters.model}
-            isEmpty={!messageList.length}
+            isEmpty={true}
             handleSubmit={handleSendMessage}
             addMessage={handleNewMessage}
             handleAbortFetch={handleStopConversation}

src/pages/playground/components/input-list.tsx

@@ -7,14 +7,19 @@ import '../style/input-list.less';
 
 interface InputListProps {
   ref?: any;
-  textList: { text: string; uid: number | string; name: string }[];
+  extra?: (data: any) => React.ReactNode;
+  textList: {
+    text: string;
+    uid: number | string;
+    name: string;
+  }[];
   onChange?: (
     textList: { text: string; uid: number | string; name: string }[]
   ) => void;
 }
 
 const InputList: React.FC<InputListProps> = forwardRef(
-  ({ textList, onChange }, ref) => {
+  ({ textList, onChange, extra }, ref) => {
     const intl = useIntl();
     const messageId = useRef(0);
 
@@ -82,6 +87,7 @@ const InputList: React.FC<InputListProps> = forwardRef(
                   ></Button>
                 </Tooltip>
               </span>
+              {extra?.(text)}
             </div>
           );
         })}

src/pages/playground/components/message-input.tsx

@@ -2,9 +2,9 @@ import IconFont from '@/components/icon-font';
 import HotKeys, { KeyMap } from '@/config/hotkeys';
 import { ClearOutlined, SendOutlined, SwapOutlined } from '@ant-design/icons';
 import { useIntl } from '@umijs/max';
-import { Button, Divider, Input, Select, Tooltip } from 'antd';
+import { Button, Checkbox, Divider, Input, Select, Tooltip } from 'antd';
 import _ from 'lodash';
-import { useCallback, useMemo, useRef, useState } from 'react';
+import React, { useCallback, useMemo, useRef, useState } from 'react';
 import { useHotkeys } from 'react-hotkeys-hook';
 import { Roles } from '../config';
 import { MessageItem } from '../config/types';
@@ -15,7 +15,14 @@ import UploadImg from './upload-img';
 
 type CurrentMessage = Omit<MessageItem, 'uid'>;
 
-type ActionType = 'clear' | 'layout' | 'role' | 'upload' | 'add' | 'paste';
+type ActionType =
+  | 'clear'
+  | 'layout'
+  | 'role'
+  | 'upload'
+  | 'add'
+  | 'paste'
+  | 'check';
 
 const layoutOptions = [
   {
@@ -57,7 +64,7 @@ const layoutOptions = [
 ];
 
 interface MessageInputProps {
-  modelList: Global.BaseOption<string>[];
+  modelList?: Global.BaseOption<string>[];
   handleSubmit: (params: CurrentMessage) => void;
   handleAbortFetch: () => void;
   updateLayout?: (value: { span: number; count: number }) => void;
@@ -67,6 +74,7 @@ interface MessageInputProps {
       instanceId: symbol;
     })[]
   ) => void;
+  onCheck?: (e: any) => void;
   submitIcon?: React.ReactNode;
   presetPrompt?: (list: CurrentMessage[]) => void;
   addMessage?: (message: CurrentMessage) => void;
@@ -75,11 +83,12 @@ interface MessageInputProps {
   showModelSelection?: boolean;
   disabled: boolean;
   isEmpty?: boolean;
-  scope: string;
   placeholer?: string;
   shouldResetMessage?: boolean;
   style?: React.CSSProperties;
   actions?: ActionType[];
+  checkLabel?: React.ReactNode;
+  defaultSize?: { minRows: number; maxRows: number };
 }
 
 const MessageInput: React.FC<MessageInputProps> = ({
@@ -90,16 +99,18 @@ const MessageInput: React.FC<MessageInputProps> = ({
   clearAll,
   updateLayout,
   addMessage,
+  onCheck,
   loading,
   modelList,
   showModelSelection,
   disabled,
   isEmpty,
-  scope,
   submitIcon,
   placeholer,
   tools,
   style,
+  checkLabel,
+  defaultSize = { minRows: 3, maxRows: 8 },
   shouldResetMessage = true,
   actions = ['clear', 'layout', 'role', 'upload', 'add', 'paste']
 }) => {
@@ -338,7 +349,6 @@ const MessageInput: React.FC<MessageInputProps> = ({
     <div className="messageInput" style={{ ...style }}>
       <div className="tool-bar">
         <div className="actions">
-          {tools}
           {
             <>
               {actions.includes('role') && (
@@ -362,6 +372,12 @@ const MessageInput: React.FC<MessageInputProps> = ({
               )}
             </>
           }
+          {tools}
+          {actions.includes('check') && (
+            <Checkbox onChange={onCheck} defaultChecked={true}>
+              {checkLabel}
+            </Checkbox>
+          )}
           {actions.includes('clear') && (
             <Tooltip
               title={intl.formatMessage({ id: 'playground.toolbar.clearmsg' })}
@@ -374,6 +390,7 @@ const MessageInput: React.FC<MessageInputProps> = ({
               ></Button>
             </Tooltip>
           )}
+
           {actions.includes('layout') && updateLayout && (
             <>
               <Divider type="vertical" style={{ margin: 0 }} />
@@ -478,7 +495,10 @@ const MessageInput: React.FC<MessageInputProps> = ({
         ) : (
           <TextArea
             ref={inputRef}
-            autoSize={{ minRows: 3, maxRows: 8 }}
+            autoSize={{
+              minRows: defaultSize.minRows,
+              maxRows: defaultSize.maxRows
+            }}
             onChange={handleInputChange}
             value={message.content}
             size="large"

src/pages/playground/components/multiple-chat/message-content.tsx

@@ -5,10 +5,6 @@ import ContentItem from './content-item';
 
 interface MessageContentProps {
   loading?: boolean;
-  spans?: {
-    span: number;
-    count: number;
-  };
   actions?: MessageItemAction[];
   editable?: boolean;
   messageList: MessageItem[];

src/pages/playground/components/reference-params.tsx

@@ -8,6 +8,7 @@ import '../style/reference-params.less';
 interface ReferenceParamsProps {
   showOutput?: boolean;
   scaleable?: boolean;
+  fields?: string[];
   usage: {
     error?: boolean;
     errorMessage?: string;
@@ -22,7 +23,12 @@ interface ReferenceParamsProps {
 
 const ReferenceParams = (props: ReferenceParamsProps) => {
   const intl = useIntl();
-  const { usage, showOutput = true, scaleable } = props;
+  const {
+    usage,
+    showOutput = true,
+    scaleable,
+    fields = ['completion_tokens', 'prompt_tokens']
+  } = props;
   if (!usage || _.isEmpty(usage)) {
     return null;
   }
@@ -60,14 +66,18 @@ const ReferenceParams = (props: ReferenceParamsProps) => {
         <Tooltip
           title={
             <Space>
-              <span>
+              {fields.includes('completion_tokens') && (
-                {intl.formatMessage({ id: 'playground.completion' })}:{' '}
+                <span>
-                {usage.completion_tokens}
+                  {intl.formatMessage({ id: 'playground.completion' })}:{' '}
-              </span>
+                  {usage.completion_tokens}
-              <span>
+                </span>
-                {intl.formatMessage({ id: 'playground.prompt' })}:{' '}
+              )}
-                {usage.prompt_tokens}
+              {fields.includes('prompt_tokens') && (
-              </span>
+                <span>
+                  {intl.formatMessage({ id: 'playground.prompt' })}:{' '}
+                  {usage.prompt_tokens}
+                </span>
+              )}
             </Space>
           }
         >

src/pages/playground/components/rerank-message.tsx

@@ -6,8 +6,13 @@ import '../style/rerank-message.less';
 
 interface RerankMessageProps {
   header?: React.ReactNode;
-  dataList: { title?: string; content: any; uid: number | string }[];
+  dataList: {
+    title?: React.ReactNode;
+    content: string | any[];
+    uid: number | string;
+  }[];
 }
+
 const RerankMessage: React.FC<RerankMessageProps> = ({ header, dataList }) => {
   if (!dataList || dataList.length === 0) {
     return null;
@@ -51,13 +56,13 @@ const RerankMessage: React.FC<RerankMessageProps> = ({ header, dataList }) => {
                           type="line"
                           status="normal"
                           percentPosition={{ align: 'end', type: 'outer' }}
-                          strokeColor="rgb(22 119 255 / 27%)"
+                          strokeColor={`linear-gradient(90deg, #388bff 0%, #fff ${sItem.normalizValue}%)`}
                           trailColor="transparent"
                           percent={sItem.normalizValue}
                           style={{
                             position: 'absolute',
                             left: 0,
-                            bottom: 0,
+                            bottom: -4,
                             lineHeight: '12px'
                           }}
                         ></Progress>

src/pages/playground/components/reranker-params.tsx

@@ -36,6 +36,7 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = ({
   initialValues,
   paramsConfig,
   modelList,
+  params,
   showModelSelector = true
 }) => {
   const intl = useIntl();
@@ -204,7 +205,7 @@ const ParamsSettings: React.FC<ParamsSettingsProps> = ({
       }
       return null;
     });
-  }, [paramsConfig]);
+  }, [paramsConfig, params]);
 
   return (
     <Form

src/pages/playground/components/thumb-img.tsx

@@ -1,5 +1,6 @@
 import AutoImage from '@/components/auto-image';
 import { CloseCircleOutlined } from '@ant-design/icons';
+import { Spin } from 'antd';
 import _ from 'lodash';
 import React, { useCallback } from 'react';
 import '../style/thumb-img.less';
@@ -7,11 +8,13 @@ import '../style/thumb-img.less';
 const ThumbImg: React.FC<{
   dataList: any[];
   editable?: boolean;
-  onDelete: (uid: number) => void;
+  onDelete?: (uid: number) => void;
-}> = ({ dataList, editable, onDelete }) => {
+  loading?: boolean;
+  style?: React.CSSProperties;
+}> = ({ dataList, editable, onDelete, loading, style }) => {
   const handleOnDelete = useCallback(
     (uid: number) => {
-      onDelete(uid);
+      onDelete?.(uid);
     },
     [onDelete]
   );
@@ -21,30 +24,54 @@ const ThumbImg: React.FC<{
   }
 
   return (
-    <div className="thumb-list-wrap">
+    <>
-      {_.map(dataList, (item: any) => {
+      {
-        return (
+        <div className="thumb-list-wrap" style={{ ...style }}>
-          <span
+          {_.map(dataList, (item: any) => {
-            key={item.uid}
+            return (
-            className="thumb-img"
+              <span
-            style={{
+                key={item.uid}
-              width: item.width,
+                className="thumb-img"
-              height: item.height
+                style={{
-            }}
+                  width: item.width || 100,
-          >
+                  height: item.height || 100
-            <span className="img">
+                }}
-              <AutoImage src={item.dataUrl} height={100} />
+              >
-            </span>
+                <span className="img">
+                  {loading ? (
+                    <span
+                      style={{
+                        width: '100%',
+                        height: '100%',
+                        display: 'flex',
+                        border: '1px solid var(--ant-color-split)',
+                        borderRadius: 'var(--border-radius-base)'
+                      }}
+                    >
+                      <Spin
+                        className="flex-center justify-center"
+                        style={{ width: '100%', height: '100%' }}
+                      ></Spin>
+                    </span>
+                  ) : (
+                    <AutoImage src={item.dataUrl} height={item.height || 100} />
+                  )}
+                </span>
 
-            {editable && (
+                {editable && (
-              <span className="del" onClick={() => handleOnDelete(item.uid)}>
+                  <span
-                <CloseCircleOutlined />
+                    className="del"
+                    onClick={() => handleOnDelete(item.uid)}
+                  >
+                    <CloseCircleOutlined />
+                  </span>
+                )}
               </span>
-            )}
+            );
-          </span>
+          })}
-        );
+        </div>
-      })}
+      }
-    </div>
+    </>
   );
 };
 

src/pages/playground/config/params-config.ts

@@ -66,6 +66,31 @@ export const TTSParamsConfig: ParamsSchema[] = [
   }
 ];
 
+export const RealtimeParamsConfig: ParamsSchema[] = [
+  {
+    type: 'Select',
+    name: 'language',
+    options: [
+      { label: 'Auto', value: 'auto' },
+      { label: 'English', value: 'en' },
+      { label: '中文', value: 'zh' },
+      { label: '日本語', value: 'ja' },
+      { label: 'Français', value: 'fr' },
+      { label: 'Deutsch', value: 'de' }
+    ],
+    label: {
+      text: 'Language',
+      isLocalized: false
+    },
+    rules: [
+      {
+        required: true,
+        message: 'Language is required'
+      }
+    ]
+  }
+];
+
 export const ImageParamsConfig: ParamsSchema[] = [
   {
     type: 'InputNumber',

src/pages/playground/config/types.ts

@@ -3,12 +3,14 @@ export interface ModelSelectionItem extends Global.BaseOption<string> {
   instanceId: symbol;
   type?: string;
 }
+
 export type MessageItemAction = 'upload' | 'delete' | 'copy';
+
 export interface MessageItem {
   content: string;
   imgs?: { uid: string | number; dataUrl: string }[];
   role: string;
-  title?: string;
+  title?: React.ReactNode;
   uid: number;
 }
 

src/pages/playground/speech.tsx

@@ -30,12 +30,12 @@ const Playground: React.FC = () => {
   const [loaded, setLoaded] = useState(false);
   const optionsList = [
     {
-      label: 'TTS',
+      label: 'Text To Speech',
       value: TabsValueMap.Tab1,
       icon: <AudioOutlined />
     },
     {
-      label: 'Realtime',
+      label: 'Speech To Text',
       value: TabsValueMap.Tab2,
       icon: <IconFont type={'icon-audio'}></IconFont>
     }

src/pages/playground/style/audio-input.less

@@ -0,0 +1,20 @@
+.audio-input {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  background-color: var(--ant-color-fill-quaternary);
+  border-radius: 4px;
+  padding: 16px;
+
+  .ant-upload {
+    border: none;
+    background-color: transparent !important;
+  }
+
+  .btns {
+    .anticon {
+      font-size: 24px;
+    }
+  }
+}

src/pages/playground/style/input-list.less

@@ -4,6 +4,7 @@
   gap: 12px;
 
   .input-item {
+    position: relative;
     display: flex;
     align-items: center;
     justify-content: space-between;
@@ -13,28 +14,42 @@
     transition: background-color 0.3s ease;
     border-radius: var(--border-radius-base);
     border: 1px solid var(--ant-color-border);
-    // &:focus-within {
-    //   border-color: var(--ant-input-active-border-color);
-    //   box-shadow: var(--ant-input-active-shadow);
-    //   outline: 0;
-    //   background-color: var(--ant-input-active-bg);
-    // }
 
     .btn-group {
       display: none;
     }
 
     &:hover {
-      background-color: var(--ant-color-fill-tertiary);
+      //background-color: var(--ant-color-fill-tertiary);
+      &::before {
+        content: '';
+        display: flex;
+        position: absolute;
+        left: 0;
+        bottom: 0;
+        right: 0;
+        top: 0;
+        background-color: var(--ant-color-fill-tertiary);
+        z-index: 5;
+        pointer-events: none;
+      }
 
       .btn-group {
         display: flex;
         gap: 8px;
       }
+
+      .hover-hidden {
+        display: none;
+      }
     }
 
     &:focus-within {
-      background-color: transparent;
+      //background-color: transparent;
+
+      &::before {
+        background-color: transparent;
+      }
     }
 
     .input-wrap {

src/pages/playground/style/rerank-message.less

@@ -31,6 +31,7 @@
   }
 
   .content-item-text {
+    position: relative;
     margin-bottom: 0;
     display: flex;
     align-items: center;

src/pages/playground/style/speech-to-text.less

@@ -0,0 +1,42 @@
+.speech-to-text {
+  padding: 32px;
+  height: 100%;
+  display: flex;
+  gap: 30px;
+  flex-direction: column;
+
+  .tips-text {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    font-size: var(--font-size-large);
+    height: 66px;
+    padding: 0;
+    gap: 10px;
+  }
+
+  .speech-box {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    gap: 120px;
+    flex: 1;
+    min-height: 160px;
+
+    .ant-upload {
+      border: none;
+      background-color: transparent !important;
+    }
+
+    .ant-btn {
+      width: 60px;
+      height: 60px;
+    }
+
+    .btns {
+      .anticon {
+        font-size: 24px;
+      }
+    }
+  }
+}

src/utils/index.ts

@@ -156,6 +156,26 @@ export const platformCall = () => {
   };
 };
 
+export const formatTime = (seconds: number) => {
+  if (isNaN(seconds) || !seconds || seconds === Infinity) {
+    return '00:00';
+  }
+  const hrs = Math.floor(seconds / 3600);
+  const mins = Math.floor((seconds % 3600) / 60);
+  const secs = Math.floor(seconds % 60);
+
+  const formatted = [
+    hrs.toString().padStart(2, '0'),
+    mins.toString().padStart(2, '0'),
+    secs.toString().padStart(2, '0')
+  ];
+
+  if (hrs > 0) {
+    return `${formatted[0]}:${formatted[1]}:${formatted[2]}`;
+  }
+  return `${formatted[1]}:${formatted[2]}`;
+};
+
 export const formatNumber = (num: number) => {
   if (!num) {
     return '0';

src/utils/load-audio-file.ts

@@ -0,0 +1,48 @@
+import { convertFileSize } from './index';
+
+export const loadAudioData = async (data: any, type: string) => {
+  return new Promise((resolve, reject) => {
+    try {
+      const audioBlob = new Blob([data], { type: type });
+      const fileSize = convertFileSize(audioBlob.size);
+
+      const audio = document.createElement('audio');
+      const url = URL.createObjectURL(audioBlob);
+      audio.src = url;
+
+      audio.addEventListener('loadedmetadata', () => {
+        const duration = audio.duration;
+        resolve({ size: fileSize, duration: Math.ceil(duration), url: url });
+      });
+
+      audio.addEventListener('ended', () => {
+        URL.revokeObjectURL(audio.src);
+      });
+    } catch (error) {
+      reject(error);
+    }
+  });
+};
+
+export const readAudioFile = async (file: File) => {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = async function (e: any) {
+      try {
+        // const size = convertFileSize(file.size);
+        console.log('file====', file);
+        const arrayBuffer = e.target.result;
+        const audioData = await loadAudioData(arrayBuffer, file.type);
+        resolve({
+          ...(audioData || {}),
+          name: file.name
+        });
+      } catch (error) {
+        reject(error);
+      }
+    };
+
+    reader.onerror = (error) => reject(error);
+    reader.readAsArrayBuffer(file);
+  });
+};