chore: speech to text

This commit is contained in:
jialin
2024-11-22 10:00:34 +08:00
parent f24aef300b
commit c3473673da
47 changed files with 3364 additions and 904 deletions
+10
View File
@@ -0,0 +1,10 @@
.canvas-wrap {
display: flex;
justify-content: center;
align-items: center;
text-align: center;
canvas {
display: block;
}
}
+121
View File
@@ -0,0 +1,121 @@
import React, { useEffect } from 'react';
import './index.less';
interface AudioAnimationProps {
width: number;
height: number;
analyserData: {
data: Uint8Array;
analyser: any;
};
}
const AudioAnimation: React.FC<AudioAnimationProps> = (props) => {
const { width, height, analyserData } = props;
const canvasRef = React.useRef<HTMLCanvasElement>(null);
const animationId = React.useRef<number>(0);
const isScaled = React.useRef<boolean>(false);
const oscillationOffset = React.useRef(0);
const direction = React.useRef(1);
const startAudioVisualization = () => {
if (!canvasRef.current || !analyserData.data?.length) return;
const canvas = canvasRef.current;
const canvasCtx = canvas.getContext('2d');
if (!canvasCtx) return;
const WIDTH = (canvas.width = width * 2);
const HEIGHT = (canvas.height = height * 2);
if (!isScaled.current) {
canvasCtx.scale(2, 2);
isScaled.current = true;
}
const barWidth = 3;
const barSpacing = 2;
const centerX = HEIGHT / 2;
const centerLine = Math.floor(HEIGHT / 2);
const jitterAmplitude = 60; // 最大抖动幅度
const minJitter = 15; // 最小抖动幅度
const frameInterval = 2;
let frameCount = 0;
canvasCtx.fillStyle = '#0073EF';
const draw = () => {
frameCount++;
if (frameCount % frameInterval !== 0) {
animationId.current = requestAnimationFrame(draw);
return;
}
analyserData.analyser?.current?.getByteFrequencyData(analyserData.data);
canvasCtx.clearRect(0, 0, WIDTH, HEIGHT);
const barCount = analyserData.data.length;
const totalWidth = barCount * (barWidth + barSpacing) - barSpacing;
let x = centerX - totalWidth / 2 + oscillationOffset.current;
oscillationOffset.current += direction.current * 0.5;
if (oscillationOffset.current > 20 || oscillationOffset.current < -20) {
direction.current *= -1;
}
for (let i = 0; i < barCount; i++) {
const baseHeight = Math.floor(analyserData.data[i] / 2);
const jitter =
minJitter +
Math.round((Math.random() - 0.5) * (jitterAmplitude - minJitter));
const barHeight = baseHeight + jitter;
const topY = Math.round(centerLine - barHeight / 2);
const bottomY = Math.round(centerLine + barHeight / 2);
canvasCtx.beginPath();
canvasCtx.moveTo(x, bottomY);
canvasCtx.lineTo(x, topY + 2);
canvasCtx.arcTo(x + barWidth, topY + 2, x + barWidth, bottomY, 2);
canvasCtx.lineTo(x + barWidth, bottomY);
canvasCtx.closePath();
canvasCtx.fill();
x += barWidth + barSpacing;
}
animationId.current = requestAnimationFrame(draw);
};
draw();
};
useEffect(() => {
if (!analyserData.data?.length || !analyserData.analyser.current) {
canvasRef.current
?.getContext('2d')
?.clearRect(0, 0, width * 2, height * 2);
cancelAnimationFrame(animationId.current);
animationId.current = 0;
return;
}
startAudioVisualization();
return () => {
if (animationId.current) cancelAnimationFrame(animationId.current);
};
}, [analyserData, width, height]);
return (
<div
className="canvas-wrap"
style={{
width: '100%',
height: height
}}
>
<canvas ref={canvasRef} style={{ width, height }}></canvas>
</div>
);
};
export default React.memo(AudioAnimation);
+53
View File
@@ -0,0 +1,53 @@
.player-wrap {
width: 100%;
display: flex;
background-color: var(--ant-color-fill-quaternary);
border-radius: 36px;
.player-ui {
padding: 5px 10px;
flex: 1;
display: flex;
justify-content: flex-start;
align-items: center;
}
.play-content {
display: flex;
justify-content: flex-start;
align-items: center;
flex: 1;
}
.time {
&.current {
margin-left: 6px;
}
}
.progress-bar {
margin-inline: 10px;
flex: 1;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
.slider {
width: 100%;
}
.file-name {
line-height: 20px;
height: 20px;
}
.ant-slider-horizontal {
margin-block: 2px;
}
}
.speaker {
margin-left: 10px;
}
}
+191
View File
@@ -0,0 +1,191 @@
import { formatTime } from '@/utils/index';
import { PauseCircleFilled, PlayCircleFilled } from '@ant-design/icons';
import { Button, Slider } from 'antd';
import React, {
forwardRef,
useCallback,
useEffect,
useImperativeHandle
} from 'react';
import IconFont from '../icon-font';
import './index.less';
interface AudioPlayerProps {
autoplay?: boolean;
url: string;
speed?: number;
ref?: any;
name: string;
height?: number;
width?: number;
duration?: number;
}
const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
const { autoplay = false, speed: defaultSpeed = 1 } = props;
const audioRef = React.useRef<HTMLAudioElement>(null);
const [audioState, setAudioState] = React.useState<{
currentTime: number;
duration: number;
}>({
currentTime: 0,
duration: 0
});
const [playOn, setPlayOn] = React.useState<boolean>(false);
const [speakerOn, setSpeakerOn] = React.useState<boolean>(false);
const [volume, setVolume] = React.useState<number>(0.5);
const [speed, setSpeed] = React.useState<number>(defaultSpeed);
const timer = React.useRef<any>(null);
useImperativeHandle(ref, () => ({
play: () => {
audioRef.current?.play();
},
pause: () => {
audioRef.current?.pause();
}
}));
const handleAudioOnPlay = useCallback(() => {
timer.current = setInterval(() => {
setAudioState((prestate) => {
return {
currentTime: Math.ceil(audioRef.current?.currentTime || 0),
duration:
prestate.duration || Math.ceil(audioRef.current?.duration || 0)
};
});
if (audioRef.current?.paused || audioRef.current?.ended) {
clearInterval(timer.current);
setPlayOn(false);
setAudioState((prestate: any) => {
return {
currentTime: 0,
duration: prestate.duration
};
});
}
}, 500);
}, []);
const handlePlay = useCallback(() => {
if (playOn) {
audioRef.current?.pause();
} else {
audioRef.current?.play();
}
setPlayOn(!playOn);
}, [playOn]);
const initPlayerConfig = useCallback(() => {
// set volume
audioRef.current!.volume = volume;
// set playback rate
audioRef.current!.playbackRate = speed;
}, []);
const handleLoadedMetadata = useCallback(
(data: any) => {
const duration = Math.ceil(audioRef.current?.duration || 0);
setAudioState({
currentTime: 0,
duration:
duration && duration !== Infinity ? duration : props.duration || 0
});
setPlayOn(autoplay);
},
[autoplay, props.duration]
);
const handleCurrentChange = useCallback((val: number) => {
audioRef.current!.currentTime = val;
setAudioState((prestate) => {
return {
currentTime: val,
duration: prestate.duration
};
});
}, []);
useEffect(() => {
if (audioRef.current) {
initPlayerConfig();
}
}, [audioRef.current]);
useEffect(() => {
return () => {
clearInterval(timer.current);
};
}, []);
return (
<div className="player-wrap" style={{ width: props.width || '100%' }}>
<div className="player-ui">
<span className="play-btn">
<Button
size="middle"
type="text"
onClick={handlePlay}
icon={
!playOn ? (
<PlayCircleFilled
style={{ fontSize: '22px' }}
></PlayCircleFilled>
) : (
<PauseCircleFilled
style={{ fontSize: '22px' }}
></PauseCircleFilled>
)
}
></Button>
</span>
<div className="play-content">
<span className="time current">
{' '}
{formatTime(audioState.currentTime)}
</span>
<div className="progress-bar">
<span className="file-name">{props.name}</span>
<div className="slider">
<Slider
tooltip={{ open: false }}
min={0}
max={audioState.duration}
value={audioState.currentTime}
onChange={handleCurrentChange}
/>
</div>
<span>{props.speed ? `${props.speed}x` : '1x'}</span>
</div>
<span className="time">{formatTime(audioState.duration)}</span>
</div>
<span className="speaker">
<Button
size="middle"
type="text"
icon={
<IconFont
type="icon-SpeakerHigh"
style={{ fontSize: '22px' }}
></IconFont>
}
></Button>
</span>
</div>
<audio
controls
autoPlay={autoplay}
src={props.url}
ref={audioRef}
preload="metadata"
style={{ display: 'none' }}
onPlay={handleAudioOnPlay}
onLoadedMetadata={handleLoadedMetadata}
></audio>
</div>
);
});
export default React.memo(AudioPlayer);
+1
View File
@@ -84,6 +84,7 @@ export const title = {
textStyle: {
fontSize: 12,
color: '#000'
// fontWeight: 500
},
text: ''
};
+4 -1
View File
@@ -13,9 +13,11 @@ const options: any = {
top: -1,
bottom: -1,
left: -1,
containLabel: true
containLabel: true,
borderRadius: 4
},
xAxis: {
scale: false,
slient: true,
splitNumber: 15,
splitLine: {
@@ -34,6 +36,7 @@ const options: any = {
}
},
yAxis: {
scale: false,
slient: true,
splitNumber: 10,
splitLine: {
+1 -1
View File
@@ -1,7 +1,7 @@
import { createFromIconfontCN } from '@ant-design/icons';
const IconFont = createFromIconfontCN({
scriptUrl: '//at.alicdn.com/t/c/font_4613488_flbkvujyhg4.js'
scriptUrl: '//at.alicdn.com/t/c/font_4613488_f5wastzhj2w.js'
});
export default IconFont;
@@ -0,0 +1,47 @@
import React, {
forwardRef,
useEffect,
useImperativeHandle,
useRef
} from 'react';
import useWavesurfer from './hooks/use-wavesurfer';
interface AudioPlayerProps {
autoplay: boolean;
audioUrl: string;
speed: number;
ref?: any;
height?: number;
width?: number;
}
const AudioPlayer: React.FC<AudioPlayerProps> = forwardRef((props, ref) => {
const { autoplay, audioUrl, speed = 1, ...rest } = props;
const container = useRef<HTMLDivElement>(null);
const { createWavesurfer, play, pause, destroyWavesurfer } = useWavesurfer({
container,
autoplay: autoplay,
url: audioUrl,
audioRate: speed,
...rest
});
useImperativeHandle(ref, () => {
return {
play,
pause
};
});
useEffect(() => {
if (container.current) {
createWavesurfer();
}
return () => {
destroyWavesurfer();
};
}, [container.current]);
return <div ref={container} className="audio-container"></div>;
});
export default React.memo(AudioPlayer);
@@ -0,0 +1,68 @@
import { useRef } from 'react';
import WaveSurfer from 'wavesurfer.js';
interface Options {
container: React.RefObject<HTMLDivElement>;
waveColor?: string;
progressColor?: string;
url: string;
barWidth?: number;
barGap?: number;
barRadius?: number;
autoplay?: boolean;
audioRate?: number;
}
const useWavesurfer = (options: Options) => {
const wavesurfer = useRef<WaveSurfer | null>(null);
const { container, url, ...rest } = options;
const createWavesurfer = () => {
if (!container.current) {
return;
}
if (wavesurfer.current) {
wavesurfer.current.destroy();
}
wavesurfer.current = WaveSurfer.create({
container: container.current,
waveColor: '#4096ff',
progressColor: 'rgb(100, 0, 100)',
url: url,
height: 60,
barWidth: 2,
barGap: 1,
barRadius: 2,
interact: true,
cursorWidth: 0,
...rest
});
};
const destroyWavesurfer = () => {
if (wavesurfer.current) {
wavesurfer.current.destroy();
}
};
const play = () => {
if (wavesurfer.current) {
wavesurfer.current.play();
}
};
const pause = () => {
if (wavesurfer.current) {
wavesurfer.current.pause();
}
};
return {
createWavesurfer,
play,
pause,
destroyWavesurfer
};
};
export default useWavesurfer;
Binary file not shown.
+20
View File
@@ -0,0 +1,20 @@
import React from 'react';
import SpeechItem from './speech-item';
interface SpeechContentProps {
dataList: any[];
loading?: boolean;
}
const SpeechContent: React.FC<SpeechContentProps> = (props) => {
console.log('SpeechContent', props);
return (
<div>
{props.dataList.map((item) => (
<SpeechItem key={item.uid} {...item} />
))}
</div>
);
};
export default React.memo(SpeechContent);
@@ -0,0 +1,87 @@
import IconFont from '@/components/icon-font';
import {
DownloadOutlined,
FileTextOutlined,
PlayCircleOutlined
} from '@ant-design/icons';
import { Button, Tooltip } from 'antd';
import React, { useRef, useState } from 'react';
import AudioPlayer from './audio-player';
import './styles/index.less';
const audioUrl = require('./ih.mp4');
interface SpeechContentProps {
prompt: string;
autoplay: boolean;
voice: string;
format: string;
speed: number;
}
const SpeechItem: React.FC<SpeechContentProps> = (props) => {
console.log('porps=======', props);
const [collapsed, setCollapsed] = useState(false);
const ref = useRef<HTMLAudioElement>(null);
const handlePlay = () => {
ref.current?.play();
};
const handleCollapse = () => {
setCollapsed(!collapsed);
};
return (
<div>
<div className="speech-item">
{/* <audio controls autoPlay={true} src={require('./ih.mp4')}></audio> */}
<div className="voice">
<IconFont type="icon-user_voice" className="font-size-16" />
<span className="text">{props.voice}</span>
</div>
<div className="wrapper">
<AudioPlayer {...props} audioUrl={audioUrl} ref={ref}></AudioPlayer>
</div>
</div>
<div className="speech-actions">
<span className="tags">
<span className="item">{props.format}</span>
<span className="item splitor"></span>
<span className="item">{props.speed}x</span>
</span>
<div className="actions">
<Tooltip title="Play">
<Button
onClick={handlePlay}
icon={<PlayCircleOutlined />}
type="text"
size="small"
></Button>
</Tooltip>
<Tooltip title="Download">
<Button
icon={<DownloadOutlined />}
type="text"
size="small"
></Button>
</Tooltip>
<Tooltip title="Show Prompt">
<Button
icon={<FileTextOutlined />}
type="text"
size="small"
onClick={handleCollapse}
></Button>
</Tooltip>
</div>
</div>
{collapsed && (
<div className="prompt-box">
<div className="prompt">{props.prompt}</div>
</div>
)}
</div>
);
};
export default React.memo(SpeechItem);
@@ -0,0 +1,93 @@
.speech-item {
display: flex;
justify-content: flex-start;
align-items: center;
.voice {
width: 80px;
display: flex;
justify-content: flex-start;
align-items: center;
gap: 5px;
.text {
display: flex;
padding: 2px 4px;
border-radius: 4px;
border: 1px solid var(--ant-color-border);
}
}
.wrapper {
flex: 1;
display: flex;
justify-content: flex-start;
align-items: center;
.audio-container {
flex: 1;
}
.format {
display: flex;
padding-left: 5px;
}
}
audio {
flex: 1;
}
}
.prompt-box {
padding-left: 80px;
.prompt {
margin-top: 16px;
padding: 10px;
border-radius: var(--border-radius-base);
background-color: var(--ant-color-fill-quaternary);
}
}
.speech-actions {
display: flex;
justify-content: space-between;
margin-top: 10px;
padding-left: 80px;
.actions {
display: flex;
justify-content: flex-start;
align-items: center;
gap: 15px;
.anticon {
font-size: 14px;
}
}
.tags {
display: flex;
justify-content: flex-start;
align-items: center;
.item {
display: flex;
justify-content: center;
align-items: center;
padding: 2px;
border-radius: 4px;
color: var(--ant-color-text-tertiary);
}
.splitor {
display: flex;
width: 1px;
height: 1px;
border-radius: 2px;
margin: 0 8px;
background-color: var(--ant-color-fill-content-hover);
}
}
}
+54
View File
@@ -0,0 +1,54 @@
import { UploadOutlined } from '@ant-design/icons';
import { Button, Tooltip, Upload } from 'antd';
import React from 'react';
interface UploadAudioProps {
accept?: string;
maxCount?: number;
type?: 'text' | 'primary' | 'default';
onChange?: (data: { file: any; fileList: any[] }) => void;
}
const UploadAudio: React.FC<UploadAudioProps> = (props) => {
const beforeUpload = (file: any) => {
return true;
};
const handleOnChange = React.useCallback(
(data: { file: any; fileList: any }) => {
console.log('handleOnChange', data);
props.onChange?.(data);
},
[]
);
return (
<Tooltip title={`Upload an audio file, support for ${props.accept}`}>
<Upload
beforeUpload={beforeUpload}
onChange={handleOnChange}
accept={props.accept}
multiple={false}
maxCount={1}
showUploadList={false}
fileList={[]}
>
<div
style={{
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
gap: 16
}}
>
<Button
icon={<UploadOutlined />}
type={props.type ?? 'text'}
shape="circle"
></Button>
</div>
</Upload>
</Tooltip>
);
};
export default React.memo(UploadAudio);