Skip to content

Commit 8b4ca13

Browse files
committedNov 8, 2024·
feat: voice print
1 parent a4c9eaf commit 8b4ca13

File tree

5 files changed

+166
-22
lines changed

5 files changed

+166
-22
lines changed
 

‎app/components/realtime-chat/realtime-chat.module.scss

-6
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,6 @@
3131
box-sizing: border-box;
3232
padding: 0 20px;
3333
}
34-
.icon-center {
35-
display: flex;
36-
justify-content: center;
37-
align-items: center;
38-
gap: 4px;
39-
}
4034

4135
.icon-left,
4236
.icon-right {

‎app/components/realtime-chat/realtime-chat.tsx

+29-16
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
} from "rt-client";
2121
import { AudioHandler } from "@/app/lib/audio";
2222
import { uploadImage } from "@/app/utils/chat";
23+
import { VoicePrint } from "@/app/components/voice-print";
2324

2425
interface RealtimeChatProps {
2526
onClose?: () => void;
@@ -41,6 +42,7 @@ export function RealtimeChat({
4142
const [isConnecting, setIsConnecting] = useState(false);
4243
const [modality, setModality] = useState("audio");
4344
const [useVAD, setUseVAD] = useState(true);
45+
const [frequencies, setFrequencies] = useState<Uint8Array | undefined>();
4446

4547
const clientRef = useRef<RTClient | null>(null);
4648
const audioHandlerRef = useRef<AudioHandler | null>(null);
@@ -272,29 +274,39 @@ export function RealtimeChat({
272274
console.error(error);
273275
});
274276

275-
// TODO demo to get frequency. will pass audioHandlerRef.current to child component draw.
276-
// TODO try using requestAnimationFrame
277-
const interval = setInterval(() => {
278-
if (audioHandlerRef.current) {
279-
const data = audioHandlerRef.current.getByteFrequencyData();
280-
console.log("getByteFrequencyData", data);
281-
}
282-
}, 1000);
283-
284277
return () => {
285278
if (isRecording) {
286279
toggleRecording();
287280
}
288-
audioHandlerRef.current
289-
?.close()
290-
.catch(console.error)
291-
.finally(() => {
292-
clearInterval(interval);
293-
});
281+
audioHandlerRef.current?.close().catch(console.error);
294282
disconnect();
295283
};
296284
}, []);
297285

286+
useEffect(() => {
287+
let animationFrameId: number;
288+
289+
if (isConnected && isRecording) {
290+
const animationFrame = () => {
291+
if (audioHandlerRef.current) {
292+
const freqData = audioHandlerRef.current.getByteFrequencyData();
293+
setFrequencies(freqData);
294+
}
295+
animationFrameId = requestAnimationFrame(animationFrame);
296+
};
297+
298+
animationFrameId = requestAnimationFrame(animationFrame);
299+
} else {
300+
setFrequencies(undefined);
301+
}
302+
303+
return () => {
304+
if (animationFrameId) {
305+
cancelAnimationFrame(animationFrameId);
306+
}
307+
};
308+
}, [isConnected, isRecording]);
309+
298310
// update session params
299311
useEffect(() => {
300312
clientRef.current?.configure({ voice });
@@ -318,8 +330,9 @@ export function RealtimeChat({
318330
[styles["pulse"]]: isRecording,
319331
})}
320332
>
321-
<div className={styles["icon-center"]}></div>
333+
<VoicePrint frequencies={frequencies} isActive={isRecording} />
322334
</div>
335+
323336
<div className={styles["bottom-icons"]}>
324337
<div>
325338
<IconButton

‎app/components/voice-print/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from "./voice-print";
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
.voice-print {
2+
width: 100%;
3+
height: 60px;
4+
margin: 20px 0;
5+
6+
canvas {
7+
width: 100%;
8+
height: 100%;
9+
filter: brightness(1.2); // 增加整体亮度
10+
}
11+
}
+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import { useEffect, useRef, useState } from "react";
2+
import styles from "./voice-print.module.scss";
3+
4+
interface VoicePrintProps {
5+
frequencies?: Uint8Array;
6+
isActive?: boolean;
7+
}
8+
9+
export function VoicePrint({ frequencies, isActive }: VoicePrintProps) {
10+
const canvasRef = useRef<HTMLCanvasElement>(null);
11+
const [history, setHistory] = useState<number[][]>([]);
12+
const historyLengthRef = useRef(10); // 保存10帧历史数据
13+
14+
useEffect(() => {
15+
const canvas = canvasRef.current;
16+
if (!canvas) return;
17+
18+
const ctx = canvas.getContext("2d");
19+
if (!ctx) return;
20+
21+
// 设置canvas尺寸
22+
const dpr = window.devicePixelRatio || 1;
23+
canvas.width = canvas.offsetWidth * dpr;
24+
canvas.height = canvas.offsetHeight * dpr;
25+
ctx.scale(dpr, dpr);
26+
27+
// 清空画布
28+
ctx.clearRect(0, 0, canvas.width, canvas.height);
29+
30+
if (!frequencies || !isActive) {
31+
setHistory([]); // 重置历史数据
32+
return;
33+
}
34+
35+
// 更新历史数据
36+
const freqArray = Array.from(frequencies);
37+
setHistory((prev) => {
38+
const newHistory = [...prev, freqArray];
39+
if (newHistory.length > historyLengthRef.current) {
40+
newHistory.shift();
41+
}
42+
return newHistory;
43+
});
44+
45+
// 绘制声纹
46+
const points: [number, number][] = [];
47+
const centerY = canvas.height / 2;
48+
const width = canvas.width;
49+
const sliceWidth = width / (frequencies.length - 1);
50+
51+
// 绘制主波形
52+
ctx.beginPath();
53+
ctx.moveTo(0, centerY);
54+
55+
// 使用历史数据计算平均值实现平滑效果
56+
for (let i = 0; i < frequencies.length; i++) {
57+
const x = i * sliceWidth;
58+
let avgFrequency = frequencies[i];
59+
60+
// 计算历史数据的平均值
61+
if (history.length > 0) {
62+
const historicalValues = history.map((h) => h[i] || 0);
63+
avgFrequency =
64+
(avgFrequency + historicalValues.reduce((a, b) => a + b, 0)) /
65+
(history.length + 1);
66+
}
67+
68+
// 使用三角函数使波形更自然
69+
const normalized = avgFrequency / 255.0;
70+
const height = normalized * (canvas.height / 2);
71+
const y = centerY + height * Math.sin(i * 0.2 + Date.now() * 0.002);
72+
73+
points.push([x, y]);
74+
75+
if (i === 0) {
76+
ctx.moveTo(x, y);
77+
} else {
78+
// 使用贝塞尔曲线使波形更平滑
79+
const prevPoint = points[i - 1];
80+
const midX = (prevPoint[0] + x) / 2;
81+
ctx.quadraticCurveTo(
82+
prevPoint[0],
83+
prevPoint[1],
84+
midX,
85+
(prevPoint[1] + y) / 2,
86+
);
87+
}
88+
}
89+
90+
// 绘制对称的下半部分
91+
for (let i = points.length - 1; i >= 0; i--) {
92+
const [x, y] = points[i];
93+
const symmetricY = centerY - (y - centerY);
94+
if (i === points.length - 1) {
95+
ctx.lineTo(x, symmetricY);
96+
} else {
97+
const nextPoint = points[i + 1];
98+
const midX = (nextPoint[0] + x) / 2;
99+
ctx.quadraticCurveTo(
100+
nextPoint[0],
101+
centerY - (nextPoint[1] - centerY),
102+
midX,
103+
centerY - ((nextPoint[1] + y) / 2 - centerY),
104+
);
105+
}
106+
}
107+
108+
ctx.closePath();
109+
110+
// 设置渐变色和透明度
111+
const gradient = ctx.createLinearGradient(0, 0, canvas.width, 0);
112+
gradient.addColorStop(0, "rgba(100, 180, 255, 0.95)");
113+
gradient.addColorStop(0.5, "rgba(140, 200, 255, 0.9)");
114+
gradient.addColorStop(1, "rgba(180, 220, 255, 0.95)");
115+
116+
ctx.fillStyle = gradient;
117+
ctx.fill();
118+
}, [frequencies, isActive, history]);
119+
120+
return (
121+
<div className={styles["voice-print"]}>
122+
<canvas ref={canvasRef} />
123+
</div>
124+
);
125+
}

0 commit comments

Comments
 (0)
Please sign in to comment.