feat(UI): add speech indicator
- add a audio monitor - when any peer speaks, and green outline is shown around the camera card
This commit is contained in:
parent
b787f04914
commit
03a36c00e6
105
resources/js/audio-monitor.js
Normal file
105
resources/js/audio-monitor.js
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
/**
|
||||||
|
* Real-Time Voice Activity Detector (VAD) & Active Speaking Border Engine
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Easily configurable Tailwind CSS classes applied when a participant is actively speaking
|
||||||
|
export const SPEAKING_ACTIVE_CLASSES = [
|
||||||
|
'border-emerald-500/80',
|
||||||
|
];
|
||||||
|
|
||||||
|
export class AudioActivityMonitor {
|
||||||
|
constructor(speakingClasses = SPEAKING_ACTIVE_CLASSES) {
|
||||||
|
this.audioCtx = null;
|
||||||
|
this.monitors = new Map(); // key -> { source, analyser, animId, targetElemId }
|
||||||
|
this.speakingClasses = speakingClasses;
|
||||||
|
}
|
||||||
|
|
||||||
|
getAudioContext() {
|
||||||
|
if (!this.audioCtx) {
|
||||||
|
const AudioCtx = window.AudioContext || window.webkitAudioContext;
|
||||||
|
if (AudioCtx) {
|
||||||
|
this.audioCtx = new AudioCtx();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.audioCtx && this.audioCtx.state === 'suspended') {
|
||||||
|
this.audioCtx.resume().catch(() => {});
|
||||||
|
}
|
||||||
|
return this.audioCtx;
|
||||||
|
}
|
||||||
|
|
||||||
|
attach(key, mediaStream, targetElemId, threshold = 12) {
|
||||||
|
this.detach(key);
|
||||||
|
if (!mediaStream || !mediaStream.getAudioTracks || mediaStream.getAudioTracks().length === 0) return;
|
||||||
|
|
||||||
|
const ctx = this.getAudioContext();
|
||||||
|
if (!ctx) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const source = ctx.createMediaStreamSource(mediaStream);
|
||||||
|
const analyser = ctx.createAnalyser();
|
||||||
|
analyser.fftSize = 256;
|
||||||
|
analyser.smoothingTimeConstant = 0.3;
|
||||||
|
source.connect(analyser);
|
||||||
|
|
||||||
|
const bufferLength = analyser.frequencyBinCount;
|
||||||
|
const dataArray = new Uint8Array(bufferLength);
|
||||||
|
|
||||||
|
let isSpeaking = false;
|
||||||
|
let silenceTimer = null;
|
||||||
|
|
||||||
|
const checkAudio = () => {
|
||||||
|
const entry = this.monitors.get(key);
|
||||||
|
if (!entry) return;
|
||||||
|
|
||||||
|
analyser.getByteFrequencyData(dataArray);
|
||||||
|
let sum = 0;
|
||||||
|
for (let i = 0; i < bufferLength; i++) {
|
||||||
|
sum += dataArray[i];
|
||||||
|
}
|
||||||
|
const average = sum / bufferLength;
|
||||||
|
|
||||||
|
const elem = document.getElementById(targetElemId);
|
||||||
|
if (average > threshold) {
|
||||||
|
if (!isSpeaking) {
|
||||||
|
isSpeaking = true;
|
||||||
|
if (silenceTimer) clearTimeout(silenceTimer);
|
||||||
|
if (elem) {
|
||||||
|
elem.classList.add(...this.speakingClasses);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (silenceTimer) clearTimeout(silenceTimer);
|
||||||
|
}
|
||||||
|
silenceTimer = setTimeout(() => {
|
||||||
|
isSpeaking = false;
|
||||||
|
if (elem) {
|
||||||
|
elem.classList.remove(...this.speakingClasses);
|
||||||
|
}
|
||||||
|
}, 350);
|
||||||
|
}
|
||||||
|
|
||||||
|
entry.animId = requestAnimationFrame(checkAudio);
|
||||||
|
};
|
||||||
|
|
||||||
|
const animId = requestAnimationFrame(checkAudio);
|
||||||
|
this.monitors.set(key, { source, analyser, animId, targetElemId });
|
||||||
|
} catch (e) {
|
||||||
|
console.log('VAD attach notice:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
detach(key) {
|
||||||
|
if (this.monitors.has(key)) {
|
||||||
|
const m = this.monitors.get(key);
|
||||||
|
if (m.animId) cancelAnimationFrame(m.animId);
|
||||||
|
try { if (m.source) m.source.disconnect(); } catch(e) {}
|
||||||
|
try { if (m.analyser) m.analyser.disconnect(); } catch(e) {}
|
||||||
|
const elem = document.getElementById(m.targetElemId);
|
||||||
|
if (elem) {
|
||||||
|
elem.classList.remove(...this.speakingClasses);
|
||||||
|
}
|
||||||
|
this.monitors.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const globalAudioMonitor = new AudioActivityMonitor();
|
||||||
@ -5,6 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { safePlayMediaStream, getInitials } from './interview-call.js';
|
import { safePlayMediaStream, getInitials } from './interview-call.js';
|
||||||
|
import { globalAudioMonitor, SPEAKING_ACTIVE_CLASSES } from './audio-monitor.js';
|
||||||
import { initMeteredIceServers } from './metered.js';
|
import { initMeteredIceServers } from './metered.js';
|
||||||
|
|
||||||
let editor = null;
|
let editor = null;
|
||||||
@ -504,6 +505,7 @@ export function addOrUpdateCandidateInterviewerTile(peerId, stream, labelName =
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function removeCandidateInterviewerTile(peerId) {
|
export function removeCandidateInterviewerTile(peerId) {
|
||||||
|
globalAudioMonitor.detach('cand-iv-video-' + peerId);
|
||||||
const stream = candidateInterviewerTiles.get(peerId);
|
const stream = candidateInterviewerTiles.get(peerId);
|
||||||
if (stream && stream.active && latestCallStatus !== 'ended') {
|
if (stream && stream.active && latestCallStatus !== 'ended') {
|
||||||
return;
|
return;
|
||||||
|
|||||||
@ -5,6 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { initMeteredIceServers } from './metered.js';
|
import { initMeteredIceServers } from './metered.js';
|
||||||
|
import { globalAudioMonitor, AudioActivityMonitor, SPEAKING_ACTIVE_CLASSES } from './audio-monitor.js';
|
||||||
|
|
||||||
// --- Global Audio Ringtone Synthesizer Engine ---
|
// --- Global Audio Ringtone Synthesizer Engine ---
|
||||||
export class CallRingtoneEngine {
|
export class CallRingtoneEngine {
|
||||||
@ -1464,7 +1465,7 @@ export function blockCandidateAction() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export { initMeteredIceServers };
|
export { initMeteredIceServers, globalAudioMonitor, AudioActivityMonitor, SPEAKING_ACTIVE_CLASSES };
|
||||||
|
|
||||||
export function safePlayMediaStream(videoElem, stream, isSelf = false) {
|
export function safePlayMediaStream(videoElem, stream, isSelf = false) {
|
||||||
if (!videoElem || !stream) return;
|
if (!videoElem || !stream) return;
|
||||||
@ -1475,6 +1476,33 @@ export function safePlayMediaStream(videoElem, stream, isSelf = false) {
|
|||||||
if (videoElem.srcObject !== stream) {
|
if (videoElem.srcObject !== stream) {
|
||||||
videoElem.srcObject = stream;
|
videoElem.srcObject = stream;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Attach dynamic emerald speaking border VAD monitor
|
||||||
|
if (!isScreenShare && stream.getAudioTracks && stream.getAudioTracks().length > 0) {
|
||||||
|
let targetId = null;
|
||||||
|
let monitorKey = videoElem.id;
|
||||||
|
|
||||||
|
if (videoElem.id === 'interviewer-cand-video') {
|
||||||
|
targetId = 'cand-video-wrapper';
|
||||||
|
} else if (videoElem.id === 'interviewer-self-video') {
|
||||||
|
targetId = 'self-video-wrapper';
|
||||||
|
} else if (videoElem.id === 'webcam') {
|
||||||
|
targetId = 'cand-self-video-box';
|
||||||
|
} else if (videoElem.id === 'interviewer-video-default') {
|
||||||
|
targetId = 'interviewer-placeholder-box';
|
||||||
|
} else if (videoElem.id.startsWith('panelist-video-')) {
|
||||||
|
const pid = videoElem.id.replace('panelist-video-', '');
|
||||||
|
targetId = 'panelist-tile-' + pid;
|
||||||
|
} else if (videoElem.id.startsWith('cand-iv-video-')) {
|
||||||
|
const pid = videoElem.id.replace('cand-iv-video-', '');
|
||||||
|
targetId = 'cand-iv-tile-' + pid;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (targetId) {
|
||||||
|
globalAudioMonitor.attach(monitorKey, stream, targetId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const playPromise = videoElem.play();
|
const playPromise = videoElem.play();
|
||||||
if (playPromise !== undefined) {
|
if (playPromise !== undefined) {
|
||||||
playPromise.catch(err => {
|
playPromise.catch(err => {
|
||||||
@ -1512,7 +1540,7 @@ export function addOrUpdatePanelistTile(peerId, stream, name = 'Panelist', micOn
|
|||||||
if (!tile) {
|
if (!tile) {
|
||||||
tile = document.createElement('div');
|
tile = document.createElement('div');
|
||||||
tile.id = 'panelist-tile-' + peerId;
|
tile.id = 'panelist-tile-' + peerId;
|
||||||
tile.className = 'video-tile relative w-[220px] aspect-video shrink-0 bg-[#0d1220] border border-[#1b2233] rounded-[10px] overflow-hidden flex flex-col items-center justify-center transition-all';
|
tile.className = 'video-tile relative w-[220px] aspect-video shrink-0 bg-[#0d1220] border border-[#1b2233] rounded-[10px] overflow-hidden flex flex-col items-center justify-center transition-all duration-200';
|
||||||
|
|
||||||
tile.innerHTML = `
|
tile.innerHTML = `
|
||||||
<video id="panelist-video-${peerId}" autoplay playsinline class="w-full h-full object-cover origin-center transition-transform duration-200" style="transform: scaleX(-1);"></video>
|
<video id="panelist-video-${peerId}" autoplay playsinline class="w-full h-full object-cover origin-center transition-transform duration-200" style="transform: scaleX(-1);"></video>
|
||||||
@ -1567,6 +1595,7 @@ export function addOrUpdatePanelistTile(peerId, stream, name = 'Panelist', micOn
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function removePanelistTile(peerId) {
|
export function removePanelistTile(peerId) {
|
||||||
|
globalAudioMonitor.detach('panelist-video-' + peerId);
|
||||||
const tile = document.getElementById('panelist-tile-' + peerId);
|
const tile = document.getElementById('panelist-tile-' + peerId);
|
||||||
if (tile) tile.remove();
|
if (tile) tile.remove();
|
||||||
panelistMeshTiles.delete(peerId);
|
panelistMeshTiles.delete(peerId);
|
||||||
|
|||||||
@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
<aside class="w-[340px] bg-slate-900/90 border-l border-white/10 p-4 flex flex-col gap-4 overflow-y-auto shrink-0 select-none">
|
<aside class="w-[340px] bg-slate-900/90 border-l border-white/10 p-4 flex flex-col gap-4 overflow-y-auto shrink-0 select-none">
|
||||||
<!-- Candidate Camera Feed -->
|
<!-- Candidate Camera Feed -->
|
||||||
<div class="w-full aspect-[16/10] bg-black rounded-xl border-2 border-white/15 overflow-hidden relative shrink-0">
|
<div id="cand-self-video-box" class="w-full aspect-[16/10] bg-black rounded-xl border-2 border-white/15 overflow-hidden relative shrink-0 transition-all duration-200">
|
||||||
<video id="webcam" autoplay muted playsinline class="w-full h-full object-cover -scale-x-100"></video>
|
<video id="webcam" autoplay muted playsinline class="w-full h-full object-cover -scale-x-100"></video>
|
||||||
<div id="webcam-avatar-placeholder" class="absolute inset-0 hidden flex-col items-center justify-center bg-slate-900 text-white z-10">
|
<div id="webcam-avatar-placeholder" class="absolute inset-0 hidden flex-col items-center justify-center bg-slate-900 text-white z-10">
|
||||||
<div class="w-14 h-14 rounded-full bg-gradient-to-br from-indigo-500 to-sky-600 flex items-center justify-center text-xl font-bold font-outfit shadow-lg shadow-indigo-500/40">
|
<div class="w-14 h-14 rounded-full bg-gradient-to-br from-indigo-500 to-sky-600 flex items-center justify-center text-xl font-bold font-outfit shadow-lg shadow-indigo-500/40">
|
||||||
@ -22,7 +22,7 @@
|
|||||||
|
|
||||||
<!-- Remote Interviewer Panelist Video Grid (Multi-Party WebRTC Mesh) -->
|
<!-- Remote Interviewer Panelist Video Grid (Multi-Party WebRTC Mesh) -->
|
||||||
<div id="interviewer-mesh-grid" class="flex flex-col gap-2.5 w-full">
|
<div id="interviewer-mesh-grid" class="flex flex-col gap-2.5 w-full">
|
||||||
<div id="interviewer-placeholder-box" class="w-full aspect-[16/10] bg-black rounded-xl border-2 border-indigo-500/40 overflow-hidden relative shrink-0">
|
<div id="interviewer-placeholder-box" class="w-full aspect-[16/10] bg-black rounded-xl border-2 border-indigo-500/40 overflow-hidden relative shrink-0 transition-all duration-200">
|
||||||
<video id="interviewer-video-default" autoplay playsinline class="w-full h-full object-cover bg-slate-950 hidden -scale-x-100"></video>
|
<video id="interviewer-video-default" autoplay playsinline class="w-full h-full object-cover bg-slate-950 hidden -scale-x-100"></video>
|
||||||
<div id="interviewer-video-placeholder" class="absolute inset-0 flex flex-col items-center justify-center bg-slate-900/95 text-slate-400 text-xs text-center p-2.5 z-10">
|
<div id="interviewer-video-placeholder" class="absolute inset-0 flex flex-col items-center justify-center bg-slate-900/95 text-slate-400 text-xs text-center p-2.5 z-10">
|
||||||
<div id="interviewer-avatar-circle" class="w-14 h-14 rounded-full bg-gradient-to-br from-sky-400 to-indigo-500 flex items-center justify-center text-xl font-bold font-outfit text-white mb-1.5 shadow-lg shadow-sky-500/30">
|
<div id="interviewer-avatar-circle" class="w-14 h-14 rounded-full bg-gradient-to-br from-sky-400 to-indigo-500 flex items-center justify-center text-xl font-bold font-outfit text-white mb-1.5 shadow-lg shadow-sky-500/30">
|
||||||
|
|||||||
@ -21,7 +21,7 @@
|
|||||||
'focused' => false,
|
'focused' => false,
|
||||||
])
|
])
|
||||||
|
|
||||||
<div id="{{ $id }}" {{ $attributes->twMerge(['class' => 'video-tile relative aspect-[16/10] bg-[#0d1220] border rounded-[10px] overflow-hidden flex flex-col items-center justify-center transition-all ' . ($focused ? 'border-[rgba(75,130,247,0.35)] shadow-[0_0_0_1px_rgba(75,130,247,0.35)]' : 'border-[#1b2233]')]) }}>
|
<div id="{{ $id }}" {{ $attributes->twMerge(['class' => 'video-tile relative aspect-[16/10] bg-[#0d1220] border-2 rounded-xl overflow-hidden flex flex-col items-center justify-center transition-all duration-200 ' . ($focused ? 'border-[rgba(75,130,247,0.35)] shadow-[0_0_0_1px_rgba(75,130,247,0.35)]' : 'border-[#1b2233]')]) }}>
|
||||||
@if($showZoom || $showFullscreen)
|
@if($showZoom || $showFullscreen)
|
||||||
<div class="tile-tools absolute top-2 right-2 z-20 flex gap-1">
|
<div class="tile-tools absolute top-2 right-2 z-20 flex gap-1">
|
||||||
@if($showZoom)
|
@if($showZoom)
|
||||||
|
|||||||
@ -101,7 +101,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="flex items-center gap-3 overflow-x-auto pb-1.5">
|
<div class="flex items-center gap-4 overflow-x-auto p-2">
|
||||||
<!-- Candidate Camera PIP Container (Active when screen share is on) -->
|
<!-- Candidate Camera PIP Container (Active when screen share is on) -->
|
||||||
<div id="cand-pip-slot" class="hidden w-55 aspect-video shrink-0"></div>
|
<div id="cand-pip-slot" class="hidden w-55 aspect-video shrink-0"></div>
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user