/** * Recording Compositor & Multi-Track Audio Mixer Engine * * Creates a unified 1920x1080 composite video stream from multiple WebRTC video sources * (Candidate Webcam, Screen Share, Self Interviewer, Remote Mesh Panelists) * matching the interviewer layout: * - Large Main Area: Candidate Webcam (or Screen Share when active) * - Right Sidebar: Stacked Peers (and Candidate Webcam at top when screen share is active) * - Camera-off Avatar Placeholders with initials and badges * - Bottom Overflow Row: Placed horizontally below main area if peers > 3 * - AudioContext multi-track audio mixing */ export class RecordingCompositor { /** * @param {{width:number, height:number, fps: number, getParticipantsData: () => {}}} options */ constructor(options = {}) { this.width = options.width || 1280; this.height = options.height || (options.width ? Math.round((options.width * 9) / 16) : 720); this.fps = options.fps || 30; this.canvas = document.createElement("canvas"); this.canvas.width = this.width; this.canvas.height = this.height; this.ctx = this.canvas.getContext("2d", { alpha: false }); this.audioCtx = null; this.audioDestination = null; this.audioSourceNodes = []; this.mixedAudioTrack = null; this.animationFrameId = null; this.isRunning = false; this.startTime = null; this.speakingStates = new Map(); // Metadata provider function this.getParticipantsData = options.getParticipantsData || (() => ({})); } start() { if (this.isRunning) return; this.isRunning = true; this.startTime = Date.now(); this.initAudioMixer(); this.renderLoop(); } stop() { this.isRunning = false; if (this.animationFrameId) { cancelAnimationFrame(this.animationFrameId); this.animationFrameId = null; } if (this.audioSourceNodes.length > 0) { this.audioSourceNodes.forEach((node) => { try { node.disconnect(); } catch (e) {} }); this.audioSourceNodes = []; } if (this.audioCtx) { try { this.audioCtx.close(); } catch (e) {} this.audioCtx = null; } this.audioDestination = null; this.mixedAudioTrack = null; } getStream() { const videoTrack = this.canvas .captureStream(this.fps) .getVideoTracks()[0]; const tracks = []; if (videoTrack) tracks.push(videoTrack); if (this.mixedAudioTrack) tracks.push(this.mixedAudioTrack); return new MediaStream(tracks); } setSpeakingState(key, isSpeaking) { this.speakingStates.set(key, Boolean(isSpeaking)); } initAudioMixer() { try { const AudioCtxClass = window.AudioContext || window.webkitAudioContext; if (!AudioCtxClass) return; this.audioCtx = new AudioCtxClass(); this.audioDestination = this.audioCtx.createMediaStreamDestination(); const data = this.getParticipantsData(); const audioStreams = []; // 1. Candidate Audio if ( data.candidateStream && data.candidateStream.getAudioTracks().length > 0 ) { audioStreams.push(data.candidateStream); } // 2. Interviewer Self Audio if ( data.selfStream && data.selfStream.getAudioTracks().length > 0 ) { audioStreams.push(data.selfStream); } // 3. Panelist Mesh Audios if (data.panelistStreams && Array.isArray(data.panelistStreams)) { data.panelistStreams.forEach((ps) => { if ( ps && ps.stream && ps.stream.getAudioTracks().length > 0 ) { audioStreams.push(ps.stream); } }); } audioStreams.forEach((stream) => { try { const source = this.audioCtx.createMediaStreamSource(stream); source.connect(this.audioDestination); this.audioSourceNodes.push(source); } catch (e) {} }); this.mixedAudioTrack = this.audioDestination.stream.getAudioTracks()[0] || null; } catch (e) { console.warn("[RecordingCompositor] Audio mixing error:", e); } } renderLoop() { if (!this.isRunning) return; this.renderFrame(); this.animationFrameId = requestAnimationFrame(() => this.renderLoop()); } renderFrame() { const ctx = this.ctx; const W = this.width; const H = this.height; // Dark background ctx.fillStyle = "#090d16"; ctx.fillRect(0, 0, W, H); const data = this.getParticipantsData(); // 1. Candidate Feed const candVideo = document.getElementById("interviewer-cand-video"); const isCandCamOn = data.candidateCamOn !== false; const isCandMicOn = data.candidateMicOn !== false; const candidateFeed = { id: "candidate", type: "candidate", name: data.candidateName || "Candidate", initials: this.extractInitials(data.candidateName || "Candidate"), video: candVideo, hasVideo: isCandCamOn && this.isVideoPlaying(candVideo), isCamOn: isCandCamOn, isMicOn: isCandMicOn, isSpeaking: this.speakingStates.get("interviewer-cand-video") || false, badge: "Candidate", }; // 2. Screen Share Feed const screenVideo = document.getElementById("interviewer-screen-video"); const isScreenSharing = this.isScreenShareActive(screenVideo); const screenFeed = { id: "screen", type: "screen", name: "Candidate Live Screen", initials: "SCR", video: screenVideo, hasVideo: isScreenSharing, isCamOn: true, isMicOn: false, isSpeaking: false, badge: "Shared Screen", }; // 3. Self Interviewer Feed const selfVideo = document.getElementById("interviewer-self-video"); const isSelfCamOn = data.selfCamOn !== false; const isSelfMicOn = data.selfMicOn !== false; const selfFeed = { id: "self", type: "interviewer", name: data.selfName ? `${data.selfName} (You)` : "Interviewer (You)", initials: this.extractInitials(data.selfName || "IV"), video: selfVideo, hasVideo: isSelfCamOn && this.isVideoPlaying(selfVideo), isCamOn: isSelfCamOn, isMicOn: isSelfMicOn, isSpeaking: false, badge: "Interviewer", }; // 4. Panelist Feeds const panelistFeeds = []; if (data.panelists && Array.isArray(data.panelists)) { data.panelists.forEach((p) => { const pVid = document.getElementById( "panelist-video-" + p.peerId, ); const isCamOn = p.camOn !== false; const isMicOn = p.micOn !== false; panelistFeeds.push({ id: p.peerId, type: "panelist", name: p.name || "Panelist", initials: this.extractInitials(p.name || "Panelist"), video: pVid, hasVideo: isCamOn && this.isVideoPlaying(pVid), isCamOn: isCamOn, isMicOn: isMicOn, isSpeaking: false, badge: p.role || "Panelist", }); }); } // Layout determination let mainFeed; let secondaryFeeds = []; if (isScreenSharing) { // When screen share is active: // Main = Screen Share // Right Sidebar Top = Candidate Webcam // Followed by Self and Panelists mainFeed = screenFeed; secondaryFeeds = [candidateFeed, selfFeed, ...panelistFeeds]; } else { // Normal Call: // Main = Candidate Webcam // Right Sidebar = Self and Panelists mainFeed = candidateFeed; secondaryFeeds = [selfFeed, ...panelistFeeds]; } // Compute dynamic geometry with strict 16:9 aspect ratio preservation for every tile const padX = Math.max(8, Math.round(W * 0.015)); const padY = Math.max(8, Math.round(H * 0.02)); const gap = Math.max(6, Math.round(W * 0.01)); const totalSecondary = secondaryFeeds.length; if (totalSecondary === 0) { // Case 0: Only Main Feed (Full Canvas 16:9 with padding) const maxAvailW = W - 2 * padX; const maxAvailH = H - 2 * padY; let mainW, mainH; if (maxAvailW / maxAvailH > 16 / 9) { mainH = maxAvailH; mainW = Math.round(mainH * (16 / 9)); } else { mainW = maxAvailW; mainH = Math.round(mainW * (9 / 16)); } const mainX = Math.round((W - mainW) / 2); const mainY = Math.round((H - mainH) / 2); this.renderTile( ctx, mainFeed, { x: mainX, y: mainY, w: mainW, h: mainH }, true, ); } else if (totalSecondary === 1) { // Case 1: Main + 1 Sidebar Tile (Both 16:9) const availW = W - 2 * padX - gap; const availH = H - 2 * padY; let sideW = Math.round(availW * 0.26); let sideH = Math.round(sideW * (9 / 16)); let mainW = availW - sideW; let mainH = Math.round(mainW * (9 / 16)); if (mainH > availH) { mainH = availH; mainW = Math.round(mainH * (16 / 9)); sideW = Math.min(availW - mainW, Math.round(availH * (16 / 9))); sideH = Math.round(sideW * (9 / 16)); } const mainX = padX; const mainY = Math.round((H - mainH) / 2); const sideX = padX + mainW + gap; const sideY = Math.round((H - sideH) / 2); this.renderTile( ctx, mainFeed, { x: mainX, y: mainY, w: mainW, h: mainH }, true, ); this.renderTile( ctx, secondaryFeeds[0], { x: sideX, y: sideY, w: sideW, h: sideH }, false, ); } else if (totalSecondary === 2) { // Case 2: Main + 2 Sidebar Tiles stacked vertically (All 16:9) const availW = W - 2 * padX - gap; const availH = H - 2 * padY; let sideW = Math.round(availW * 0.28); let sideH = Math.round(sideW * (9 / 16)); if (2 * sideH + gap > availH) { sideH = Math.floor((availH - gap) / 2); sideW = Math.round(sideH * (16 / 9)); } const totalSideH = 2 * sideH + gap; let mainW = availW - sideW; let mainH = Math.round(mainW * (9 / 16)); if (mainH > availH) { mainH = availH; mainW = Math.round(mainH * (16 / 9)); } const mainX = padX; const mainY = Math.round((H - mainH) / 2); const sideX = W - padX - sideW; const sideYStart = Math.round((H - totalSideH) / 2); this.renderTile( ctx, mainFeed, { x: mainX, y: mainY, w: mainW, h: mainH }, true, ); this.renderTile( ctx, secondaryFeeds[0], { x: sideX, y: sideYStart, w: sideW, h: sideH }, false, ); this.renderTile( ctx, secondaryFeeds[1], { x: sideX, y: sideYStart + sideH + gap, w: sideW, h: sideH }, false, ); } else if (totalSecondary === 3) { // Case 3: Main + 3 Sidebar Tiles stacked vertically (All 16:9) const availH = H - 2 * padY; const sideH = Math.floor((availH - 2 * gap) / 3); const sideW = Math.round(sideH * (16 / 9)); const totalSideH = 3 * sideH + 2 * gap; const topY = Math.round((H - totalSideH) / 2); const sideX = W - padX - sideW; let mainW = sideX - gap - padX; let mainH = Math.round(mainW * (9 / 16)); if (mainH > totalSideH) { mainH = totalSideH; mainW = Math.round(mainH * (16 / 9)); } const mainX = padX; const mainY = Math.round((H - mainH) / 2); this.renderTile( ctx, mainFeed, { x: mainX, y: mainY, w: mainW, h: mainH }, true, ); for (let i = 0; i < 3; i++) { const feed = secondaryFeeds[i]; const tileY = topY + i * (sideH + gap); this.renderTile( ctx, feed, { x: sideX, y: tileY, w: sideW, h: sideH }, false, ); } } else { // Case 4: Main + 3 Sidebar Tiles + Bottom Overflow Row (All 16:9) const availH = H - 2 * padY; const sideH = Math.floor((availH - 2 * gap) / 3); const sideW = Math.round(sideH * (16 / 9)); const totalSideH = 3 * sideH + 2 * gap; const topY = Math.round((H - totalSideH) / 2); const sideX = W - padX - sideW; const mainAreaW = sideX - gap - padX; const targetMainH = Math.round((totalSideH - gap) * 0.68); let mainW = Math.round(targetMainH * (16 / 9)); let mainH = targetMainH; if (mainW > mainAreaW) { mainW = mainAreaW; mainH = Math.round(mainW * (9 / 16)); } const mainX = padX + Math.round((mainAreaW - mainW) / 2); const mainY = topY; // 1. Render Main this.renderTile( ctx, mainFeed, { x: mainX, y: mainY, w: mainW, h: mainH }, true, ); // 2. Render 3 Sidebar Tiles for (let i = 0; i < 3; i++) { const feed = secondaryFeeds[i]; const tileY = topY + i * (sideH + gap); this.renderTile( ctx, feed, { x: sideX, y: tileY, w: sideW, h: sideH }, false, ); } // 3. Render Bottom Overflow Tiles (each 16:9) const overflowFeeds = secondaryFeeds.slice(3); const numOverflow = overflowFeeds.length; const botMaxH = totalSideH - mainH - gap; const botYBase = topY + mainH + gap; const candBotW = Math.floor( (mainAreaW - (numOverflow - 1) * gap) / numOverflow, ); const candBotH = Math.round(candBotW * (9 / 16)); let botW, botH; if (candBotH > botMaxH) { botH = botMaxH; botW = Math.round(botH * (16 / 9)); } else { botW = candBotW; botH = candBotH; } const totalBotW = numOverflow * botW + (numOverflow - 1) * gap; const botXStart = padX + Math.round((mainAreaW - totalBotW) / 2); const botY = botYBase + Math.round((botMaxH - botH) / 2); overflowFeeds.forEach((feed, j) => { const tileX = botXStart + j * (botW + gap); this.renderTile( ctx, feed, { x: tileX, y: botY, w: botW, h: botH }, false, ); }); } } renderTile(ctx, feed, rect, isMain = false) { const { x, y, w, h } = rect; const radius = Math.max(6, Math.round(Math.min(w, h) * 0.035)); ctx.save(); this.drawRoundedClip(ctx, x, y, w, h, radius); // Base tile background ctx.fillStyle = "#0d1220"; ctx.fillRect(x, y, w, h); if (feed.hasVideo && feed.video) { // Draw video frame if (feed.type === "screen") { this.drawVideoContain(ctx, feed.video, x, y, w, h); } else { this.drawVideoCover(ctx, feed.video, x, y, w, h); } } else { // Draw avatar placeholder this.drawPlaceholder(ctx, feed, x, y, w, h, isMain); } ctx.restore(); // Sleek subtle border ctx.save(); ctx.lineWidth = 1; ctx.strokeStyle = "rgba(255, 255, 255, 0.08)"; this.drawRoundedStroke(ctx, x, y, w, h, radius); ctx.restore(); // Overlay Pill Tag (Bottom-Left) this.drawPillTag(ctx, feed, x, y, w, h); } drawVideoCover(ctx, video, x, y, w, h) { try { const vw = video.videoWidth || 640; const vh = video.videoHeight || 480; const targetRatio = w / h; const videoRatio = vw / vh; let sx = 0, sy = 0, sw = vw, sh = vh; if (videoRatio > targetRatio) { // Video is wider than 16:9 target: crop horizontally sw = Math.round(vh * targetRatio); sx = Math.round((vw - sw) / 2); } else { // Video is taller than 16:9 target: crop vertically sh = Math.round(vw / targetRatio); sy = Math.round((vh - sh) / 2); } ctx.drawImage(video, sx, sy, sw, sh, x, y, w, h); } catch (e) {} } drawVideoContain(ctx, video, x, y, w, h) { try { const vw = video.videoWidth || 1920; const vh = video.videoHeight || 1080; const targetRatio = w / h; const videoRatio = vw / vh; let dw = w; let dh = h; let dx = x; let dy = y; if (videoRatio > targetRatio) { // Video is wider than tile dh = Math.round(w / videoRatio); dy = Math.round(y + (h - dh) / 2); } else { // Video is taller than tile dw = Math.round(h * videoRatio); dx = Math.round(x + (w - dw) / 2); } ctx.drawImage(video, 0, 0, vw, vh, dx, dy, dw, dh); } catch (e) {} } drawPlaceholder(ctx, feed, x, y, w, h, isMain = false) { const cx = x + w / 2; const cy = y + h / 2 - (isMain ? Math.round(h * 0.03) : Math.round(h * 0.02)); const circleRadius = isMain ? Math.min(Math.round(h * 0.18), Math.round(w * 0.12), 54) : Math.min(Math.round(h * 0.2), Math.round(w * 0.14), 36); // Circular gradient avatar const gradient = ctx.createLinearGradient( cx - circleRadius, cy - circleRadius, cx + circleRadius, cy + circleRadius, ); if (feed.type === "candidate") { gradient.addColorStop(0, "#10b981"); gradient.addColorStop(1, "#059669"); } else { gradient.addColorStop(0, "#5b8bff"); gradient.addColorStop(1, "#3b63e0"); } ctx.fillStyle = gradient; ctx.beginPath(); ctx.arc(cx, cy, Math.max(1, circleRadius), 0, Math.PI * 2); ctx.fill(); // Avatar Initials Text const initialsFont = Math.max(10, Math.round(circleRadius * 0.85)); ctx.fillStyle = "#ffffff"; ctx.font = `bold ${initialsFont}px sans-serif`; ctx.textAlign = "center"; ctx.textBaseline = "middle"; ctx.fillText(feed.initials || "IV", cx, cy); // Participant Name const nameFont = Math.max(10, Math.round(isMain ? Math.min(20, h * 0.045) : Math.min(14, h * 0.06))); ctx.fillStyle = "#ffffff"; ctx.font = `600 ${nameFont}px sans-serif`; ctx.textBaseline = "top"; ctx.fillText(feed.name, cx, cy + circleRadius + Math.max(6, Math.round(h * 0.02))); // Subtitle status const subFont = Math.max(9, Math.round(isMain ? Math.min(14, h * 0.032) : Math.min(11, h * 0.045))); ctx.fillStyle = "#94a3b8"; ctx.font = `400 ${subFont}px sans-serif`; ctx.fillText( feed.isCamOn ? "Connecting video..." : "Camera turned off", cx, cy + circleRadius + Math.max(6, Math.round(h * 0.02)) + nameFont + Math.max(4, Math.round(h * 0.01)), ); } drawPillTag(ctx, feed, tileX, tileY, tileW, tileH) { const text = feed.name; const fontSize = Math.max(9, Math.min(12, Math.round(tileH * 0.045))); ctx.font = `600 ${fontSize}px sans-serif`; const textMetrics = ctx.measureText(text); const tagPaddingX = Math.max(6, Math.round(fontSize * 0.7)); const tagW = textMetrics.width + tagPaddingX * 2; const tagH = Math.max(16, Math.round(fontSize * 1.8)); const marginX = Math.max(6, Math.round(tileW * 0.02)); const marginY = Math.max(6, Math.round(tileH * 0.03)); const tagX = tileX + marginX; const tagY = tileY + tileH - marginY - tagH; const tagRadius = Math.max(3, Math.round(tagH * 0.25)); ctx.save(); // Background ctx.fillStyle = "rgba(9, 13, 22, 0.8)"; this.drawRoundedFill(ctx, tagX, tagY, tagW, tagH, tagRadius); // Border ctx.strokeStyle = "rgba(255, 255, 255, 0.12)"; ctx.lineWidth = 1; this.drawRoundedStroke(ctx, tagX, tagY, tagW, tagH, tagRadius); // Name text ctx.fillStyle = "#e2e8f0"; ctx.textBaseline = "middle"; ctx.textAlign = "left"; ctx.fillText(text, tagX + tagPaddingX, tagY + tagH / 2); ctx.restore(); } isScreenShareActive(video) { if (!video) return false; const stream = video.srcObject; if (!stream || !stream.active) return false; const tracks = stream.getVideoTracks ? stream.getVideoTracks() : []; if (tracks.length === 0) return false; const hasLiveTrack = tracks.some( (t) => t.readyState === "live" && t.enabled, ); if (!hasLiveTrack) return false; if (video.paused && typeof video.play === "function") { video.play().catch(() => {}); } return true; } isVideoPlaying(video) { if (!video) return false; const stream = video.srcObject; if (!stream || !stream.active) return false; const tracks = stream.getVideoTracks ? stream.getVideoTracks() : []; if (tracks.length === 0) return false; const hasLiveTrack = tracks.some( (t) => t.readyState === "live" && t.enabled, ); if (!hasLiveTrack) return false; if (video.paused && typeof video.play === "function") { video.play().catch(() => {}); } return Boolean( video.videoWidth > 0 || video.readyState >= 1 || hasLiveTrack, ); } extractInitials(name) { if (!name) return "CD"; const parts = name.trim().split(/\s+/); if (parts.length >= 2) { return (parts[0][0] + parts[parts.length - 1][0]).toUpperCase(); } return name.slice(0, 2).toUpperCase(); } roundedRectPath(ctx, x, y, w, h, r) { if (typeof ctx.roundRect === "function") { ctx.beginPath(); ctx.roundRect(x, y, w, h, r); return; } ctx.beginPath(); ctx.moveTo(x + r, y); ctx.lineTo(x + w - r, y); ctx.arcTo(x + w, y, x + w, y + r, r); ctx.lineTo(x + w, y + h - r); ctx.arcTo(x + w, y + h, x + w - r, y + h, r); ctx.lineTo(x + r, y + h); ctx.arcTo(x, y + h, x, y + h - r, r); ctx.lineTo(x, y + r); ctx.arcTo(x, y, x + r, y, r); ctx.closePath(); } drawRoundedClip(ctx, x, y, w, h, r) { this.roundedRectPath(ctx, x, y, w, h, r); ctx.clip(); } drawRoundedStroke(ctx, x, y, w, h, r) { this.roundedRectPath(ctx, x, y, w, h, r); ctx.stroke(); } drawRoundedFill(ctx, x, y, w, h, r) { this.roundedRectPath(ctx, x, y, w, h, r); ctx.fill(); } }