sls/resources/js/recording-compositor.js
kushal.saha 9fdd101bfc fix: crop recorded video
- Calculate video dimensions dynamically in the compositor.
- Ensure recorded videos are rendered with the correct size and aspect ratio.
2026-08-24 06:24:01 +00:00

741 lines
25 KiB
JavaScript

/**
* Recording Compositor & Multi-Track Audio Mixer Engine
*
* Creates a unified 1920x1080 composite video stream from multiple WebRTC video sources
* (Candidate Webcam, Screen Share, Self Interviewer, Remote Mesh Panelists)
* matching the interviewer layout:
* - Large Main Area: Candidate Webcam (or Screen Share when active)
* - Right Sidebar: Stacked Peers (and Candidate Webcam at top when screen share is active)
* - Camera-off Avatar Placeholders with initials and badges
* - Bottom Overflow Row: Placed horizontally below main area if peers > 3
* - AudioContext multi-track audio mixing
*/
export class RecordingCompositor {
/**
* @param {{width:number, height:number, fps: number, getParticipantsData: () => {}}} options
*/
constructor(options = {}) {
this.width = options.width || 1280;
this.height = options.height || (options.width ? Math.round((options.width * 9) / 16) : 720);
this.fps = options.fps || 30;
this.canvas = document.createElement("canvas");
this.canvas.width = this.width;
this.canvas.height = this.height;
this.ctx = this.canvas.getContext("2d", { alpha: false });
this.audioCtx = null;
this.audioDestination = null;
this.audioSourceNodes = [];
this.mixedAudioTrack = null;
this.animationFrameId = null;
this.isRunning = false;
this.startTime = null;
this.speakingStates = new Map();
// Metadata provider function
this.getParticipantsData = options.getParticipantsData || (() => ({}));
}
start() {
if (this.isRunning) return;
this.isRunning = true;
this.startTime = Date.now();
this.initAudioMixer();
this.renderLoop();
}
stop() {
this.isRunning = false;
if (this.animationFrameId) {
cancelAnimationFrame(this.animationFrameId);
this.animationFrameId = null;
}
if (this.audioSourceNodes.length > 0) {
this.audioSourceNodes.forEach((node) => {
try {
node.disconnect();
} catch (e) {}
});
this.audioSourceNodes = [];
}
if (this.audioCtx) {
try {
this.audioCtx.close();
} catch (e) {}
this.audioCtx = null;
}
this.audioDestination = null;
this.mixedAudioTrack = null;
}
getStream() {
const videoTrack = this.canvas
.captureStream(this.fps)
.getVideoTracks()[0];
const tracks = [];
if (videoTrack) tracks.push(videoTrack);
if (this.mixedAudioTrack) tracks.push(this.mixedAudioTrack);
return new MediaStream(tracks);
}
setSpeakingState(key, isSpeaking) {
this.speakingStates.set(key, Boolean(isSpeaking));
}
initAudioMixer() {
try {
const AudioCtxClass =
window.AudioContext || window.webkitAudioContext;
if (!AudioCtxClass) return;
this.audioCtx = new AudioCtxClass();
this.audioDestination =
this.audioCtx.createMediaStreamDestination();
const data = this.getParticipantsData();
const audioStreams = [];
// 1. Candidate Audio
if (
data.candidateStream &&
data.candidateStream.getAudioTracks().length > 0
) {
audioStreams.push(data.candidateStream);
}
// 2. Interviewer Self Audio
if (
data.selfStream &&
data.selfStream.getAudioTracks().length > 0
) {
audioStreams.push(data.selfStream);
}
// 3. Panelist Mesh Audios
if (data.panelistStreams && Array.isArray(data.panelistStreams)) {
data.panelistStreams.forEach((ps) => {
if (
ps &&
ps.stream &&
ps.stream.getAudioTracks().length > 0
) {
audioStreams.push(ps.stream);
}
});
}
audioStreams.forEach((stream) => {
try {
const source =
this.audioCtx.createMediaStreamSource(stream);
source.connect(this.audioDestination);
this.audioSourceNodes.push(source);
} catch (e) {}
});
this.mixedAudioTrack =
this.audioDestination.stream.getAudioTracks()[0] || null;
} catch (e) {
console.warn("[RecordingCompositor] Audio mixing error:", e);
}
}
renderLoop() {
if (!this.isRunning) return;
this.renderFrame();
this.animationFrameId = requestAnimationFrame(() => this.renderLoop());
}
renderFrame() {
const ctx = this.ctx;
const W = this.width;
const H = this.height;
// Dark background
ctx.fillStyle = "#090d16";
ctx.fillRect(0, 0, W, H);
const data = this.getParticipantsData();
// 1. Candidate Feed
const candVideo = document.getElementById("interviewer-cand-video");
const isCandCamOn = data.candidateCamOn !== false;
const isCandMicOn = data.candidateMicOn !== false;
const candidateFeed = {
id: "candidate",
type: "candidate",
name: data.candidateName || "Candidate",
initials: this.extractInitials(data.candidateName || "Candidate"),
video: candVideo,
hasVideo: isCandCamOn && this.isVideoPlaying(candVideo),
isCamOn: isCandCamOn,
isMicOn: isCandMicOn,
isSpeaking:
this.speakingStates.get("interviewer-cand-video") || false,
badge: "Candidate",
};
// 2. Screen Share Feed
const screenVideo = document.getElementById("interviewer-screen-video");
const isScreenSharing = this.isScreenShareActive(screenVideo);
const screenFeed = {
id: "screen",
type: "screen",
name: "Candidate Live Screen",
initials: "SCR",
video: screenVideo,
hasVideo: isScreenSharing,
isCamOn: true,
isMicOn: false,
isSpeaking: false,
badge: "Shared Screen",
};
// 3. Self Interviewer Feed
const selfVideo = document.getElementById("interviewer-self-video");
const isSelfCamOn = data.selfCamOn !== false;
const isSelfMicOn = data.selfMicOn !== false;
const selfFeed = {
id: "self",
type: "interviewer",
name: data.selfName
? `${data.selfName} (You)`
: "Interviewer (You)",
initials: this.extractInitials(data.selfName || "IV"),
video: selfVideo,
hasVideo: isSelfCamOn && this.isVideoPlaying(selfVideo),
isCamOn: isSelfCamOn,
isMicOn: isSelfMicOn,
isSpeaking: false,
badge: "Interviewer",
};
// 4. Panelist Feeds
const panelistFeeds = [];
if (data.panelists && Array.isArray(data.panelists)) {
data.panelists.forEach((p) => {
const pVid = document.getElementById(
"panelist-video-" + p.peerId,
);
const isCamOn = p.camOn !== false;
const isMicOn = p.micOn !== false;
panelistFeeds.push({
id: p.peerId,
type: "panelist",
name: p.name || "Panelist",
initials: this.extractInitials(p.name || "Panelist"),
video: pVid,
hasVideo: isCamOn && this.isVideoPlaying(pVid),
isCamOn: isCamOn,
isMicOn: isMicOn,
isSpeaking: false,
badge: p.role || "Panelist",
});
});
}
// Layout determination
let mainFeed;
let secondaryFeeds = [];
if (isScreenSharing) {
// When screen share is active:
// Main = Screen Share
// Right Sidebar Top = Candidate Webcam
// Followed by Self and Panelists
mainFeed = screenFeed;
secondaryFeeds = [candidateFeed, selfFeed, ...panelistFeeds];
} else {
// Normal Call:
// Main = Candidate Webcam
// Right Sidebar = Self and Panelists
mainFeed = candidateFeed;
secondaryFeeds = [selfFeed, ...panelistFeeds];
}
// Compute dynamic geometry with strict 16:9 aspect ratio preservation for every tile
const padX = Math.max(8, Math.round(W * 0.015));
const padY = Math.max(8, Math.round(H * 0.02));
const gap = Math.max(6, Math.round(W * 0.01));
const totalSecondary = secondaryFeeds.length;
if (totalSecondary === 0) {
// Case 0: Only Main Feed (Full Canvas 16:9 with padding)
const maxAvailW = W - 2 * padX;
const maxAvailH = H - 2 * padY;
let mainW, mainH;
if (maxAvailW / maxAvailH > 16 / 9) {
mainH = maxAvailH;
mainW = Math.round(mainH * (16 / 9));
} else {
mainW = maxAvailW;
mainH = Math.round(mainW * (9 / 16));
}
const mainX = Math.round((W - mainW) / 2);
const mainY = Math.round((H - mainH) / 2);
this.renderTile(
ctx,
mainFeed,
{ x: mainX, y: mainY, w: mainW, h: mainH },
true,
);
} else if (totalSecondary === 1) {
// Case 1: Main + 1 Sidebar Tile (Both 16:9)
const availW = W - 2 * padX - gap;
const availH = H - 2 * padY;
let sideW = Math.round(availW * 0.26);
let sideH = Math.round(sideW * (9 / 16));
let mainW = availW - sideW;
let mainH = Math.round(mainW * (9 / 16));
if (mainH > availH) {
mainH = availH;
mainW = Math.round(mainH * (16 / 9));
sideW = Math.min(availW - mainW, Math.round(availH * (16 / 9)));
sideH = Math.round(sideW * (9 / 16));
}
const mainX = padX;
const mainY = Math.round((H - mainH) / 2);
const sideX = padX + mainW + gap;
const sideY = Math.round((H - sideH) / 2);
this.renderTile(
ctx,
mainFeed,
{ x: mainX, y: mainY, w: mainW, h: mainH },
true,
);
this.renderTile(
ctx,
secondaryFeeds[0],
{ x: sideX, y: sideY, w: sideW, h: sideH },
false,
);
} else if (totalSecondary === 2) {
// Case 2: Main + 2 Sidebar Tiles stacked vertically (All 16:9)
const availW = W - 2 * padX - gap;
const availH = H - 2 * padY;
let sideW = Math.round(availW * 0.28);
let sideH = Math.round(sideW * (9 / 16));
if (2 * sideH + gap > availH) {
sideH = Math.floor((availH - gap) / 2);
sideW = Math.round(sideH * (16 / 9));
}
const totalSideH = 2 * sideH + gap;
let mainW = availW - sideW;
let mainH = Math.round(mainW * (9 / 16));
if (mainH > availH) {
mainH = availH;
mainW = Math.round(mainH * (16 / 9));
}
const mainX = padX;
const mainY = Math.round((H - mainH) / 2);
const sideX = W - padX - sideW;
const sideYStart = Math.round((H - totalSideH) / 2);
this.renderTile(
ctx,
mainFeed,
{ x: mainX, y: mainY, w: mainW, h: mainH },
true,
);
this.renderTile(
ctx,
secondaryFeeds[0],
{ x: sideX, y: sideYStart, w: sideW, h: sideH },
false,
);
this.renderTile(
ctx,
secondaryFeeds[1],
{ x: sideX, y: sideYStart + sideH + gap, w: sideW, h: sideH },
false,
);
} else if (totalSecondary === 3) {
// Case 3: Main + 3 Sidebar Tiles stacked vertically (All 16:9)
const availH = H - 2 * padY;
const sideH = Math.floor((availH - 2 * gap) / 3);
const sideW = Math.round(sideH * (16 / 9));
const totalSideH = 3 * sideH + 2 * gap;
const topY = Math.round((H - totalSideH) / 2);
const sideX = W - padX - sideW;
let mainW = sideX - gap - padX;
let mainH = Math.round(mainW * (9 / 16));
if (mainH > totalSideH) {
mainH = totalSideH;
mainW = Math.round(mainH * (16 / 9));
}
const mainX = padX;
const mainY = Math.round((H - mainH) / 2);
this.renderTile(
ctx,
mainFeed,
{ x: mainX, y: mainY, w: mainW, h: mainH },
true,
);
for (let i = 0; i < 3; i++) {
const feed = secondaryFeeds[i];
const tileY = topY + i * (sideH + gap);
this.renderTile(
ctx,
feed,
{ x: sideX, y: tileY, w: sideW, h: sideH },
false,
);
}
} else {
// Case 4: Main + 3 Sidebar Tiles + Bottom Overflow Row (All 16:9)
const availH = H - 2 * padY;
const sideH = Math.floor((availH - 2 * gap) / 3);
const sideW = Math.round(sideH * (16 / 9));
const totalSideH = 3 * sideH + 2 * gap;
const topY = Math.round((H - totalSideH) / 2);
const sideX = W - padX - sideW;
const mainAreaW = sideX - gap - padX;
const targetMainH = Math.round((totalSideH - gap) * 0.68);
let mainW = Math.round(targetMainH * (16 / 9));
let mainH = targetMainH;
if (mainW > mainAreaW) {
mainW = mainAreaW;
mainH = Math.round(mainW * (9 / 16));
}
const mainX = padX + Math.round((mainAreaW - mainW) / 2);
const mainY = topY;
// 1. Render Main
this.renderTile(
ctx,
mainFeed,
{ x: mainX, y: mainY, w: mainW, h: mainH },
true,
);
// 2. Render 3 Sidebar Tiles
for (let i = 0; i < 3; i++) {
const feed = secondaryFeeds[i];
const tileY = topY + i * (sideH + gap);
this.renderTile(
ctx,
feed,
{ x: sideX, y: tileY, w: sideW, h: sideH },
false,
);
}
// 3. Render Bottom Overflow Tiles (each 16:9)
const overflowFeeds = secondaryFeeds.slice(3);
const numOverflow = overflowFeeds.length;
const botMaxH = totalSideH - mainH - gap;
const botYBase = topY + mainH + gap;
const candBotW = Math.floor(
(mainAreaW - (numOverflow - 1) * gap) / numOverflow,
);
const candBotH = Math.round(candBotW * (9 / 16));
let botW, botH;
if (candBotH > botMaxH) {
botH = botMaxH;
botW = Math.round(botH * (16 / 9));
} else {
botW = candBotW;
botH = candBotH;
}
const totalBotW = numOverflow * botW + (numOverflow - 1) * gap;
const botXStart = padX + Math.round((mainAreaW - totalBotW) / 2);
const botY = botYBase + Math.round((botMaxH - botH) / 2);
overflowFeeds.forEach((feed, j) => {
const tileX = botXStart + j * (botW + gap);
this.renderTile(
ctx,
feed,
{ x: tileX, y: botY, w: botW, h: botH },
false,
);
});
}
}
renderTile(ctx, feed, rect, isMain = false) {
const { x, y, w, h } = rect;
const radius = Math.max(6, Math.round(Math.min(w, h) * 0.035));
ctx.save();
this.drawRoundedClip(ctx, x, y, w, h, radius);
// Base tile background
ctx.fillStyle = "#0d1220";
ctx.fillRect(x, y, w, h);
if (feed.hasVideo && feed.video) {
// Draw video frame
if (feed.type === "screen") {
this.drawVideoContain(ctx, feed.video, x, y, w, h);
} else {
this.drawVideoCover(ctx, feed.video, x, y, w, h);
}
} else {
// Draw avatar placeholder
this.drawPlaceholder(ctx, feed, x, y, w, h, isMain);
}
ctx.restore();
// Sleek subtle border
ctx.save();
ctx.lineWidth = 1;
ctx.strokeStyle = "rgba(255, 255, 255, 0.08)";
this.drawRoundedStroke(ctx, x, y, w, h, radius);
ctx.restore();
// Overlay Pill Tag (Bottom-Left)
this.drawPillTag(ctx, feed, x, y, w, h);
}
drawVideoCover(ctx, video, x, y, w, h) {
try {
const vw = video.videoWidth || 640;
const vh = video.videoHeight || 480;
const targetRatio = w / h;
const videoRatio = vw / vh;
let sx = 0,
sy = 0,
sw = vw,
sh = vh;
if (videoRatio > targetRatio) {
// Video is wider than 16:9 target: crop horizontally
sw = Math.round(vh * targetRatio);
sx = Math.round((vw - sw) / 2);
} else {
// Video is taller than 16:9 target: crop vertically
sh = Math.round(vw / targetRatio);
sy = Math.round((vh - sh) / 2);
}
ctx.drawImage(video, sx, sy, sw, sh, x, y, w, h);
} catch (e) {}
}
drawVideoContain(ctx, video, x, y, w, h) {
try {
const vw = video.videoWidth || 1920;
const vh = video.videoHeight || 1080;
const targetRatio = w / h;
const videoRatio = vw / vh;
let dw = w;
let dh = h;
let dx = x;
let dy = y;
if (videoRatio > targetRatio) {
// Video is wider than tile
dh = Math.round(w / videoRatio);
dy = Math.round(y + (h - dh) / 2);
} else {
// Video is taller than tile
dw = Math.round(h * videoRatio);
dx = Math.round(x + (w - dw) / 2);
}
ctx.drawImage(video, 0, 0, vw, vh, dx, dy, dw, dh);
} catch (e) {}
}
drawPlaceholder(ctx, feed, x, y, w, h, isMain = false) {
const cx = x + w / 2;
const cy = y + h / 2 - (isMain ? Math.round(h * 0.03) : Math.round(h * 0.02));
const circleRadius = isMain
? Math.min(Math.round(h * 0.18), Math.round(w * 0.12), 54)
: Math.min(Math.round(h * 0.2), Math.round(w * 0.14), 36);
// Circular gradient avatar
const gradient = ctx.createLinearGradient(
cx - circleRadius,
cy - circleRadius,
cx + circleRadius,
cy + circleRadius,
);
if (feed.type === "candidate") {
gradient.addColorStop(0, "#10b981");
gradient.addColorStop(1, "#059669");
} else {
gradient.addColorStop(0, "#5b8bff");
gradient.addColorStop(1, "#3b63e0");
}
ctx.fillStyle = gradient;
ctx.beginPath();
ctx.arc(cx, cy, Math.max(1, circleRadius), 0, Math.PI * 2);
ctx.fill();
// Avatar Initials Text
const initialsFont = Math.max(10, Math.round(circleRadius * 0.85));
ctx.fillStyle = "#ffffff";
ctx.font = `bold ${initialsFont}px sans-serif`;
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.fillText(feed.initials || "IV", cx, cy);
// Participant Name
const nameFont = Math.max(10, Math.round(isMain ? Math.min(20, h * 0.045) : Math.min(14, h * 0.06)));
ctx.fillStyle = "#ffffff";
ctx.font = `600 ${nameFont}px sans-serif`;
ctx.textBaseline = "top";
ctx.fillText(feed.name, cx, cy + circleRadius + Math.max(6, Math.round(h * 0.02)));
// Subtitle status
const subFont = Math.max(9, Math.round(isMain ? Math.min(14, h * 0.032) : Math.min(11, h * 0.045)));
ctx.fillStyle = "#94a3b8";
ctx.font = `400 ${subFont}px sans-serif`;
ctx.fillText(
feed.isCamOn ? "Connecting video..." : "Camera turned off",
cx,
cy + circleRadius + Math.max(6, Math.round(h * 0.02)) + nameFont + Math.max(4, Math.round(h * 0.01)),
);
}
drawPillTag(ctx, feed, tileX, tileY, tileW, tileH) {
const text = feed.name;
const fontSize = Math.max(9, Math.min(12, Math.round(tileH * 0.045)));
ctx.font = `600 ${fontSize}px sans-serif`;
const textMetrics = ctx.measureText(text);
const tagPaddingX = Math.max(6, Math.round(fontSize * 0.7));
const tagW = textMetrics.width + tagPaddingX * 2;
const tagH = Math.max(16, Math.round(fontSize * 1.8));
const marginX = Math.max(6, Math.round(tileW * 0.02));
const marginY = Math.max(6, Math.round(tileH * 0.03));
const tagX = tileX + marginX;
const tagY = tileY + tileH - marginY - tagH;
const tagRadius = Math.max(3, Math.round(tagH * 0.25));
ctx.save();
// Background
ctx.fillStyle = "rgba(9, 13, 22, 0.8)";
this.drawRoundedFill(ctx, tagX, tagY, tagW, tagH, tagRadius);
// Border
ctx.strokeStyle = "rgba(255, 255, 255, 0.12)";
ctx.lineWidth = 1;
this.drawRoundedStroke(ctx, tagX, tagY, tagW, tagH, tagRadius);
// Name text
ctx.fillStyle = "#e2e8f0";
ctx.textBaseline = "middle";
ctx.textAlign = "left";
ctx.fillText(text, tagX + tagPaddingX, tagY + tagH / 2);
ctx.restore();
}
isScreenShareActive(video) {
if (!video) return false;
const stream = video.srcObject;
if (!stream || !stream.active) return false;
const tracks = stream.getVideoTracks ? stream.getVideoTracks() : [];
if (tracks.length === 0) return false;
const hasLiveTrack = tracks.some(
(t) => t.readyState === "live" && t.enabled,
);
if (!hasLiveTrack) return false;
if (video.paused && typeof video.play === "function") {
video.play().catch(() => {});
}
return true;
}
isVideoPlaying(video) {
if (!video) return false;
const stream = video.srcObject;
if (!stream || !stream.active) return false;
const tracks = stream.getVideoTracks ? stream.getVideoTracks() : [];
if (tracks.length === 0) return false;
const hasLiveTrack = tracks.some(
(t) => t.readyState === "live" && t.enabled,
);
if (!hasLiveTrack) return false;
if (video.paused && typeof video.play === "function") {
video.play().catch(() => {});
}
return Boolean(
video.videoWidth > 0 || video.readyState >= 1 || hasLiveTrack,
);
}
extractInitials(name) {
if (!name) return "CD";
const parts = name.trim().split(/\s+/);
if (parts.length >= 2) {
return (parts[0][0] + parts[parts.length - 1][0]).toUpperCase();
}
return name.slice(0, 2).toUpperCase();
}
roundedRectPath(ctx, x, y, w, h, r) {
if (typeof ctx.roundRect === "function") {
ctx.beginPath();
ctx.roundRect(x, y, w, h, r);
return;
}
ctx.beginPath();
ctx.moveTo(x + r, y);
ctx.lineTo(x + w - r, y);
ctx.arcTo(x + w, y, x + w, y + r, r);
ctx.lineTo(x + w, y + h - r);
ctx.arcTo(x + w, y + h, x + w - r, y + h, r);
ctx.lineTo(x + r, y + h);
ctx.arcTo(x, y + h, x, y + h - r, r);
ctx.lineTo(x, y + r);
ctx.arcTo(x, y, x + r, y, r);
ctx.closePath();
}
drawRoundedClip(ctx, x, y, w, h, r) {
this.roundedRectPath(ctx, x, y, w, h, r);
ctx.clip();
}
drawRoundedStroke(ctx, x, y, w, h, r) {
this.roundedRectPath(ctx, x, y, w, h, r);
ctx.stroke();
}
drawRoundedFill(ctx, x, y, w, h, r) {
this.roundedRectPath(ctx, x, y, w, h, r);
ctx.fill();
}
}