/* * Candidate-side proctoring pipeline. * MediaPipe runs locally; no camera frame is sent to the application server. * Events are deliberately emitted as structured objects so the logger can be * replaced by an API adapter later without changing detection code. */ // Load MediaPipe's published browser bundle directly. This avoids CDN // transform endpoints that may return text/plain or a 404 through proxies. const VISION_MODULE_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22-rc.20250304/vision_bundle.mjs'; const VISION_MODULE_FALLBACK_URL = 'https://unpkg.com/@mediapipe/tasks-vision@0.10.22-rc.20250304/vision_bundle.mjs'; const WASM_ROOT = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22-rc.20250304/wasm'; const FACE_MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task'; const OBJECT_MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float32/1/efficientdet_lite0.tflite'; const SAMPLE_INTERVAL_MS = 200; const NO_FACE_GRACE_MS = 1000; const LOOKING_AWAY_GRACE_MS = 700; const PROLONGED_LOOKING_AWAY_MS = 10000; const PROLONGED_LOOKING_AWAY_RECHECK_MS = 120000; const PHONE_EVENT_COOLDOWN_MS = 3000; const SNAPSHOT_QUALITY = 0.72; const HEAD_YAW_THRESHOLD = 0.34; const HEAD_PITCH_UP_THRESHOLD = 0.2; const HEAD_PITCH_DOWN_THRESHOLD = 0.74; const GAZE_HORIZONTAL_LEFT_THRESHOLD = 0.28; const GAZE_HORIZONTAL_RIGHT_THRESHOLD = 0.72; const GAZE_DOWN_THRESHOLD = 0.62; class StructuredEventLogger { constructor(sessionId) { this.sessionId = sessionId; this.events = []; } record(type, confidence, details, snapshot) { const event = { schema: 'candidate-proctoring/v1', session_id: this.sessionId, event_id: `${type}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, type, timestamp: new Date().toISOString(), confidence: Number(Math.max(0, Math.min(1, confidence || 0)).toFixed(4)), details: details || {}, snapshot: snapshot || null, }; this.events.push(event); console.log('[proctoring:event]', event); return event; } report(startedAt, endedAt, durations) { const report = { schema: 'candidate-proctoring/report-v1', session_id: this.sessionId, started_at: startedAt, ended_at: endedAt, generated_at: new Date().toISOString(), review_required: true, summary: { looking_away_ms: Math.round(durations.lookingAwayMs), no_face_ms: Math.round(durations.noFaceMs), multiple_faces_ms: Math.round(durations.multipleFacesMs), prolonged_looking_away_event_count: this.events.filter((event) => event.type === 'prolonged_looking_away').length, phone_event_count: this.events.filter((event) => event.type === 'phone_detected').length, }, events: this.events, }; console.log('[proctoring:report]', report); return report; } } class WarningPresenter { constructor() { this.element = null; } show(message) { if (!this.element) { this.element = document.createElement('div'); this.element.setAttribute('role', 'status'); this.element.style.cssText = 'position:fixed;top:70px;left:50%;transform:translateX(-50%);z-index:9999;padding:12px 24px;border-radius:12px;background:rgba(239,68,68,.95);color:#fff;font:700 14px Instrument Sans,sans-serif;box-shadow:0 10px 30px rgba(239,68,68,.45);pointer-events:none;'; document.body.appendChild(this.element); } this.element.textContent = `Proctoring notice: ${message}`; this.element.style.display = 'block'; clearTimeout(this.hideTimer); this.hideTimer = setTimeout(() => { this.element.style.display = 'none'; }, 2500); } } class SnapshotService { constructor(video, canvas) { this.video = video; this.canvas = canvas; } capture() { if (!this.video || this.video.readyState < 2 || !this.video.videoWidth) return null; const width = Math.min(640, this.video.videoWidth); const height = Math.round(width * this.video.videoHeight / this.video.videoWidth); this.canvas.width = width; this.canvas.height = height; this.canvas.getContext('2d').drawImage(this.video, 0, 0, width, height); return this.canvas.toDataURL('image/jpeg', SNAPSHOT_QUALITY); } } class FacePoseDetector { constructor(landmarker) { this.landmarker = landmarker; } detect(video, timestamp) { const result = this.landmarker.detectForVideo(video, timestamp); const faces = result.faceLandmarks || []; return { count: faces.length, pose: faces[0] ? this.estimatePose(faces[0]) : null, gaze: faces[0] ? this.estimateGaze(faces[0]) : null, }; } estimatePose(landmarks) { const point = (index) => landmarks[index]; const leftEye = point(33); const rightEye = point(263); const nose = point(1); const mouth = point(13); const eyeMidX = (leftEye.x + rightEye.x) / 2; const eyeMidY = (leftEye.y + rightEye.y) / 2; const eyeDistance = Math.max(0.001, Math.abs(rightEye.x - leftEye.x)); const yaw = (nose.x - eyeMidX) / eyeDistance; const pitch = (nose.y - eyeMidY) / Math.max(0.001, Math.abs(mouth.y - eyeMidY)); return { yaw, pitch, confidence: Math.min(1, 0.5 + Math.abs(yaw) * 0.4 + Math.abs(pitch - 0.5) * 0.2) }; } estimateGaze(landmarks) { const leftEye = this.estimateEyeGaze(landmarks, { corners: [33, 133], top: [159, 158], bottom: [145, 153], iris: [468, 469, 470, 471, 472], }); const rightEye = this.estimateEyeGaze(landmarks, { corners: [362, 263], top: [386, 385], bottom: [374, 380], iris: [473, 474, 475, 476, 477], }); const eyes = [leftEye, rightEye].filter(Boolean); if (!eyes.length) return null; const horizontal = eyes.reduce((total, eye) => total + eye.horizontal, 0) / eyes.length; const vertical = eyes.reduce((total, eye) => total + eye.vertical, 0) / eyes.length; const confidence = Math.min(1, 0.45 + Math.max(Math.abs(horizontal - 0.5), Math.abs(vertical - 0.5)) * 1.4); return { horizontal, vertical, confidence }; } estimateEyeGaze(landmarks, config) { const points = [...config.corners, ...config.top, ...config.bottom, ...config.iris].map((index) => landmarks[index]); if (points.some((point) => !point)) return null; const cornerA = landmarks[config.corners[0]]; const cornerB = landmarks[config.corners[1]]; const topY = this.averageCoordinate(landmarks, config.top, 'y'); const bottomY = this.averageCoordinate(landmarks, config.bottom, 'y'); const irisX = this.averageCoordinate(landmarks, config.iris, 'x'); const irisY = this.averageCoordinate(landmarks, config.iris, 'y'); const leftX = Math.min(cornerA.x, cornerB.x); const rightX = Math.max(cornerA.x, cornerB.x); return { horizontal: this.clamp01((irisX - leftX) / Math.max(0.001, rightX - leftX)), vertical: this.clamp01((irisY - topY) / Math.max(0.001, bottomY - topY)), }; } averageCoordinate(landmarks, indexes, coordinate) { return indexes.reduce((total, index) => total + landmarks[index][coordinate], 0) / indexes.length; } clamp01(value) { return Math.max(0, Math.min(1, value)); } } class LookingAwayClassifier { classify(face) { const reasons = []; const pose = face.pose; const gaze = face.gaze; if (pose && Math.abs(pose.yaw) > HEAD_YAW_THRESHOLD) reasons.push('head_yaw'); if (pose && pose.pitch < HEAD_PITCH_UP_THRESHOLD) reasons.push('head_up'); if (pose && pose.pitch > HEAD_PITCH_DOWN_THRESHOLD) reasons.push('head_down'); if (gaze && gaze.horizontal < GAZE_HORIZONTAL_LEFT_THRESHOLD) reasons.push('eye_gaze_left'); if (gaze && gaze.horizontal > GAZE_HORIZONTAL_RIGHT_THRESHOLD) reasons.push('eye_gaze_right'); if (gaze && gaze.vertical > GAZE_DOWN_THRESHOLD) reasons.push('eye_gaze_down'); return { active: reasons.length > 0, confidence: this.confidence(reasons, pose, gaze), details: { reasons, yaw: pose?.yaw ?? null, pitch: pose?.pitch ?? null, gaze_horizontal: gaze?.horizontal ?? null, gaze_vertical: gaze?.vertical ?? null, }, }; } confidence(reasons, pose, gaze) { if (!reasons.length) return 0; return Math.max( reasons.some((reason) => reason.startsWith('head_')) ? pose?.confidence || 0 : 0, reasons.some((reason) => reason.startsWith('eye_')) ? gaze?.confidence || 0 : 0, ); } } class PhoneDetector { constructor(detector) { this.detector = detector; } detect(video, timestamp) { const detections = this.detector.detectForVideo(video, timestamp).detections || []; const phone = detections .map((detection) => ({ detection, category: detection.categories?.[0] })) .find(({ category }) => category && /cell phone|mobile phone|phone/i.test(category.categoryName || '') && (category.score || 0) >= 0.45); return phone ? { confidence: phone.category.score, label: phone.category.categoryName } : null; } } class DurationTracker { constructor() { this.activeSince = { lookingAway: null, noFace: null, multipleFaces: null }; this.total = { lookingAwayMs: 0, noFaceMs: 0, multipleFacesMs: 0 }; } update(name, active, now) { const startedAt = this.activeSince[name]; if (active && startedAt === null) this.activeSince[name] = now; if (!active && startedAt !== null) { this.total[`${name}Ms`] += now - startedAt; this.activeSince[name] = null; } } finish(now) { Object.keys(this.activeSince).forEach((name) => this.update(name, false, now)); return { ...this.total }; } } class ProlongedLookingAwayTracker { constructor({ thresholdMs, recheckMs }) { this.thresholdMs = thresholdMs; this.recheckMs = recheckMs; this.startedAt = null; this.nextCheckAt = null; } update(active, now) { if (!active) { this.reset(); return null; } if (this.startedAt === null) this.startedAt = now; const durationMs = now - this.startedAt; if (durationMs < this.thresholdMs || (this.nextCheckAt !== null && now < this.nextCheckAt)) return null; const recheck = this.nextCheckAt !== null; this.nextCheckAt = now + this.recheckMs; return { duration_ms: Math.round(durationMs), threshold_ms: this.thresholdMs, recheck, recheck_interval_ms: this.recheckMs, next_check_at: new Date(Date.now() + this.recheckMs).toISOString(), }; } reset() { this.startedAt = null; this.nextCheckAt = null; } } class CandidateProctoring { constructor({ sessionId, video, canvas }) { this.sessionId = sessionId; this.video = video; this.canvas = canvas; this.logger = new StructuredEventLogger(sessionId); this.warning = new WarningPresenter(); this.snapshot = new SnapshotService(video, canvas); this.durations = new DurationTracker(); this.lookingAwayClassifier = new LookingAwayClassifier(); this.prolongedLookingAway = new ProlongedLookingAwayTracker({ thresholdMs: PROLONGED_LOOKING_AWAY_MS, recheckMs: PROLONGED_LOOKING_AWAY_RECHECK_MS, }); this.startedAt = new Date().toISOString(); this.lastSampleAt = 0; this.lastPhoneEventAt = 0; this.lastStates = { lookingAway: false, noFace: false, multipleFaces: false }; this.running = false; this.modelsReady = false; this.reported = false; } async start() { if (this.running) return; try { const vision = await this.importVisionModule(); const fileset = await vision.FilesetResolver.forVisionTasks(WASM_ROOT); const [faceLandmarker, objectDetector] = await this.withAmdLoaderDisabled(() => Promise.all([ this.createModel(vision.FaceLandmarker, fileset, { modelAssetPath: FACE_MODEL_URL, runningMode: 'VIDEO', numFaces: 3, minFaceDetectionConfidence: 0.5, minFacePresenceConfidence: 0.5, minTrackingConfidence: 0.5, }), this.createModel(vision.ObjectDetector, fileset, { modelAssetPath: OBJECT_MODEL_URL, runningMode: 'VIDEO', maxResults: 5, scoreThreshold: 0.35, }), ])); this.faceDetector = new FacePoseDetector(faceLandmarker); this.phoneDetector = new PhoneDetector(objectDetector); this.modelsReady = true; this.running = true; this.scheduleNextSample(); console.log('[proctoring:ready]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId }); } catch (error) { console.error('[proctoring:error]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'model_initialization_failed', message: error.message }); } } async importVisionModule() { try { return await import(/* @vite-ignore */ VISION_MODULE_URL); } catch (primaryError) { console.warn('[proctoring:warning]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'vision_cdn_fallback', message: primaryError.message, }); return import(/* @vite-ignore */ VISION_MODULE_FALLBACK_URL); } } async withAmdLoaderDisabled(callback) { // Monaco's RequireJS loader sees MediaPipe's internally injected // vision_wasm_internal.js and reports "one anonymous define". Keep // the two loaders isolated only for model creation, then restore all // globals so the editor continues to work normally. const amdGlobals = ['define', 'require', 'requirejs']; const saved = amdGlobals.map((name) => ({ name, value: globalThis[name] })); amdGlobals.forEach((name) => { try { delete globalThis[name]; } catch (error) { globalThis[name] = undefined; } }); try { return await callback(); } finally { saved.forEach(({ name, value }) => { if (value === undefined) delete globalThis[name]; else globalThis[name] = value; }); } } async createModel(Model, fileset, options) { try { return await Model.createFromOptions(fileset, { ...options, baseOptions: { modelAssetPath: options.modelAssetPath, delegate: 'GPU' } }); } catch (gpuError) { console.warn('[proctoring:warning]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'gpu_delegate_unavailable', message: gpuError.message }); return Model.createFromOptions(fileset, { ...options, baseOptions: { modelAssetPath: options.modelAssetPath, delegate: 'CPU' } }); } } scheduleNextSample() { if (!this.running) return; setTimeout(() => this.sample(), SAMPLE_INTERVAL_MS); } sample() { if (!this.running) return; if (this.video.readyState < 2 || !this.video.videoWidth) return this.scheduleNextSample(); const timestamp = performance.now(); try { const face = this.faceDetector.detect(this.video, timestamp); const phone = this.phoneDetector.detect(this.video, timestamp); this.processFace(face, timestamp); this.processPhone(phone, timestamp); } catch (error) { console.error('[proctoring:error]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'frame_processing_failed', message: error.message }); } this.lastSampleAt = timestamp; this.scheduleNextSample(); } processFace(result, now) { const noFace = result.count === 0; const multipleFaces = result.count > 1; const lookingAwayResult = this.lookingAwayClassifier.classify(result); this.durations.update('noFace', noFace, now); this.durations.update('multipleFaces', multipleFaces, now); this.durations.update('lookingAway', lookingAwayResult.active, now); this.transition('noFace', noFace, noFace ? 0.95 : 0, { face_count: result.count }, NO_FACE_GRACE_MS, 'Candidate face is not visible'); this.transition('multipleFaces', multipleFaces, multipleFaces ? 0.95 : 0, { face_count: result.count }, 0, 'Multiple faces detected'); this.transition('lookingAway', lookingAwayResult.active, lookingAwayResult.confidence, lookingAwayResult.details, LOOKING_AWAY_GRACE_MS, 'Please look toward the camera'); this.trackProlongedLookingAway(lookingAwayResult, now); } transition(name, active, confidence, details, graceMs, message) { if (active === this.lastStates[name]) return; if (active && graceMs > 0 && this.durations.activeSince[name] && performance.now() - this.durations.activeSince[name] < graceMs) return; this.lastStates[name] = active; if (active) { this.warning.show(message); this.logger.record(name === 'lookingAway' ? 'looking_away' : name === 'noFace' ? 'face_missing' : 'multiple_faces', confidence, details, this.snapshot.capture()); } } trackProlongedLookingAway(lookingAwayResult, now) { const prolongedEvent = this.prolongedLookingAway.update(lookingAwayResult.active, now); if (!prolongedEvent) return; this.warning.show(`Candidate has been looking away for over ${Math.round(PROLONGED_LOOKING_AWAY_MS / 1000)} seconds`); this.logger.record('prolonged_looking_away', lookingAwayResult.confidence, { ...prolongedEvent, ...lookingAwayResult.details, }, this.snapshot.capture()); } processPhone(phone, now) { if (!phone || now - this.lastPhoneEventAt < PHONE_EVENT_COOLDOWN_MS) return; this.lastPhoneEventAt = now; this.warning.show('Possible phone detected in camera frame'); this.logger.record('phone_detected', phone.confidence, { label: phone.label }, this.snapshot.capture()); } stop() { if (this.reported) return null; this.running = false; this.reported = true; return this.logger.report(this.startedAt, new Date().toISOString(), this.durations.finish(performance.now())); } } function bootCandidateProctoring() { const video = document.getElementById('webcam'); const canvas = document.getElementById('proctor-canvas') || document.createElement('canvas'); const sessionId = document.body.dataset.interviewId; if (!video || !sessionId || !navigator.mediaDevices) return; const proctor = new CandidateProctoring({ sessionId, video, canvas }); window.candidateProctoring = proctor; const startWhenReady = () => proctor.start(); if (video.readyState >= 2) startWhenReady(); video.addEventListener('loadeddata', startWhenReady, { once: true }); const callStatus = document.getElementById('call-status-badge'); if (callStatus) { const statusObserver = new MutationObserver(() => { if (/ended/i.test(callStatus.textContent || '')) proctor.stop(); }); statusObserver.observe(callStatus, { childList: true, characterData: true, subtree: true }); } window.addEventListener('pagehide', () => proctor.stop(), { once: true }); } if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', bootCandidateProctoring, { once: true }); else bootCandidateProctoring();