484 lines
20 KiB
JavaScript
484 lines
20 KiB
JavaScript
/*
|
|
* Candidate-side proctoring pipeline.
|
|
* MediaPipe runs locally; no camera frame is sent to the application server.
|
|
* Events are deliberately emitted as structured objects so the logger can be
|
|
* replaced by an API adapter later without changing detection code.
|
|
*/
|
|
|
|
// Load MediaPipe's published browser bundle directly. This avoids CDN
|
|
// transform endpoints that may return text/plain or a 404 through proxies.
|
|
const VISION_MODULE_URL = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22-rc.20250304/vision_bundle.mjs';
|
|
const VISION_MODULE_FALLBACK_URL = 'https://unpkg.com/@mediapipe/tasks-vision@0.10.22-rc.20250304/vision_bundle.mjs';
|
|
const WASM_ROOT = 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.22-rc.20250304/wasm';
|
|
const FACE_MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task';
|
|
const OBJECT_MODEL_URL = 'https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float32/1/efficientdet_lite0.tflite';
|
|
const SAMPLE_INTERVAL_MS = 200;
|
|
const NO_FACE_GRACE_MS = 1000;
|
|
const LOOKING_AWAY_GRACE_MS = 700;
|
|
const PROLONGED_LOOKING_AWAY_MS = 10000;
|
|
const PROLONGED_LOOKING_AWAY_RECHECK_MS = 120000;
|
|
const PHONE_EVENT_COOLDOWN_MS = 3000;
|
|
const SNAPSHOT_QUALITY = 0.72;
|
|
const HEAD_YAW_THRESHOLD = 0.34;
|
|
const HEAD_PITCH_UP_THRESHOLD = 0.2;
|
|
const HEAD_PITCH_DOWN_THRESHOLD = 0.74;
|
|
const GAZE_HORIZONTAL_LEFT_THRESHOLD = 0.28;
|
|
const GAZE_HORIZONTAL_RIGHT_THRESHOLD = 0.72;
|
|
const GAZE_DOWN_THRESHOLD = 0.62;
|
|
|
|
class StructuredEventLogger {
|
|
constructor(sessionId) {
|
|
this.sessionId = sessionId;
|
|
this.events = [];
|
|
}
|
|
|
|
record(type, confidence, details, snapshot) {
|
|
const event = {
|
|
schema: 'candidate-proctoring/v1',
|
|
session_id: this.sessionId,
|
|
event_id: `${type}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
type,
|
|
timestamp: new Date().toISOString(),
|
|
confidence: Number(Math.max(0, Math.min(1, confidence || 0)).toFixed(4)),
|
|
details: details || {},
|
|
snapshot: snapshot || null,
|
|
};
|
|
this.events.push(event);
|
|
console.log('[proctoring:event]', event);
|
|
return event;
|
|
}
|
|
|
|
report(startedAt, endedAt, durations) {
|
|
const report = {
|
|
schema: 'candidate-proctoring/report-v1',
|
|
session_id: this.sessionId,
|
|
started_at: startedAt,
|
|
ended_at: endedAt,
|
|
generated_at: new Date().toISOString(),
|
|
review_required: true,
|
|
summary: {
|
|
looking_away_ms: Math.round(durations.lookingAwayMs),
|
|
no_face_ms: Math.round(durations.noFaceMs),
|
|
multiple_faces_ms: Math.round(durations.multipleFacesMs),
|
|
prolonged_looking_away_event_count: this.events.filter((event) => event.type === 'prolonged_looking_away').length,
|
|
phone_event_count: this.events.filter((event) => event.type === 'phone_detected').length,
|
|
},
|
|
events: this.events,
|
|
};
|
|
console.log('[proctoring:report]', report);
|
|
return report;
|
|
}
|
|
}
|
|
|
|
class WarningPresenter {
|
|
constructor() {
|
|
this.element = null;
|
|
}
|
|
|
|
show(message) {
|
|
if (!this.element) {
|
|
this.element = document.createElement('div');
|
|
this.element.setAttribute('role', 'status');
|
|
this.element.style.cssText = 'position:fixed;top:70px;left:50%;transform:translateX(-50%);z-index:9999;padding:12px 24px;border-radius:12px;background:rgba(239,68,68,.95);color:#fff;font:700 14px Instrument Sans,sans-serif;box-shadow:0 10px 30px rgba(239,68,68,.45);pointer-events:none;';
|
|
document.body.appendChild(this.element);
|
|
}
|
|
this.element.textContent = `Proctoring notice: ${message}`;
|
|
this.element.style.display = 'block';
|
|
clearTimeout(this.hideTimer);
|
|
this.hideTimer = setTimeout(() => { this.element.style.display = 'none'; }, 2500);
|
|
}
|
|
}
|
|
|
|
class SnapshotService {
|
|
constructor(video, canvas) {
|
|
this.video = video;
|
|
this.canvas = canvas;
|
|
}
|
|
|
|
capture() {
|
|
if (!this.video || this.video.readyState < 2 || !this.video.videoWidth) return null;
|
|
const width = Math.min(640, this.video.videoWidth);
|
|
const height = Math.round(width * this.video.videoHeight / this.video.videoWidth);
|
|
this.canvas.width = width;
|
|
this.canvas.height = height;
|
|
this.canvas.getContext('2d').drawImage(this.video, 0, 0, width, height);
|
|
return this.canvas.toDataURL('image/jpeg', SNAPSHOT_QUALITY);
|
|
}
|
|
}
|
|
|
|
class FacePoseDetector {
|
|
constructor(landmarker) {
|
|
this.landmarker = landmarker;
|
|
}
|
|
|
|
detect(video, timestamp) {
|
|
const result = this.landmarker.detectForVideo(video, timestamp);
|
|
const faces = result.faceLandmarks || [];
|
|
return {
|
|
count: faces.length,
|
|
pose: faces[0] ? this.estimatePose(faces[0]) : null,
|
|
gaze: faces[0] ? this.estimateGaze(faces[0]) : null,
|
|
};
|
|
}
|
|
|
|
estimatePose(landmarks) {
|
|
const point = (index) => landmarks[index];
|
|
const leftEye = point(33);
|
|
const rightEye = point(263);
|
|
const nose = point(1);
|
|
const mouth = point(13);
|
|
const eyeMidX = (leftEye.x + rightEye.x) / 2;
|
|
const eyeMidY = (leftEye.y + rightEye.y) / 2;
|
|
const eyeDistance = Math.max(0.001, Math.abs(rightEye.x - leftEye.x));
|
|
const yaw = (nose.x - eyeMidX) / eyeDistance;
|
|
const pitch = (nose.y - eyeMidY) / Math.max(0.001, Math.abs(mouth.y - eyeMidY));
|
|
return { yaw, pitch, confidence: Math.min(1, 0.5 + Math.abs(yaw) * 0.4 + Math.abs(pitch - 0.5) * 0.2) };
|
|
}
|
|
|
|
estimateGaze(landmarks) {
|
|
const leftEye = this.estimateEyeGaze(landmarks, {
|
|
corners: [33, 133],
|
|
top: [159, 158],
|
|
bottom: [145, 153],
|
|
iris: [468, 469, 470, 471, 472],
|
|
});
|
|
const rightEye = this.estimateEyeGaze(landmarks, {
|
|
corners: [362, 263],
|
|
top: [386, 385],
|
|
bottom: [374, 380],
|
|
iris: [473, 474, 475, 476, 477],
|
|
});
|
|
const eyes = [leftEye, rightEye].filter(Boolean);
|
|
if (!eyes.length) return null;
|
|
|
|
const horizontal = eyes.reduce((total, eye) => total + eye.horizontal, 0) / eyes.length;
|
|
const vertical = eyes.reduce((total, eye) => total + eye.vertical, 0) / eyes.length;
|
|
const confidence = Math.min(1, 0.45 + Math.max(Math.abs(horizontal - 0.5), Math.abs(vertical - 0.5)) * 1.4);
|
|
|
|
return { horizontal, vertical, confidence };
|
|
}
|
|
|
|
estimateEyeGaze(landmarks, config) {
|
|
const points = [...config.corners, ...config.top, ...config.bottom, ...config.iris].map((index) => landmarks[index]);
|
|
if (points.some((point) => !point)) return null;
|
|
|
|
const cornerA = landmarks[config.corners[0]];
|
|
const cornerB = landmarks[config.corners[1]];
|
|
const topY = this.averageCoordinate(landmarks, config.top, 'y');
|
|
const bottomY = this.averageCoordinate(landmarks, config.bottom, 'y');
|
|
const irisX = this.averageCoordinate(landmarks, config.iris, 'x');
|
|
const irisY = this.averageCoordinate(landmarks, config.iris, 'y');
|
|
const leftX = Math.min(cornerA.x, cornerB.x);
|
|
const rightX = Math.max(cornerA.x, cornerB.x);
|
|
|
|
return {
|
|
horizontal: this.clamp01((irisX - leftX) / Math.max(0.001, rightX - leftX)),
|
|
vertical: this.clamp01((irisY - topY) / Math.max(0.001, bottomY - topY)),
|
|
};
|
|
}
|
|
|
|
averageCoordinate(landmarks, indexes, coordinate) {
|
|
return indexes.reduce((total, index) => total + landmarks[index][coordinate], 0) / indexes.length;
|
|
}
|
|
|
|
clamp01(value) {
|
|
return Math.max(0, Math.min(1, value));
|
|
}
|
|
}
|
|
|
|
class LookingAwayClassifier {
|
|
classify(face) {
|
|
const reasons = [];
|
|
const pose = face.pose;
|
|
const gaze = face.gaze;
|
|
|
|
if (pose && Math.abs(pose.yaw) > HEAD_YAW_THRESHOLD) reasons.push('head_yaw');
|
|
if (pose && pose.pitch < HEAD_PITCH_UP_THRESHOLD) reasons.push('head_up');
|
|
if (pose && pose.pitch > HEAD_PITCH_DOWN_THRESHOLD) reasons.push('head_down');
|
|
if (gaze && gaze.horizontal < GAZE_HORIZONTAL_LEFT_THRESHOLD) reasons.push('eye_gaze_left');
|
|
if (gaze && gaze.horizontal > GAZE_HORIZONTAL_RIGHT_THRESHOLD) reasons.push('eye_gaze_right');
|
|
if (gaze && gaze.vertical > GAZE_DOWN_THRESHOLD) reasons.push('eye_gaze_down');
|
|
|
|
return {
|
|
active: reasons.length > 0,
|
|
confidence: this.confidence(reasons, pose, gaze),
|
|
details: {
|
|
reasons,
|
|
yaw: pose?.yaw ?? null,
|
|
pitch: pose?.pitch ?? null,
|
|
gaze_horizontal: gaze?.horizontal ?? null,
|
|
gaze_vertical: gaze?.vertical ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
confidence(reasons, pose, gaze) {
|
|
if (!reasons.length) return 0;
|
|
return Math.max(
|
|
reasons.some((reason) => reason.startsWith('head_')) ? pose?.confidence || 0 : 0,
|
|
reasons.some((reason) => reason.startsWith('eye_')) ? gaze?.confidence || 0 : 0,
|
|
);
|
|
}
|
|
}
|
|
|
|
class PhoneDetector {
|
|
constructor(detector) {
|
|
this.detector = detector;
|
|
}
|
|
|
|
detect(video, timestamp) {
|
|
const detections = this.detector.detectForVideo(video, timestamp).detections || [];
|
|
const phone = detections
|
|
.map((detection) => ({ detection, category: detection.categories?.[0] }))
|
|
.find(({ category }) => category && /cell phone|mobile phone|phone/i.test(category.categoryName || '') && (category.score || 0) >= 0.45);
|
|
return phone ? { confidence: phone.category.score, label: phone.category.categoryName } : null;
|
|
}
|
|
}
|
|
|
|
class DurationTracker {
|
|
constructor() {
|
|
this.activeSince = { lookingAway: null, noFace: null, multipleFaces: null };
|
|
this.total = { lookingAwayMs: 0, noFaceMs: 0, multipleFacesMs: 0 };
|
|
}
|
|
|
|
update(name, active, now) {
|
|
const startedAt = this.activeSince[name];
|
|
if (active && startedAt === null) this.activeSince[name] = now;
|
|
if (!active && startedAt !== null) {
|
|
this.total[`${name}Ms`] += now - startedAt;
|
|
this.activeSince[name] = null;
|
|
}
|
|
}
|
|
|
|
finish(now) {
|
|
Object.keys(this.activeSince).forEach((name) => this.update(name, false, now));
|
|
return { ...this.total };
|
|
}
|
|
}
|
|
|
|
class ProlongedLookingAwayTracker {
|
|
constructor({ thresholdMs, recheckMs }) {
|
|
this.thresholdMs = thresholdMs;
|
|
this.recheckMs = recheckMs;
|
|
this.startedAt = null;
|
|
this.nextCheckAt = null;
|
|
}
|
|
|
|
update(active, now) {
|
|
if (!active) {
|
|
this.reset();
|
|
return null;
|
|
}
|
|
|
|
if (this.startedAt === null) this.startedAt = now;
|
|
|
|
const durationMs = now - this.startedAt;
|
|
if (durationMs < this.thresholdMs || (this.nextCheckAt !== null && now < this.nextCheckAt)) return null;
|
|
|
|
const recheck = this.nextCheckAt !== null;
|
|
this.nextCheckAt = now + this.recheckMs;
|
|
|
|
return {
|
|
duration_ms: Math.round(durationMs),
|
|
threshold_ms: this.thresholdMs,
|
|
recheck,
|
|
recheck_interval_ms: this.recheckMs,
|
|
next_check_at: new Date(Date.now() + this.recheckMs).toISOString(),
|
|
};
|
|
}
|
|
|
|
reset() {
|
|
this.startedAt = null;
|
|
this.nextCheckAt = null;
|
|
}
|
|
}
|
|
|
|
class CandidateProctoring {
|
|
constructor({ sessionId, video, canvas }) {
|
|
this.sessionId = sessionId;
|
|
this.video = video;
|
|
this.canvas = canvas;
|
|
this.logger = new StructuredEventLogger(sessionId);
|
|
this.warning = new WarningPresenter();
|
|
this.snapshot = new SnapshotService(video, canvas);
|
|
this.durations = new DurationTracker();
|
|
this.lookingAwayClassifier = new LookingAwayClassifier();
|
|
this.prolongedLookingAway = new ProlongedLookingAwayTracker({
|
|
thresholdMs: PROLONGED_LOOKING_AWAY_MS,
|
|
recheckMs: PROLONGED_LOOKING_AWAY_RECHECK_MS,
|
|
});
|
|
this.startedAt = new Date().toISOString();
|
|
this.lastSampleAt = 0;
|
|
this.lastPhoneEventAt = 0;
|
|
this.lastStates = { lookingAway: false, noFace: false, multipleFaces: false };
|
|
this.running = false;
|
|
this.modelsReady = false;
|
|
this.reported = false;
|
|
}
|
|
|
|
async start() {
|
|
if (this.running) return;
|
|
try {
|
|
const vision = await this.importVisionModule();
|
|
const fileset = await vision.FilesetResolver.forVisionTasks(WASM_ROOT);
|
|
const [faceLandmarker, objectDetector] = await this.withAmdLoaderDisabled(() => Promise.all([
|
|
this.createModel(vision.FaceLandmarker, fileset, {
|
|
modelAssetPath: FACE_MODEL_URL,
|
|
runningMode: 'VIDEO', numFaces: 3, minFaceDetectionConfidence: 0.5,
|
|
minFacePresenceConfidence: 0.5, minTrackingConfidence: 0.5,
|
|
}),
|
|
this.createModel(vision.ObjectDetector, fileset, {
|
|
modelAssetPath: OBJECT_MODEL_URL,
|
|
runningMode: 'VIDEO', maxResults: 5, scoreThreshold: 0.35,
|
|
}),
|
|
]));
|
|
this.faceDetector = new FacePoseDetector(faceLandmarker);
|
|
this.phoneDetector = new PhoneDetector(objectDetector);
|
|
this.modelsReady = true;
|
|
this.running = true;
|
|
this.scheduleNextSample();
|
|
console.log('[proctoring:ready]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId });
|
|
} catch (error) {
|
|
console.error('[proctoring:error]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'model_initialization_failed', message: error.message });
|
|
}
|
|
}
|
|
|
|
async importVisionModule() {
|
|
try {
|
|
return await import(/* @vite-ignore */ VISION_MODULE_URL);
|
|
} catch (primaryError) {
|
|
console.warn('[proctoring:warning]', {
|
|
schema: 'candidate-proctoring/v1', session_id: this.sessionId,
|
|
code: 'vision_cdn_fallback', message: primaryError.message,
|
|
});
|
|
return import(/* @vite-ignore */ VISION_MODULE_FALLBACK_URL);
|
|
}
|
|
}
|
|
|
|
async withAmdLoaderDisabled(callback) {
|
|
// Monaco's RequireJS loader sees MediaPipe's internally injected
|
|
// vision_wasm_internal.js and reports "one anonymous define". Keep
|
|
// the two loaders isolated only for model creation, then restore all
|
|
// globals so the editor continues to work normally.
|
|
const amdGlobals = ['define', 'require', 'requirejs'];
|
|
const saved = amdGlobals.map((name) => ({ name, value: globalThis[name] }));
|
|
amdGlobals.forEach((name) => {
|
|
try {
|
|
delete globalThis[name];
|
|
} catch (error) {
|
|
globalThis[name] = undefined;
|
|
}
|
|
});
|
|
|
|
try {
|
|
return await callback();
|
|
} finally {
|
|
saved.forEach(({ name, value }) => {
|
|
if (value === undefined) delete globalThis[name];
|
|
else globalThis[name] = value;
|
|
});
|
|
}
|
|
}
|
|
|
|
async createModel(Model, fileset, options) {
|
|
try {
|
|
return await Model.createFromOptions(fileset, { ...options, baseOptions: { modelAssetPath: options.modelAssetPath, delegate: 'GPU' } });
|
|
} catch (gpuError) {
|
|
console.warn('[proctoring:warning]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'gpu_delegate_unavailable', message: gpuError.message });
|
|
return Model.createFromOptions(fileset, { ...options, baseOptions: { modelAssetPath: options.modelAssetPath, delegate: 'CPU' } });
|
|
}
|
|
}
|
|
|
|
scheduleNextSample() {
|
|
if (!this.running) return;
|
|
setTimeout(() => this.sample(), SAMPLE_INTERVAL_MS);
|
|
}
|
|
|
|
sample() {
|
|
if (!this.running) return;
|
|
if (this.video.readyState < 2 || !this.video.videoWidth) return this.scheduleNextSample();
|
|
const timestamp = performance.now();
|
|
try {
|
|
const face = this.faceDetector.detect(this.video, timestamp);
|
|
const phone = this.phoneDetector.detect(this.video, timestamp);
|
|
this.processFace(face, timestamp);
|
|
this.processPhone(phone, timestamp);
|
|
} catch (error) {
|
|
console.error('[proctoring:error]', { schema: 'candidate-proctoring/v1', session_id: this.sessionId, code: 'frame_processing_failed', message: error.message });
|
|
}
|
|
this.lastSampleAt = timestamp;
|
|
this.scheduleNextSample();
|
|
}
|
|
|
|
processFace(result, now) {
|
|
const noFace = result.count === 0;
|
|
const multipleFaces = result.count > 1;
|
|
const lookingAwayResult = this.lookingAwayClassifier.classify(result);
|
|
this.durations.update('noFace', noFace, now);
|
|
this.durations.update('multipleFaces', multipleFaces, now);
|
|
this.durations.update('lookingAway', lookingAwayResult.active, now);
|
|
this.transition('noFace', noFace, noFace ? 0.95 : 0, { face_count: result.count }, NO_FACE_GRACE_MS, 'Candidate face is not visible');
|
|
this.transition('multipleFaces', multipleFaces, multipleFaces ? 0.95 : 0, { face_count: result.count }, 0, 'Multiple faces detected');
|
|
this.transition('lookingAway', lookingAwayResult.active, lookingAwayResult.confidence, lookingAwayResult.details, LOOKING_AWAY_GRACE_MS, 'Please look toward the camera');
|
|
this.trackProlongedLookingAway(lookingAwayResult, now);
|
|
}
|
|
|
|
transition(name, active, confidence, details, graceMs, message) {
|
|
if (active === this.lastStates[name]) return;
|
|
if (active && graceMs > 0 && this.durations.activeSince[name] && performance.now() - this.durations.activeSince[name] < graceMs) return;
|
|
this.lastStates[name] = active;
|
|
if (active) {
|
|
this.warning.show(message);
|
|
this.logger.record(name === 'lookingAway' ? 'looking_away' : name === 'noFace' ? 'face_missing' : 'multiple_faces', confidence, details, this.snapshot.capture());
|
|
}
|
|
}
|
|
|
|
trackProlongedLookingAway(lookingAwayResult, now) {
|
|
const prolongedEvent = this.prolongedLookingAway.update(lookingAwayResult.active, now);
|
|
if (!prolongedEvent) return;
|
|
|
|
this.warning.show(`Candidate has been looking away for over ${Math.round(PROLONGED_LOOKING_AWAY_MS / 1000)} seconds`);
|
|
this.logger.record('prolonged_looking_away', lookingAwayResult.confidence, {
|
|
...prolongedEvent,
|
|
...lookingAwayResult.details,
|
|
}, this.snapshot.capture());
|
|
}
|
|
|
|
processPhone(phone, now) {
|
|
if (!phone || now - this.lastPhoneEventAt < PHONE_EVENT_COOLDOWN_MS) return;
|
|
this.lastPhoneEventAt = now;
|
|
this.warning.show('Possible phone detected in camera frame');
|
|
this.logger.record('phone_detected', phone.confidence, { label: phone.label }, this.snapshot.capture());
|
|
}
|
|
|
|
stop() {
|
|
if (this.reported) return null;
|
|
this.running = false;
|
|
this.reported = true;
|
|
return this.logger.report(this.startedAt, new Date().toISOString(), this.durations.finish(performance.now()));
|
|
}
|
|
}
|
|
|
|
function bootCandidateProctoring() {
|
|
const video = document.getElementById('webcam');
|
|
const canvas = document.getElementById('proctor-canvas') || document.createElement('canvas');
|
|
const sessionId = document.body.dataset.interviewId;
|
|
if (!video || !sessionId || !navigator.mediaDevices) return;
|
|
const proctor = new CandidateProctoring({ sessionId, video, canvas });
|
|
window.candidateProctoring = proctor;
|
|
const startWhenReady = () => proctor.start();
|
|
if (video.readyState >= 2) startWhenReady();
|
|
video.addEventListener('loadeddata', startWhenReady, { once: true });
|
|
const callStatus = document.getElementById('call-status-badge');
|
|
if (callStatus) {
|
|
const statusObserver = new MutationObserver(() => {
|
|
if (/ended/i.test(callStatus.textContent || '')) proctor.stop();
|
|
});
|
|
statusObserver.observe(callStatus, { childList: true, characterData: true, subtree: true });
|
|
}
|
|
window.addEventListener('pagehide', () => proctor.stop(), { once: true });
|
|
}
|
|
|
|
if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', bootCandidateProctoring, { once: true });
|
|
else bootCandidateProctoring();
|