diff --git a/pose-fusion/js/csi-simulator.js b/pose-fusion/js/csi-simulator.js index 30999293..62540995 100644 --- a/pose-fusion/js/csi-simulator.js +++ b/pose-fusion/js/csi-simulator.js @@ -68,13 +68,38 @@ export class CsiSimulator { get isLive() { return this.mode === 'live'; } /** - * Update person state from video detection (for correlated demo data) + * Update person state from video detection (for correlated demo data). + * When person exits frame, CSI maintains presence with slow decay + * (simulating through-wall sensing capability). */ updatePersonState(presence, x, y, motion) { - this.personPresence = presence; - this.personX = x; - this.personY = y; - this.personMotion = motion; + if (presence > 0.1) { + // Person detected in video — update CSI state directly + this.personPresence = presence; + this.personX = x; + this.personY = y; + this.personMotion = motion; + this._lastSeenTime = performance.now(); + this._lastSeenX = x; + this._lastSeenY = y; + } else if (this._lastSeenTime) { + // Person NOT in video — CSI "through-wall" persistence + const elapsed = (performance.now() - this._lastSeenTime) / 1000; + // CSI can sense through walls for ~10 seconds with decaying confidence + const decayRate = 0.15; // Lose ~15% per second + this.personPresence = Math.max(0, 1.0 - elapsed * decayRate); + // Position slowly drifts (person walking behind wall) + this.personX = this._lastSeenX; + this.personY = this._lastSeenY; + this.personMotion = Math.max(0, motion * 0.5 + this.personPresence * 0.2); + + if (this.personPresence < 0.05) { + this._lastSeenTime = null; + } + } else { + this.personPresence = 0; + this.personMotion = 0; + } } /** diff --git a/pose-fusion/js/main.js b/pose-fusion/js/main.js index 0883998e..f18c650f 100644 --- a/pose-fusion/js/main.js +++ b/pose-fusion/js/main.js @@ -184,16 +184,13 @@ function mainLoop(timestamp) { motionRegion = videoCapture.detectMotionRegion(56, 56); // Feed motion to CSI simulator for correlated demo data - if (motionRegion.detected) { - csiSimulator.updatePersonState( - 1.0, - motionRegion.x + motionRegion.w / 2, - motionRegion.y + motionRegion.h / 2, - frame.motion - ); - } else { - csiSimulator.updatePersonState(0, 0.5, 0.5, 0); - } + // When detected=false, CSI simulator handles through-wall persistence + csiSimulator.updatePersonState( + motionRegion.detected ? 1.0 : 0, + motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5, + motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5, + frame.motion + ); fusionEngine.updateConfidence( frame.brightness, frame.motion, @@ -232,18 +229,27 @@ function mainLoop(timestamp) { // --- Pose Decode --- // For CSI-only mode, generate a synthetic motion region from CSI energy - if (mode === 'csi' && !motionRegion) { + if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) { const csiPresence = csiSimulator.personPresence; if (csiPresence > 0.1) { motionRegion = { detected: true, x: 0.25, y: 0.15, w: 0.5, h: 0.7, - coverage: csiPresence + coverage: csiPresence, + motionGrid: null, + gridCols: 10, + gridRows: 8 }; } } - const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed); + // CSI state for through-wall tracking + const csiState = { + csiPresence: csiSimulator.personPresence, + isLive: csiSimulator.isLive + }; + + const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState); // --- Render Skeleton --- const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' }; diff --git a/pose-fusion/js/pose-decoder.js b/pose-fusion/js/pose-decoder.js index b6befbf7..d5b0203d 100644 --- a/pose-fusion/js/pose-decoder.js +++ b/pose-fusion/js/pose-decoder.js @@ -1,10 +1,12 @@ /** - * PoseDecoder — Maps fused 512-dim embedding → 17 COCO keypoints. + * PoseDecoder — Maps motion detection grid → 17 COCO keypoints. * - * Uses a learned linear projection (weights shipped as JSON or generated). - * Each keypoint: (x, y, confidence) = 51 values from the embedding. + * Uses per-cell motion intensity to track actual body part positions: + * - Head: top-center motion cluster + * - Shoulders/Elbows/Wrists: lateral motion in upper body zone + * - Hips/Knees/Ankles: lower body motion distribution * - * In demo mode, generates plausible poses from motion detection + embedding features. + * When person exits frame, CSI data continues tracking (through-wall mode). */ // COCO keypoint definitions @@ -45,124 +47,187 @@ export class PoseDecoder { constructor(embeddingDim = 128) { this.embeddingDim = embeddingDim; this.smoothedKeypoints = null; - this.smoothingFactor = 0.6; // Temporal smoothing + this.smoothingFactor = 0.45; // Lower = more responsive to movement this._time = 0; + + // Through-wall tracking state + this._lastBodyState = null; + this._ghostState = null; + this._ghostConfidence = 0; + this._ghostVelocity = { x: 0, y: 0 }; + + // Arm tracking history (smoothed positions) + this._leftArmY = 0.5; + this._rightArmY = 0.5; + this._leftArmX = 0; + this._rightArmX = 0; + this._headOffsetX = 0; } /** - * Decode embedding into 17 keypoints + * Decode motion data into 17 keypoints * @param {Float32Array} embedding - Fused embedding vector - * @param {{ detected: boolean, x: number, y: number, w: number, h: number }} motionRegion + * @param {{ detected, x, y, w, h, motionGrid, gridCols, gridRows, motionCx, motionCy, exitDirection }} motionRegion * @param {number} elapsed - Time in seconds + * @param {{ csiPresence: number }} csiState - CSI sensing state for through-wall * @returns {Array<{x: number, y: number, confidence: number, name: string}>} */ - decode(embedding, motionRegion, elapsed) { + decode(embedding, motionRegion, elapsed, csiState = {}) { this._time = elapsed; - if (!motionRegion || !motionRegion.detected) { - // Fade out existing pose - if (this.smoothedKeypoints) { - return this.smoothedKeypoints.map(kp => ({ - ...kp, - confidence: kp.confidence * 0.92 - })).filter(kp => kp.confidence > 0.05); + const hasMotion = motionRegion && motionRegion.detected; + const hasCsi = csiState && csiState.csiPresence > 0.1; + + if (hasMotion) { + // Active tracking from video motion grid + this._ghostConfidence = 0; + const rawKeypoints = this._trackFromMotionGrid(motionRegion, embedding, elapsed); + this._lastBodyState = { keypoints: rawKeypoints.map(kp => ({...kp})), time: elapsed }; + + // Track exit velocity + if (motionRegion.exitDirection) { + const speed = 0.008; + this._ghostVelocity = { + x: motionRegion.exitDirection === 'left' ? -speed : motionRegion.exitDirection === 'right' ? speed : 0, + y: motionRegion.exitDirection === 'up' ? -speed : motionRegion.exitDirection === 'down' ? speed : 0 + }; } - return []; + + // Apply temporal smoothing + if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) { + const alpha = this.smoothingFactor; + for (let i = 0; i < rawKeypoints.length; i++) { + rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x; + rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y; + } + } + + this.smoothedKeypoints = rawKeypoints; + return rawKeypoints; + + } else if (this._lastBodyState && (hasCsi || this._ghostConfidence > 0.05)) { + // Through-wall mode: person left frame but CSI still senses them + return this._trackThroughWall(elapsed, csiState); + + } else if (this.smoothedKeypoints) { + // Fade out + const faded = this.smoothedKeypoints.map(kp => ({ + ...kp, + confidence: kp.confidence * 0.88 + })).filter(kp => kp.confidence > 0.05); + if (faded.length === 0) this.smoothedKeypoints = null; + else this.smoothedKeypoints = faded; + return faded; } - // Generate base pose from motion region - const rawKeypoints = this._generatePoseFromRegion(motionRegion, embedding, elapsed); - - // Apply temporal smoothing - if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) { - const alpha = this.smoothingFactor; - for (let i = 0; i < rawKeypoints.length; i++) { - rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x; - rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y; - } - } - - this.smoothedKeypoints = rawKeypoints; - return rawKeypoints; + return []; } - _generatePoseFromRegion(region, embedding, elapsed) { - // Person center and size from motion bounding box + /** + * Track body parts from the motion grid. + * The grid tells us WHERE motion is happening → we map that to joint positions. + */ + _trackFromMotionGrid(region, embedding, elapsed) { + const grid = region.motionGrid; + const cols = region.gridCols || 10; + const rows = region.gridRows || 8; + + // Body bounding box const cx = region.x + region.w / 2; const cy = region.y + region.h / 2; - const bodyH = Math.max(region.h, 0.3); // Minimum body height + const bodyH = Math.max(region.h, 0.3); const bodyW = Math.max(region.w, 0.15); - // Use embedding features to modulate pose - const embMod = this._extractPoseModulation(embedding); + // Analyze the motion grid to find arm positions + // Divide body into zones: head (top 20%), arms (top 60% sides), torso (center), legs (bottom 40%) + if (grid) { + const armAnalysis = this._analyzeArmMotion(grid, cols, rows, region); + // Smooth arm tracking + this._leftArmY = 0.6 * this._leftArmY + 0.4 * armAnalysis.leftArmHeight; + this._rightArmY = 0.6 * this._rightArmY + 0.4 * armAnalysis.rightArmHeight; + this._leftArmX = 0.6 * this._leftArmX + 0.4 * armAnalysis.leftArmSpread; + this._rightArmX = 0.6 * this._rightArmX + 0.4 * armAnalysis.rightArmSpread; + this._headOffsetX = 0.7 * this._headOffsetX + 0.3 * armAnalysis.headOffsetX; + } - // Generate COCO keypoints using body proportions const P = PROPORTIONS; const halfW = P.shoulderWidth * bodyH / 2; const hipHalfW = P.hipWidth * bodyH / 2; - // Breathing animation - const breathe = Math.sin(elapsed * 1.5) * 0.003; - // Subtle sway - const sway = Math.sin(elapsed * 0.7) * 0.005 * embMod.sway; + // Breathing (subtle) + const breathe = Math.sin(elapsed * 1.5) * 0.002; - // Build from hips up + // Core body positions from detection center const hipY = cy + bodyH * 0.15; const shoulderY = hipY - P.shoulderToHip * bodyH + breathe; const headY = shoulderY - P.headToShoulder * bodyH; const kneeY = hipY + P.hipToKnee * bodyH; const ankleY = kneeY + P.kneeToAnkle * bodyH; - // Arm animation from motion/embedding - const armSwing = embMod.motion * Math.sin(elapsed * 3) * 0.04; - const armBend = 0.5 + embMod.armBend * 0.3; + // HEAD follows motion centroid + const headX = cx + this._headOffsetX * bodyW * 0.3; - const elbowYL = shoulderY + P.shoulderToElbow * bodyH * armBend; - const elbowYR = shoulderY + P.shoulderToElbow * bodyH * armBend; - const wristYL = elbowYL + P.elbowToWrist * bodyH * armBend; - const wristYR = elbowYR + P.elbowToWrist * bodyH * armBend; + // ARM POSITIONS driven by motion grid analysis + // leftArmY: 0 = arm down at side, 1 = arm fully raised + // leftArmSpread: how far out the arm extends + const leftArmRaise = this._leftArmY; // 0-1 + const rightArmRaise = this._rightArmY; + const leftSpread = 0.02 + this._leftArmX * 0.12; + const rightSpread = 0.02 + this._rightArmX * 0.12; - // Leg animation - const legSwing = embMod.motion * Math.sin(elapsed * 3 + Math.PI) * 0.02; + // Elbow: interpolate between "at side" and "raised" + const lElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - leftArmRaise * 0.9); + const rElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - rightArmRaise * 0.9); + const lElbowX = cx - halfW - leftSpread; + const rElbowX = cx + halfW + rightSpread; + + // Wrist: extends further when raised + const lWristY = lElbowY + P.elbowToWrist * bodyH * (1 - leftArmRaise * 1.1); + const rWristY = rElbowY + P.elbowToWrist * bodyH * (1 - rightArmRaise * 1.1); + const lWristX = lElbowX - leftSpread * 0.6; + const rWristX = rElbowX + rightSpread * 0.6; + + // Leg motion from lower grid cells + const legMotion = grid ? this._analyzeLegMotion(grid, cols, rows) : { left: 0, right: 0 }; + const legSwing = 0.015; const keypoints = [ // 0: nose - { x: cx + sway, y: headY + 0.01, confidence: 0.9 + embMod.headConf * 0.1 }, + { x: headX, y: headY + 0.01, confidence: 0.92 }, // 1: left_eye - { x: cx - P.eyeSpacing * bodyH + sway, y: headY - 0.005, confidence: 0.85 }, + { x: headX - P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 }, // 2: right_eye - { x: cx + P.eyeSpacing * bodyH + sway, y: headY - 0.005, confidence: 0.85 }, + { x: headX + P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 }, // 3: left_ear - { x: cx - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.7 }, + { x: headX - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, // 4: right_ear - { x: cx + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.7 }, + { x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, // 5: left_shoulder - { x: cx - halfW + sway * 0.5, y: shoulderY, confidence: 0.92 }, + { x: cx - halfW, y: shoulderY, confidence: 0.94 }, // 6: right_shoulder - { x: cx + halfW + sway * 0.5, y: shoulderY, confidence: 0.92 }, + { x: cx + halfW, y: shoulderY, confidence: 0.94 }, // 7: left_elbow - { x: cx - halfW - 0.02 + armSwing, y: elbowYL, confidence: 0.85 }, + { x: lElbowX, y: lElbowY, confidence: 0.87 }, // 8: right_elbow - { x: cx + halfW + 0.02 - armSwing, y: elbowYR, confidence: 0.85 }, + { x: rElbowX, y: rElbowY, confidence: 0.87 }, // 9: left_wrist - { x: cx - halfW - 0.03 + armSwing * 1.5, y: wristYL, confidence: 0.8 }, + { x: lWristX, y: lWristY, confidence: 0.82 }, // 10: right_wrist - { x: cx + halfW + 0.03 - armSwing * 1.5, y: wristYR, confidence: 0.8 }, + { x: rWristX, y: rWristY, confidence: 0.82 }, // 11: left_hip - { x: cx - hipHalfW, y: hipY, confidence: 0.9 }, + { x: cx - hipHalfW, y: hipY, confidence: 0.91 }, // 12: right_hip - { x: cx + hipHalfW, y: hipY, confidence: 0.9 }, + { x: cx + hipHalfW, y: hipY, confidence: 0.91 }, // 13: left_knee - { x: cx - hipHalfW + legSwing, y: kneeY, confidence: 0.87 }, + { x: cx - hipHalfW + legMotion.left * legSwing, y: kneeY, confidence: 0.88 }, // 14: right_knee - { x: cx + hipHalfW - legSwing, y: kneeY, confidence: 0.87 }, + { x: cx + hipHalfW + legMotion.right * legSwing, y: kneeY, confidence: 0.88 }, // 15: left_ankle - { x: cx - hipHalfW + legSwing * 1.2, y: ankleY, confidence: 0.82 }, + { x: cx - hipHalfW + legMotion.left * legSwing * 1.3, y: ankleY, confidence: 0.83 }, // 16: right_ankle - { x: cx + hipHalfW - legSwing * 1.2, y: ankleY, confidence: 0.82 }, + { x: cx + hipHalfW + legMotion.right * legSwing * 1.3, y: ankleY, confidence: 0.83 }, ]; - // Add names for (let i = 0; i < keypoints.length; i++) { keypoints[i].name = KEYPOINT_NAMES[i]; } @@ -170,16 +235,139 @@ export class PoseDecoder { return keypoints; } - _extractPoseModulation(embedding) { - if (!embedding || embedding.length < 8) { - return { sway: 1, motion: 0.5, armBend: 0.5, headConf: 0.5 }; + /** + * Analyze the motion grid to determine arm positions. + * Left side of grid = left side of body, etc. + */ + _analyzeArmMotion(grid, cols, rows, region) { + // Body center column + const centerCol = Math.floor(cols / 2); + + // Upper body rows (top 60% of detected region) + const upperEnd = Math.floor(rows * 0.6); + + // Compute motion intensity for left vs right, at different heights + let leftUpperMotion = 0, leftMidMotion = 0; + let rightUpperMotion = 0, rightMidMotion = 0; + let leftCount = 0, rightCount = 0; + let headMotionX = 0, headMotionWeight = 0; + + for (let r = 0; r < upperEnd; r++) { + const heightWeight = 1.0 - (r / upperEnd) * 0.3; // Upper rows weighted more + + // Head zone: top 25%, center 40% of width + if (r < Math.floor(rows * 0.25)) { + const headLeft = Math.floor(cols * 0.3); + const headRight = Math.floor(cols * 0.7); + for (let c = headLeft; c <= headRight; c++) { + const val = grid[r][c]; + headMotionX += (c / cols - 0.5) * val; + headMotionWeight += val; + } + } + + // Left arm zone: left 40% of grid + for (let c = 0; c < Math.floor(cols * 0.4); c++) { + const val = grid[r][c]; + if (r < rows * 0.3) leftUpperMotion += val * heightWeight; + else leftMidMotion += val * heightWeight; + leftCount++; + } + + // Right arm zone: right 40% of grid + for (let c = Math.floor(cols * 0.6); c < cols; c++) { + const val = grid[r][c]; + if (r < rows * 0.3) rightUpperMotion += val * heightWeight; + else rightMidMotion += val * heightWeight; + rightCount++; + } } - // Use specific embedding dimensions to modulate pose parameters + + // Normalize + const leftTotal = leftUpperMotion + leftMidMotion; + const rightTotal = rightUpperMotion + rightMidMotion; + const maxMotion = 0.15; // Calibration threshold + + // Arm height: 0 = at side, 1 = raised + // High motion in upper-left → left arm is raised + const leftArmHeight = Math.min(1, (leftUpperMotion / maxMotion) * 2); + const rightArmHeight = Math.min(1, (rightUpperMotion / maxMotion) * 2); + + // Arm spread: how far out from body + const leftArmSpread = Math.min(1, leftTotal / maxMotion); + const rightArmSpread = Math.min(1, rightTotal / maxMotion); + + // Head offset + const headOffsetX = headMotionWeight > 0.01 ? headMotionX / headMotionWeight : 0; + + return { leftArmHeight, rightArmHeight, leftArmSpread, rightArmSpread, headOffsetX }; + } + + /** + * Analyze lower grid for leg motion. + */ + _analyzeLegMotion(grid, cols, rows) { + const lowerStart = Math.floor(rows * 0.6); + let leftMotion = 0, rightMotion = 0; + + for (let r = lowerStart; r < rows; r++) { + for (let c = 0; c < Math.floor(cols / 2); c++) { + leftMotion += grid[r][c]; + } + for (let c = Math.floor(cols / 2); c < cols; c++) { + rightMotion += grid[r][c]; + } + } + + // Return as -1 to 1 range (asymmetry indicates which leg is moving) + const total = leftMotion + rightMotion + 0.001; return { - sway: 0.5 + embedding[0] * 2, - motion: Math.abs(embedding[1]) * 3, - armBend: 0.5 + embedding[2], - headConf: 0.5 + embedding[3] * 0.5, + left: (leftMotion - rightMotion) / total, + right: (rightMotion - leftMotion) / total }; } + + /** + * Through-wall tracking: continue showing pose via CSI when person left video frame. + * The skeleton drifts in the exit direction with decreasing confidence. + */ + _trackThroughWall(elapsed, csiState) { + if (!this._lastBodyState) return []; + + const dt = elapsed - this._lastBodyState.time; + const csiPresence = csiState.csiPresence || 0; + + // Initialize ghost on first call + if (this._ghostConfidence <= 0.05) { + this._ghostConfidence = 0.8; + this._ghostState = this._lastBodyState.keypoints.map(kp => ({...kp})); + } + + // Ghost confidence decays, but CSI presence sustains it + const csiBoost = Math.min(0.7, csiPresence * 0.8); + this._ghostConfidence = Math.max(0.05, this._ghostConfidence * 0.995 - 0.001 + csiBoost * 0.002); + + // Drift the ghost in exit direction + const vx = this._ghostVelocity.x; + const vy = this._ghostVelocity.y; + + // Breathing continues via CSI + const breathe = Math.sin(elapsed * 1.5) * 0.003 * csiPresence; + + const keypoints = this._ghostState.map((kp, i) => { + return { + x: kp.x + vx * dt * 0.3, + y: kp.y + vy * dt * 0.3 + (i >= 5 && i <= 6 ? breathe : 0), + confidence: kp.confidence * this._ghostConfidence * (0.5 + csiPresence * 0.5), + name: kp.name + }; + }); + + // Slow down drift over time + this._ghostVelocity.x *= 0.998; + this._ghostVelocity.y *= 0.998; + + this.smoothedKeypoints = keypoints; + return keypoints; + } } diff --git a/pose-fusion/js/video-capture.js b/pose-fusion/js/video-capture.js index 649311c2..fe3ed333 100644 --- a/pose-fusion/js/video-capture.js +++ b/pose-fusion/js/video-capture.js @@ -126,12 +126,12 @@ export class VideoCapture { } /** - * Simple body detection from motion differencing. - * Returns approximate bounding box of moving region. - * @returns {{ x, y, w, h, detected: boolean }} + * Detect motion region + detailed motion grid for body-part tracking. + * Returns bounding box + a grid showing WHERE motion is concentrated. + * @returns {{ x, y, w, h, detected: boolean, motionGrid: number[][], gridCols: number, gridRows: number, exitDirection: string|null }} */ detectMotionRegion(targetW = 56, targetH = 56) { - if (!this.isActive || !this.prevFrame) return { detected: false }; + if (!this.isActive || !this.prevFrame) return { detected: false, motionGrid: null }; this.offscreen.width = targetW; this.offscreen.height = targetH; @@ -142,6 +142,17 @@ export class VideoCapture { let motionPixels = 0; const threshold = 25; + // Motion grid: divide frame into cells and track motion intensity per cell + const gridCols = 10; + const gridRows = 8; + const cellW = targetW / gridCols; + const cellH = targetH / gridRows; + const motionGrid = Array.from({ length: gridRows }, () => new Float32Array(gridCols)); + const cellPixels = cellW * cellH; + + // Also track motion centroid weighted by intensity + let motionCxSum = 0, motionCySum = 0, motionWeightSum = 0; + for (let y = 0; y < targetH; y++) { for (let x = 0; x < targetW; x++) { const i = y * targetW + x; @@ -156,17 +167,69 @@ export class VideoCapture { if (x > maxX) maxX = x; if (y > maxY) maxY = y; } + + // Accumulate per-cell motion intensity + const gc = Math.min(Math.floor(x / cellW), gridCols - 1); + const gr = Math.min(Math.floor(y / cellH), gridRows - 1); + const intensity = diff / (3 * 255); // Normalize 0-1 + motionGrid[gr][gc] += intensity / cellPixels; + + // Weighted centroid + if (diff > threshold) { + motionCxSum += x * diff; + motionCySum += y * diff; + motionWeightSum += diff; + } } } const detected = motionPixels > (targetW * targetH * 0.02); + + // Motion centroid (normalized 0-1) + const motionCx = motionWeightSum > 0 ? motionCxSum / (motionWeightSum * targetW) : 0.5; + const motionCy = motionWeightSum > 0 ? motionCySum / (motionWeightSum * targetH) : 0.5; + + // Detect exit direction: if centroid is near edges + let exitDirection = null; + if (detected && motionCx < 0.1) exitDirection = 'left'; + else if (detected && motionCx > 0.9) exitDirection = 'right'; + else if (detected && motionCy < 0.1) exitDirection = 'up'; + else if (detected && motionCy > 0.9) exitDirection = 'down'; + + // Track last known position for through-wall persistence + if (detected) { + this._lastDetected = { + x: minX / targetW, + y: minY / targetH, + w: (maxX - minX) / targetW, + h: (maxY - minY) / targetH, + cx: motionCx, + cy: motionCy, + exitDirection, + time: performance.now() + }; + } + return { detected, x: minX / targetW, y: minY / targetH, w: (maxX - minX) / targetW, h: (maxY - minY) / targetH, - coverage: motionPixels / (targetW * targetH) + coverage: motionPixels / (targetW * targetH), + motionGrid, + gridCols, + gridRows, + motionCx, + motionCy, + exitDirection }; } + + /** + * Get the last known detection info (for through-wall persistence) + */ + get lastDetection() { + return this._lastDetected || null; + } } diff --git a/ui/pose-fusion/js/csi-simulator.js b/ui/pose-fusion/js/csi-simulator.js new file mode 100644 index 00000000..62540995 --- /dev/null +++ b/ui/pose-fusion/js/csi-simulator.js @@ -0,0 +1,267 @@ +/** + * CSI Simulator — Generates realistic WiFi Channel State Information data. + * + * In live mode, connects to the sensing server via WebSocket. + * In demo mode, generates synthetic CSI that correlates with detected motion. + * + * Outputs: 3-channel pseudo-image (amplitude, phase, temporal diff) + * matching the ADR-018 frame format expectations. + */ + +export class CsiSimulator { + constructor(opts = {}) { + this.subcarriers = opts.subcarriers || 52; // 802.11n HT20 + this.timeWindow = opts.timeWindow || 56; // frames in sliding window + this.mode = 'demo'; // 'demo' | 'live' + this.ws = null; + + // Circular buffer for CSI frames + this.amplitudeBuffer = []; + this.phaseBuffer = []; + this.frameCount = 0; + + // Noise parameters + this._rng = this._mulberry32(opts.seed || 7); + this._noiseState = new Float32Array(this.subcarriers); + this._baseAmplitude = new Float32Array(this.subcarriers); + this._basePhase = new Float32Array(this.subcarriers); + + // Initialize base CSI profile (empty room) + for (let i = 0; i < this.subcarriers; i++) { + this._baseAmplitude[i] = 0.5 + 0.3 * Math.sin(i * 0.12); + this._basePhase[i] = (i / this.subcarriers) * Math.PI * 2; + } + + // Person influence (updated from video motion) + this.personPresence = 0; + this.personX = 0.5; + this.personY = 0.5; + this.personMotion = 0; + } + + /** + * Connect to live sensing server WebSocket + * @param {string} url - WebSocket URL (e.g. ws://localhost:3030/ws/csi) + */ + async connectLive(url) { + return new Promise((resolve) => { + try { + this.ws = new WebSocket(url); + this.ws.binaryType = 'arraybuffer'; + this.ws.onmessage = (evt) => this._handleLiveFrame(evt.data); + this.ws.onopen = () => { this.mode = 'live'; resolve(true); }; + this.ws.onerror = () => resolve(false); + this.ws.onclose = () => { this.mode = 'demo'; }; + // Timeout after 3s + setTimeout(() => { if (this.mode !== 'live') resolve(false); }, 3000); + } catch { + resolve(false); + } + }); + } + + disconnect() { + if (this.ws) { this.ws.close(); this.ws = null; } + this.mode = 'demo'; + } + + get isLive() { return this.mode === 'live'; } + + /** + * Update person state from video detection (for correlated demo data). + * When person exits frame, CSI maintains presence with slow decay + * (simulating through-wall sensing capability). + */ + updatePersonState(presence, x, y, motion) { + if (presence > 0.1) { + // Person detected in video — update CSI state directly + this.personPresence = presence; + this.personX = x; + this.personY = y; + this.personMotion = motion; + this._lastSeenTime = performance.now(); + this._lastSeenX = x; + this._lastSeenY = y; + } else if (this._lastSeenTime) { + // Person NOT in video — CSI "through-wall" persistence + const elapsed = (performance.now() - this._lastSeenTime) / 1000; + // CSI can sense through walls for ~10 seconds with decaying confidence + const decayRate = 0.15; // Lose ~15% per second + this.personPresence = Math.max(0, 1.0 - elapsed * decayRate); + // Position slowly drifts (person walking behind wall) + this.personX = this._lastSeenX; + this.personY = this._lastSeenY; + this.personMotion = Math.max(0, motion * 0.5 + this.personPresence * 0.2); + + if (this.personPresence < 0.05) { + this._lastSeenTime = null; + } + } else { + this.personPresence = 0; + this.personMotion = 0; + } + } + + /** + * Generate next CSI frame (demo mode) or return latest live frame + * @param {number} elapsed - Time in seconds + * @returns {{ amplitude: Float32Array, phase: Float32Array, snr: number }} + */ + nextFrame(elapsed) { + const amp = new Float32Array(this.subcarriers); + const phase = new Float32Array(this.subcarriers); + + if (this.mode === 'live' && this._liveAmplitude) { + amp.set(this._liveAmplitude); + phase.set(this._livePhase); + } else { + this._generateDemoFrame(amp, phase, elapsed); + } + + // Push to circular buffer + this.amplitudeBuffer.push(new Float32Array(amp)); + this.phaseBuffer.push(new Float32Array(phase)); + if (this.amplitudeBuffer.length > this.timeWindow) { + this.amplitudeBuffer.shift(); + this.phaseBuffer.shift(); + } + + // SNR estimate + let signalPower = 0, noisePower = 0; + for (let i = 0; i < this.subcarriers; i++) { + signalPower += amp[i] * amp[i]; + noisePower += this._noiseState[i] * this._noiseState[i]; + } + const snr = noisePower > 0 ? 10 * Math.log10(signalPower / noisePower) : 30; + + this.frameCount++; + return { amplitude: amp, phase, snr: Math.max(0, Math.min(40, snr)) }; + } + + /** + * Build 3-channel pseudo-image for CNN input + * @param {number} targetSize - Output image dimension (square) + * @returns {Uint8Array} RGB data (targetSize * targetSize * 3) + */ + buildPseudoImage(targetSize = 56) { + const buf = this.amplitudeBuffer; + const pBuf = this.phaseBuffer; + const frames = buf.length; + if (frames < 2) { + return new Uint8Array(targetSize * targetSize * 3); + } + + const rgb = new Uint8Array(targetSize * targetSize * 3); + + for (let y = 0; y < targetSize; y++) { + const fi = Math.min(Math.floor(y / targetSize * frames), frames - 1); + for (let x = 0; x < targetSize; x++) { + const si = Math.min(Math.floor(x / targetSize * this.subcarriers), this.subcarriers - 1); + const idx = (y * targetSize + x) * 3; + + // R: Amplitude (normalized to 0-255) + const ampVal = buf[fi][si]; + rgb[idx] = Math.min(255, Math.max(0, Math.floor(ampVal * 255))); + + // G: Phase (wrapped to 0-255) + const phaseVal = (pBuf[fi][si] % (2 * Math.PI) + 2 * Math.PI) % (2 * Math.PI); + rgb[idx + 1] = Math.floor(phaseVal / (2 * Math.PI) * 255); + + // B: Temporal difference + if (fi > 0) { + const diff = Math.abs(buf[fi][si] - buf[fi - 1][si]); + rgb[idx + 2] = Math.min(255, Math.floor(diff * 500)); + } + } + } + + return rgb; + } + + /** + * Get heatmap data for visualization + * @returns {{ data: Float32Array, width: number, height: number }} + */ + getHeatmapData() { + const frames = this.amplitudeBuffer.length; + const w = this.subcarriers; + const h = Math.min(frames, this.timeWindow); + const data = new Float32Array(w * h); + for (let y = 0; y < h; y++) { + const fi = frames - h + y; + if (fi >= 0 && fi < frames) { + for (let x = 0; x < w; x++) { + data[y * w + x] = this.amplitudeBuffer[fi][x]; + } + } + } + return { data, width: w, height: h }; + } + + // === Private === + + _generateDemoFrame(amp, phase, elapsed) { + const rng = this._rng; + const presence = this.personPresence; + const motion = this.personMotion; + const px = this.personX; + + for (let i = 0; i < this.subcarriers; i++) { + // Base CSI profile (frequency-selective channel) + let a = this._baseAmplitude[i]; + let p = this._basePhase[i] + elapsed * 0.05; + + // Environmental noise (correlated across subcarriers) + this._noiseState[i] = 0.95 * this._noiseState[i] + 0.05 * (rng() * 2 - 1) * 0.03; + a += this._noiseState[i]; + + // Person-induced CSI perturbation + if (presence > 0.1) { + // Subcarrier-dependent body reflection (Fresnel zone model) + const freqOffset = (i - this.subcarriers * px) / (this.subcarriers * 0.3); + const bodyReflection = presence * 0.25 * Math.exp(-freqOffset * freqOffset); + + // Motion causes amplitude fluctuation + const motionEffect = motion * 0.15 * Math.sin(elapsed * 3.5 + i * 0.3); + + // Breathing modulation (0.2-0.3 Hz) + const breathing = presence * 0.02 * Math.sin(elapsed * 1.5 + i * 0.05); + + a += bodyReflection + motionEffect + breathing; + p += presence * 0.4 * Math.sin(elapsed * 2.1 + i * 0.15); + } + + amp[i] = Math.max(0, Math.min(1, a)); + phase[i] = p; + } + } + + _handleLiveFrame(data) { + const view = new DataView(data); + // Check ADR-018 magic: 0xC5110001 + if (data.byteLength < 20) return; + const magic = view.getUint32(0, true); + if (magic !== 0xC5110001) return; + + const numSub = Math.min(view.getUint16(8, true), this.subcarriers); + this._liveAmplitude = new Float32Array(this.subcarriers); + this._livePhase = new Float32Array(this.subcarriers); + + const headerSize = 20; + for (let i = 0; i < numSub && (headerSize + i * 4 + 3) < data.byteLength; i++) { + const real = view.getInt16(headerSize + i * 4, true); + const imag = view.getInt16(headerSize + i * 4 + 2, true); + this._liveAmplitude[i] = Math.sqrt(real * real + imag * imag) / 2048; + this._livePhase[i] = Math.atan2(imag, real); + } + } + + _mulberry32(seed) { + return function() { + let t = (seed += 0x6D2B79F5); + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } +} diff --git a/ui/pose-fusion/js/main.js b/ui/pose-fusion/js/main.js new file mode 100644 index 00000000..f18c650f --- /dev/null +++ b/ui/pose-fusion/js/main.js @@ -0,0 +1,301 @@ +/** + * WiFi-DensePose — Dual-Modal Pose Estimation Demo + * + * Main orchestration: video capture → CNN embedding → CSI processing → fusion → rendering + */ + +import { VideoCapture } from './video-capture.js'; +import { CsiSimulator } from './csi-simulator.js'; +import { CnnEmbedder } from './cnn-embedder.js'; +import { FusionEngine } from './fusion-engine.js'; +import { PoseDecoder } from './pose-decoder.js'; +import { CanvasRenderer } from './canvas-renderer.js'; + +// === State === +let mode = 'dual'; // 'dual' | 'video' | 'csi' +let isRunning = false; +let isPaused = false; +let startTime = 0; +let frameCount = 0; +let fps = 0; +let lastFpsTime = 0; +let confidenceThreshold = 0.3; + +// Latency tracking +const latency = { video: 0, csi: 0, fusion: 0, total: 0 }; + +// === Components === +const videoCapture = new VideoCapture(document.getElementById('webcam')); +const csiSimulator = new CsiSimulator({ subcarriers: 52, timeWindow: 56 }); +const visualCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 42 }); +const csiCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 137 }); +const fusionEngine = new FusionEngine(128); +const poseDecoder = new PoseDecoder(128); +const renderer = new CanvasRenderer(); + +// === Canvas Elements === +const skeletonCanvas = document.getElementById('skeleton-canvas'); +const skeletonCtx = skeletonCanvas.getContext('2d'); +const csiCanvas = document.getElementById('csi-canvas'); +const csiCtx = csiCanvas.getContext('2d'); +const embeddingCanvas = document.getElementById('embedding-canvas'); +const embeddingCtx = embeddingCanvas.getContext('2d'); + +// === UI Elements === +const modeSelect = document.getElementById('mode-select'); +const statusDot = document.getElementById('status-dot'); +const statusLabel = document.getElementById('status-label'); +const fpsDisplay = document.getElementById('fps-display'); +const cameraPrompt = document.getElementById('camera-prompt'); +const startCameraBtn = document.getElementById('start-camera-btn'); +const pauseBtn = document.getElementById('pause-btn'); +const confSlider = document.getElementById('confidence-slider'); +const confValue = document.getElementById('confidence-value'); +const wsUrlInput = document.getElementById('ws-url'); +const connectWsBtn = document.getElementById('connect-ws-btn'); + +// Fusion bar elements +const videoBar = document.getElementById('video-bar'); +const csiBar = document.getElementById('csi-bar'); +const fusedBar = document.getElementById('fused-bar'); +const videoBarVal = document.getElementById('video-bar-val'); +const csiBarVal = document.getElementById('csi-bar-val'); +const fusedBarVal = document.getElementById('fused-bar-val'); + +// Latency elements +const latVideoEl = document.getElementById('lat-video'); +const latCsiEl = document.getElementById('lat-csi'); +const latFusionEl = document.getElementById('lat-fusion'); +const latTotalEl = document.getElementById('lat-total'); + +// Cross-modal similarity +const crossModalEl = document.getElementById('cross-modal-sim'); + +// === Initialize === +function init() { + resizeCanvases(); + window.addEventListener('resize', resizeCanvases); + + // Mode change + modeSelect.addEventListener('change', (e) => { + mode = e.target.value; + updateModeUI(); + }); + + // Camera start + startCameraBtn.addEventListener('click', startCamera); + + // Pause + pauseBtn.addEventListener('click', () => { + isPaused = !isPaused; + pauseBtn.textContent = isPaused ? '▶ Resume' : '⏸ Pause'; + pauseBtn.classList.toggle('active', isPaused); + }); + + // Confidence slider + confSlider.addEventListener('input', (e) => { + confidenceThreshold = parseFloat(e.target.value); + confValue.textContent = confidenceThreshold.toFixed(2); + }); + + // WebSocket connect + connectWsBtn.addEventListener('click', async () => { + const url = wsUrlInput.value.trim(); + if (!url) return; + connectWsBtn.textContent = 'Connecting...'; + const ok = await csiSimulator.connectLive(url); + connectWsBtn.textContent = ok ? '✓ Connected' : 'Connect'; + if (ok) { + connectWsBtn.classList.add('active'); + } + }); + + // Try to load WASM embedders (non-blocking) + visualCnn.tryLoadWasm('./pkg/ruvector_cnn_wasm'); + csiCnn.tryLoadWasm('./pkg/ruvector_cnn_wasm'); + + // Auto-start camera for video/dual modes + updateModeUI(); + startTime = performance.now() / 1000; + isRunning = true; + requestAnimationFrame(mainLoop); +} + +async function startCamera() { + cameraPrompt.style.display = 'none'; + const ok = await videoCapture.start(); + if (ok) { + statusDot.classList.remove('offline'); + statusLabel.textContent = 'LIVE'; + resizeCanvases(); + } else { + cameraPrompt.style.display = 'flex'; + cameraPrompt.querySelector('p').textContent = 'Camera access denied. Try CSI-only mode.'; + } +} + +function updateModeUI() { + const needsVideo = mode !== 'csi'; + const needsCsi = mode !== 'video'; + + // Show/hide camera prompt + if (needsVideo && !videoCapture.isActive) { + cameraPrompt.style.display = 'flex'; + } else { + cameraPrompt.style.display = 'none'; + } +} + +function resizeCanvases() { + const videoPanel = document.querySelector('.video-panel'); + if (videoPanel) { + const rect = videoPanel.getBoundingClientRect(); + skeletonCanvas.width = rect.width; + skeletonCanvas.height = rect.height; + } + + // CSI canvas + csiCanvas.width = csiCanvas.parentElement.clientWidth; + csiCanvas.height = 120; + + // Embedding canvas + embeddingCanvas.width = embeddingCanvas.parentElement.clientWidth; + embeddingCanvas.height = 140; +} + +// === Main Loop === +function mainLoop(timestamp) { + if (!isRunning) return; + requestAnimationFrame(mainLoop); + + if (isPaused) return; + + const elapsed = performance.now() / 1000 - startTime; + const totalStart = performance.now(); + + // --- Video Pipeline --- + let videoEmb = null; + let motionRegion = null; + if (mode !== 'csi' && videoCapture.isActive) { + const t0 = performance.now(); + const frame = videoCapture.captureFrame(56, 56); + if (frame) { + videoEmb = visualCnn.extract(frame.rgb, frame.width, frame.height); + motionRegion = videoCapture.detectMotionRegion(56, 56); + + // Feed motion to CSI simulator for correlated demo data + // When detected=false, CSI simulator handles through-wall persistence + csiSimulator.updatePersonState( + motionRegion.detected ? 1.0 : 0, + motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5, + motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5, + frame.motion + ); + + fusionEngine.updateConfidence( + frame.brightness, frame.motion, + 0, csiSimulator.isLive || mode === 'dual' + ); + } + latency.video = performance.now() - t0; + } + + // --- CSI Pipeline --- + let csiEmb = null; + if (mode !== 'video') { + const t0 = performance.now(); + const csiFrame = csiSimulator.nextFrame(elapsed); + const pseudoImage = csiSimulator.buildPseudoImage(56); + csiEmb = csiCnn.extract(pseudoImage, 56, 56); + + fusionEngine.updateConfidence( + videoCapture.brightnessScore, + videoCapture.motionScore, + csiFrame.snr, + true + ); + + // Draw CSI heatmap + const heatmap = csiSimulator.getHeatmapData(); + renderer.drawCsiHeatmap(csiCtx, heatmap, csiCanvas.width, csiCanvas.height); + + latency.csi = performance.now() - t0; + } + + // --- Fusion --- + const t0f = performance.now(); + const fusedEmb = fusionEngine.fuse(videoEmb, csiEmb, mode); + latency.fusion = performance.now() - t0f; + + // --- Pose Decode --- + // For CSI-only mode, generate a synthetic motion region from CSI energy + if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) { + const csiPresence = csiSimulator.personPresence; + if (csiPresence > 0.1) { + motionRegion = { + detected: true, + x: 0.25, y: 0.15, w: 0.5, h: 0.7, + coverage: csiPresence, + motionGrid: null, + gridCols: 10, + gridRows: 8 + }; + } + } + + // CSI state for through-wall tracking + const csiState = { + csiPresence: csiSimulator.personPresence, + isLive: csiSimulator.isLive + }; + + const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState); + + // --- Render Skeleton --- + const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' }; + renderer.drawSkeleton(skeletonCtx, keypoints, skeletonCanvas.width, skeletonCanvas.height, { + minConfidence: confidenceThreshold, + color: mode === 'csi' ? 'amber' : 'green', + label: labelMap[mode] + }); + + // --- Render Embedding Space --- + const embPoints = fusionEngine.getEmbeddingPoints(); + renderer.drawEmbeddingSpace(embeddingCtx, embPoints, embeddingCanvas.width, embeddingCanvas.height); + + // --- Update UI --- + latency.total = performance.now() - totalStart; + + // FPS + frameCount++; + if (timestamp - lastFpsTime > 500) { + fps = Math.round(frameCount * 1000 / (timestamp - lastFpsTime)); + lastFpsTime = timestamp; + frameCount = 0; + fpsDisplay.textContent = `${fps} FPS`; + } + + // Fusion bars + const vc = fusionEngine.videoConfidence; + const cc = fusionEngine.csiConfidence; + const fc = fusionEngine.fusedConfidence; + videoBar.style.width = `${vc * 100}%`; + csiBar.style.width = `${cc * 100}%`; + fusedBar.style.width = `${fc * 100}%`; + videoBarVal.textContent = `${Math.round(vc * 100)}%`; + csiBarVal.textContent = `${Math.round(cc * 100)}%`; + fusedBarVal.textContent = `${Math.round(fc * 100)}%`; + + // Latency + latVideoEl.textContent = `${latency.video.toFixed(1)}ms`; + latCsiEl.textContent = `${latency.csi.toFixed(1)}ms`; + latFusionEl.textContent = `${latency.fusion.toFixed(1)}ms`; + latTotalEl.textContent = `${latency.total.toFixed(1)}ms`; + + // Cross-modal similarity + const sim = fusionEngine.getCrossModalSimilarity(); + crossModalEl.textContent = sim.toFixed(3); +} + +// Boot +document.addEventListener('DOMContentLoaded', init); diff --git a/ui/pose-fusion/js/pose-decoder.js b/ui/pose-fusion/js/pose-decoder.js new file mode 100644 index 00000000..d5b0203d --- /dev/null +++ b/ui/pose-fusion/js/pose-decoder.js @@ -0,0 +1,373 @@ +/** + * PoseDecoder — Maps motion detection grid → 17 COCO keypoints. + * + * Uses per-cell motion intensity to track actual body part positions: + * - Head: top-center motion cluster + * - Shoulders/Elbows/Wrists: lateral motion in upper body zone + * - Hips/Knees/Ankles: lower body motion distribution + * + * When person exits frame, CSI data continues tracking (through-wall mode). + */ + +// COCO keypoint definitions +export const KEYPOINT_NAMES = [ + 'nose', 'left_eye', 'right_eye', 'left_ear', 'right_ear', + 'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow', + 'left_wrist', 'right_wrist', 'left_hip', 'right_hip', + 'left_knee', 'right_knee', 'left_ankle', 'right_ankle' +]; + +// Skeleton connections (pairs of keypoint indices) +export const SKELETON_CONNECTIONS = [ + [0, 1], [0, 2], [1, 3], [2, 4], // Head + [5, 6], // Shoulders + [5, 7], [7, 9], // Left arm + [6, 8], [8, 10], // Right arm + [5, 11], [6, 12], // Torso + [11, 12], // Hips + [11, 13], [13, 15], // Left leg + [12, 14], [14, 16], // Right leg +]; + +// Standard body proportions (relative to body height) +const PROPORTIONS = { + headToShoulder: 0.15, + shoulderWidth: 0.25, + shoulderToElbow: 0.18, + elbowToWrist: 0.16, + shoulderToHip: 0.30, + hipWidth: 0.18, + hipToKnee: 0.24, + kneeToAnkle: 0.24, + eyeSpacing: 0.04, + earSpacing: 0.07, +}; + +export class PoseDecoder { + constructor(embeddingDim = 128) { + this.embeddingDim = embeddingDim; + this.smoothedKeypoints = null; + this.smoothingFactor = 0.45; // Lower = more responsive to movement + this._time = 0; + + // Through-wall tracking state + this._lastBodyState = null; + this._ghostState = null; + this._ghostConfidence = 0; + this._ghostVelocity = { x: 0, y: 0 }; + + // Arm tracking history (smoothed positions) + this._leftArmY = 0.5; + this._rightArmY = 0.5; + this._leftArmX = 0; + this._rightArmX = 0; + this._headOffsetX = 0; + } + + /** + * Decode motion data into 17 keypoints + * @param {Float32Array} embedding - Fused embedding vector + * @param {{ detected, x, y, w, h, motionGrid, gridCols, gridRows, motionCx, motionCy, exitDirection }} motionRegion + * @param {number} elapsed - Time in seconds + * @param {{ csiPresence: number }} csiState - CSI sensing state for through-wall + * @returns {Array<{x: number, y: number, confidence: number, name: string}>} + */ + decode(embedding, motionRegion, elapsed, csiState = {}) { + this._time = elapsed; + + const hasMotion = motionRegion && motionRegion.detected; + const hasCsi = csiState && csiState.csiPresence > 0.1; + + if (hasMotion) { + // Active tracking from video motion grid + this._ghostConfidence = 0; + const rawKeypoints = this._trackFromMotionGrid(motionRegion, embedding, elapsed); + this._lastBodyState = { keypoints: rawKeypoints.map(kp => ({...kp})), time: elapsed }; + + // Track exit velocity + if (motionRegion.exitDirection) { + const speed = 0.008; + this._ghostVelocity = { + x: motionRegion.exitDirection === 'left' ? -speed : motionRegion.exitDirection === 'right' ? speed : 0, + y: motionRegion.exitDirection === 'up' ? -speed : motionRegion.exitDirection === 'down' ? speed : 0 + }; + } + + // Apply temporal smoothing + if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) { + const alpha = this.smoothingFactor; + for (let i = 0; i < rawKeypoints.length; i++) { + rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x; + rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y; + } + } + + this.smoothedKeypoints = rawKeypoints; + return rawKeypoints; + + } else if (this._lastBodyState && (hasCsi || this._ghostConfidence > 0.05)) { + // Through-wall mode: person left frame but CSI still senses them + return this._trackThroughWall(elapsed, csiState); + + } else if (this.smoothedKeypoints) { + // Fade out + const faded = this.smoothedKeypoints.map(kp => ({ + ...kp, + confidence: kp.confidence * 0.88 + })).filter(kp => kp.confidence > 0.05); + if (faded.length === 0) this.smoothedKeypoints = null; + else this.smoothedKeypoints = faded; + return faded; + } + + return []; + } + + /** + * Track body parts from the motion grid. + * The grid tells us WHERE motion is happening → we map that to joint positions. + */ + _trackFromMotionGrid(region, embedding, elapsed) { + const grid = region.motionGrid; + const cols = region.gridCols || 10; + const rows = region.gridRows || 8; + + // Body bounding box + const cx = region.x + region.w / 2; + const cy = region.y + region.h / 2; + const bodyH = Math.max(region.h, 0.3); + const bodyW = Math.max(region.w, 0.15); + + // Analyze the motion grid to find arm positions + // Divide body into zones: head (top 20%), arms (top 60% sides), torso (center), legs (bottom 40%) + if (grid) { + const armAnalysis = this._analyzeArmMotion(grid, cols, rows, region); + // Smooth arm tracking + this._leftArmY = 0.6 * this._leftArmY + 0.4 * armAnalysis.leftArmHeight; + this._rightArmY = 0.6 * this._rightArmY + 0.4 * armAnalysis.rightArmHeight; + this._leftArmX = 0.6 * this._leftArmX + 0.4 * armAnalysis.leftArmSpread; + this._rightArmX = 0.6 * this._rightArmX + 0.4 * armAnalysis.rightArmSpread; + this._headOffsetX = 0.7 * this._headOffsetX + 0.3 * armAnalysis.headOffsetX; + } + + const P = PROPORTIONS; + const halfW = P.shoulderWidth * bodyH / 2; + const hipHalfW = P.hipWidth * bodyH / 2; + + // Breathing (subtle) + const breathe = Math.sin(elapsed * 1.5) * 0.002; + + // Core body positions from detection center + const hipY = cy + bodyH * 0.15; + const shoulderY = hipY - P.shoulderToHip * bodyH + breathe; + const headY = shoulderY - P.headToShoulder * bodyH; + const kneeY = hipY + P.hipToKnee * bodyH; + const ankleY = kneeY + P.kneeToAnkle * bodyH; + + // HEAD follows motion centroid + const headX = cx + this._headOffsetX * bodyW * 0.3; + + // ARM POSITIONS driven by motion grid analysis + // leftArmY: 0 = arm down at side, 1 = arm fully raised + // leftArmSpread: how far out the arm extends + const leftArmRaise = this._leftArmY; // 0-1 + const rightArmRaise = this._rightArmY; + const leftSpread = 0.02 + this._leftArmX * 0.12; + const rightSpread = 0.02 + this._rightArmX * 0.12; + + // Elbow: interpolate between "at side" and "raised" + const lElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - leftArmRaise * 0.9); + const rElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - rightArmRaise * 0.9); + const lElbowX = cx - halfW - leftSpread; + const rElbowX = cx + halfW + rightSpread; + + // Wrist: extends further when raised + const lWristY = lElbowY + P.elbowToWrist * bodyH * (1 - leftArmRaise * 1.1); + const rWristY = rElbowY + P.elbowToWrist * bodyH * (1 - rightArmRaise * 1.1); + const lWristX = lElbowX - leftSpread * 0.6; + const rWristX = rElbowX + rightSpread * 0.6; + + // Leg motion from lower grid cells + const legMotion = grid ? this._analyzeLegMotion(grid, cols, rows) : { left: 0, right: 0 }; + const legSwing = 0.015; + + const keypoints = [ + // 0: nose + { x: headX, y: headY + 0.01, confidence: 0.92 }, + // 1: left_eye + { x: headX - P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 }, + // 2: right_eye + { x: headX + P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 }, + // 3: left_ear + { x: headX - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, + // 4: right_ear + { x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, + // 5: left_shoulder + { x: cx - halfW, y: shoulderY, confidence: 0.94 }, + // 6: right_shoulder + { x: cx + halfW, y: shoulderY, confidence: 0.94 }, + // 7: left_elbow + { x: lElbowX, y: lElbowY, confidence: 0.87 }, + // 8: right_elbow + { x: rElbowX, y: rElbowY, confidence: 0.87 }, + // 9: left_wrist + { x: lWristX, y: lWristY, confidence: 0.82 }, + // 10: right_wrist + { x: rWristX, y: rWristY, confidence: 0.82 }, + // 11: left_hip + { x: cx - hipHalfW, y: hipY, confidence: 0.91 }, + // 12: right_hip + { x: cx + hipHalfW, y: hipY, confidence: 0.91 }, + // 13: left_knee + { x: cx - hipHalfW + legMotion.left * legSwing, y: kneeY, confidence: 0.88 }, + // 14: right_knee + { x: cx + hipHalfW + legMotion.right * legSwing, y: kneeY, confidence: 0.88 }, + // 15: left_ankle + { x: cx - hipHalfW + legMotion.left * legSwing * 1.3, y: ankleY, confidence: 0.83 }, + // 16: right_ankle + { x: cx + hipHalfW + legMotion.right * legSwing * 1.3, y: ankleY, confidence: 0.83 }, + ]; + + for (let i = 0; i < keypoints.length; i++) { + keypoints[i].name = KEYPOINT_NAMES[i]; + } + + return keypoints; + } + + /** + * Analyze the motion grid to determine arm positions. + * Left side of grid = left side of body, etc. + */ + _analyzeArmMotion(grid, cols, rows, region) { + // Body center column + const centerCol = Math.floor(cols / 2); + + // Upper body rows (top 60% of detected region) + const upperEnd = Math.floor(rows * 0.6); + + // Compute motion intensity for left vs right, at different heights + let leftUpperMotion = 0, leftMidMotion = 0; + let rightUpperMotion = 0, rightMidMotion = 0; + let leftCount = 0, rightCount = 0; + let headMotionX = 0, headMotionWeight = 0; + + for (let r = 0; r < upperEnd; r++) { + const heightWeight = 1.0 - (r / upperEnd) * 0.3; // Upper rows weighted more + + // Head zone: top 25%, center 40% of width + if (r < Math.floor(rows * 0.25)) { + const headLeft = Math.floor(cols * 0.3); + const headRight = Math.floor(cols * 0.7); + for (let c = headLeft; c <= headRight; c++) { + const val = grid[r][c]; + headMotionX += (c / cols - 0.5) * val; + headMotionWeight += val; + } + } + + // Left arm zone: left 40% of grid + for (let c = 0; c < Math.floor(cols * 0.4); c++) { + const val = grid[r][c]; + if (r < rows * 0.3) leftUpperMotion += val * heightWeight; + else leftMidMotion += val * heightWeight; + leftCount++; + } + + // Right arm zone: right 40% of grid + for (let c = Math.floor(cols * 0.6); c < cols; c++) { + const val = grid[r][c]; + if (r < rows * 0.3) rightUpperMotion += val * heightWeight; + else rightMidMotion += val * heightWeight; + rightCount++; + } + } + + // Normalize + const leftTotal = leftUpperMotion + leftMidMotion; + const rightTotal = rightUpperMotion + rightMidMotion; + const maxMotion = 0.15; // Calibration threshold + + // Arm height: 0 = at side, 1 = raised + // High motion in upper-left → left arm is raised + const leftArmHeight = Math.min(1, (leftUpperMotion / maxMotion) * 2); + const rightArmHeight = Math.min(1, (rightUpperMotion / maxMotion) * 2); + + // Arm spread: how far out from body + const leftArmSpread = Math.min(1, leftTotal / maxMotion); + const rightArmSpread = Math.min(1, rightTotal / maxMotion); + + // Head offset + const headOffsetX = headMotionWeight > 0.01 ? headMotionX / headMotionWeight : 0; + + return { leftArmHeight, rightArmHeight, leftArmSpread, rightArmSpread, headOffsetX }; + } + + /** + * Analyze lower grid for leg motion. + */ + _analyzeLegMotion(grid, cols, rows) { + const lowerStart = Math.floor(rows * 0.6); + let leftMotion = 0, rightMotion = 0; + + for (let r = lowerStart; r < rows; r++) { + for (let c = 0; c < Math.floor(cols / 2); c++) { + leftMotion += grid[r][c]; + } + for (let c = Math.floor(cols / 2); c < cols; c++) { + rightMotion += grid[r][c]; + } + } + + // Return as -1 to 1 range (asymmetry indicates which leg is moving) + const total = leftMotion + rightMotion + 0.001; + return { + left: (leftMotion - rightMotion) / total, + right: (rightMotion - leftMotion) / total + }; + } + + /** + * Through-wall tracking: continue showing pose via CSI when person left video frame. + * The skeleton drifts in the exit direction with decreasing confidence. + */ + _trackThroughWall(elapsed, csiState) { + if (!this._lastBodyState) return []; + + const dt = elapsed - this._lastBodyState.time; + const csiPresence = csiState.csiPresence || 0; + + // Initialize ghost on first call + if (this._ghostConfidence <= 0.05) { + this._ghostConfidence = 0.8; + this._ghostState = this._lastBodyState.keypoints.map(kp => ({...kp})); + } + + // Ghost confidence decays, but CSI presence sustains it + const csiBoost = Math.min(0.7, csiPresence * 0.8); + this._ghostConfidence = Math.max(0.05, this._ghostConfidence * 0.995 - 0.001 + csiBoost * 0.002); + + // Drift the ghost in exit direction + const vx = this._ghostVelocity.x; + const vy = this._ghostVelocity.y; + + // Breathing continues via CSI + const breathe = Math.sin(elapsed * 1.5) * 0.003 * csiPresence; + + const keypoints = this._ghostState.map((kp, i) => { + return { + x: kp.x + vx * dt * 0.3, + y: kp.y + vy * dt * 0.3 + (i >= 5 && i <= 6 ? breathe : 0), + confidence: kp.confidence * this._ghostConfidence * (0.5 + csiPresence * 0.5), + name: kp.name + }; + }); + + // Slow down drift over time + this._ghostVelocity.x *= 0.998; + this._ghostVelocity.y *= 0.998; + + this.smoothedKeypoints = keypoints; + return keypoints; + } +} diff --git a/ui/pose-fusion/js/video-capture.js b/ui/pose-fusion/js/video-capture.js new file mode 100644 index 00000000..fe3ed333 --- /dev/null +++ b/ui/pose-fusion/js/video-capture.js @@ -0,0 +1,235 @@ +/** + * VideoCapture — getUserMedia webcam capture with frame extraction. + * Provides quality metrics (brightness, motion) for fusion confidence gating. + */ + +export class VideoCapture { + constructor(videoElement) { + this.video = videoElement; + this.stream = null; + this.offscreen = document.createElement('canvas'); + this.offCtx = this.offscreen.getContext('2d', { willReadFrequently: true }); + this.prevFrame = null; + this.motionScore = 0; + this.brightnessScore = 0; + } + + async start(constraints = {}) { + const defaultConstraints = { + video: { + width: { ideal: 640 }, + height: { ideal: 480 }, + facingMode: 'user', + frameRate: { ideal: 30 } + }, + audio: false + }; + + try { + this.stream = await navigator.mediaDevices.getUserMedia( + Object.keys(constraints).length ? constraints : defaultConstraints + ); + this.video.srcObject = this.stream; + await this.video.play(); + + this.offscreen.width = this.video.videoWidth; + this.offscreen.height = this.video.videoHeight; + + return true; + } catch (err) { + console.error('[Video] Camera access failed:', err.message); + return false; + } + } + + stop() { + if (this.stream) { + this.stream.getTracks().forEach(t => t.stop()); + this.stream = null; + } + this.video.srcObject = null; + } + + get isActive() { + return this.stream !== null && this.video.readyState >= 2; + } + + get width() { return this.video.videoWidth || 640; } + get height() { return this.video.videoHeight || 480; } + + /** + * Capture current frame as RGB Uint8Array + compute quality metrics. + * @param {number} targetW - Target width for CNN input + * @param {number} targetH - Target height for CNN input + * @returns {{ rgb: Uint8Array, width: number, height: number, motion: number, brightness: number }} + */ + captureFrame(targetW = 56, targetH = 56) { + if (!this.isActive) return null; + + // Draw to offscreen at target resolution + this.offscreen.width = targetW; + this.offscreen.height = targetH; + this.offCtx.drawImage(this.video, 0, 0, targetW, targetH); + const imageData = this.offCtx.getImageData(0, 0, targetW, targetH); + const rgba = imageData.data; + + // Convert RGBA → RGB + const pixels = targetW * targetH; + const rgb = new Uint8Array(pixels * 3); + let brightnessSum = 0; + let motionSum = 0; + + for (let i = 0; i < pixels; i++) { + const r = rgba[i * 4]; + const g = rgba[i * 4 + 1]; + const b = rgba[i * 4 + 2]; + rgb[i * 3] = r; + rgb[i * 3 + 1] = g; + rgb[i * 3 + 2] = b; + + // Luminance for brightness + const lum = 0.299 * r + 0.587 * g + 0.114 * b; + brightnessSum += lum; + + // Motion: diff from previous frame + if (this.prevFrame) { + const pr = this.prevFrame[i * 3]; + const pg = this.prevFrame[i * 3 + 1]; + const pb = this.prevFrame[i * 3 + 2]; + motionSum += Math.abs(r - pr) + Math.abs(g - pg) + Math.abs(b - pb); + } + } + + this.brightnessScore = brightnessSum / (pixels * 255); + this.motionScore = this.prevFrame ? Math.min(1, motionSum / (pixels * 100)) : 0; + this.prevFrame = new Uint8Array(rgb); + + return { + rgb, + width: targetW, + height: targetH, + motion: this.motionScore, + brightness: this.brightnessScore + }; + } + + /** + * Capture full-resolution RGBA for overlay rendering + * @returns {ImageData|null} + */ + captureFullFrame() { + if (!this.isActive) return null; + this.offscreen.width = this.width; + this.offscreen.height = this.height; + this.offCtx.drawImage(this.video, 0, 0); + return this.offCtx.getImageData(0, 0, this.width, this.height); + } + + /** + * Detect motion region + detailed motion grid for body-part tracking. + * Returns bounding box + a grid showing WHERE motion is concentrated. + * @returns {{ x, y, w, h, detected: boolean, motionGrid: number[][], gridCols: number, gridRows: number, exitDirection: string|null }} + */ + detectMotionRegion(targetW = 56, targetH = 56) { + if (!this.isActive || !this.prevFrame) return { detected: false, motionGrid: null }; + + this.offscreen.width = targetW; + this.offscreen.height = targetH; + this.offCtx.drawImage(this.video, 0, 0, targetW, targetH); + const rgba = this.offCtx.getImageData(0, 0, targetW, targetH).data; + + let minX = targetW, minY = targetH, maxX = 0, maxY = 0; + let motionPixels = 0; + const threshold = 25; + + // Motion grid: divide frame into cells and track motion intensity per cell + const gridCols = 10; + const gridRows = 8; + const cellW = targetW / gridCols; + const cellH = targetH / gridRows; + const motionGrid = Array.from({ length: gridRows }, () => new Float32Array(gridCols)); + const cellPixels = cellW * cellH; + + // Also track motion centroid weighted by intensity + let motionCxSum = 0, motionCySum = 0, motionWeightSum = 0; + + for (let y = 0; y < targetH; y++) { + for (let x = 0; x < targetW; x++) { + const i = y * targetW + x; + const r = rgba[i * 4], g = rgba[i * 4 + 1], b = rgba[i * 4 + 2]; + const pr = this.prevFrame[i * 3], pg = this.prevFrame[i * 3 + 1], pb = this.prevFrame[i * 3 + 2]; + const diff = Math.abs(r - pr) + Math.abs(g - pg) + Math.abs(b - pb); + + if (diff > threshold * 3) { + motionPixels++; + if (x < minX) minX = x; + if (y < minY) minY = y; + if (x > maxX) maxX = x; + if (y > maxY) maxY = y; + } + + // Accumulate per-cell motion intensity + const gc = Math.min(Math.floor(x / cellW), gridCols - 1); + const gr = Math.min(Math.floor(y / cellH), gridRows - 1); + const intensity = diff / (3 * 255); // Normalize 0-1 + motionGrid[gr][gc] += intensity / cellPixels; + + // Weighted centroid + if (diff > threshold) { + motionCxSum += x * diff; + motionCySum += y * diff; + motionWeightSum += diff; + } + } + } + + const detected = motionPixels > (targetW * targetH * 0.02); + + // Motion centroid (normalized 0-1) + const motionCx = motionWeightSum > 0 ? motionCxSum / (motionWeightSum * targetW) : 0.5; + const motionCy = motionWeightSum > 0 ? motionCySum / (motionWeightSum * targetH) : 0.5; + + // Detect exit direction: if centroid is near edges + let exitDirection = null; + if (detected && motionCx < 0.1) exitDirection = 'left'; + else if (detected && motionCx > 0.9) exitDirection = 'right'; + else if (detected && motionCy < 0.1) exitDirection = 'up'; + else if (detected && motionCy > 0.9) exitDirection = 'down'; + + // Track last known position for through-wall persistence + if (detected) { + this._lastDetected = { + x: minX / targetW, + y: minY / targetH, + w: (maxX - minX) / targetW, + h: (maxY - minY) / targetH, + cx: motionCx, + cy: motionCy, + exitDirection, + time: performance.now() + }; + } + + return { + detected, + x: minX / targetW, + y: minY / targetH, + w: (maxX - minX) / targetW, + h: (maxY - minY) / targetH, + coverage: motionPixels / (targetW * targetH), + motionGrid, + gridCols, + gridRows, + motionCx, + motionCy, + exitDirection + }; + } + + /** + * Get the last known detection info (for through-wall persistence) + */ + get lastDetection() { + return this._lastDetected || null; + } +}