fix: motion-responsive skeleton + through-wall tracking
Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
ruv
parent
a387894689
commit
aeea8b2124
|
|
@ -68,13 +68,38 @@ export class CsiSimulator {
|
||||||
get isLive() { return this.mode === 'live'; }
|
get isLive() { return this.mode === 'live'; }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update person state from video detection (for correlated demo data)
|
* Update person state from video detection (for correlated demo data).
|
||||||
|
* When person exits frame, CSI maintains presence with slow decay
|
||||||
|
* (simulating through-wall sensing capability).
|
||||||
*/
|
*/
|
||||||
updatePersonState(presence, x, y, motion) {
|
updatePersonState(presence, x, y, motion) {
|
||||||
this.personPresence = presence;
|
if (presence > 0.1) {
|
||||||
this.personX = x;
|
// Person detected in video — update CSI state directly
|
||||||
this.personY = y;
|
this.personPresence = presence;
|
||||||
this.personMotion = motion;
|
this.personX = x;
|
||||||
|
this.personY = y;
|
||||||
|
this.personMotion = motion;
|
||||||
|
this._lastSeenTime = performance.now();
|
||||||
|
this._lastSeenX = x;
|
||||||
|
this._lastSeenY = y;
|
||||||
|
} else if (this._lastSeenTime) {
|
||||||
|
// Person NOT in video — CSI "through-wall" persistence
|
||||||
|
const elapsed = (performance.now() - this._lastSeenTime) / 1000;
|
||||||
|
// CSI can sense through walls for ~10 seconds with decaying confidence
|
||||||
|
const decayRate = 0.15; // Lose ~15% per second
|
||||||
|
this.personPresence = Math.max(0, 1.0 - elapsed * decayRate);
|
||||||
|
// Position slowly drifts (person walking behind wall)
|
||||||
|
this.personX = this._lastSeenX;
|
||||||
|
this.personY = this._lastSeenY;
|
||||||
|
this.personMotion = Math.max(0, motion * 0.5 + this.personPresence * 0.2);
|
||||||
|
|
||||||
|
if (this.personPresence < 0.05) {
|
||||||
|
this._lastSeenTime = null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.personPresence = 0;
|
||||||
|
this.personMotion = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -184,16 +184,13 @@ function mainLoop(timestamp) {
|
||||||
motionRegion = videoCapture.detectMotionRegion(56, 56);
|
motionRegion = videoCapture.detectMotionRegion(56, 56);
|
||||||
|
|
||||||
// Feed motion to CSI simulator for correlated demo data
|
// Feed motion to CSI simulator for correlated demo data
|
||||||
if (motionRegion.detected) {
|
// When detected=false, CSI simulator handles through-wall persistence
|
||||||
csiSimulator.updatePersonState(
|
csiSimulator.updatePersonState(
|
||||||
1.0,
|
motionRegion.detected ? 1.0 : 0,
|
||||||
motionRegion.x + motionRegion.w / 2,
|
motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5,
|
||||||
motionRegion.y + motionRegion.h / 2,
|
motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5,
|
||||||
frame.motion
|
frame.motion
|
||||||
);
|
);
|
||||||
} else {
|
|
||||||
csiSimulator.updatePersonState(0, 0.5, 0.5, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
fusionEngine.updateConfidence(
|
fusionEngine.updateConfidence(
|
||||||
frame.brightness, frame.motion,
|
frame.brightness, frame.motion,
|
||||||
|
|
@ -232,18 +229,27 @@ function mainLoop(timestamp) {
|
||||||
|
|
||||||
// --- Pose Decode ---
|
// --- Pose Decode ---
|
||||||
// For CSI-only mode, generate a synthetic motion region from CSI energy
|
// For CSI-only mode, generate a synthetic motion region from CSI energy
|
||||||
if (mode === 'csi' && !motionRegion) {
|
if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) {
|
||||||
const csiPresence = csiSimulator.personPresence;
|
const csiPresence = csiSimulator.personPresence;
|
||||||
if (csiPresence > 0.1) {
|
if (csiPresence > 0.1) {
|
||||||
motionRegion = {
|
motionRegion = {
|
||||||
detected: true,
|
detected: true,
|
||||||
x: 0.25, y: 0.15, w: 0.5, h: 0.7,
|
x: 0.25, y: 0.15, w: 0.5, h: 0.7,
|
||||||
coverage: csiPresence
|
coverage: csiPresence,
|
||||||
|
motionGrid: null,
|
||||||
|
gridCols: 10,
|
||||||
|
gridRows: 8
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed);
|
// CSI state for through-wall tracking
|
||||||
|
const csiState = {
|
||||||
|
csiPresence: csiSimulator.personPresence,
|
||||||
|
isLive: csiSimulator.isLive
|
||||||
|
};
|
||||||
|
|
||||||
|
const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState);
|
||||||
|
|
||||||
// --- Render Skeleton ---
|
// --- Render Skeleton ---
|
||||||
const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' };
|
const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' };
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,12 @@
|
||||||
/**
|
/**
|
||||||
* PoseDecoder — Maps fused 512-dim embedding → 17 COCO keypoints.
|
* PoseDecoder — Maps motion detection grid → 17 COCO keypoints.
|
||||||
*
|
*
|
||||||
* Uses a learned linear projection (weights shipped as JSON or generated).
|
* Uses per-cell motion intensity to track actual body part positions:
|
||||||
* Each keypoint: (x, y, confidence) = 51 values from the embedding.
|
* - Head: top-center motion cluster
|
||||||
|
* - Shoulders/Elbows/Wrists: lateral motion in upper body zone
|
||||||
|
* - Hips/Knees/Ankles: lower body motion distribution
|
||||||
*
|
*
|
||||||
* In demo mode, generates plausible poses from motion detection + embedding features.
|
* When person exits frame, CSI data continues tracking (through-wall mode).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// COCO keypoint definitions
|
// COCO keypoint definitions
|
||||||
|
|
@ -45,124 +47,187 @@ export class PoseDecoder {
|
||||||
constructor(embeddingDim = 128) {
|
constructor(embeddingDim = 128) {
|
||||||
this.embeddingDim = embeddingDim;
|
this.embeddingDim = embeddingDim;
|
||||||
this.smoothedKeypoints = null;
|
this.smoothedKeypoints = null;
|
||||||
this.smoothingFactor = 0.6; // Temporal smoothing
|
this.smoothingFactor = 0.45; // Lower = more responsive to movement
|
||||||
this._time = 0;
|
this._time = 0;
|
||||||
|
|
||||||
|
// Through-wall tracking state
|
||||||
|
this._lastBodyState = null;
|
||||||
|
this._ghostState = null;
|
||||||
|
this._ghostConfidence = 0;
|
||||||
|
this._ghostVelocity = { x: 0, y: 0 };
|
||||||
|
|
||||||
|
// Arm tracking history (smoothed positions)
|
||||||
|
this._leftArmY = 0.5;
|
||||||
|
this._rightArmY = 0.5;
|
||||||
|
this._leftArmX = 0;
|
||||||
|
this._rightArmX = 0;
|
||||||
|
this._headOffsetX = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Decode embedding into 17 keypoints
|
* Decode motion data into 17 keypoints
|
||||||
* @param {Float32Array} embedding - Fused embedding vector
|
* @param {Float32Array} embedding - Fused embedding vector
|
||||||
* @param {{ detected: boolean, x: number, y: number, w: number, h: number }} motionRegion
|
* @param {{ detected, x, y, w, h, motionGrid, gridCols, gridRows, motionCx, motionCy, exitDirection }} motionRegion
|
||||||
* @param {number} elapsed - Time in seconds
|
* @param {number} elapsed - Time in seconds
|
||||||
|
* @param {{ csiPresence: number }} csiState - CSI sensing state for through-wall
|
||||||
* @returns {Array<{x: number, y: number, confidence: number, name: string}>}
|
* @returns {Array<{x: number, y: number, confidence: number, name: string}>}
|
||||||
*/
|
*/
|
||||||
decode(embedding, motionRegion, elapsed) {
|
decode(embedding, motionRegion, elapsed, csiState = {}) {
|
||||||
this._time = elapsed;
|
this._time = elapsed;
|
||||||
|
|
||||||
if (!motionRegion || !motionRegion.detected) {
|
const hasMotion = motionRegion && motionRegion.detected;
|
||||||
// Fade out existing pose
|
const hasCsi = csiState && csiState.csiPresence > 0.1;
|
||||||
if (this.smoothedKeypoints) {
|
|
||||||
return this.smoothedKeypoints.map(kp => ({
|
if (hasMotion) {
|
||||||
...kp,
|
// Active tracking from video motion grid
|
||||||
confidence: kp.confidence * 0.92
|
this._ghostConfidence = 0;
|
||||||
})).filter(kp => kp.confidence > 0.05);
|
const rawKeypoints = this._trackFromMotionGrid(motionRegion, embedding, elapsed);
|
||||||
|
this._lastBodyState = { keypoints: rawKeypoints.map(kp => ({...kp})), time: elapsed };
|
||||||
|
|
||||||
|
// Track exit velocity
|
||||||
|
if (motionRegion.exitDirection) {
|
||||||
|
const speed = 0.008;
|
||||||
|
this._ghostVelocity = {
|
||||||
|
x: motionRegion.exitDirection === 'left' ? -speed : motionRegion.exitDirection === 'right' ? speed : 0,
|
||||||
|
y: motionRegion.exitDirection === 'up' ? -speed : motionRegion.exitDirection === 'down' ? speed : 0
|
||||||
|
};
|
||||||
}
|
}
|
||||||
return [];
|
|
||||||
|
// Apply temporal smoothing
|
||||||
|
if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) {
|
||||||
|
const alpha = this.smoothingFactor;
|
||||||
|
for (let i = 0; i < rawKeypoints.length; i++) {
|
||||||
|
rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x;
|
||||||
|
rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.smoothedKeypoints = rawKeypoints;
|
||||||
|
return rawKeypoints;
|
||||||
|
|
||||||
|
} else if (this._lastBodyState && (hasCsi || this._ghostConfidence > 0.05)) {
|
||||||
|
// Through-wall mode: person left frame but CSI still senses them
|
||||||
|
return this._trackThroughWall(elapsed, csiState);
|
||||||
|
|
||||||
|
} else if (this.smoothedKeypoints) {
|
||||||
|
// Fade out
|
||||||
|
const faded = this.smoothedKeypoints.map(kp => ({
|
||||||
|
...kp,
|
||||||
|
confidence: kp.confidence * 0.88
|
||||||
|
})).filter(kp => kp.confidence > 0.05);
|
||||||
|
if (faded.length === 0) this.smoothedKeypoints = null;
|
||||||
|
else this.smoothedKeypoints = faded;
|
||||||
|
return faded;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate base pose from motion region
|
return [];
|
||||||
const rawKeypoints = this._generatePoseFromRegion(motionRegion, embedding, elapsed);
|
|
||||||
|
|
||||||
// Apply temporal smoothing
|
|
||||||
if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) {
|
|
||||||
const alpha = this.smoothingFactor;
|
|
||||||
for (let i = 0; i < rawKeypoints.length; i++) {
|
|
||||||
rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x;
|
|
||||||
rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
this.smoothedKeypoints = rawKeypoints;
|
|
||||||
return rawKeypoints;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_generatePoseFromRegion(region, embedding, elapsed) {
|
/**
|
||||||
// Person center and size from motion bounding box
|
* Track body parts from the motion grid.
|
||||||
|
* The grid tells us WHERE motion is happening → we map that to joint positions.
|
||||||
|
*/
|
||||||
|
_trackFromMotionGrid(region, embedding, elapsed) {
|
||||||
|
const grid = region.motionGrid;
|
||||||
|
const cols = region.gridCols || 10;
|
||||||
|
const rows = region.gridRows || 8;
|
||||||
|
|
||||||
|
// Body bounding box
|
||||||
const cx = region.x + region.w / 2;
|
const cx = region.x + region.w / 2;
|
||||||
const cy = region.y + region.h / 2;
|
const cy = region.y + region.h / 2;
|
||||||
const bodyH = Math.max(region.h, 0.3); // Minimum body height
|
const bodyH = Math.max(region.h, 0.3);
|
||||||
const bodyW = Math.max(region.w, 0.15);
|
const bodyW = Math.max(region.w, 0.15);
|
||||||
|
|
||||||
// Use embedding features to modulate pose
|
// Analyze the motion grid to find arm positions
|
||||||
const embMod = this._extractPoseModulation(embedding);
|
// Divide body into zones: head (top 20%), arms (top 60% sides), torso (center), legs (bottom 40%)
|
||||||
|
if (grid) {
|
||||||
|
const armAnalysis = this._analyzeArmMotion(grid, cols, rows, region);
|
||||||
|
// Smooth arm tracking
|
||||||
|
this._leftArmY = 0.6 * this._leftArmY + 0.4 * armAnalysis.leftArmHeight;
|
||||||
|
this._rightArmY = 0.6 * this._rightArmY + 0.4 * armAnalysis.rightArmHeight;
|
||||||
|
this._leftArmX = 0.6 * this._leftArmX + 0.4 * armAnalysis.leftArmSpread;
|
||||||
|
this._rightArmX = 0.6 * this._rightArmX + 0.4 * armAnalysis.rightArmSpread;
|
||||||
|
this._headOffsetX = 0.7 * this._headOffsetX + 0.3 * armAnalysis.headOffsetX;
|
||||||
|
}
|
||||||
|
|
||||||
// Generate COCO keypoints using body proportions
|
|
||||||
const P = PROPORTIONS;
|
const P = PROPORTIONS;
|
||||||
const halfW = P.shoulderWidth * bodyH / 2;
|
const halfW = P.shoulderWidth * bodyH / 2;
|
||||||
const hipHalfW = P.hipWidth * bodyH / 2;
|
const hipHalfW = P.hipWidth * bodyH / 2;
|
||||||
|
|
||||||
// Breathing animation
|
// Breathing (subtle)
|
||||||
const breathe = Math.sin(elapsed * 1.5) * 0.003;
|
const breathe = Math.sin(elapsed * 1.5) * 0.002;
|
||||||
// Subtle sway
|
|
||||||
const sway = Math.sin(elapsed * 0.7) * 0.005 * embMod.sway;
|
|
||||||
|
|
||||||
// Build from hips up
|
// Core body positions from detection center
|
||||||
const hipY = cy + bodyH * 0.15;
|
const hipY = cy + bodyH * 0.15;
|
||||||
const shoulderY = hipY - P.shoulderToHip * bodyH + breathe;
|
const shoulderY = hipY - P.shoulderToHip * bodyH + breathe;
|
||||||
const headY = shoulderY - P.headToShoulder * bodyH;
|
const headY = shoulderY - P.headToShoulder * bodyH;
|
||||||
const kneeY = hipY + P.hipToKnee * bodyH;
|
const kneeY = hipY + P.hipToKnee * bodyH;
|
||||||
const ankleY = kneeY + P.kneeToAnkle * bodyH;
|
const ankleY = kneeY + P.kneeToAnkle * bodyH;
|
||||||
|
|
||||||
// Arm animation from motion/embedding
|
// HEAD follows motion centroid
|
||||||
const armSwing = embMod.motion * Math.sin(elapsed * 3) * 0.04;
|
const headX = cx + this._headOffsetX * bodyW * 0.3;
|
||||||
const armBend = 0.5 + embMod.armBend * 0.3;
|
|
||||||
|
|
||||||
const elbowYL = shoulderY + P.shoulderToElbow * bodyH * armBend;
|
// ARM POSITIONS driven by motion grid analysis
|
||||||
const elbowYR = shoulderY + P.shoulderToElbow * bodyH * armBend;
|
// leftArmY: 0 = arm down at side, 1 = arm fully raised
|
||||||
const wristYL = elbowYL + P.elbowToWrist * bodyH * armBend;
|
// leftArmSpread: how far out the arm extends
|
||||||
const wristYR = elbowYR + P.elbowToWrist * bodyH * armBend;
|
const leftArmRaise = this._leftArmY; // 0-1
|
||||||
|
const rightArmRaise = this._rightArmY;
|
||||||
|
const leftSpread = 0.02 + this._leftArmX * 0.12;
|
||||||
|
const rightSpread = 0.02 + this._rightArmX * 0.12;
|
||||||
|
|
||||||
// Leg animation
|
// Elbow: interpolate between "at side" and "raised"
|
||||||
const legSwing = embMod.motion * Math.sin(elapsed * 3 + Math.PI) * 0.02;
|
const lElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - leftArmRaise * 0.9);
|
||||||
|
const rElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - rightArmRaise * 0.9);
|
||||||
|
const lElbowX = cx - halfW - leftSpread;
|
||||||
|
const rElbowX = cx + halfW + rightSpread;
|
||||||
|
|
||||||
|
// Wrist: extends further when raised
|
||||||
|
const lWristY = lElbowY + P.elbowToWrist * bodyH * (1 - leftArmRaise * 1.1);
|
||||||
|
const rWristY = rElbowY + P.elbowToWrist * bodyH * (1 - rightArmRaise * 1.1);
|
||||||
|
const lWristX = lElbowX - leftSpread * 0.6;
|
||||||
|
const rWristX = rElbowX + rightSpread * 0.6;
|
||||||
|
|
||||||
|
// Leg motion from lower grid cells
|
||||||
|
const legMotion = grid ? this._analyzeLegMotion(grid, cols, rows) : { left: 0, right: 0 };
|
||||||
|
const legSwing = 0.015;
|
||||||
|
|
||||||
const keypoints = [
|
const keypoints = [
|
||||||
// 0: nose
|
// 0: nose
|
||||||
{ x: cx + sway, y: headY + 0.01, confidence: 0.9 + embMod.headConf * 0.1 },
|
{ x: headX, y: headY + 0.01, confidence: 0.92 },
|
||||||
// 1: left_eye
|
// 1: left_eye
|
||||||
{ x: cx - P.eyeSpacing * bodyH + sway, y: headY - 0.005, confidence: 0.85 },
|
{ x: headX - P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 },
|
||||||
// 2: right_eye
|
// 2: right_eye
|
||||||
{ x: cx + P.eyeSpacing * bodyH + sway, y: headY - 0.005, confidence: 0.85 },
|
{ x: headX + P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 },
|
||||||
// 3: left_ear
|
// 3: left_ear
|
||||||
{ x: cx - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.7 },
|
{ x: headX - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 },
|
||||||
// 4: right_ear
|
// 4: right_ear
|
||||||
{ x: cx + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.7 },
|
{ x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 },
|
||||||
// 5: left_shoulder
|
// 5: left_shoulder
|
||||||
{ x: cx - halfW + sway * 0.5, y: shoulderY, confidence: 0.92 },
|
{ x: cx - halfW, y: shoulderY, confidence: 0.94 },
|
||||||
// 6: right_shoulder
|
// 6: right_shoulder
|
||||||
{ x: cx + halfW + sway * 0.5, y: shoulderY, confidence: 0.92 },
|
{ x: cx + halfW, y: shoulderY, confidence: 0.94 },
|
||||||
// 7: left_elbow
|
// 7: left_elbow
|
||||||
{ x: cx - halfW - 0.02 + armSwing, y: elbowYL, confidence: 0.85 },
|
{ x: lElbowX, y: lElbowY, confidence: 0.87 },
|
||||||
// 8: right_elbow
|
// 8: right_elbow
|
||||||
{ x: cx + halfW + 0.02 - armSwing, y: elbowYR, confidence: 0.85 },
|
{ x: rElbowX, y: rElbowY, confidence: 0.87 },
|
||||||
// 9: left_wrist
|
// 9: left_wrist
|
||||||
{ x: cx - halfW - 0.03 + armSwing * 1.5, y: wristYL, confidence: 0.8 },
|
{ x: lWristX, y: lWristY, confidence: 0.82 },
|
||||||
// 10: right_wrist
|
// 10: right_wrist
|
||||||
{ x: cx + halfW + 0.03 - armSwing * 1.5, y: wristYR, confidence: 0.8 },
|
{ x: rWristX, y: rWristY, confidence: 0.82 },
|
||||||
// 11: left_hip
|
// 11: left_hip
|
||||||
{ x: cx - hipHalfW, y: hipY, confidence: 0.9 },
|
{ x: cx - hipHalfW, y: hipY, confidence: 0.91 },
|
||||||
// 12: right_hip
|
// 12: right_hip
|
||||||
{ x: cx + hipHalfW, y: hipY, confidence: 0.9 },
|
{ x: cx + hipHalfW, y: hipY, confidence: 0.91 },
|
||||||
// 13: left_knee
|
// 13: left_knee
|
||||||
{ x: cx - hipHalfW + legSwing, y: kneeY, confidence: 0.87 },
|
{ x: cx - hipHalfW + legMotion.left * legSwing, y: kneeY, confidence: 0.88 },
|
||||||
// 14: right_knee
|
// 14: right_knee
|
||||||
{ x: cx + hipHalfW - legSwing, y: kneeY, confidence: 0.87 },
|
{ x: cx + hipHalfW + legMotion.right * legSwing, y: kneeY, confidence: 0.88 },
|
||||||
// 15: left_ankle
|
// 15: left_ankle
|
||||||
{ x: cx - hipHalfW + legSwing * 1.2, y: ankleY, confidence: 0.82 },
|
{ x: cx - hipHalfW + legMotion.left * legSwing * 1.3, y: ankleY, confidence: 0.83 },
|
||||||
// 16: right_ankle
|
// 16: right_ankle
|
||||||
{ x: cx + hipHalfW - legSwing * 1.2, y: ankleY, confidence: 0.82 },
|
{ x: cx + hipHalfW + legMotion.right * legSwing * 1.3, y: ankleY, confidence: 0.83 },
|
||||||
];
|
];
|
||||||
|
|
||||||
// Add names
|
|
||||||
for (let i = 0; i < keypoints.length; i++) {
|
for (let i = 0; i < keypoints.length; i++) {
|
||||||
keypoints[i].name = KEYPOINT_NAMES[i];
|
keypoints[i].name = KEYPOINT_NAMES[i];
|
||||||
}
|
}
|
||||||
|
|
@ -170,16 +235,139 @@ export class PoseDecoder {
|
||||||
return keypoints;
|
return keypoints;
|
||||||
}
|
}
|
||||||
|
|
||||||
_extractPoseModulation(embedding) {
|
/**
|
||||||
if (!embedding || embedding.length < 8) {
|
* Analyze the motion grid to determine arm positions.
|
||||||
return { sway: 1, motion: 0.5, armBend: 0.5, headConf: 0.5 };
|
* Left side of grid = left side of body, etc.
|
||||||
|
*/
|
||||||
|
_analyzeArmMotion(grid, cols, rows, region) {
|
||||||
|
// Body center column
|
||||||
|
const centerCol = Math.floor(cols / 2);
|
||||||
|
|
||||||
|
// Upper body rows (top 60% of detected region)
|
||||||
|
const upperEnd = Math.floor(rows * 0.6);
|
||||||
|
|
||||||
|
// Compute motion intensity for left vs right, at different heights
|
||||||
|
let leftUpperMotion = 0, leftMidMotion = 0;
|
||||||
|
let rightUpperMotion = 0, rightMidMotion = 0;
|
||||||
|
let leftCount = 0, rightCount = 0;
|
||||||
|
let headMotionX = 0, headMotionWeight = 0;
|
||||||
|
|
||||||
|
for (let r = 0; r < upperEnd; r++) {
|
||||||
|
const heightWeight = 1.0 - (r / upperEnd) * 0.3; // Upper rows weighted more
|
||||||
|
|
||||||
|
// Head zone: top 25%, center 40% of width
|
||||||
|
if (r < Math.floor(rows * 0.25)) {
|
||||||
|
const headLeft = Math.floor(cols * 0.3);
|
||||||
|
const headRight = Math.floor(cols * 0.7);
|
||||||
|
for (let c = headLeft; c <= headRight; c++) {
|
||||||
|
const val = grid[r][c];
|
||||||
|
headMotionX += (c / cols - 0.5) * val;
|
||||||
|
headMotionWeight += val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Left arm zone: left 40% of grid
|
||||||
|
for (let c = 0; c < Math.floor(cols * 0.4); c++) {
|
||||||
|
const val = grid[r][c];
|
||||||
|
if (r < rows * 0.3) leftUpperMotion += val * heightWeight;
|
||||||
|
else leftMidMotion += val * heightWeight;
|
||||||
|
leftCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Right arm zone: right 40% of grid
|
||||||
|
for (let c = Math.floor(cols * 0.6); c < cols; c++) {
|
||||||
|
const val = grid[r][c];
|
||||||
|
if (r < rows * 0.3) rightUpperMotion += val * heightWeight;
|
||||||
|
else rightMidMotion += val * heightWeight;
|
||||||
|
rightCount++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Use specific embedding dimensions to modulate pose parameters
|
|
||||||
|
// Normalize
|
||||||
|
const leftTotal = leftUpperMotion + leftMidMotion;
|
||||||
|
const rightTotal = rightUpperMotion + rightMidMotion;
|
||||||
|
const maxMotion = 0.15; // Calibration threshold
|
||||||
|
|
||||||
|
// Arm height: 0 = at side, 1 = raised
|
||||||
|
// High motion in upper-left → left arm is raised
|
||||||
|
const leftArmHeight = Math.min(1, (leftUpperMotion / maxMotion) * 2);
|
||||||
|
const rightArmHeight = Math.min(1, (rightUpperMotion / maxMotion) * 2);
|
||||||
|
|
||||||
|
// Arm spread: how far out from body
|
||||||
|
const leftArmSpread = Math.min(1, leftTotal / maxMotion);
|
||||||
|
const rightArmSpread = Math.min(1, rightTotal / maxMotion);
|
||||||
|
|
||||||
|
// Head offset
|
||||||
|
const headOffsetX = headMotionWeight > 0.01 ? headMotionX / headMotionWeight : 0;
|
||||||
|
|
||||||
|
return { leftArmHeight, rightArmHeight, leftArmSpread, rightArmSpread, headOffsetX };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyze lower grid for leg motion.
|
||||||
|
*/
|
||||||
|
_analyzeLegMotion(grid, cols, rows) {
|
||||||
|
const lowerStart = Math.floor(rows * 0.6);
|
||||||
|
let leftMotion = 0, rightMotion = 0;
|
||||||
|
|
||||||
|
for (let r = lowerStart; r < rows; r++) {
|
||||||
|
for (let c = 0; c < Math.floor(cols / 2); c++) {
|
||||||
|
leftMotion += grid[r][c];
|
||||||
|
}
|
||||||
|
for (let c = Math.floor(cols / 2); c < cols; c++) {
|
||||||
|
rightMotion += grid[r][c];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return as -1 to 1 range (asymmetry indicates which leg is moving)
|
||||||
|
const total = leftMotion + rightMotion + 0.001;
|
||||||
return {
|
return {
|
||||||
sway: 0.5 + embedding[0] * 2,
|
left: (leftMotion - rightMotion) / total,
|
||||||
motion: Math.abs(embedding[1]) * 3,
|
right: (rightMotion - leftMotion) / total
|
||||||
armBend: 0.5 + embedding[2],
|
|
||||||
headConf: 0.5 + embedding[3] * 0.5,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Through-wall tracking: continue showing pose via CSI when person left video frame.
|
||||||
|
* The skeleton drifts in the exit direction with decreasing confidence.
|
||||||
|
*/
|
||||||
|
_trackThroughWall(elapsed, csiState) {
|
||||||
|
if (!this._lastBodyState) return [];
|
||||||
|
|
||||||
|
const dt = elapsed - this._lastBodyState.time;
|
||||||
|
const csiPresence = csiState.csiPresence || 0;
|
||||||
|
|
||||||
|
// Initialize ghost on first call
|
||||||
|
if (this._ghostConfidence <= 0.05) {
|
||||||
|
this._ghostConfidence = 0.8;
|
||||||
|
this._ghostState = this._lastBodyState.keypoints.map(kp => ({...kp}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ghost confidence decays, but CSI presence sustains it
|
||||||
|
const csiBoost = Math.min(0.7, csiPresence * 0.8);
|
||||||
|
this._ghostConfidence = Math.max(0.05, this._ghostConfidence * 0.995 - 0.001 + csiBoost * 0.002);
|
||||||
|
|
||||||
|
// Drift the ghost in exit direction
|
||||||
|
const vx = this._ghostVelocity.x;
|
||||||
|
const vy = this._ghostVelocity.y;
|
||||||
|
|
||||||
|
// Breathing continues via CSI
|
||||||
|
const breathe = Math.sin(elapsed * 1.5) * 0.003 * csiPresence;
|
||||||
|
|
||||||
|
const keypoints = this._ghostState.map((kp, i) => {
|
||||||
|
return {
|
||||||
|
x: kp.x + vx * dt * 0.3,
|
||||||
|
y: kp.y + vy * dt * 0.3 + (i >= 5 && i <= 6 ? breathe : 0),
|
||||||
|
confidence: kp.confidence * this._ghostConfidence * (0.5 + csiPresence * 0.5),
|
||||||
|
name: kp.name
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Slow down drift over time
|
||||||
|
this._ghostVelocity.x *= 0.998;
|
||||||
|
this._ghostVelocity.y *= 0.998;
|
||||||
|
|
||||||
|
this.smoothedKeypoints = keypoints;
|
||||||
|
return keypoints;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -126,12 +126,12 @@ export class VideoCapture {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Simple body detection from motion differencing.
|
* Detect motion region + detailed motion grid for body-part tracking.
|
||||||
* Returns approximate bounding box of moving region.
|
* Returns bounding box + a grid showing WHERE motion is concentrated.
|
||||||
* @returns {{ x, y, w, h, detected: boolean }}
|
* @returns {{ x, y, w, h, detected: boolean, motionGrid: number[][], gridCols: number, gridRows: number, exitDirection: string|null }}
|
||||||
*/
|
*/
|
||||||
detectMotionRegion(targetW = 56, targetH = 56) {
|
detectMotionRegion(targetW = 56, targetH = 56) {
|
||||||
if (!this.isActive || !this.prevFrame) return { detected: false };
|
if (!this.isActive || !this.prevFrame) return { detected: false, motionGrid: null };
|
||||||
|
|
||||||
this.offscreen.width = targetW;
|
this.offscreen.width = targetW;
|
||||||
this.offscreen.height = targetH;
|
this.offscreen.height = targetH;
|
||||||
|
|
@ -142,6 +142,17 @@ export class VideoCapture {
|
||||||
let motionPixels = 0;
|
let motionPixels = 0;
|
||||||
const threshold = 25;
|
const threshold = 25;
|
||||||
|
|
||||||
|
// Motion grid: divide frame into cells and track motion intensity per cell
|
||||||
|
const gridCols = 10;
|
||||||
|
const gridRows = 8;
|
||||||
|
const cellW = targetW / gridCols;
|
||||||
|
const cellH = targetH / gridRows;
|
||||||
|
const motionGrid = Array.from({ length: gridRows }, () => new Float32Array(gridCols));
|
||||||
|
const cellPixels = cellW * cellH;
|
||||||
|
|
||||||
|
// Also track motion centroid weighted by intensity
|
||||||
|
let motionCxSum = 0, motionCySum = 0, motionWeightSum = 0;
|
||||||
|
|
||||||
for (let y = 0; y < targetH; y++) {
|
for (let y = 0; y < targetH; y++) {
|
||||||
for (let x = 0; x < targetW; x++) {
|
for (let x = 0; x < targetW; x++) {
|
||||||
const i = y * targetW + x;
|
const i = y * targetW + x;
|
||||||
|
|
@ -156,17 +167,69 @@ export class VideoCapture {
|
||||||
if (x > maxX) maxX = x;
|
if (x > maxX) maxX = x;
|
||||||
if (y > maxY) maxY = y;
|
if (y > maxY) maxY = y;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Accumulate per-cell motion intensity
|
||||||
|
const gc = Math.min(Math.floor(x / cellW), gridCols - 1);
|
||||||
|
const gr = Math.min(Math.floor(y / cellH), gridRows - 1);
|
||||||
|
const intensity = diff / (3 * 255); // Normalize 0-1
|
||||||
|
motionGrid[gr][gc] += intensity / cellPixels;
|
||||||
|
|
||||||
|
// Weighted centroid
|
||||||
|
if (diff > threshold) {
|
||||||
|
motionCxSum += x * diff;
|
||||||
|
motionCySum += y * diff;
|
||||||
|
motionWeightSum += diff;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const detected = motionPixels > (targetW * targetH * 0.02);
|
const detected = motionPixels > (targetW * targetH * 0.02);
|
||||||
|
|
||||||
|
// Motion centroid (normalized 0-1)
|
||||||
|
const motionCx = motionWeightSum > 0 ? motionCxSum / (motionWeightSum * targetW) : 0.5;
|
||||||
|
const motionCy = motionWeightSum > 0 ? motionCySum / (motionWeightSum * targetH) : 0.5;
|
||||||
|
|
||||||
|
// Detect exit direction: if centroid is near edges
|
||||||
|
let exitDirection = null;
|
||||||
|
if (detected && motionCx < 0.1) exitDirection = 'left';
|
||||||
|
else if (detected && motionCx > 0.9) exitDirection = 'right';
|
||||||
|
else if (detected && motionCy < 0.1) exitDirection = 'up';
|
||||||
|
else if (detected && motionCy > 0.9) exitDirection = 'down';
|
||||||
|
|
||||||
|
// Track last known position for through-wall persistence
|
||||||
|
if (detected) {
|
||||||
|
this._lastDetected = {
|
||||||
|
x: minX / targetW,
|
||||||
|
y: minY / targetH,
|
||||||
|
w: (maxX - minX) / targetW,
|
||||||
|
h: (maxY - minY) / targetH,
|
||||||
|
cx: motionCx,
|
||||||
|
cy: motionCy,
|
||||||
|
exitDirection,
|
||||||
|
time: performance.now()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
detected,
|
detected,
|
||||||
x: minX / targetW,
|
x: minX / targetW,
|
||||||
y: minY / targetH,
|
y: minY / targetH,
|
||||||
w: (maxX - minX) / targetW,
|
w: (maxX - minX) / targetW,
|
||||||
h: (maxY - minY) / targetH,
|
h: (maxY - minY) / targetH,
|
||||||
coverage: motionPixels / (targetW * targetH)
|
coverage: motionPixels / (targetW * targetH),
|
||||||
|
motionGrid,
|
||||||
|
gridCols,
|
||||||
|
gridRows,
|
||||||
|
motionCx,
|
||||||
|
motionCy,
|
||||||
|
exitDirection
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the last known detection info (for through-wall persistence)
|
||||||
|
*/
|
||||||
|
get lastDetection() {
|
||||||
|
return this._lastDetected || null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,267 @@
|
||||||
|
/**
|
||||||
|
* CSI Simulator — Generates realistic WiFi Channel State Information data.
|
||||||
|
*
|
||||||
|
* In live mode, connects to the sensing server via WebSocket.
|
||||||
|
* In demo mode, generates synthetic CSI that correlates with detected motion.
|
||||||
|
*
|
||||||
|
* Outputs: 3-channel pseudo-image (amplitude, phase, temporal diff)
|
||||||
|
* matching the ADR-018 frame format expectations.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class CsiSimulator {
|
||||||
|
constructor(opts = {}) {
|
||||||
|
this.subcarriers = opts.subcarriers || 52; // 802.11n HT20
|
||||||
|
this.timeWindow = opts.timeWindow || 56; // frames in sliding window
|
||||||
|
this.mode = 'demo'; // 'demo' | 'live'
|
||||||
|
this.ws = null;
|
||||||
|
|
||||||
|
// Circular buffer for CSI frames
|
||||||
|
this.amplitudeBuffer = [];
|
||||||
|
this.phaseBuffer = [];
|
||||||
|
this.frameCount = 0;
|
||||||
|
|
||||||
|
// Noise parameters
|
||||||
|
this._rng = this._mulberry32(opts.seed || 7);
|
||||||
|
this._noiseState = new Float32Array(this.subcarriers);
|
||||||
|
this._baseAmplitude = new Float32Array(this.subcarriers);
|
||||||
|
this._basePhase = new Float32Array(this.subcarriers);
|
||||||
|
|
||||||
|
// Initialize base CSI profile (empty room)
|
||||||
|
for (let i = 0; i < this.subcarriers; i++) {
|
||||||
|
this._baseAmplitude[i] = 0.5 + 0.3 * Math.sin(i * 0.12);
|
||||||
|
this._basePhase[i] = (i / this.subcarriers) * Math.PI * 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Person influence (updated from video motion)
|
||||||
|
this.personPresence = 0;
|
||||||
|
this.personX = 0.5;
|
||||||
|
this.personY = 0.5;
|
||||||
|
this.personMotion = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect to live sensing server WebSocket
|
||||||
|
* @param {string} url - WebSocket URL (e.g. ws://localhost:3030/ws/csi)
|
||||||
|
*/
|
||||||
|
async connectLive(url) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
try {
|
||||||
|
this.ws = new WebSocket(url);
|
||||||
|
this.ws.binaryType = 'arraybuffer';
|
||||||
|
this.ws.onmessage = (evt) => this._handleLiveFrame(evt.data);
|
||||||
|
this.ws.onopen = () => { this.mode = 'live'; resolve(true); };
|
||||||
|
this.ws.onerror = () => resolve(false);
|
||||||
|
this.ws.onclose = () => { this.mode = 'demo'; };
|
||||||
|
// Timeout after 3s
|
||||||
|
setTimeout(() => { if (this.mode !== 'live') resolve(false); }, 3000);
|
||||||
|
} catch {
|
||||||
|
resolve(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
disconnect() {
|
||||||
|
if (this.ws) { this.ws.close(); this.ws = null; }
|
||||||
|
this.mode = 'demo';
|
||||||
|
}
|
||||||
|
|
||||||
|
get isLive() { return this.mode === 'live'; }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update person state from video detection (for correlated demo data).
|
||||||
|
* When person exits frame, CSI maintains presence with slow decay
|
||||||
|
* (simulating through-wall sensing capability).
|
||||||
|
*/
|
||||||
|
updatePersonState(presence, x, y, motion) {
|
||||||
|
if (presence > 0.1) {
|
||||||
|
// Person detected in video — update CSI state directly
|
||||||
|
this.personPresence = presence;
|
||||||
|
this.personX = x;
|
||||||
|
this.personY = y;
|
||||||
|
this.personMotion = motion;
|
||||||
|
this._lastSeenTime = performance.now();
|
||||||
|
this._lastSeenX = x;
|
||||||
|
this._lastSeenY = y;
|
||||||
|
} else if (this._lastSeenTime) {
|
||||||
|
// Person NOT in video — CSI "through-wall" persistence
|
||||||
|
const elapsed = (performance.now() - this._lastSeenTime) / 1000;
|
||||||
|
// CSI can sense through walls for ~10 seconds with decaying confidence
|
||||||
|
const decayRate = 0.15; // Lose ~15% per second
|
||||||
|
this.personPresence = Math.max(0, 1.0 - elapsed * decayRate);
|
||||||
|
// Position slowly drifts (person walking behind wall)
|
||||||
|
this.personX = this._lastSeenX;
|
||||||
|
this.personY = this._lastSeenY;
|
||||||
|
this.personMotion = Math.max(0, motion * 0.5 + this.personPresence * 0.2);
|
||||||
|
|
||||||
|
if (this.personPresence < 0.05) {
|
||||||
|
this._lastSeenTime = null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.personPresence = 0;
|
||||||
|
this.personMotion = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate next CSI frame (demo mode) or return latest live frame
|
||||||
|
* @param {number} elapsed - Time in seconds
|
||||||
|
* @returns {{ amplitude: Float32Array, phase: Float32Array, snr: number }}
|
||||||
|
*/
|
||||||
|
nextFrame(elapsed) {
|
||||||
|
const amp = new Float32Array(this.subcarriers);
|
||||||
|
const phase = new Float32Array(this.subcarriers);
|
||||||
|
|
||||||
|
if (this.mode === 'live' && this._liveAmplitude) {
|
||||||
|
amp.set(this._liveAmplitude);
|
||||||
|
phase.set(this._livePhase);
|
||||||
|
} else {
|
||||||
|
this._generateDemoFrame(amp, phase, elapsed);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push to circular buffer
|
||||||
|
this.amplitudeBuffer.push(new Float32Array(amp));
|
||||||
|
this.phaseBuffer.push(new Float32Array(phase));
|
||||||
|
if (this.amplitudeBuffer.length > this.timeWindow) {
|
||||||
|
this.amplitudeBuffer.shift();
|
||||||
|
this.phaseBuffer.shift();
|
||||||
|
}
|
||||||
|
|
||||||
|
// SNR estimate
|
||||||
|
let signalPower = 0, noisePower = 0;
|
||||||
|
for (let i = 0; i < this.subcarriers; i++) {
|
||||||
|
signalPower += amp[i] * amp[i];
|
||||||
|
noisePower += this._noiseState[i] * this._noiseState[i];
|
||||||
|
}
|
||||||
|
const snr = noisePower > 0 ? 10 * Math.log10(signalPower / noisePower) : 30;
|
||||||
|
|
||||||
|
this.frameCount++;
|
||||||
|
return { amplitude: amp, phase, snr: Math.max(0, Math.min(40, snr)) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build 3-channel pseudo-image for CNN input
|
||||||
|
* @param {number} targetSize - Output image dimension (square)
|
||||||
|
* @returns {Uint8Array} RGB data (targetSize * targetSize * 3)
|
||||||
|
*/
|
||||||
|
buildPseudoImage(targetSize = 56) {
|
||||||
|
const buf = this.amplitudeBuffer;
|
||||||
|
const pBuf = this.phaseBuffer;
|
||||||
|
const frames = buf.length;
|
||||||
|
if (frames < 2) {
|
||||||
|
return new Uint8Array(targetSize * targetSize * 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
const rgb = new Uint8Array(targetSize * targetSize * 3);
|
||||||
|
|
||||||
|
for (let y = 0; y < targetSize; y++) {
|
||||||
|
const fi = Math.min(Math.floor(y / targetSize * frames), frames - 1);
|
||||||
|
for (let x = 0; x < targetSize; x++) {
|
||||||
|
const si = Math.min(Math.floor(x / targetSize * this.subcarriers), this.subcarriers - 1);
|
||||||
|
const idx = (y * targetSize + x) * 3;
|
||||||
|
|
||||||
|
// R: Amplitude (normalized to 0-255)
|
||||||
|
const ampVal = buf[fi][si];
|
||||||
|
rgb[idx] = Math.min(255, Math.max(0, Math.floor(ampVal * 255)));
|
||||||
|
|
||||||
|
// G: Phase (wrapped to 0-255)
|
||||||
|
const phaseVal = (pBuf[fi][si] % (2 * Math.PI) + 2 * Math.PI) % (2 * Math.PI);
|
||||||
|
rgb[idx + 1] = Math.floor(phaseVal / (2 * Math.PI) * 255);
|
||||||
|
|
||||||
|
// B: Temporal difference
|
||||||
|
if (fi > 0) {
|
||||||
|
const diff = Math.abs(buf[fi][si] - buf[fi - 1][si]);
|
||||||
|
rgb[idx + 2] = Math.min(255, Math.floor(diff * 500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rgb;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get heatmap data for visualization
|
||||||
|
* @returns {{ data: Float32Array, width: number, height: number }}
|
||||||
|
*/
|
||||||
|
getHeatmapData() {
|
||||||
|
const frames = this.amplitudeBuffer.length;
|
||||||
|
const w = this.subcarriers;
|
||||||
|
const h = Math.min(frames, this.timeWindow);
|
||||||
|
const data = new Float32Array(w * h);
|
||||||
|
for (let y = 0; y < h; y++) {
|
||||||
|
const fi = frames - h + y;
|
||||||
|
if (fi >= 0 && fi < frames) {
|
||||||
|
for (let x = 0; x < w; x++) {
|
||||||
|
data[y * w + x] = this.amplitudeBuffer[fi][x];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { data, width: w, height: h };
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Private ===
|
||||||
|
|
||||||
|
_generateDemoFrame(amp, phase, elapsed) {
|
||||||
|
const rng = this._rng;
|
||||||
|
const presence = this.personPresence;
|
||||||
|
const motion = this.personMotion;
|
||||||
|
const px = this.personX;
|
||||||
|
|
||||||
|
for (let i = 0; i < this.subcarriers; i++) {
|
||||||
|
// Base CSI profile (frequency-selective channel)
|
||||||
|
let a = this._baseAmplitude[i];
|
||||||
|
let p = this._basePhase[i] + elapsed * 0.05;
|
||||||
|
|
||||||
|
// Environmental noise (correlated across subcarriers)
|
||||||
|
this._noiseState[i] = 0.95 * this._noiseState[i] + 0.05 * (rng() * 2 - 1) * 0.03;
|
||||||
|
a += this._noiseState[i];
|
||||||
|
|
||||||
|
// Person-induced CSI perturbation
|
||||||
|
if (presence > 0.1) {
|
||||||
|
// Subcarrier-dependent body reflection (Fresnel zone model)
|
||||||
|
const freqOffset = (i - this.subcarriers * px) / (this.subcarriers * 0.3);
|
||||||
|
const bodyReflection = presence * 0.25 * Math.exp(-freqOffset * freqOffset);
|
||||||
|
|
||||||
|
// Motion causes amplitude fluctuation
|
||||||
|
const motionEffect = motion * 0.15 * Math.sin(elapsed * 3.5 + i * 0.3);
|
||||||
|
|
||||||
|
// Breathing modulation (0.2-0.3 Hz)
|
||||||
|
const breathing = presence * 0.02 * Math.sin(elapsed * 1.5 + i * 0.05);
|
||||||
|
|
||||||
|
a += bodyReflection + motionEffect + breathing;
|
||||||
|
p += presence * 0.4 * Math.sin(elapsed * 2.1 + i * 0.15);
|
||||||
|
}
|
||||||
|
|
||||||
|
amp[i] = Math.max(0, Math.min(1, a));
|
||||||
|
phase[i] = p;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_handleLiveFrame(data) {
|
||||||
|
const view = new DataView(data);
|
||||||
|
// Check ADR-018 magic: 0xC5110001
|
||||||
|
if (data.byteLength < 20) return;
|
||||||
|
const magic = view.getUint32(0, true);
|
||||||
|
if (magic !== 0xC5110001) return;
|
||||||
|
|
||||||
|
const numSub = Math.min(view.getUint16(8, true), this.subcarriers);
|
||||||
|
this._liveAmplitude = new Float32Array(this.subcarriers);
|
||||||
|
this._livePhase = new Float32Array(this.subcarriers);
|
||||||
|
|
||||||
|
const headerSize = 20;
|
||||||
|
for (let i = 0; i < numSub && (headerSize + i * 4 + 3) < data.byteLength; i++) {
|
||||||
|
const real = view.getInt16(headerSize + i * 4, true);
|
||||||
|
const imag = view.getInt16(headerSize + i * 4 + 2, true);
|
||||||
|
this._liveAmplitude[i] = Math.sqrt(real * real + imag * imag) / 2048;
|
||||||
|
this._livePhase[i] = Math.atan2(imag, real);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_mulberry32(seed) {
|
||||||
|
return function() {
|
||||||
|
let t = (seed += 0x6D2B79F5);
|
||||||
|
t = Math.imul(t ^ (t >>> 15), t | 1);
|
||||||
|
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
||||||
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,301 @@
|
||||||
|
/**
|
||||||
|
* WiFi-DensePose — Dual-Modal Pose Estimation Demo
|
||||||
|
*
|
||||||
|
* Main orchestration: video capture → CNN embedding → CSI processing → fusion → rendering
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { VideoCapture } from './video-capture.js';
|
||||||
|
import { CsiSimulator } from './csi-simulator.js';
|
||||||
|
import { CnnEmbedder } from './cnn-embedder.js';
|
||||||
|
import { FusionEngine } from './fusion-engine.js';
|
||||||
|
import { PoseDecoder } from './pose-decoder.js';
|
||||||
|
import { CanvasRenderer } from './canvas-renderer.js';
|
||||||
|
|
||||||
|
// === State ===
|
||||||
|
let mode = 'dual'; // 'dual' | 'video' | 'csi'
|
||||||
|
let isRunning = false;
|
||||||
|
let isPaused = false;
|
||||||
|
let startTime = 0;
|
||||||
|
let frameCount = 0;
|
||||||
|
let fps = 0;
|
||||||
|
let lastFpsTime = 0;
|
||||||
|
let confidenceThreshold = 0.3;
|
||||||
|
|
||||||
|
// Latency tracking
|
||||||
|
const latency = { video: 0, csi: 0, fusion: 0, total: 0 };
|
||||||
|
|
||||||
|
// === Components ===
|
||||||
|
const videoCapture = new VideoCapture(document.getElementById('webcam'));
|
||||||
|
const csiSimulator = new CsiSimulator({ subcarriers: 52, timeWindow: 56 });
|
||||||
|
const visualCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 42 });
|
||||||
|
const csiCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 137 });
|
||||||
|
const fusionEngine = new FusionEngine(128);
|
||||||
|
const poseDecoder = new PoseDecoder(128);
|
||||||
|
const renderer = new CanvasRenderer();
|
||||||
|
|
||||||
|
// === Canvas Elements ===
|
||||||
|
const skeletonCanvas = document.getElementById('skeleton-canvas');
|
||||||
|
const skeletonCtx = skeletonCanvas.getContext('2d');
|
||||||
|
const csiCanvas = document.getElementById('csi-canvas');
|
||||||
|
const csiCtx = csiCanvas.getContext('2d');
|
||||||
|
const embeddingCanvas = document.getElementById('embedding-canvas');
|
||||||
|
const embeddingCtx = embeddingCanvas.getContext('2d');
|
||||||
|
|
||||||
|
// === UI Elements ===
|
||||||
|
const modeSelect = document.getElementById('mode-select');
|
||||||
|
const statusDot = document.getElementById('status-dot');
|
||||||
|
const statusLabel = document.getElementById('status-label');
|
||||||
|
const fpsDisplay = document.getElementById('fps-display');
|
||||||
|
const cameraPrompt = document.getElementById('camera-prompt');
|
||||||
|
const startCameraBtn = document.getElementById('start-camera-btn');
|
||||||
|
const pauseBtn = document.getElementById('pause-btn');
|
||||||
|
const confSlider = document.getElementById('confidence-slider');
|
||||||
|
const confValue = document.getElementById('confidence-value');
|
||||||
|
const wsUrlInput = document.getElementById('ws-url');
|
||||||
|
const connectWsBtn = document.getElementById('connect-ws-btn');
|
||||||
|
|
||||||
|
// Fusion bar elements
|
||||||
|
const videoBar = document.getElementById('video-bar');
|
||||||
|
const csiBar = document.getElementById('csi-bar');
|
||||||
|
const fusedBar = document.getElementById('fused-bar');
|
||||||
|
const videoBarVal = document.getElementById('video-bar-val');
|
||||||
|
const csiBarVal = document.getElementById('csi-bar-val');
|
||||||
|
const fusedBarVal = document.getElementById('fused-bar-val');
|
||||||
|
|
||||||
|
// Latency elements
|
||||||
|
const latVideoEl = document.getElementById('lat-video');
|
||||||
|
const latCsiEl = document.getElementById('lat-csi');
|
||||||
|
const latFusionEl = document.getElementById('lat-fusion');
|
||||||
|
const latTotalEl = document.getElementById('lat-total');
|
||||||
|
|
||||||
|
// Cross-modal similarity
|
||||||
|
const crossModalEl = document.getElementById('cross-modal-sim');
|
||||||
|
|
||||||
|
// === Initialize ===
|
||||||
|
function init() {
|
||||||
|
resizeCanvases();
|
||||||
|
window.addEventListener('resize', resizeCanvases);
|
||||||
|
|
||||||
|
// Mode change
|
||||||
|
modeSelect.addEventListener('change', (e) => {
|
||||||
|
mode = e.target.value;
|
||||||
|
updateModeUI();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Camera start
|
||||||
|
startCameraBtn.addEventListener('click', startCamera);
|
||||||
|
|
||||||
|
// Pause
|
||||||
|
pauseBtn.addEventListener('click', () => {
|
||||||
|
isPaused = !isPaused;
|
||||||
|
pauseBtn.textContent = isPaused ? '▶ Resume' : '⏸ Pause';
|
||||||
|
pauseBtn.classList.toggle('active', isPaused);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Confidence slider
|
||||||
|
confSlider.addEventListener('input', (e) => {
|
||||||
|
confidenceThreshold = parseFloat(e.target.value);
|
||||||
|
confValue.textContent = confidenceThreshold.toFixed(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
// WebSocket connect
|
||||||
|
connectWsBtn.addEventListener('click', async () => {
|
||||||
|
const url = wsUrlInput.value.trim();
|
||||||
|
if (!url) return;
|
||||||
|
connectWsBtn.textContent = 'Connecting...';
|
||||||
|
const ok = await csiSimulator.connectLive(url);
|
||||||
|
connectWsBtn.textContent = ok ? '✓ Connected' : 'Connect';
|
||||||
|
if (ok) {
|
||||||
|
connectWsBtn.classList.add('active');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Try to load WASM embedders (non-blocking)
|
||||||
|
visualCnn.tryLoadWasm('./pkg/ruvector_cnn_wasm');
|
||||||
|
csiCnn.tryLoadWasm('./pkg/ruvector_cnn_wasm');
|
||||||
|
|
||||||
|
// Auto-start camera for video/dual modes
|
||||||
|
updateModeUI();
|
||||||
|
startTime = performance.now() / 1000;
|
||||||
|
isRunning = true;
|
||||||
|
requestAnimationFrame(mainLoop);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startCamera() {
|
||||||
|
cameraPrompt.style.display = 'none';
|
||||||
|
const ok = await videoCapture.start();
|
||||||
|
if (ok) {
|
||||||
|
statusDot.classList.remove('offline');
|
||||||
|
statusLabel.textContent = 'LIVE';
|
||||||
|
resizeCanvases();
|
||||||
|
} else {
|
||||||
|
cameraPrompt.style.display = 'flex';
|
||||||
|
cameraPrompt.querySelector('p').textContent = 'Camera access denied. Try CSI-only mode.';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateModeUI() {
|
||||||
|
const needsVideo = mode !== 'csi';
|
||||||
|
const needsCsi = mode !== 'video';
|
||||||
|
|
||||||
|
// Show/hide camera prompt
|
||||||
|
if (needsVideo && !videoCapture.isActive) {
|
||||||
|
cameraPrompt.style.display = 'flex';
|
||||||
|
} else {
|
||||||
|
cameraPrompt.style.display = 'none';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resizeCanvases() {
|
||||||
|
const videoPanel = document.querySelector('.video-panel');
|
||||||
|
if (videoPanel) {
|
||||||
|
const rect = videoPanel.getBoundingClientRect();
|
||||||
|
skeletonCanvas.width = rect.width;
|
||||||
|
skeletonCanvas.height = rect.height;
|
||||||
|
}
|
||||||
|
|
||||||
|
// CSI canvas
|
||||||
|
csiCanvas.width = csiCanvas.parentElement.clientWidth;
|
||||||
|
csiCanvas.height = 120;
|
||||||
|
|
||||||
|
// Embedding canvas
|
||||||
|
embeddingCanvas.width = embeddingCanvas.parentElement.clientWidth;
|
||||||
|
embeddingCanvas.height = 140;
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Main Loop ===
|
||||||
|
function mainLoop(timestamp) {
|
||||||
|
if (!isRunning) return;
|
||||||
|
requestAnimationFrame(mainLoop);
|
||||||
|
|
||||||
|
if (isPaused) return;
|
||||||
|
|
||||||
|
const elapsed = performance.now() / 1000 - startTime;
|
||||||
|
const totalStart = performance.now();
|
||||||
|
|
||||||
|
// --- Video Pipeline ---
|
||||||
|
let videoEmb = null;
|
||||||
|
let motionRegion = null;
|
||||||
|
if (mode !== 'csi' && videoCapture.isActive) {
|
||||||
|
const t0 = performance.now();
|
||||||
|
const frame = videoCapture.captureFrame(56, 56);
|
||||||
|
if (frame) {
|
||||||
|
videoEmb = visualCnn.extract(frame.rgb, frame.width, frame.height);
|
||||||
|
motionRegion = videoCapture.detectMotionRegion(56, 56);
|
||||||
|
|
||||||
|
// Feed motion to CSI simulator for correlated demo data
|
||||||
|
// When detected=false, CSI simulator handles through-wall persistence
|
||||||
|
csiSimulator.updatePersonState(
|
||||||
|
motionRegion.detected ? 1.0 : 0,
|
||||||
|
motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5,
|
||||||
|
motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5,
|
||||||
|
frame.motion
|
||||||
|
);
|
||||||
|
|
||||||
|
fusionEngine.updateConfidence(
|
||||||
|
frame.brightness, frame.motion,
|
||||||
|
0, csiSimulator.isLive || mode === 'dual'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
latency.video = performance.now() - t0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- CSI Pipeline ---
|
||||||
|
let csiEmb = null;
|
||||||
|
if (mode !== 'video') {
|
||||||
|
const t0 = performance.now();
|
||||||
|
const csiFrame = csiSimulator.nextFrame(elapsed);
|
||||||
|
const pseudoImage = csiSimulator.buildPseudoImage(56);
|
||||||
|
csiEmb = csiCnn.extract(pseudoImage, 56, 56);
|
||||||
|
|
||||||
|
fusionEngine.updateConfidence(
|
||||||
|
videoCapture.brightnessScore,
|
||||||
|
videoCapture.motionScore,
|
||||||
|
csiFrame.snr,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
// Draw CSI heatmap
|
||||||
|
const heatmap = csiSimulator.getHeatmapData();
|
||||||
|
renderer.drawCsiHeatmap(csiCtx, heatmap, csiCanvas.width, csiCanvas.height);
|
||||||
|
|
||||||
|
latency.csi = performance.now() - t0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Fusion ---
|
||||||
|
const t0f = performance.now();
|
||||||
|
const fusedEmb = fusionEngine.fuse(videoEmb, csiEmb, mode);
|
||||||
|
latency.fusion = performance.now() - t0f;
|
||||||
|
|
||||||
|
// --- Pose Decode ---
|
||||||
|
// For CSI-only mode, generate a synthetic motion region from CSI energy
|
||||||
|
if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) {
|
||||||
|
const csiPresence = csiSimulator.personPresence;
|
||||||
|
if (csiPresence > 0.1) {
|
||||||
|
motionRegion = {
|
||||||
|
detected: true,
|
||||||
|
x: 0.25, y: 0.15, w: 0.5, h: 0.7,
|
||||||
|
coverage: csiPresence,
|
||||||
|
motionGrid: null,
|
||||||
|
gridCols: 10,
|
||||||
|
gridRows: 8
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CSI state for through-wall tracking
|
||||||
|
const csiState = {
|
||||||
|
csiPresence: csiSimulator.personPresence,
|
||||||
|
isLive: csiSimulator.isLive
|
||||||
|
};
|
||||||
|
|
||||||
|
const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState);
|
||||||
|
|
||||||
|
// --- Render Skeleton ---
|
||||||
|
const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' };
|
||||||
|
renderer.drawSkeleton(skeletonCtx, keypoints, skeletonCanvas.width, skeletonCanvas.height, {
|
||||||
|
minConfidence: confidenceThreshold,
|
||||||
|
color: mode === 'csi' ? 'amber' : 'green',
|
||||||
|
label: labelMap[mode]
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- Render Embedding Space ---
|
||||||
|
const embPoints = fusionEngine.getEmbeddingPoints();
|
||||||
|
renderer.drawEmbeddingSpace(embeddingCtx, embPoints, embeddingCanvas.width, embeddingCanvas.height);
|
||||||
|
|
||||||
|
// --- Update UI ---
|
||||||
|
latency.total = performance.now() - totalStart;
|
||||||
|
|
||||||
|
// FPS
|
||||||
|
frameCount++;
|
||||||
|
if (timestamp - lastFpsTime > 500) {
|
||||||
|
fps = Math.round(frameCount * 1000 / (timestamp - lastFpsTime));
|
||||||
|
lastFpsTime = timestamp;
|
||||||
|
frameCount = 0;
|
||||||
|
fpsDisplay.textContent = `${fps} FPS`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fusion bars
|
||||||
|
const vc = fusionEngine.videoConfidence;
|
||||||
|
const cc = fusionEngine.csiConfidence;
|
||||||
|
const fc = fusionEngine.fusedConfidence;
|
||||||
|
videoBar.style.width = `${vc * 100}%`;
|
||||||
|
csiBar.style.width = `${cc * 100}%`;
|
||||||
|
fusedBar.style.width = `${fc * 100}%`;
|
||||||
|
videoBarVal.textContent = `${Math.round(vc * 100)}%`;
|
||||||
|
csiBarVal.textContent = `${Math.round(cc * 100)}%`;
|
||||||
|
fusedBarVal.textContent = `${Math.round(fc * 100)}%`;
|
||||||
|
|
||||||
|
// Latency
|
||||||
|
latVideoEl.textContent = `${latency.video.toFixed(1)}ms`;
|
||||||
|
latCsiEl.textContent = `${latency.csi.toFixed(1)}ms`;
|
||||||
|
latFusionEl.textContent = `${latency.fusion.toFixed(1)}ms`;
|
||||||
|
latTotalEl.textContent = `${latency.total.toFixed(1)}ms`;
|
||||||
|
|
||||||
|
// Cross-modal similarity
|
||||||
|
const sim = fusionEngine.getCrossModalSimilarity();
|
||||||
|
crossModalEl.textContent = sim.toFixed(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Boot
|
||||||
|
document.addEventListener('DOMContentLoaded', init);
|
||||||
|
|
@ -0,0 +1,373 @@
|
||||||
|
/**
|
||||||
|
* PoseDecoder — Maps motion detection grid → 17 COCO keypoints.
|
||||||
|
*
|
||||||
|
* Uses per-cell motion intensity to track actual body part positions:
|
||||||
|
* - Head: top-center motion cluster
|
||||||
|
* - Shoulders/Elbows/Wrists: lateral motion in upper body zone
|
||||||
|
* - Hips/Knees/Ankles: lower body motion distribution
|
||||||
|
*
|
||||||
|
* When person exits frame, CSI data continues tracking (through-wall mode).
|
||||||
|
*/
|
||||||
|
|
||||||
|
// COCO keypoint definitions
|
||||||
|
export const KEYPOINT_NAMES = [
|
||||||
|
'nose', 'left_eye', 'right_eye', 'left_ear', 'right_ear',
|
||||||
|
'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow',
|
||||||
|
'left_wrist', 'right_wrist', 'left_hip', 'right_hip',
|
||||||
|
'left_knee', 'right_knee', 'left_ankle', 'right_ankle'
|
||||||
|
];
|
||||||
|
|
||||||
|
// Skeleton connections (pairs of keypoint indices)
|
||||||
|
export const SKELETON_CONNECTIONS = [
|
||||||
|
[0, 1], [0, 2], [1, 3], [2, 4], // Head
|
||||||
|
[5, 6], // Shoulders
|
||||||
|
[5, 7], [7, 9], // Left arm
|
||||||
|
[6, 8], [8, 10], // Right arm
|
||||||
|
[5, 11], [6, 12], // Torso
|
||||||
|
[11, 12], // Hips
|
||||||
|
[11, 13], [13, 15], // Left leg
|
||||||
|
[12, 14], [14, 16], // Right leg
|
||||||
|
];
|
||||||
|
|
||||||
|
// Standard body proportions (relative to body height)
|
||||||
|
const PROPORTIONS = {
|
||||||
|
headToShoulder: 0.15,
|
||||||
|
shoulderWidth: 0.25,
|
||||||
|
shoulderToElbow: 0.18,
|
||||||
|
elbowToWrist: 0.16,
|
||||||
|
shoulderToHip: 0.30,
|
||||||
|
hipWidth: 0.18,
|
||||||
|
hipToKnee: 0.24,
|
||||||
|
kneeToAnkle: 0.24,
|
||||||
|
eyeSpacing: 0.04,
|
||||||
|
earSpacing: 0.07,
|
||||||
|
};
|
||||||
|
|
||||||
|
export class PoseDecoder {
|
||||||
|
constructor(embeddingDim = 128) {
|
||||||
|
this.embeddingDim = embeddingDim;
|
||||||
|
this.smoothedKeypoints = null;
|
||||||
|
this.smoothingFactor = 0.45; // Lower = more responsive to movement
|
||||||
|
this._time = 0;
|
||||||
|
|
||||||
|
// Through-wall tracking state
|
||||||
|
this._lastBodyState = null;
|
||||||
|
this._ghostState = null;
|
||||||
|
this._ghostConfidence = 0;
|
||||||
|
this._ghostVelocity = { x: 0, y: 0 };
|
||||||
|
|
||||||
|
// Arm tracking history (smoothed positions)
|
||||||
|
this._leftArmY = 0.5;
|
||||||
|
this._rightArmY = 0.5;
|
||||||
|
this._leftArmX = 0;
|
||||||
|
this._rightArmX = 0;
|
||||||
|
this._headOffsetX = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decode motion data into 17 keypoints
|
||||||
|
* @param {Float32Array} embedding - Fused embedding vector
|
||||||
|
* @param {{ detected, x, y, w, h, motionGrid, gridCols, gridRows, motionCx, motionCy, exitDirection }} motionRegion
|
||||||
|
* @param {number} elapsed - Time in seconds
|
||||||
|
* @param {{ csiPresence: number }} csiState - CSI sensing state for through-wall
|
||||||
|
* @returns {Array<{x: number, y: number, confidence: number, name: string}>}
|
||||||
|
*/
|
||||||
|
decode(embedding, motionRegion, elapsed, csiState = {}) {
|
||||||
|
this._time = elapsed;
|
||||||
|
|
||||||
|
const hasMotion = motionRegion && motionRegion.detected;
|
||||||
|
const hasCsi = csiState && csiState.csiPresence > 0.1;
|
||||||
|
|
||||||
|
if (hasMotion) {
|
||||||
|
// Active tracking from video motion grid
|
||||||
|
this._ghostConfidence = 0;
|
||||||
|
const rawKeypoints = this._trackFromMotionGrid(motionRegion, embedding, elapsed);
|
||||||
|
this._lastBodyState = { keypoints: rawKeypoints.map(kp => ({...kp})), time: elapsed };
|
||||||
|
|
||||||
|
// Track exit velocity
|
||||||
|
if (motionRegion.exitDirection) {
|
||||||
|
const speed = 0.008;
|
||||||
|
this._ghostVelocity = {
|
||||||
|
x: motionRegion.exitDirection === 'left' ? -speed : motionRegion.exitDirection === 'right' ? speed : 0,
|
||||||
|
y: motionRegion.exitDirection === 'up' ? -speed : motionRegion.exitDirection === 'down' ? speed : 0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply temporal smoothing
|
||||||
|
if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) {
|
||||||
|
const alpha = this.smoothingFactor;
|
||||||
|
for (let i = 0; i < rawKeypoints.length; i++) {
|
||||||
|
rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x;
|
||||||
|
rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.smoothedKeypoints = rawKeypoints;
|
||||||
|
return rawKeypoints;
|
||||||
|
|
||||||
|
} else if (this._lastBodyState && (hasCsi || this._ghostConfidence > 0.05)) {
|
||||||
|
// Through-wall mode: person left frame but CSI still senses them
|
||||||
|
return this._trackThroughWall(elapsed, csiState);
|
||||||
|
|
||||||
|
} else if (this.smoothedKeypoints) {
|
||||||
|
// Fade out
|
||||||
|
const faded = this.smoothedKeypoints.map(kp => ({
|
||||||
|
...kp,
|
||||||
|
confidence: kp.confidence * 0.88
|
||||||
|
})).filter(kp => kp.confidence > 0.05);
|
||||||
|
if (faded.length === 0) this.smoothedKeypoints = null;
|
||||||
|
else this.smoothedKeypoints = faded;
|
||||||
|
return faded;
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Track body parts from the motion grid.
|
||||||
|
* The grid tells us WHERE motion is happening → we map that to joint positions.
|
||||||
|
*/
|
||||||
|
_trackFromMotionGrid(region, embedding, elapsed) {
|
||||||
|
const grid = region.motionGrid;
|
||||||
|
const cols = region.gridCols || 10;
|
||||||
|
const rows = region.gridRows || 8;
|
||||||
|
|
||||||
|
// Body bounding box
|
||||||
|
const cx = region.x + region.w / 2;
|
||||||
|
const cy = region.y + region.h / 2;
|
||||||
|
const bodyH = Math.max(region.h, 0.3);
|
||||||
|
const bodyW = Math.max(region.w, 0.15);
|
||||||
|
|
||||||
|
// Analyze the motion grid to find arm positions
|
||||||
|
// Divide body into zones: head (top 20%), arms (top 60% sides), torso (center), legs (bottom 40%)
|
||||||
|
if (grid) {
|
||||||
|
const armAnalysis = this._analyzeArmMotion(grid, cols, rows, region);
|
||||||
|
// Smooth arm tracking
|
||||||
|
this._leftArmY = 0.6 * this._leftArmY + 0.4 * armAnalysis.leftArmHeight;
|
||||||
|
this._rightArmY = 0.6 * this._rightArmY + 0.4 * armAnalysis.rightArmHeight;
|
||||||
|
this._leftArmX = 0.6 * this._leftArmX + 0.4 * armAnalysis.leftArmSpread;
|
||||||
|
this._rightArmX = 0.6 * this._rightArmX + 0.4 * armAnalysis.rightArmSpread;
|
||||||
|
this._headOffsetX = 0.7 * this._headOffsetX + 0.3 * armAnalysis.headOffsetX;
|
||||||
|
}
|
||||||
|
|
||||||
|
const P = PROPORTIONS;
|
||||||
|
const halfW = P.shoulderWidth * bodyH / 2;
|
||||||
|
const hipHalfW = P.hipWidth * bodyH / 2;
|
||||||
|
|
||||||
|
// Breathing (subtle)
|
||||||
|
const breathe = Math.sin(elapsed * 1.5) * 0.002;
|
||||||
|
|
||||||
|
// Core body positions from detection center
|
||||||
|
const hipY = cy + bodyH * 0.15;
|
||||||
|
const shoulderY = hipY - P.shoulderToHip * bodyH + breathe;
|
||||||
|
const headY = shoulderY - P.headToShoulder * bodyH;
|
||||||
|
const kneeY = hipY + P.hipToKnee * bodyH;
|
||||||
|
const ankleY = kneeY + P.kneeToAnkle * bodyH;
|
||||||
|
|
||||||
|
// HEAD follows motion centroid
|
||||||
|
const headX = cx + this._headOffsetX * bodyW * 0.3;
|
||||||
|
|
||||||
|
// ARM POSITIONS driven by motion grid analysis
|
||||||
|
// leftArmY: 0 = arm down at side, 1 = arm fully raised
|
||||||
|
// leftArmSpread: how far out the arm extends
|
||||||
|
const leftArmRaise = this._leftArmY; // 0-1
|
||||||
|
const rightArmRaise = this._rightArmY;
|
||||||
|
const leftSpread = 0.02 + this._leftArmX * 0.12;
|
||||||
|
const rightSpread = 0.02 + this._rightArmX * 0.12;
|
||||||
|
|
||||||
|
// Elbow: interpolate between "at side" and "raised"
|
||||||
|
const lElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - leftArmRaise * 0.9);
|
||||||
|
const rElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - rightArmRaise * 0.9);
|
||||||
|
const lElbowX = cx - halfW - leftSpread;
|
||||||
|
const rElbowX = cx + halfW + rightSpread;
|
||||||
|
|
||||||
|
// Wrist: extends further when raised
|
||||||
|
const lWristY = lElbowY + P.elbowToWrist * bodyH * (1 - leftArmRaise * 1.1);
|
||||||
|
const rWristY = rElbowY + P.elbowToWrist * bodyH * (1 - rightArmRaise * 1.1);
|
||||||
|
const lWristX = lElbowX - leftSpread * 0.6;
|
||||||
|
const rWristX = rElbowX + rightSpread * 0.6;
|
||||||
|
|
||||||
|
// Leg motion from lower grid cells
|
||||||
|
const legMotion = grid ? this._analyzeLegMotion(grid, cols, rows) : { left: 0, right: 0 };
|
||||||
|
const legSwing = 0.015;
|
||||||
|
|
||||||
|
const keypoints = [
|
||||||
|
// 0: nose
|
||||||
|
{ x: headX, y: headY + 0.01, confidence: 0.92 },
|
||||||
|
// 1: left_eye
|
||||||
|
{ x: headX - P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 },
|
||||||
|
// 2: right_eye
|
||||||
|
{ x: headX + P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 },
|
||||||
|
// 3: left_ear
|
||||||
|
{ x: headX - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 },
|
||||||
|
// 4: right_ear
|
||||||
|
{ x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 },
|
||||||
|
// 5: left_shoulder
|
||||||
|
{ x: cx - halfW, y: shoulderY, confidence: 0.94 },
|
||||||
|
// 6: right_shoulder
|
||||||
|
{ x: cx + halfW, y: shoulderY, confidence: 0.94 },
|
||||||
|
// 7: left_elbow
|
||||||
|
{ x: lElbowX, y: lElbowY, confidence: 0.87 },
|
||||||
|
// 8: right_elbow
|
||||||
|
{ x: rElbowX, y: rElbowY, confidence: 0.87 },
|
||||||
|
// 9: left_wrist
|
||||||
|
{ x: lWristX, y: lWristY, confidence: 0.82 },
|
||||||
|
// 10: right_wrist
|
||||||
|
{ x: rWristX, y: rWristY, confidence: 0.82 },
|
||||||
|
// 11: left_hip
|
||||||
|
{ x: cx - hipHalfW, y: hipY, confidence: 0.91 },
|
||||||
|
// 12: right_hip
|
||||||
|
{ x: cx + hipHalfW, y: hipY, confidence: 0.91 },
|
||||||
|
// 13: left_knee
|
||||||
|
{ x: cx - hipHalfW + legMotion.left * legSwing, y: kneeY, confidence: 0.88 },
|
||||||
|
// 14: right_knee
|
||||||
|
{ x: cx + hipHalfW + legMotion.right * legSwing, y: kneeY, confidence: 0.88 },
|
||||||
|
// 15: left_ankle
|
||||||
|
{ x: cx - hipHalfW + legMotion.left * legSwing * 1.3, y: ankleY, confidence: 0.83 },
|
||||||
|
// 16: right_ankle
|
||||||
|
{ x: cx + hipHalfW + legMotion.right * legSwing * 1.3, y: ankleY, confidence: 0.83 },
|
||||||
|
];
|
||||||
|
|
||||||
|
for (let i = 0; i < keypoints.length; i++) {
|
||||||
|
keypoints[i].name = KEYPOINT_NAMES[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
return keypoints;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyze the motion grid to determine arm positions.
|
||||||
|
* Left side of grid = left side of body, etc.
|
||||||
|
*/
|
||||||
|
_analyzeArmMotion(grid, cols, rows, region) {
|
||||||
|
// Body center column
|
||||||
|
const centerCol = Math.floor(cols / 2);
|
||||||
|
|
||||||
|
// Upper body rows (top 60% of detected region)
|
||||||
|
const upperEnd = Math.floor(rows * 0.6);
|
||||||
|
|
||||||
|
// Compute motion intensity for left vs right, at different heights
|
||||||
|
let leftUpperMotion = 0, leftMidMotion = 0;
|
||||||
|
let rightUpperMotion = 0, rightMidMotion = 0;
|
||||||
|
let leftCount = 0, rightCount = 0;
|
||||||
|
let headMotionX = 0, headMotionWeight = 0;
|
||||||
|
|
||||||
|
for (let r = 0; r < upperEnd; r++) {
|
||||||
|
const heightWeight = 1.0 - (r / upperEnd) * 0.3; // Upper rows weighted more
|
||||||
|
|
||||||
|
// Head zone: top 25%, center 40% of width
|
||||||
|
if (r < Math.floor(rows * 0.25)) {
|
||||||
|
const headLeft = Math.floor(cols * 0.3);
|
||||||
|
const headRight = Math.floor(cols * 0.7);
|
||||||
|
for (let c = headLeft; c <= headRight; c++) {
|
||||||
|
const val = grid[r][c];
|
||||||
|
headMotionX += (c / cols - 0.5) * val;
|
||||||
|
headMotionWeight += val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Left arm zone: left 40% of grid
|
||||||
|
for (let c = 0; c < Math.floor(cols * 0.4); c++) {
|
||||||
|
const val = grid[r][c];
|
||||||
|
if (r < rows * 0.3) leftUpperMotion += val * heightWeight;
|
||||||
|
else leftMidMotion += val * heightWeight;
|
||||||
|
leftCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Right arm zone: right 40% of grid
|
||||||
|
for (let c = Math.floor(cols * 0.6); c < cols; c++) {
|
||||||
|
const val = grid[r][c];
|
||||||
|
if (r < rows * 0.3) rightUpperMotion += val * heightWeight;
|
||||||
|
else rightMidMotion += val * heightWeight;
|
||||||
|
rightCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize
|
||||||
|
const leftTotal = leftUpperMotion + leftMidMotion;
|
||||||
|
const rightTotal = rightUpperMotion + rightMidMotion;
|
||||||
|
const maxMotion = 0.15; // Calibration threshold
|
||||||
|
|
||||||
|
// Arm height: 0 = at side, 1 = raised
|
||||||
|
// High motion in upper-left → left arm is raised
|
||||||
|
const leftArmHeight = Math.min(1, (leftUpperMotion / maxMotion) * 2);
|
||||||
|
const rightArmHeight = Math.min(1, (rightUpperMotion / maxMotion) * 2);
|
||||||
|
|
||||||
|
// Arm spread: how far out from body
|
||||||
|
const leftArmSpread = Math.min(1, leftTotal / maxMotion);
|
||||||
|
const rightArmSpread = Math.min(1, rightTotal / maxMotion);
|
||||||
|
|
||||||
|
// Head offset
|
||||||
|
const headOffsetX = headMotionWeight > 0.01 ? headMotionX / headMotionWeight : 0;
|
||||||
|
|
||||||
|
return { leftArmHeight, rightArmHeight, leftArmSpread, rightArmSpread, headOffsetX };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyze lower grid for leg motion.
|
||||||
|
*/
|
||||||
|
_analyzeLegMotion(grid, cols, rows) {
|
||||||
|
const lowerStart = Math.floor(rows * 0.6);
|
||||||
|
let leftMotion = 0, rightMotion = 0;
|
||||||
|
|
||||||
|
for (let r = lowerStart; r < rows; r++) {
|
||||||
|
for (let c = 0; c < Math.floor(cols / 2); c++) {
|
||||||
|
leftMotion += grid[r][c];
|
||||||
|
}
|
||||||
|
for (let c = Math.floor(cols / 2); c < cols; c++) {
|
||||||
|
rightMotion += grid[r][c];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return as -1 to 1 range (asymmetry indicates which leg is moving)
|
||||||
|
const total = leftMotion + rightMotion + 0.001;
|
||||||
|
return {
|
||||||
|
left: (leftMotion - rightMotion) / total,
|
||||||
|
right: (rightMotion - leftMotion) / total
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Through-wall tracking: continue showing pose via CSI when person left video frame.
|
||||||
|
* The skeleton drifts in the exit direction with decreasing confidence.
|
||||||
|
*/
|
||||||
|
_trackThroughWall(elapsed, csiState) {
|
||||||
|
if (!this._lastBodyState) return [];
|
||||||
|
|
||||||
|
const dt = elapsed - this._lastBodyState.time;
|
||||||
|
const csiPresence = csiState.csiPresence || 0;
|
||||||
|
|
||||||
|
// Initialize ghost on first call
|
||||||
|
if (this._ghostConfidence <= 0.05) {
|
||||||
|
this._ghostConfidence = 0.8;
|
||||||
|
this._ghostState = this._lastBodyState.keypoints.map(kp => ({...kp}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ghost confidence decays, but CSI presence sustains it
|
||||||
|
const csiBoost = Math.min(0.7, csiPresence * 0.8);
|
||||||
|
this._ghostConfidence = Math.max(0.05, this._ghostConfidence * 0.995 - 0.001 + csiBoost * 0.002);
|
||||||
|
|
||||||
|
// Drift the ghost in exit direction
|
||||||
|
const vx = this._ghostVelocity.x;
|
||||||
|
const vy = this._ghostVelocity.y;
|
||||||
|
|
||||||
|
// Breathing continues via CSI
|
||||||
|
const breathe = Math.sin(elapsed * 1.5) * 0.003 * csiPresence;
|
||||||
|
|
||||||
|
const keypoints = this._ghostState.map((kp, i) => {
|
||||||
|
return {
|
||||||
|
x: kp.x + vx * dt * 0.3,
|
||||||
|
y: kp.y + vy * dt * 0.3 + (i >= 5 && i <= 6 ? breathe : 0),
|
||||||
|
confidence: kp.confidence * this._ghostConfidence * (0.5 + csiPresence * 0.5),
|
||||||
|
name: kp.name
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Slow down drift over time
|
||||||
|
this._ghostVelocity.x *= 0.998;
|
||||||
|
this._ghostVelocity.y *= 0.998;
|
||||||
|
|
||||||
|
this.smoothedKeypoints = keypoints;
|
||||||
|
return keypoints;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,235 @@
|
||||||
|
/**
|
||||||
|
* VideoCapture — getUserMedia webcam capture with frame extraction.
|
||||||
|
* Provides quality metrics (brightness, motion) for fusion confidence gating.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export class VideoCapture {
|
||||||
|
constructor(videoElement) {
|
||||||
|
this.video = videoElement;
|
||||||
|
this.stream = null;
|
||||||
|
this.offscreen = document.createElement('canvas');
|
||||||
|
this.offCtx = this.offscreen.getContext('2d', { willReadFrequently: true });
|
||||||
|
this.prevFrame = null;
|
||||||
|
this.motionScore = 0;
|
||||||
|
this.brightnessScore = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
async start(constraints = {}) {
|
||||||
|
const defaultConstraints = {
|
||||||
|
video: {
|
||||||
|
width: { ideal: 640 },
|
||||||
|
height: { ideal: 480 },
|
||||||
|
facingMode: 'user',
|
||||||
|
frameRate: { ideal: 30 }
|
||||||
|
},
|
||||||
|
audio: false
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.stream = await navigator.mediaDevices.getUserMedia(
|
||||||
|
Object.keys(constraints).length ? constraints : defaultConstraints
|
||||||
|
);
|
||||||
|
this.video.srcObject = this.stream;
|
||||||
|
await this.video.play();
|
||||||
|
|
||||||
|
this.offscreen.width = this.video.videoWidth;
|
||||||
|
this.offscreen.height = this.video.videoHeight;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[Video] Camera access failed:', err.message);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stop() {
|
||||||
|
if (this.stream) {
|
||||||
|
this.stream.getTracks().forEach(t => t.stop());
|
||||||
|
this.stream = null;
|
||||||
|
}
|
||||||
|
this.video.srcObject = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
get isActive() {
|
||||||
|
return this.stream !== null && this.video.readyState >= 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
get width() { return this.video.videoWidth || 640; }
|
||||||
|
get height() { return this.video.videoHeight || 480; }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Capture current frame as RGB Uint8Array + compute quality metrics.
|
||||||
|
* @param {number} targetW - Target width for CNN input
|
||||||
|
* @param {number} targetH - Target height for CNN input
|
||||||
|
* @returns {{ rgb: Uint8Array, width: number, height: number, motion: number, brightness: number }}
|
||||||
|
*/
|
||||||
|
captureFrame(targetW = 56, targetH = 56) {
|
||||||
|
if (!this.isActive) return null;
|
||||||
|
|
||||||
|
// Draw to offscreen at target resolution
|
||||||
|
this.offscreen.width = targetW;
|
||||||
|
this.offscreen.height = targetH;
|
||||||
|
this.offCtx.drawImage(this.video, 0, 0, targetW, targetH);
|
||||||
|
const imageData = this.offCtx.getImageData(0, 0, targetW, targetH);
|
||||||
|
const rgba = imageData.data;
|
||||||
|
|
||||||
|
// Convert RGBA → RGB
|
||||||
|
const pixels = targetW * targetH;
|
||||||
|
const rgb = new Uint8Array(pixels * 3);
|
||||||
|
let brightnessSum = 0;
|
||||||
|
let motionSum = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < pixels; i++) {
|
||||||
|
const r = rgba[i * 4];
|
||||||
|
const g = rgba[i * 4 + 1];
|
||||||
|
const b = rgba[i * 4 + 2];
|
||||||
|
rgb[i * 3] = r;
|
||||||
|
rgb[i * 3 + 1] = g;
|
||||||
|
rgb[i * 3 + 2] = b;
|
||||||
|
|
||||||
|
// Luminance for brightness
|
||||||
|
const lum = 0.299 * r + 0.587 * g + 0.114 * b;
|
||||||
|
brightnessSum += lum;
|
||||||
|
|
||||||
|
// Motion: diff from previous frame
|
||||||
|
if (this.prevFrame) {
|
||||||
|
const pr = this.prevFrame[i * 3];
|
||||||
|
const pg = this.prevFrame[i * 3 + 1];
|
||||||
|
const pb = this.prevFrame[i * 3 + 2];
|
||||||
|
motionSum += Math.abs(r - pr) + Math.abs(g - pg) + Math.abs(b - pb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.brightnessScore = brightnessSum / (pixels * 255);
|
||||||
|
this.motionScore = this.prevFrame ? Math.min(1, motionSum / (pixels * 100)) : 0;
|
||||||
|
this.prevFrame = new Uint8Array(rgb);
|
||||||
|
|
||||||
|
return {
|
||||||
|
rgb,
|
||||||
|
width: targetW,
|
||||||
|
height: targetH,
|
||||||
|
motion: this.motionScore,
|
||||||
|
brightness: this.brightnessScore
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Capture full-resolution RGBA for overlay rendering
|
||||||
|
* @returns {ImageData|null}
|
||||||
|
*/
|
||||||
|
captureFullFrame() {
|
||||||
|
if (!this.isActive) return null;
|
||||||
|
this.offscreen.width = this.width;
|
||||||
|
this.offscreen.height = this.height;
|
||||||
|
this.offCtx.drawImage(this.video, 0, 0);
|
||||||
|
return this.offCtx.getImageData(0, 0, this.width, this.height);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect motion region + detailed motion grid for body-part tracking.
|
||||||
|
* Returns bounding box + a grid showing WHERE motion is concentrated.
|
||||||
|
* @returns {{ x, y, w, h, detected: boolean, motionGrid: number[][], gridCols: number, gridRows: number, exitDirection: string|null }}
|
||||||
|
*/
|
||||||
|
detectMotionRegion(targetW = 56, targetH = 56) {
|
||||||
|
if (!this.isActive || !this.prevFrame) return { detected: false, motionGrid: null };
|
||||||
|
|
||||||
|
this.offscreen.width = targetW;
|
||||||
|
this.offscreen.height = targetH;
|
||||||
|
this.offCtx.drawImage(this.video, 0, 0, targetW, targetH);
|
||||||
|
const rgba = this.offCtx.getImageData(0, 0, targetW, targetH).data;
|
||||||
|
|
||||||
|
let minX = targetW, minY = targetH, maxX = 0, maxY = 0;
|
||||||
|
let motionPixels = 0;
|
||||||
|
const threshold = 25;
|
||||||
|
|
||||||
|
// Motion grid: divide frame into cells and track motion intensity per cell
|
||||||
|
const gridCols = 10;
|
||||||
|
const gridRows = 8;
|
||||||
|
const cellW = targetW / gridCols;
|
||||||
|
const cellH = targetH / gridRows;
|
||||||
|
const motionGrid = Array.from({ length: gridRows }, () => new Float32Array(gridCols));
|
||||||
|
const cellPixels = cellW * cellH;
|
||||||
|
|
||||||
|
// Also track motion centroid weighted by intensity
|
||||||
|
let motionCxSum = 0, motionCySum = 0, motionWeightSum = 0;
|
||||||
|
|
||||||
|
for (let y = 0; y < targetH; y++) {
|
||||||
|
for (let x = 0; x < targetW; x++) {
|
||||||
|
const i = y * targetW + x;
|
||||||
|
const r = rgba[i * 4], g = rgba[i * 4 + 1], b = rgba[i * 4 + 2];
|
||||||
|
const pr = this.prevFrame[i * 3], pg = this.prevFrame[i * 3 + 1], pb = this.prevFrame[i * 3 + 2];
|
||||||
|
const diff = Math.abs(r - pr) + Math.abs(g - pg) + Math.abs(b - pb);
|
||||||
|
|
||||||
|
if (diff > threshold * 3) {
|
||||||
|
motionPixels++;
|
||||||
|
if (x < minX) minX = x;
|
||||||
|
if (y < minY) minY = y;
|
||||||
|
if (x > maxX) maxX = x;
|
||||||
|
if (y > maxY) maxY = y;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Accumulate per-cell motion intensity
|
||||||
|
const gc = Math.min(Math.floor(x / cellW), gridCols - 1);
|
||||||
|
const gr = Math.min(Math.floor(y / cellH), gridRows - 1);
|
||||||
|
const intensity = diff / (3 * 255); // Normalize 0-1
|
||||||
|
motionGrid[gr][gc] += intensity / cellPixels;
|
||||||
|
|
||||||
|
// Weighted centroid
|
||||||
|
if (diff > threshold) {
|
||||||
|
motionCxSum += x * diff;
|
||||||
|
motionCySum += y * diff;
|
||||||
|
motionWeightSum += diff;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const detected = motionPixels > (targetW * targetH * 0.02);
|
||||||
|
|
||||||
|
// Motion centroid (normalized 0-1)
|
||||||
|
const motionCx = motionWeightSum > 0 ? motionCxSum / (motionWeightSum * targetW) : 0.5;
|
||||||
|
const motionCy = motionWeightSum > 0 ? motionCySum / (motionWeightSum * targetH) : 0.5;
|
||||||
|
|
||||||
|
// Detect exit direction: if centroid is near edges
|
||||||
|
let exitDirection = null;
|
||||||
|
if (detected && motionCx < 0.1) exitDirection = 'left';
|
||||||
|
else if (detected && motionCx > 0.9) exitDirection = 'right';
|
||||||
|
else if (detected && motionCy < 0.1) exitDirection = 'up';
|
||||||
|
else if (detected && motionCy > 0.9) exitDirection = 'down';
|
||||||
|
|
||||||
|
// Track last known position for through-wall persistence
|
||||||
|
if (detected) {
|
||||||
|
this._lastDetected = {
|
||||||
|
x: minX / targetW,
|
||||||
|
y: minY / targetH,
|
||||||
|
w: (maxX - minX) / targetW,
|
||||||
|
h: (maxY - minY) / targetH,
|
||||||
|
cx: motionCx,
|
||||||
|
cy: motionCy,
|
||||||
|
exitDirection,
|
||||||
|
time: performance.now()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
detected,
|
||||||
|
x: minX / targetW,
|
||||||
|
y: minY / targetH,
|
||||||
|
w: (maxX - minX) / targetW,
|
||||||
|
h: (maxY - minY) / targetH,
|
||||||
|
coverage: motionPixels / (targetW * targetH),
|
||||||
|
motionGrid,
|
||||||
|
gridCols,
|
||||||
|
gridRows,
|
||||||
|
motionCx,
|
||||||
|
motionCy,
|
||||||
|
exitDirection
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the last known detection info (for through-wall persistence)
|
||||||
|
*/
|
||||||
|
get lastDetection() {
|
||||||
|
return this._lastDetected || null;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue